diff --git a/kernel/api/block_op.go b/kernel/api/block_op.go index 5289a0bac..4d6420a84 100644 --- a/kernel/api/block_op.go +++ b/kernel/api/block_op.go @@ -17,14 +17,15 @@ package api import ( - "github.com/siyuan-note/siyuan/kernel/treenode" "net/http" "github.com/88250/gulu" "github.com/88250/lute" "github.com/88250/lute/ast" "github.com/gin-gonic/gin" + "github.com/siyuan-note/siyuan/kernel/filesys" "github.com/siyuan-note/siyuan/kernel/model" + "github.com/siyuan-note/siyuan/kernel/treenode" "github.com/siyuan-note/siyuan/kernel/util" ) @@ -44,7 +45,7 @@ func appendBlock(c *gin.Context) { return } if "markdown" == dataType { - luteEngine := model.NewLute() + luteEngine := util.NewLute() data = dataBlockDOM(data, luteEngine) } @@ -89,7 +90,7 @@ func prependBlock(c *gin.Context) { return } if "markdown" == dataType { - luteEngine := model.NewLute() + luteEngine := util.NewLute() data = dataBlockDOM(data, luteEngine) } @@ -150,7 +151,7 @@ func insertBlock(c *gin.Context) { } if "markdown" == dataType { - luteEngine := model.NewLute() + luteEngine := util.NewLute() data = dataBlockDOM(data, luteEngine) } @@ -197,7 +198,7 @@ func updateBlock(c *gin.Context) { return } - luteEngine := model.NewLute() + luteEngine := util.NewLute() if "markdown" == dataType { data = dataBlockDOM(data, luteEngine) } @@ -217,7 +218,7 @@ func updateBlock(c *gin.Context) { var transactions []*model.Transaction if "NodeDocument" == block.Type { - oldTree, err := model.LoadTree(block.Box, block.Path) + oldTree, err := filesys.LoadTree(block.Box, block.Path, luteEngine) if nil != err { ret.Code = -1 ret.Msg = "load tree failed: " + err.Error() diff --git a/kernel/api/filetree.go b/kernel/api/filetree.go index ff7cf44dd..45f9cfd2c 100644 --- a/kernel/api/filetree.go +++ b/kernel/api/filetree.go @@ -28,6 +28,7 @@ import ( "github.com/88250/gulu" "github.com/gin-gonic/gin" "github.com/siyuan-note/filelock" + "github.com/siyuan-note/siyuan/kernel/filesys" "github.com/siyuan-note/siyuan/kernel/model" "github.com/siyuan-note/siyuan/kernel/util" ) @@ -86,7 +87,8 @@ func heading2Doc(c *gin.Context) { } model.WaitForWritingFiles() - tree, err := model.LoadTree(targetNotebook, targetPath) + luteEngine := util.NewLute() + tree, err := filesys.LoadTree(targetNotebook, targetPath, luteEngine) if nil != err { ret.Code = -1 ret.Msg = err.Error() @@ -130,7 +132,8 @@ func li2Doc(c *gin.Context) { } model.WaitForWritingFiles() - tree, err := model.LoadTree(targetNotebook, targetPath) + luteEngine := util.NewLute() + tree, err := filesys.LoadTree(targetNotebook, targetPath, luteEngine) if nil != err { ret.Code = -1 ret.Msg = err.Error() @@ -424,7 +427,8 @@ func createDailyNote(c *gin.Context) { box := model.Conf.Box(notebook) model.WaitForWritingFiles() - tree, err := model.LoadTree(box.ID, p) + luteEngine := util.NewLute() + tree, err := filesys.LoadTree(box.ID, p, luteEngine) if nil != err { ret.Code = -1 ret.Msg = err.Error() diff --git a/kernel/api/lute.go b/kernel/api/lute.go index 472eeabcf..24edff4dd 100644 --- a/kernel/api/lute.go +++ b/kernel/api/lute.go @@ -62,7 +62,7 @@ func html2BlockDOM(c *gin.Context) { return } - luteEngine := model.NewLute() + luteEngine := util.NewLute() var unlinks []*ast.Node tree := parse.Parse("", []byte(markdown), luteEngine.ParseOptions) ast.Walk(tree.Root, func(n *ast.Node, entering bool) ast.WalkStatus { diff --git a/kernel/filesys/tree.go b/kernel/filesys/tree.go index 2d03ce271..3e06492e5 100644 --- a/kernel/filesys/tree.go +++ b/kernel/filesys/tree.go @@ -40,6 +40,7 @@ func LoadTree(boxID, p string, luteEngine *lute.Lute) (ret *parse.Tree, err erro filePath := filepath.Join(util.DataDir, boxID, p) data, err := filelock.ReadFile(filePath) if nil != err { + logging.LogErrorf("load tree [%s] failed: %s", p, err) return } @@ -50,7 +51,9 @@ func LoadTree(boxID, p string, luteEngine *lute.Lute) (ret *parse.Tree, err erro func LoadTreeByData(data []byte, boxID, p string, luteEngine *lute.Lute) (ret *parse.Tree, err error) { ret = parseJSON2Tree(boxID, p, data, luteEngine) if nil == ret { - return nil, errors.New("parse tree failed") + logging.LogErrorf("parse tree [%s] failed", p) + err = errors.New("parse tree failed") + return } ret.Path = p ret.Root.Path = p diff --git a/kernel/model/assets.go b/kernel/model/assets.go index 400f67359..1591d374c 100644 --- a/kernel/model/assets.go +++ b/kernel/model/assets.go @@ -434,7 +434,7 @@ func RemoveUnusedAssets() (ret []string) { IncSync() } - indexHistoryDir(filepath.Base(historyDir), NewLute()) + indexHistoryDir(filepath.Base(historyDir), util.NewLute()) cache.LoadAssets() return } @@ -468,7 +468,7 @@ func RemoveUnusedAsset(p string) (ret string) { ret = absPath IncSync() - indexHistoryDir(filepath.Base(historyDir), NewLute()) + indexHistoryDir(filepath.Base(historyDir), util.NewLute()) cache.RemoveAsset(p) return } @@ -504,7 +504,7 @@ func RenameAsset(oldPath, newName string) (err error) { return } - luteEngine := NewLute() + luteEngine := util.NewLute() for _, notebook := range notebooks { pages := pagedPaths(filepath.Join(util.DataDir, notebook.ID), 32) @@ -562,7 +562,7 @@ func UnusedAssets() (ret []string) { if nil != err { return } - luteEngine := NewLute() + luteEngine := util.NewLute() for _, notebook := range notebooks { dests := map[string]bool{} diff --git a/kernel/model/backlink.go b/kernel/model/backlink.go index 5cb07493a..9984a2545 100644 --- a/kernel/model/backlink.go +++ b/kernel/model/backlink.go @@ -665,7 +665,7 @@ func searchBackmention(mentionKeywords []string, keyword string, excludeBacklink } blocks := fromSQLBlocks(&sqlBlocks, strings.Join(terms, search.TermSep), beforeLen) - luteEngine := NewLute() + luteEngine := util.NewLute() var tmp []*Block for _, b := range blocks { tree := parse.Parse("", gulu.Str.ToBytes(b.Markdown), luteEngine.ParseOptions) diff --git a/kernel/model/block.go b/kernel/model/block.go index 6ed99cb6f..066c3a8f7 100644 --- a/kernel/model/block.go +++ b/kernel/model/block.go @@ -294,7 +294,7 @@ func GetHeadingDeleteTransaction(id string) (transaction *Transaction, err error nodes = append(nodes, treenode.HeadingChildren(node)...) transaction = &Transaction{} - luteEngine := NewLute() + luteEngine := util.NewLute() for _, n := range nodes { op := &Operation{} op.ID = n.ID @@ -347,7 +347,7 @@ func GetHeadingChildrenDOM(id string) (ret string) { nodes := append([]*ast.Node{}, heading) children := treenode.HeadingChildren(heading) nodes = append(nodes, children...) - luteEngine := NewLute() + luteEngine := util.NewLute() ret = renderBlockDOMByNodes(nodes, luteEngine) return } @@ -383,7 +383,7 @@ func GetHeadingLevelTransaction(id string, level int) (transaction *Transaction, } transaction = &Transaction{} - luteEngine := NewLute() + luteEngine := util.NewLute() for _, c := range childrenHeadings { op := &Operation{} op.ID = c.ID diff --git a/kernel/model/box.go b/kernel/model/box.go index 36216e69d..1410ac4f8 100644 --- a/kernel/model/box.go +++ b/kernel/model/box.go @@ -38,6 +38,7 @@ import ( "github.com/siyuan-note/filelock" "github.com/siyuan-note/logging" "github.com/siyuan-note/siyuan/kernel/conf" + "github.com/siyuan-note/siyuan/kernel/filesys" "github.com/siyuan-note/siyuan/kernel/sql" "github.com/siyuan-note/siyuan/kernel/task" "github.com/siyuan-note/siyuan/kernel/treenode" @@ -386,12 +387,13 @@ func (box *Box) renameSubTrees(tree *parse.Tree) { func (box *Box) moveTrees0(files []*FileInfo) { totals := len(files) + 5 showProgress := 64 < totals + luteEngine := util.NewLute() for i, subFile := range files { if !strings.HasSuffix(subFile.path, ".sy") { continue } - subTree, err := LoadTree(box.ID, subFile.path) // LoadTree 会重新构造 HPath + subTree, err := filesys.LoadTree(box.ID, subFile.path, luteEngine) // LoadTree 会重新构造 HPath if nil != err { continue } diff --git a/kernel/model/export_merge.go b/kernel/model/export_merge.go index e95961e91..f451f609a 100644 --- a/kernel/model/export_merge.go +++ b/kernel/model/export_merge.go @@ -19,6 +19,7 @@ package model import ( "github.com/88250/lute/ast" "github.com/88250/lute/parse" + "github.com/siyuan-note/siyuan/kernel/filesys" ) func mergeSubDocs(rootTree *parse.Tree) (ret *parse.Tree, err error) { @@ -74,7 +75,8 @@ func walkBlock(insertPoint *ast.Node, block *Block, level int) (err error) { } func loadTreeNodes(box string, p string, level int) (ret []*ast.Node, err error) { - tree, err := LoadTree(box, p) + luteEngine := NewLute() + tree, err := filesys.LoadTree(box, p, luteEngine) if nil != err { return } diff --git a/kernel/model/file.go b/kernel/model/file.go index a9fec8522..e543beea1 100644 --- a/kernel/model/file.go +++ b/kernel/model/file.go @@ -19,7 +19,6 @@ package model import ( "errors" "fmt" - "github.com/siyuan-note/siyuan/kernel/task" "math" "os" "path" @@ -31,6 +30,7 @@ import ( "unicode/utf8" "github.com/88250/gulu" + "github.com/88250/lute" "github.com/88250/lute/ast" "github.com/88250/lute/html" "github.com/88250/lute/parse" @@ -44,6 +44,7 @@ import ( "github.com/siyuan-note/siyuan/kernel/filesys" "github.com/siyuan-note/siyuan/kernel/search" "github.com/siyuan-note/siyuan/kernel/sql" + "github.com/siyuan-note/siyuan/kernel/task" "github.com/siyuan-note/siyuan/kernel/treenode" "github.com/siyuan-note/siyuan/kernel/util" ) @@ -378,7 +379,7 @@ func ListDocTree(boxID, path string, sortMode int) (ret []*File, totals int, err } func ContentStat(content string) (ret *util.BlockStatResult) { - luteEngine := NewLute() + luteEngine := util.NewLute() tree := luteEngine.BlockDOM2Tree(content) runeCnt, wordCnt, linkCnt, imgCnt, refCnt := tree.Root.Stat() return &util.BlockStatResult{ @@ -393,6 +394,7 @@ func ContentStat(content string) (ret *util.BlockStatResult) { func BlocksWordCount(ids []string) (ret *util.BlockStatResult) { ret = &util.BlockStatResult{} trees := map[string]*parse.Tree{} // 缓存 + luteEngine := util.NewLute() for _, id := range ids { bt := treenode.GetBlockTree(id) if nil == bt { @@ -402,7 +404,7 @@ func BlocksWordCount(ids []string) (ret *util.BlockStatResult) { tree := trees[bt.RootID] if nil == tree { - tree, _ = LoadTree(bt.BoxID, bt.Path) + tree, _ = filesys.LoadTree(bt.BoxID, bt.Path, luteEngine) if nil == tree { continue } @@ -927,7 +929,7 @@ func CreateDocByMd(boxID, p, title, md string, sorts []string) (tree *parse.Tree return } - luteEngine := NewLute() + luteEngine := util.NewLute() dom := luteEngine.Md2BlockDOM(md, false) tree, err = createDoc(box.ID, p, title, dom) if nil != err { @@ -946,7 +948,7 @@ func CreateWithMarkdown(boxID, hPath, md string) (id string, err error) { } WaitForWritingFiles() - luteEngine := NewLute() + luteEngine := util.NewLute() dom := luteEngine.Md2BlockDOM(md, false) id, _, err = createDocsByHPath(box.ID, hPath, dom) return @@ -958,7 +960,8 @@ func GetHPathByPath(boxID, p string) (hPath string, err error) { return } - tree, err := LoadTree(boxID, p) + luteEngine := util.NewLute() + tree, err := filesys.LoadTree(boxID, p, luteEngine) if nil != err { return } @@ -1038,8 +1041,9 @@ func MoveDocs(fromPaths []string, toBoxID, toPath string) (err error) { } WaitForWritingFiles() + luteEngine := util.NewLute() for fromPath, fromBox := range pathsBoxes { - _, err = moveDoc(fromBox, fromPath, toBox, toPath) + _, err = moveDoc(fromBox, fromPath, toBox, toPath, luteEngine) if nil != err { return } @@ -1055,7 +1059,7 @@ func MoveDocs(fromPaths []string, toBoxID, toPath string) (err error) { return } -func moveDoc(fromBox *Box, fromPath string, toBox *Box, toPath string) (newPath string, err error) { +func moveDoc(fromBox *Box, fromPath string, toBox *Box, toPath string, luteEngine *lute.Lute) (newPath string, err error) { isSameBox := fromBox.ID == toBox.ID if isSameBox { @@ -1070,7 +1074,7 @@ func moveDoc(fromBox *Box, fromPath string, toBox *Box, toPath string) (newPath } } - tree, err := LoadTree(fromBox.ID, fromPath) + tree, err := filesys.LoadTree(fromBox.ID, fromPath, luteEngine) if nil != err { err = ErrBlockNotFound return @@ -1083,9 +1087,9 @@ func moveDoc(fromBox *Box, fromPath string, toBox *Box, toPath string) (newPath if !moveToRoot { var toTree *parse.Tree if isSameBox { - toTree, err = LoadTree(fromBox.ID, toPath) + toTree, err = filesys.LoadTree(fromBox.ID, toPath, luteEngine) } else { - toTree, err = LoadTree(toBox.ID, toPath) + toTree, err = filesys.LoadTree(toBox.ID, toPath, luteEngine) } if nil != err { err = ErrBlockNotFound @@ -1136,7 +1140,7 @@ func moveDoc(fromBox *Box, fromPath string, toBox *Box, toPath string) (newPath return } - tree, err = LoadTree(fromBox.ID, newPath) + tree, err = filesys.LoadTree(fromBox.ID, newPath, luteEngine) if nil != err { return } @@ -1152,7 +1156,7 @@ func moveDoc(fromBox *Box, fromPath string, toBox *Box, toPath string) (newPath return } - tree, err = LoadTree(toBox.ID, newPath) + tree, err = filesys.LoadTree(toBox.ID, newPath, luteEngine) if nil != err { return } @@ -1180,7 +1184,8 @@ func RemoveDoc(boxID, p string) { } WaitForWritingFiles() - removeDoc(box, p) + luteEngine := util.NewLute() + removeDoc(box, p, luteEngine) IncSync() return } @@ -1192,14 +1197,15 @@ func RemoveDocs(paths []string) { paths = util.FilterSelfChildDocs(paths) pathsBoxes := getBoxesByPaths(paths) WaitForWritingFiles() + luteEngine := util.NewLute() for p, box := range pathsBoxes { - removeDoc(box, p) + removeDoc(box, p, luteEngine) } return } -func removeDoc(box *Box, p string) { - tree, _ := LoadTree(box.ID, p) +func removeDoc(box *Box, p string, luteEngine *lute.Lute) { + tree, _ := filesys.LoadTree(box.ID, p, luteEngine) if nil == tree { return } @@ -1231,7 +1237,7 @@ func removeDoc(box *Box, p string) { return } } - indexHistoryDir(filepath.Base(historyDir), NewLute()) + indexHistoryDir(filepath.Base(historyDir), util.NewLute()) if existChildren { if err = box.Remove(childrenDir); nil != err { @@ -1274,7 +1280,8 @@ func RenameDoc(boxID, p, title string) (err error) { } WaitForWritingFiles() - tree, err := LoadTree(box.ID, p) + luteEngine := util.NewLute() + tree, err := filesys.LoadTree(box.ID, p, luteEngine) if nil != err { return } @@ -1370,7 +1377,7 @@ func CreateDailyNote(boxID string) (p string, existed bool, err error) { if nil == err { tree.Root.FirstChild.Unlink() - luteEngine := NewLute() + luteEngine := util.NewLute() newTree := luteEngine.BlockDOM2Tree(dom) var children []*ast.Node for c := newTree.Root.FirstChild; nil != c; c = c.Next { @@ -1450,7 +1457,7 @@ func createDoc(boxID, p, title, dom string) (tree *parse.Tree, err error) { return } - luteEngine := NewLute() + luteEngine := util.NewLute() tree = luteEngine.BlockDOM2Tree(dom) tree.Box = boxID tree.Path = p diff --git a/kernel/model/format.go b/kernel/model/format.go index 5c47369d0..db0b26a70 100644 --- a/kernel/model/format.go +++ b/kernel/model/format.go @@ -105,5 +105,5 @@ func generateFormatHistory(tree *parse.Tree) { return } - indexHistoryDir(filepath.Base(historyDir), NewLute()) + indexHistoryDir(filepath.Base(historyDir), util.NewLute()) } diff --git a/kernel/model/graph.go b/kernel/model/graph.go index f8314affd..3a9c3bc99 100644 --- a/kernel/model/graph.go +++ b/kernel/model/graph.go @@ -18,6 +18,7 @@ package model import ( "bytes" + "github.com/siyuan-note/siyuan/kernel/util" "math" "strings" "unicode/utf8" @@ -693,7 +694,7 @@ func query2Stmt(queryStr string) (ret string) { buf.WriteString("id = '" + queryStr + "'") } else { var tags []string - luteEngine := NewLute() + luteEngine := util.NewLute() t := parse.Inline("", []byte(queryStr), luteEngine.ParseOptions) ast.Walk(t.Root, func(n *ast.Node, entering bool) ast.WalkStatus { if !entering { diff --git a/kernel/model/heading.go b/kernel/model/heading.go index 27c1d8341..9ca488212 100644 --- a/kernel/model/heading.go +++ b/kernel/model/heading.go @@ -290,7 +290,7 @@ func Heading2Doc(srcHeadingID, targetBoxID, targetPath string) (srcRootBlockID, } headingNode.RemoveIALAttr("fold") - luteEngine := NewLute() + luteEngine := util.NewLute() newTree := &parse.Tree{Root: &ast.Node{Type: ast.NodeDocument, ID: srcHeadingID}, Context: &parse.Context{ParseOption: luteEngine.ParseOptions}} children = treenode.HeadingChildren(headingNode) for _, c := range children { diff --git a/kernel/model/history.go b/kernel/model/history.go index d560dc4e7..b4a7ffcfb 100644 --- a/kernel/model/history.go +++ b/kernel/model/history.go @@ -482,7 +482,7 @@ func (box *Box) generateDocHistory0() { } } - indexHistoryDir(filepath.Base(historyDir), NewLute()) + indexHistoryDir(filepath.Base(historyDir), util.NewLute()) return } @@ -593,7 +593,7 @@ func ReindexHistory() (err error) { defer util.PushClearProgress() sql.InitHistoryDatabase(true) - lutEngine := NewLute() + lutEngine := util.NewLute() for _, historyDir := range historyDirs { if !historyDir.IsDir() { continue diff --git a/kernel/model/import.go b/kernel/model/import.go index bdb35caad..ac67b8ce0 100644 --- a/kernel/model/import.go +++ b/kernel/model/import.go @@ -53,7 +53,7 @@ import ( func HTML2Markdown(htmlStr string) (markdown string, err error) { assetDirPath := filepath.Join(util.DataDir, "assets") - luteEngine := NewLute() + luteEngine := util.NewLute() luteEngine.SetProtyleWYSIWYG(false) tree := luteEngine.HTML2Tree(htmlStr) ast.Walk(tree.Root, func(n *ast.Node, entering bool) ast.WalkStatus { @@ -387,7 +387,7 @@ func ImportSY(zipPath, boxID, toPath string) (err error) { absPath := filepath.Join(targetDir, treePath) p := strings.TrimPrefix(absPath, boxAbsPath) p = filepath.ToSlash(p) - tree, err := LoadTree(boxID, p) + tree, err := filesys.LoadTree(boxID, p, luteEngine) if nil != err { logging.LogErrorf("load tree [%s] failed: %s", treePath, err) continue diff --git a/kernel/model/index.go b/kernel/model/index.go index 36fcdb0f6..9c4aa152f 100644 --- a/kernel/model/index.go +++ b/kernel/model/index.go @@ -17,18 +17,22 @@ package model import ( + "bytes" "fmt" + "path/filepath" "runtime" "runtime/debug" "strings" "sync" "time" + "github.com/88250/gulu" + "github.com/88250/lute/ast" "github.com/88250/lute/parse" "github.com/dustin/go-humanize" - "github.com/emirpasic/gods/sets/hashset" "github.com/panjf2000/ants/v2" "github.com/siyuan-note/eventbus" + "github.com/siyuan-note/filelock" "github.com/siyuan-note/logging" "github.com/siyuan-note/siyuan/kernel/cache" "github.com/siyuan-note/siyuan/kernel/filesys" @@ -68,7 +72,7 @@ func index(boxID string) { bootProgressPart := 30.0 / float64(boxLen) / float64(len(files)) start := time.Now() - luteEngine := NewLute() + luteEngine := util.NewLute() var treeCount int var treeSize int64 i := 0 @@ -135,53 +139,72 @@ func IndexRefs() { util.SetBootDetails("Resolving refs...") util.PushStatusBar(Conf.Language(54)) - // 引用入库 util.SetBootDetails("Indexing refs...") - refBlocks := sql.GetRefExistedBlocks() - refTreeIDs := hashset.New() - for _, refBlock := range refBlocks { - refTreeIDs.Add(refBlock.RootID) - } - i := 0 - if 0 < refTreeIDs.Size() { - luteEngine := NewLute() - bootProgressPart := 10.0 / float64(refTreeIDs.Size()) - for _, box := range Conf.GetOpenedBoxes() { - sql.DeleteBoxRefsQueue(box.ID) + var defBlockIDs []string + luteEngine := util.NewLute() + boxes := Conf.GetOpenedBoxes() + for _, box := range boxes { + sql.DeleteBoxRefsQueue(box.ID) - files := box.ListFiles("/") - for _, file := range files { - if file.isdir || !strings.HasSuffix(file.name, ".sy") { + pages := pagedPaths(filepath.Join(util.DataDir, box.ID), 32) + for _, paths := range pages { + for _, treeAbsPath := range paths { + data, readErr := filelock.ReadFile(treeAbsPath) + if nil != readErr { + logging.LogWarnf("get data [path=%s] failed: %s", treeAbsPath, readErr) continue } - if file.isdir || !strings.HasSuffix(file.name, ".sy") { + if !bytes.Contains(data, []byte("TextMarkBlockRefID")) && !bytes.Contains(data, []byte("TextMarkFileAnnotationRefID")) { continue } - id := strings.TrimSuffix(file.name, ".sy") - if !refTreeIDs.Contains(id) { + p := filepath.ToSlash(strings.TrimPrefix(treeAbsPath, filepath.Join(util.DataDir, box.ID))) + tree, parseErr := filesys.LoadTreeByData(data, box.ID, p, luteEngine) + if nil != parseErr { + logging.LogWarnf("parse json to tree [%s] failed: %s", treeAbsPath, parseErr) continue } - util.IncBootProgress(bootProgressPart, "Indexing ref "+util.ShortPathForBootingDisplay(file.path)) + ast.Walk(tree.Root, func(n *ast.Node, entering bool) ast.WalkStatus { + if !entering { + return ast.WalkContinue + } - tree, err := filesys.LoadTree(box.ID, file.path, luteEngine) - if nil != err { - logging.LogErrorf("parse box [%s] tree [%s] failed", box.ID, file.path) - continue - } - - sql.InsertRefsTreeQueue(tree) - if 1 < i && 0 == i%64 { - util.PushStatusBar(fmt.Sprintf(Conf.Language(55), i)) - } - i++ + if n.IsTextMarkType("block-ref") { + defBlockIDs = append(defBlockIDs, n.TextMarkBlockRefID) + } else if n.IsTextMarkType("file-annotation-ref") { + defBlockIDs = append(defBlockIDs, n.TextMarkFileAnnotationRefID) + } + return ast.WalkContinue + }) } } } - logging.LogInfof("resolved refs [%d] in [%dms]", len(refBlocks), time.Now().Sub(start).Milliseconds()) + + defBlockIDs = gulu.Str.RemoveDuplicatedElem(defBlockIDs) + + i := 0 + size := len(defBlockIDs) + if 0 < size { + bootProgressPart := 10.0 / float64(size) + + for _, defBlockID := range defBlockIDs { + defTree, loadErr := LoadTreeByID(defBlockID) + if nil != loadErr { + continue + } + + util.IncBootProgress(bootProgressPart, "Indexing ref "+defTree.ID) + sql.InsertRefsTreeQueue(defTree) + if 1 < i && 0 == i%64 { + util.PushStatusBar(fmt.Sprintf(Conf.Language(55), i)) + } + i++ + } + } + logging.LogInfof("resolved refs [%d] in [%dms]", size, time.Now().Sub(start).Milliseconds()) util.PushStatusBar(fmt.Sprintf(Conf.Language(55), i)) } diff --git a/kernel/model/index_fix.go b/kernel/model/index_fix.go index 67a77f0fa..2167d3ae3 100644 --- a/kernel/model/index_fix.go +++ b/kernel/model/index_fix.go @@ -27,10 +27,12 @@ import ( "time" "github.com/88250/gulu" + "github.com/88250/lute" "github.com/88250/lute/ast" "github.com/88250/lute/html" "github.com/88250/lute/parse" "github.com/siyuan-note/logging" + "github.com/siyuan-note/siyuan/kernel/filesys" "github.com/siyuan-note/siyuan/kernel/sql" "github.com/siyuan-note/siyuan/kernel/task" "github.com/siyuan-note/siyuan/kernel/treenode" @@ -107,6 +109,7 @@ func fixBlockTreeByFileSys() { util.PushStatusBar(Conf.Language(58)) boxes := Conf.GetOpenedBoxes() + luteEngine := lute.New() for _, box := range boxes { boxPath := filepath.Join(util.DataDir, box.ID) var paths []string @@ -133,7 +136,7 @@ func fixBlockTreeByFileSys() { continue } - reindexTreeByPath(box.ID, p, i, size) + reindexTreeByPath(box.ID, p, i, size, luteEngine) if util.IsExiting { break } @@ -166,6 +169,7 @@ func fixDatabaseIndexByBlockTree() { func reindexTreeByUpdated(rootUpdatedMap, dbRootUpdatedMap map[string]string) { i := -1 size := len(rootUpdatedMap) + luteEngine := util.NewLute() for rootID, updated := range rootUpdatedMap { i++ @@ -176,13 +180,13 @@ func reindexTreeByUpdated(rootUpdatedMap, dbRootUpdatedMap map[string]string) { rootUpdated := dbRootUpdatedMap[rootID] if "" == rootUpdated { //logging.LogWarnf("not found tree [%s] in database, reindex it", rootID) - reindexTree(rootID, i, size) + reindexTree(rootID, i, size, luteEngine) continue } if "" == updated { // BlockTree 迁移,v2.6.3 之前没有 updated 字段 - reindexTree(rootID, i, size) + reindexTree(rootID, i, size, luteEngine) continue } @@ -190,7 +194,7 @@ func reindexTreeByUpdated(rootUpdatedMap, dbRootUpdatedMap map[string]string) { dbUpdated, _ := time.Parse("20060102150405", rootUpdated) if dbUpdated.Before(btUpdated.Add(-10 * time.Minute)) { logging.LogWarnf("tree [%s] is not up to date, reindex it", rootID) - reindexTree(rootID, i, size) + reindexTree(rootID, i, size, luteEngine) continue } @@ -231,8 +235,8 @@ func reindexTreeByUpdated(rootUpdatedMap, dbRootUpdatedMap map[string]string) { sql.BatchRemoveTreeQueue(toRemoveRootIDs) } -func reindexTreeByPath(box, p string, i, size int) { - tree, err := LoadTree(box, p) +func reindexTreeByPath(box, p string, i, size int, luteEngine *lute.Lute) { + tree, err := filesys.LoadTree(box, p, luteEngine) if nil != err { return } @@ -240,14 +244,14 @@ func reindexTreeByPath(box, p string, i, size int) { reindexTree0(tree, i, size) } -func reindexTree(rootID string, i, size int) { +func reindexTree(rootID string, i, size int, luteEngine *lute.Lute) { root := treenode.GetBlockTree(rootID) if nil == root { logging.LogWarnf("root block [%s] not found", rootID) return } - tree, err := LoadTree(root.BoxID, root.Path) + tree, err := filesys.LoadTree(root.BoxID, root.Path, luteEngine) if nil != err { if os.IsNotExist(err) { // 文件系统上没有找到该 .sy 文件,则订正块树 diff --git a/kernel/model/listitem.go b/kernel/model/listitem.go index 955de6c1a..409ee88a3 100644 --- a/kernel/model/listitem.go +++ b/kernel/model/listitem.go @@ -76,7 +76,7 @@ func ListItem2Doc(srcListItemID, targetBoxID, targetPath string) (srcRootBlockID children = append(children, newNode) } - luteEngine := NewLute() + luteEngine := util.NewLute() newTree := &parse.Tree{Root: &ast.Node{Type: ast.NodeDocument, ID: srcListItemID}, Context: &parse.Context{ParseOption: luteEngine.ParseOptions}} for _, c := range children { newTree.Root.AppendChild(c) diff --git a/kernel/model/repository.go b/kernel/model/repository.go index f73e39feb..002a9fc87 100644 --- a/kernel/model/repository.go +++ b/kernel/model/repository.go @@ -1075,7 +1075,7 @@ func processSyncMergeResult(exit, byHand bool, start time.Time, mergeResult *dej // 云端同步发生冲突时生成副本 https://github.com/siyuan-note/siyuan/issues/5687 historyDir := filepath.Join(util.HistoryDir, mergeResult.Time.Format("2006-01-02-150405")+"-sync") - luteEngine := NewLute() + luteEngine := util.NewLute() for _, file := range mergeResult.Conflicts { if !strings.HasSuffix(file.Path, ".sy") { continue diff --git a/kernel/model/sync.go b/kernel/model/sync.go index dcdc3eec8..6e0d41923 100644 --- a/kernel/model/sync.go +++ b/kernel/model/sync.go @@ -32,6 +32,7 @@ import ( "github.com/siyuan-note/dejavu/cloud" "github.com/siyuan-note/logging" "github.com/siyuan-note/siyuan/kernel/conf" + "github.com/siyuan-note/siyuan/kernel/filesys" "github.com/siyuan-note/siyuan/kernel/sql" "github.com/siyuan-note/siyuan/kernel/treenode" "github.com/siyuan-note/siyuan/kernel/util" @@ -267,6 +268,7 @@ func incReindex(upserts, removes []string) { msg := fmt.Sprintf(Conf.Language(35)) util.PushStatusBar(msg) + luteEngine := util.NewLute() // 先执行 remove,否则移动文档时 upsert 会被忽略,导致未被索引 bootProgressPart := 10 / float64(len(removes)) for _, removeFile := range removes { @@ -311,7 +313,7 @@ func incReindex(upserts, removes []string) { util.IncBootProgress(bootProgressPart, msg) util.PushStatusBar(msg) - tree, err0 := LoadTree(box, p) + tree, err0 := filesys.LoadTree(box, p, luteEngine) if nil != err0 { continue } diff --git a/kernel/model/transaction.go b/kernel/model/transaction.go index 9f84a27a7..6c4adb2e1 100644 --- a/kernel/model/transaction.go +++ b/kernel/model/transaction.go @@ -26,6 +26,7 @@ import ( "time" "github.com/88250/gulu" + "github.com/88250/lute" "github.com/88250/lute/ast" "github.com/88250/lute/editor" "github.com/88250/lute/lex" @@ -34,6 +35,7 @@ import ( "github.com/siyuan-note/filelock" "github.com/siyuan-note/logging" "github.com/siyuan-note/siyuan/kernel/cache" + "github.com/siyuan-note/siyuan/kernel/filesys" "github.com/siyuan-note/siyuan/kernel/sql" "github.com/siyuan-note/siyuan/kernel/treenode" "github.com/siyuan-note/siyuan/kernel/util" @@ -419,8 +421,7 @@ func (tx *Transaction) doPrependInsert(operation *Operation) (ret *TxErr) { } data := strings.ReplaceAll(operation.Data.(string), editor.FrontEndCaret, "") - luteEngine := NewLute() - subTree := luteEngine.BlockDOM2Tree(data) + subTree := tx.luteEngine.BlockDOM2Tree(data) insertedNode := subTree.Root.FirstChild if nil == insertedNode { return &TxErr{code: TxErrCodeBlockNotFound, msg: "invalid data tree", id: block.ID} @@ -507,8 +508,7 @@ func (tx *Transaction) doAppendInsert(operation *Operation) (ret *TxErr) { } data := strings.ReplaceAll(operation.Data.(string), editor.FrontEndCaret, "") - luteEngine := NewLute() - subTree := luteEngine.BlockDOM2Tree(data) + subTree := tx.luteEngine.BlockDOM2Tree(data) insertedNode := subTree.Root.FirstChild if nil == insertedNode { return &TxErr{code: TxErrCodeBlockNotFound, msg: "invalid data tree", id: block.ID} @@ -720,8 +720,7 @@ func (tx *Transaction) doInsert(operation *Operation) (ret *TxErr) { } data := strings.ReplaceAll(operation.Data.(string), editor.FrontEndCaret, "") - luteEngine := NewLute() - subTree := luteEngine.BlockDOM2Tree(data) + subTree := tx.luteEngine.BlockDOM2Tree(data) p := block.Path assets := getAssetsDir(filepath.Join(util.DataDir, block.BoxID), filepath.Dir(filepath.Join(util.DataDir, block.BoxID, p))) @@ -870,8 +869,7 @@ func (tx *Transaction) doUpdate(operation *Operation) (ret *TxErr) { return &TxErr{code: TxErrCodeBlockNotFound, id: id} } - luteEngine := NewLute() - subTree := luteEngine.BlockDOM2Tree(data) + subTree := tx.luteEngine.BlockDOM2Tree(data) subTree.ID, subTree.Box, subTree.Path = tree.ID, tree.Box, tree.Path oldNode := treenode.GetNodeInTree(tree, id) if nil == oldNode { @@ -1034,6 +1032,8 @@ type Transaction struct { trees map[string]*parse.Tree nodes map[string]*ast.Node + + luteEngine *lute.Lute } func (tx *Transaction) begin() (err error) { @@ -1042,6 +1042,7 @@ func (tx *Transaction) begin() (err error) { } tx.trees = map[string]*parse.Tree{} tx.nodes = map[string]*ast.Node{} + tx.luteEngine = util.NewLute() return } @@ -1077,7 +1078,7 @@ func (tx *Transaction) loadTree(id string) (ret *parse.Tree, err error) { return } - ret, err = LoadTree(box, p) + ret, err = filesys.LoadTree(box, p, tx.luteEngine) if nil != err { return } diff --git a/kernel/model/tree.go b/kernel/model/tree.go index 1218276ca..1c4a0ff6c 100644 --- a/kernel/model/tree.go +++ b/kernel/model/tree.go @@ -164,19 +164,8 @@ func loadTreeByBlockID(id string) (ret *parse.Tree, err error) { return nil, ErrBlockNotFound } - ret, err = LoadTree(bt.BoxID, bt.Path) - if nil != err { - return - } + + luteEngine := util.NewLute() + ret, err = filesys.LoadTree(bt.BoxID, bt.Path, luteEngine) return } - -func LoadTree(boxID, p string) (*parse.Tree, error) { - luteEngine := NewLute() - tree, err := filesys.LoadTree(boxID, p, luteEngine) - if nil != err { - logging.LogErrorf("load tree [%s] failed: %s", boxID+p, err) - return nil, err - } - return tree, nil -} diff --git a/kernel/sql/block_query.go b/kernel/sql/block_query.go index c5910981d..67886be1d 100644 --- a/kernel/sql/block_query.go +++ b/kernel/sql/block_query.go @@ -555,22 +555,6 @@ func GetAllChildBlocks(rootID, condition string) (ret []*Block) { return } -func GetRefExistedBlocks() (ret []*Block) { - stmt := "SELECT * FROM blocks WHERE markdown LIKE ? OR markdown LIKE ?" - rows, err := query(stmt, "%((20%", "%<<20%") - if nil != err { - logging.LogErrorf("sql query [%s] failed: %s", stmt, err) - return - } - defer rows.Close() - for rows.Next() { - if block := scanBlockRows(rows); nil != block { - ret = append(ret, block) - } - } - return -} - func GetBlock(id string) (ret *Block) { ret = getBlockCache(id) if nil != ret {