⚡ 改进重命名资源文件性能 https://github.com/siyuan-note/siyuan/issues/7322

改进建立引用索引 https://github.com/siyuan-note/siyuan/issues/7320

kernel/api/block_op.go

@@ -17,14 +17,15 @@
 package api
 
 import (
-	"github.com/siyuan-note/siyuan/kernel/treenode"
 	"net/http"
 
 	"github.com/88250/gulu"
 	"github.com/88250/lute"
 	"github.com/88250/lute/ast"
 	"github.com/gin-gonic/gin"
+	"github.com/siyuan-note/siyuan/kernel/filesys"
 	"github.com/siyuan-note/siyuan/kernel/model"
+	"github.com/siyuan-note/siyuan/kernel/treenode"
 	"github.com/siyuan-note/siyuan/kernel/util"
 )
 
@@ -44,7 +45,7 @@ func appendBlock(c *gin.Context) {
 		return
 	}
 	if "markdown" == dataType {
-		luteEngine := model.NewLute()
+		luteEngine := util.NewLute()
 		data = dataBlockDOM(data, luteEngine)
 	}
 
@@ -89,7 +90,7 @@ func prependBlock(c *gin.Context) {
 		return
 	}
 	if "markdown" == dataType {
-		luteEngine := model.NewLute()
+		luteEngine := util.NewLute()
 		data = dataBlockDOM(data, luteEngine)
 	}
 
@@ -150,7 +151,7 @@ func insertBlock(c *gin.Context) {
 	}
 
 	if "markdown" == dataType {
-		luteEngine := model.NewLute()
+		luteEngine := util.NewLute()
 		data = dataBlockDOM(data, luteEngine)
 	}
 
@@ -197,7 +198,7 @@ func updateBlock(c *gin.Context) {
 		return
 	}
 
-	luteEngine := model.NewLute()
+	luteEngine := util.NewLute()
 	if "markdown" == dataType {
 		data = dataBlockDOM(data, luteEngine)
 	}
@@ -217,7 +218,7 @@ func updateBlock(c *gin.Context) {
 
 	var transactions []*model.Transaction
 	if "NodeDocument" == block.Type {
-		oldTree, err := model.LoadTree(block.Box, block.Path)
+		oldTree, err := filesys.LoadTree(block.Box, block.Path, luteEngine)
 		if nil != err {
 			ret.Code = -1
 			ret.Msg = "load tree failed: " + err.Error()

kernel/api/filetree.go

@@ -28,6 +28,7 @@ import (
 	"github.com/88250/gulu"
 	"github.com/gin-gonic/gin"
 	"github.com/siyuan-note/filelock"
+	"github.com/siyuan-note/siyuan/kernel/filesys"
 	"github.com/siyuan-note/siyuan/kernel/model"
 	"github.com/siyuan-note/siyuan/kernel/util"
 )
@@ -86,7 +87,8 @@ func heading2Doc(c *gin.Context) {
 	}
 
 	model.WaitForWritingFiles()
-	tree, err := model.LoadTree(targetNotebook, targetPath)
+	luteEngine := util.NewLute()
+	tree, err := filesys.LoadTree(targetNotebook, targetPath, luteEngine)
 	if nil != err {
 		ret.Code = -1
 		ret.Msg = err.Error()
@@ -130,7 +132,8 @@ func li2Doc(c *gin.Context) {
 	}
 
 	model.WaitForWritingFiles()
-	tree, err := model.LoadTree(targetNotebook, targetPath)
+	luteEngine := util.NewLute()
+	tree, err := filesys.LoadTree(targetNotebook, targetPath, luteEngine)
 	if nil != err {
 		ret.Code = -1
 		ret.Msg = err.Error()
@@ -424,7 +427,8 @@ func createDailyNote(c *gin.Context) {
 
 	box := model.Conf.Box(notebook)
 	model.WaitForWritingFiles()
-	tree, err := model.LoadTree(box.ID, p)
+	luteEngine := util.NewLute()
+	tree, err := filesys.LoadTree(box.ID, p, luteEngine)
 	if nil != err {
 		ret.Code = -1
 		ret.Msg = err.Error()

kernel/api/lute.go

@@ -62,7 +62,7 @@ func html2BlockDOM(c *gin.Context) {
 		return
 	}
 
-	luteEngine := model.NewLute()
+	luteEngine := util.NewLute()
 	var unlinks []*ast.Node
 	tree := parse.Parse("", []byte(markdown), luteEngine.ParseOptions)
 	ast.Walk(tree.Root, func(n *ast.Node, entering bool) ast.WalkStatus {

kernel/filesys/tree.go

@@ -40,6 +40,7 @@ func LoadTree(boxID, p string, luteEngine *lute.Lute) (ret *parse.Tree, err erro
 	filePath := filepath.Join(util.DataDir, boxID, p)
 	data, err := filelock.ReadFile(filePath)
 	if nil != err {
+		logging.LogErrorf("load tree [%s] failed: %s", p, err)
 		return
 	}
 
@@ -50,7 +51,9 @@ func LoadTree(boxID, p string, luteEngine *lute.Lute) (ret *parse.Tree, err erro
 func LoadTreeByData(data []byte, boxID, p string, luteEngine *lute.Lute) (ret *parse.Tree, err error) {
 	ret = parseJSON2Tree(boxID, p, data, luteEngine)
 	if nil == ret {
-		return nil, errors.New("parse tree failed")
+		logging.LogErrorf("parse tree [%s] failed", p)
+		err = errors.New("parse tree failed")
+		return
 	}
 	ret.Path = p
 	ret.Root.Path = p

kernel/model/assets.go

@@ -434,7 +434,7 @@ func RemoveUnusedAssets() (ret []string) {
 		IncSync()
 	}
 
-	indexHistoryDir(filepath.Base(historyDir), NewLute())
+	indexHistoryDir(filepath.Base(historyDir), util.NewLute())
 	cache.LoadAssets()
 	return
 }
@@ -468,7 +468,7 @@ func RemoveUnusedAsset(p string) (ret string) {
 	ret = absPath
 	IncSync()
 
-	indexHistoryDir(filepath.Base(historyDir), NewLute())
+	indexHistoryDir(filepath.Base(historyDir), util.NewLute())
 	cache.RemoveAsset(p)
 	return
 }
@@ -504,7 +504,7 @@ func RenameAsset(oldPath, newName string) (err error) {
 		return
 	}
 
-	luteEngine := NewLute()
+	luteEngine := util.NewLute()
 	for _, notebook := range notebooks {
 		pages := pagedPaths(filepath.Join(util.DataDir, notebook.ID), 32)
 
@@ -562,7 +562,7 @@ func UnusedAssets() (ret []string) {
 	if nil != err {
 		return
 	}
-	luteEngine := NewLute()
+	luteEngine := util.NewLute()
 	for _, notebook := range notebooks {
 		dests := map[string]bool{}
 

kernel/model/backlink.go

@@ -665,7 +665,7 @@ func searchBackmention(mentionKeywords []string, keyword string, excludeBacklink
 	}
 	blocks := fromSQLBlocks(&sqlBlocks, strings.Join(terms, search.TermSep), beforeLen)
 
-	luteEngine := NewLute()
+	luteEngine := util.NewLute()
 	var tmp []*Block
 	for _, b := range blocks {
 		tree := parse.Parse("", gulu.Str.ToBytes(b.Markdown), luteEngine.ParseOptions)

kernel/model/block.go

@@ -294,7 +294,7 @@ func GetHeadingDeleteTransaction(id string) (transaction *Transaction, err error
 	nodes = append(nodes, treenode.HeadingChildren(node)...)
 
 	transaction = &Transaction{}
-	luteEngine := NewLute()
+	luteEngine := util.NewLute()
 	for _, n := range nodes {
 		op := &Operation{}
 		op.ID = n.ID
@@ -347,7 +347,7 @@ func GetHeadingChildrenDOM(id string) (ret string) {
 	nodes := append([]*ast.Node{}, heading)
 	children := treenode.HeadingChildren(heading)
 	nodes = append(nodes, children...)
-	luteEngine := NewLute()
+	luteEngine := util.NewLute()
 	ret = renderBlockDOMByNodes(nodes, luteEngine)
 	return
 }
@@ -383,7 +383,7 @@ func GetHeadingLevelTransaction(id string, level int) (transaction *Transaction,
 	}
 
 	transaction = &Transaction{}
-	luteEngine := NewLute()
+	luteEngine := util.NewLute()
 	for _, c := range childrenHeadings {
 		op := &Operation{}
 		op.ID = c.ID

kernel/model/box.go

@@ -38,6 +38,7 @@ import (
 	"github.com/siyuan-note/filelock"
 	"github.com/siyuan-note/logging"
 	"github.com/siyuan-note/siyuan/kernel/conf"
+	"github.com/siyuan-note/siyuan/kernel/filesys"
 	"github.com/siyuan-note/siyuan/kernel/sql"
 	"github.com/siyuan-note/siyuan/kernel/task"
 	"github.com/siyuan-note/siyuan/kernel/treenode"
@@ -386,12 +387,13 @@ func (box *Box) renameSubTrees(tree *parse.Tree) {
 func (box *Box) moveTrees0(files []*FileInfo) {
 	totals := len(files) + 5
 	showProgress := 64 < totals
+	luteEngine := util.NewLute()
 	for i, subFile := range files {
 		if !strings.HasSuffix(subFile.path, ".sy") {
 			continue
 		}
 
-		subTree, err := LoadTree(box.ID, subFile.path) // LoadTree 会重新构造 HPath
+		subTree, err := filesys.LoadTree(box.ID, subFile.path, luteEngine) // LoadTree 会重新构造 HPath
 		if nil != err {
 			continue
 		}

kernel/model/export_merge.go

@@ -19,6 +19,7 @@ package model
 import (
 	"github.com/88250/lute/ast"
 	"github.com/88250/lute/parse"
+	"github.com/siyuan-note/siyuan/kernel/filesys"
 )
 
 func mergeSubDocs(rootTree *parse.Tree) (ret *parse.Tree, err error) {
@@ -74,7 +75,8 @@ func walkBlock(insertPoint *ast.Node, block *Block, level int) (err error) {
 }
 
 func loadTreeNodes(box string, p string, level int) (ret []*ast.Node, err error) {
-	tree, err := LoadTree(box, p)
+	luteEngine := NewLute()
+	tree, err := filesys.LoadTree(box, p, luteEngine)
 	if nil != err {
 		return
 	}

kernel/model/file.go

@@ -19,7 +19,6 @@ package model
 import (
 	"errors"
 	"fmt"
-	"github.com/siyuan-note/siyuan/kernel/task"
 	"math"
 	"os"
 	"path"
@@ -31,6 +30,7 @@ import (
 	"unicode/utf8"
 
 	"github.com/88250/gulu"
+	"github.com/88250/lute"
 	"github.com/88250/lute/ast"
 	"github.com/88250/lute/html"
 	"github.com/88250/lute/parse"
@@ -44,6 +44,7 @@ import (
 	"github.com/siyuan-note/siyuan/kernel/filesys"
 	"github.com/siyuan-note/siyuan/kernel/search"
 	"github.com/siyuan-note/siyuan/kernel/sql"
+	"github.com/siyuan-note/siyuan/kernel/task"
 	"github.com/siyuan-note/siyuan/kernel/treenode"
 	"github.com/siyuan-note/siyuan/kernel/util"
 )
@@ -378,7 +379,7 @@ func ListDocTree(boxID, path string, sortMode int) (ret []*File, totals int, err
 }
 
 func ContentStat(content string) (ret *util.BlockStatResult) {
-	luteEngine := NewLute()
+	luteEngine := util.NewLute()
 	tree := luteEngine.BlockDOM2Tree(content)
 	runeCnt, wordCnt, linkCnt, imgCnt, refCnt := tree.Root.Stat()
 	return &util.BlockStatResult{
@@ -393,6 +394,7 @@ func ContentStat(content string) (ret *util.BlockStatResult) {
 func BlocksWordCount(ids []string) (ret *util.BlockStatResult) {
 	ret = &util.BlockStatResult{}
 	trees := map[string]*parse.Tree{} // 缓存
+	luteEngine := util.NewLute()
 	for _, id := range ids {
 		bt := treenode.GetBlockTree(id)
 		if nil == bt {
@@ -402,7 +404,7 @@ func BlocksWordCount(ids []string) (ret *util.BlockStatResult) {
 
 		tree := trees[bt.RootID]
 		if nil == tree {
-			tree, _ = LoadTree(bt.BoxID, bt.Path)
+			tree, _ = filesys.LoadTree(bt.BoxID, bt.Path, luteEngine)
 			if nil == tree {
 				continue
 			}
@@ -927,7 +929,7 @@ func CreateDocByMd(boxID, p, title, md string, sorts []string) (tree *parse.Tree
 		return
 	}
 
-	luteEngine := NewLute()
+	luteEngine := util.NewLute()
 	dom := luteEngine.Md2BlockDOM(md, false)
 	tree, err = createDoc(box.ID, p, title, dom)
 	if nil != err {
@@ -946,7 +948,7 @@ func CreateWithMarkdown(boxID, hPath, md string) (id string, err error) {
 	}
 
 	WaitForWritingFiles()
-	luteEngine := NewLute()
+	luteEngine := util.NewLute()
 	dom := luteEngine.Md2BlockDOM(md, false)
 	id, _, err = createDocsByHPath(box.ID, hPath, dom)
 	return
@@ -958,7 +960,8 @@ func GetHPathByPath(boxID, p string) (hPath string, err error) {
 		return
 	}
 
-	tree, err := LoadTree(boxID, p)
+	luteEngine := util.NewLute()
+	tree, err := filesys.LoadTree(boxID, p, luteEngine)
 	if nil != err {
 		return
 	}
@@ -1038,8 +1041,9 @@ func MoveDocs(fromPaths []string, toBoxID, toPath string) (err error) {
 	}
 
 	WaitForWritingFiles()
+	luteEngine := util.NewLute()
 	for fromPath, fromBox := range pathsBoxes {
-		_, err = moveDoc(fromBox, fromPath, toBox, toPath)
+		_, err = moveDoc(fromBox, fromPath, toBox, toPath, luteEngine)
 		if nil != err {
 			return
 		}
@@ -1055,7 +1059,7 @@ func MoveDocs(fromPaths []string, toBoxID, toPath string) (err error) {
 	return
 }
 
-func moveDoc(fromBox *Box, fromPath string, toBox *Box, toPath string) (newPath string, err error) {
+func moveDoc(fromBox *Box, fromPath string, toBox *Box, toPath string, luteEngine *lute.Lute) (newPath string, err error) {
 	isSameBox := fromBox.ID == toBox.ID
 
 	if isSameBox {
@@ -1070,7 +1074,7 @@ func moveDoc(fromBox *Box, fromPath string, toBox *Box, toPath string) (newPath
 		}
 	}
 
-	tree, err := LoadTree(fromBox.ID, fromPath)
+	tree, err := filesys.LoadTree(fromBox.ID, fromPath, luteEngine)
 	if nil != err {
 		err = ErrBlockNotFound
 		return
@@ -1083,9 +1087,9 @@ func moveDoc(fromBox *Box, fromPath string, toBox *Box, toPath string) (newPath
 	if !moveToRoot {
 		var toTree *parse.Tree
 		if isSameBox {
-			toTree, err = LoadTree(fromBox.ID, toPath)
+			toTree, err = filesys.LoadTree(fromBox.ID, toPath, luteEngine)
 		} else {
-			toTree, err = LoadTree(toBox.ID, toPath)
+			toTree, err = filesys.LoadTree(toBox.ID, toPath, luteEngine)
 		}
 		if nil != err {
 			err = ErrBlockNotFound
@@ -1136,7 +1140,7 @@ func moveDoc(fromBox *Box, fromPath string, toBox *Box, toPath string) (newPath
 			return
 		}
 
-		tree, err = LoadTree(fromBox.ID, newPath)
+		tree, err = filesys.LoadTree(fromBox.ID, newPath, luteEngine)
 		if nil != err {
 			return
 		}
@@ -1152,7 +1156,7 @@ func moveDoc(fromBox *Box, fromPath string, toBox *Box, toPath string) (newPath
 			return
 		}
 
-		tree, err = LoadTree(toBox.ID, newPath)
+		tree, err = filesys.LoadTree(toBox.ID, newPath, luteEngine)
 		if nil != err {
 			return
 		}
@@ -1180,7 +1184,8 @@ func RemoveDoc(boxID, p string) {
 	}
 
 	WaitForWritingFiles()
-	removeDoc(box, p)
+	luteEngine := util.NewLute()
+	removeDoc(box, p, luteEngine)
 	IncSync()
 	return
 }
@@ -1192,14 +1197,15 @@ func RemoveDocs(paths []string) {
 	paths = util.FilterSelfChildDocs(paths)
 	pathsBoxes := getBoxesByPaths(paths)
 	WaitForWritingFiles()
+	luteEngine := util.NewLute()
 	for p, box := range pathsBoxes {
-		removeDoc(box, p)
+		removeDoc(box, p, luteEngine)
 	}
 	return
 }
 
-func removeDoc(box *Box, p string) {
+func removeDoc(box *Box, p string, luteEngine *lute.Lute) {
-	tree, _ := LoadTree(box.ID, p)
+	tree, _ := filesys.LoadTree(box.ID, p, luteEngine)
 	if nil == tree {
 		return
 	}
@@ -1231,7 +1237,7 @@ func removeDoc(box *Box, p string) {
 			return
 		}
 	}
-	indexHistoryDir(filepath.Base(historyDir), NewLute())
+	indexHistoryDir(filepath.Base(historyDir), util.NewLute())
 
 	if existChildren {
 		if err = box.Remove(childrenDir); nil != err {
@@ -1274,7 +1280,8 @@ func RenameDoc(boxID, p, title string) (err error) {
 	}
 
 	WaitForWritingFiles()
-	tree, err := LoadTree(box.ID, p)
+	luteEngine := util.NewLute()
+	tree, err := filesys.LoadTree(box.ID, p, luteEngine)
 	if nil != err {
 		return
 	}
@@ -1370,7 +1377,7 @@ func CreateDailyNote(boxID string) (p string, existed bool, err error) {
 		if nil == err {
 			tree.Root.FirstChild.Unlink()
 
-			luteEngine := NewLute()
+			luteEngine := util.NewLute()
 			newTree := luteEngine.BlockDOM2Tree(dom)
 			var children []*ast.Node
 			for c := newTree.Root.FirstChild; nil != c; c = c.Next {
@@ -1450,7 +1457,7 @@ func createDoc(boxID, p, title, dom string) (tree *parse.Tree, err error) {
 		return
 	}
 
-	luteEngine := NewLute()
+	luteEngine := util.NewLute()
 	tree = luteEngine.BlockDOM2Tree(dom)
 	tree.Box = boxID
 	tree.Path = p

kernel/model/format.go

@@ -105,5 +105,5 @@ func generateFormatHistory(tree *parse.Tree) {
 		return
 	}
 
-	indexHistoryDir(filepath.Base(historyDir), NewLute())
+	indexHistoryDir(filepath.Base(historyDir), util.NewLute())
 }

kernel/model/graph.go

@@ -18,6 +18,7 @@ package model
 
 import (
 	"bytes"
+	"github.com/siyuan-note/siyuan/kernel/util"
 	"math"
 	"strings"
 	"unicode/utf8"
@@ -693,7 +694,7 @@ func query2Stmt(queryStr string) (ret string) {
 		buf.WriteString("id = '" + queryStr + "'")
 	} else {
 		var tags []string
-		luteEngine := NewLute()
+		luteEngine := util.NewLute()
 		t := parse.Inline("", []byte(queryStr), luteEngine.ParseOptions)
 		ast.Walk(t.Root, func(n *ast.Node, entering bool) ast.WalkStatus {
 			if !entering {

kernel/model/heading.go

@@ -290,7 +290,7 @@ func Heading2Doc(srcHeadingID, targetBoxID, targetPath string) (srcRootBlockID,
 	}
 	headingNode.RemoveIALAttr("fold")
 
-	luteEngine := NewLute()
+	luteEngine := util.NewLute()
 	newTree := &parse.Tree{Root: &ast.Node{Type: ast.NodeDocument, ID: srcHeadingID}, Context: &parse.Context{ParseOption: luteEngine.ParseOptions}}
 	children = treenode.HeadingChildren(headingNode)
 	for _, c := range children {

kernel/model/history.go

@@ -482,7 +482,7 @@ func (box *Box) generateDocHistory0() {
 		}
 	}
 
-	indexHistoryDir(filepath.Base(historyDir), NewLute())
+	indexHistoryDir(filepath.Base(historyDir), util.NewLute())
 	return
 }
 
@@ -593,7 +593,7 @@ func ReindexHistory() (err error) {
 	defer util.PushClearProgress()
 
 	sql.InitHistoryDatabase(true)
-	lutEngine := NewLute()
+	lutEngine := util.NewLute()
 	for _, historyDir := range historyDirs {
 		if !historyDir.IsDir() {
 			continue

kernel/model/import.go

@@ -53,7 +53,7 @@ import (
 
 func HTML2Markdown(htmlStr string) (markdown string, err error) {
 	assetDirPath := filepath.Join(util.DataDir, "assets")
-	luteEngine := NewLute()
+	luteEngine := util.NewLute()
 	luteEngine.SetProtyleWYSIWYG(false)
 	tree := luteEngine.HTML2Tree(htmlStr)
 	ast.Walk(tree.Root, func(n *ast.Node, entering bool) ast.WalkStatus {
@@ -387,7 +387,7 @@ func ImportSY(zipPath, boxID, toPath string) (err error) {
 		absPath := filepath.Join(targetDir, treePath)
 		p := strings.TrimPrefix(absPath, boxAbsPath)
 		p = filepath.ToSlash(p)
-		tree, err := LoadTree(boxID, p)
+		tree, err := filesys.LoadTree(boxID, p, luteEngine)
 		if nil != err {
 			logging.LogErrorf("load tree [%s] failed: %s", treePath, err)
 			continue

kernel/model/index.go

@@ -17,18 +17,22 @@
 package model
 
 import (
+	"bytes"
 	"fmt"
+	"path/filepath"
 	"runtime"
 	"runtime/debug"
 	"strings"
 	"sync"
 	"time"
 
+	"github.com/88250/gulu"
+	"github.com/88250/lute/ast"
 	"github.com/88250/lute/parse"
 	"github.com/dustin/go-humanize"
-	"github.com/emirpasic/gods/sets/hashset"
 	"github.com/panjf2000/ants/v2"
 	"github.com/siyuan-note/eventbus"
+	"github.com/siyuan-note/filelock"
 	"github.com/siyuan-note/logging"
 	"github.com/siyuan-note/siyuan/kernel/cache"
 	"github.com/siyuan-note/siyuan/kernel/filesys"
@@ -68,7 +72,7 @@ func index(boxID string) {
 	bootProgressPart := 30.0 / float64(boxLen) / float64(len(files))
 
 	start := time.Now()
-	luteEngine := NewLute()
+	luteEngine := util.NewLute()
 	var treeCount int
 	var treeSize int64
 	i := 0
@@ -135,53 +139,72 @@ func IndexRefs() {
 	util.SetBootDetails("Resolving refs...")
 	util.PushStatusBar(Conf.Language(54))
 
-	// 引用入库
 	util.SetBootDetails("Indexing refs...")
-	refBlocks := sql.GetRefExistedBlocks()
-	refTreeIDs := hashset.New()
-	for _, refBlock := range refBlocks {
-		refTreeIDs.Add(refBlock.RootID)
-	}
 
-	i := 0
+	var defBlockIDs []string
-	if 0 < refTreeIDs.Size() {
+	luteEngine := util.NewLute()
-		luteEngine := NewLute()
+	boxes := Conf.GetOpenedBoxes()
-		bootProgressPart := 10.0 / float64(refTreeIDs.Size())
+	for _, box := range boxes {
-		for _, box := range Conf.GetOpenedBoxes() {
+		sql.DeleteBoxRefsQueue(box.ID)
-			sql.DeleteBoxRefsQueue(box.ID)
 
-			files := box.ListFiles("/")
+		pages := pagedPaths(filepath.Join(util.DataDir, box.ID), 32)
-			for _, file := range files {
+		for _, paths := range pages {
-				if file.isdir || !strings.HasSuffix(file.name, ".sy") {
+			for _, treeAbsPath := range paths {
+				data, readErr := filelock.ReadFile(treeAbsPath)
+				if nil != readErr {
+					logging.LogWarnf("get data [path=%s] failed: %s", treeAbsPath, readErr)
 					continue
 				}
 
-				if file.isdir || !strings.HasSuffix(file.name, ".sy") {
+				if !bytes.Contains(data, []byte("TextMarkBlockRefID")) && !bytes.Contains(data, []byte("TextMarkFileAnnotationRefID")) {
 					continue
 				}
 
-				id := strings.TrimSuffix(file.name, ".sy")
+				p := filepath.ToSlash(strings.TrimPrefix(treeAbsPath, filepath.Join(util.DataDir, box.ID)))
-				if !refTreeIDs.Contains(id) {
+				tree, parseErr := filesys.LoadTreeByData(data, box.ID, p, luteEngine)
+				if nil != parseErr {
+					logging.LogWarnf("parse json to tree [%s] failed: %s", treeAbsPath, parseErr)
 					continue
 				}
 
-				util.IncBootProgress(bootProgressPart, "Indexing ref "+util.ShortPathForBootingDisplay(file.path))
+				ast.Walk(tree.Root, func(n *ast.Node, entering bool) ast.WalkStatus {
+					if !entering {
+						return ast.WalkContinue
+					}
 
-				tree, err := filesys.LoadTree(box.ID, file.path, luteEngine)
+					if n.IsTextMarkType("block-ref") {
-				if nil != err {
+						defBlockIDs = append(defBlockIDs, n.TextMarkBlockRefID)
-					logging.LogErrorf("parse box [%s] tree [%s] failed", box.ID, file.path)
+					} else if n.IsTextMarkType("file-annotation-ref") {
-					continue
+						defBlockIDs = append(defBlockIDs, n.TextMarkFileAnnotationRefID)
-				}
+					}
-
+					return ast.WalkContinue
-				sql.InsertRefsTreeQueue(tree)
+				})
-				if 1 < i && 0 == i%64 {
-					util.PushStatusBar(fmt.Sprintf(Conf.Language(55), i))
-				}
-				i++
 			}
 		}
 	}
-	logging.LogInfof("resolved refs [%d] in [%dms]", len(refBlocks), time.Now().Sub(start).Milliseconds())
+
+	defBlockIDs = gulu.Str.RemoveDuplicatedElem(defBlockIDs)
+
+	i := 0
+	size := len(defBlockIDs)
+	if 0 < size {
+		bootProgressPart := 10.0 / float64(size)
+
+		for _, defBlockID := range defBlockIDs {
+			defTree, loadErr := LoadTreeByID(defBlockID)
+			if nil != loadErr {
+				continue
+			}
+
+			util.IncBootProgress(bootProgressPart, "Indexing ref "+defTree.ID)
+			sql.InsertRefsTreeQueue(defTree)
+			if 1 < i && 0 == i%64 {
+				util.PushStatusBar(fmt.Sprintf(Conf.Language(55), i))
+			}
+			i++
+		}
+	}
+	logging.LogInfof("resolved refs [%d] in [%dms]", size, time.Now().Sub(start).Milliseconds())
 	util.PushStatusBar(fmt.Sprintf(Conf.Language(55), i))
 }
 

kernel/model/index_fix.go

@@ -27,10 +27,12 @@ import (
 	"time"
 
 	"github.com/88250/gulu"
+	"github.com/88250/lute"
 	"github.com/88250/lute/ast"
 	"github.com/88250/lute/html"
 	"github.com/88250/lute/parse"
 	"github.com/siyuan-note/logging"
+	"github.com/siyuan-note/siyuan/kernel/filesys"
 	"github.com/siyuan-note/siyuan/kernel/sql"
 	"github.com/siyuan-note/siyuan/kernel/task"
 	"github.com/siyuan-note/siyuan/kernel/treenode"
@@ -107,6 +109,7 @@ func fixBlockTreeByFileSys() {
 
 	util.PushStatusBar(Conf.Language(58))
 	boxes := Conf.GetOpenedBoxes()
+	luteEngine := lute.New()
 	for _, box := range boxes {
 		boxPath := filepath.Join(util.DataDir, box.ID)
 		var paths []string
@@ -133,7 +136,7 @@ func fixBlockTreeByFileSys() {
 				continue
 			}
 
-			reindexTreeByPath(box.ID, p, i, size)
+			reindexTreeByPath(box.ID, p, i, size, luteEngine)
 			if util.IsExiting {
 				break
 			}
@@ -166,6 +169,7 @@ func fixDatabaseIndexByBlockTree() {
 func reindexTreeByUpdated(rootUpdatedMap, dbRootUpdatedMap map[string]string) {
 	i := -1
 	size := len(rootUpdatedMap)
+	luteEngine := util.NewLute()
 	for rootID, updated := range rootUpdatedMap {
 		i++
 
@@ -176,13 +180,13 @@ func reindexTreeByUpdated(rootUpdatedMap, dbRootUpdatedMap map[string]string) {
 		rootUpdated := dbRootUpdatedMap[rootID]
 		if "" == rootUpdated {
 			//logging.LogWarnf("not found tree [%s] in database, reindex it", rootID)
-			reindexTree(rootID, i, size)
+			reindexTree(rootID, i, size, luteEngine)
 			continue
 		}
 
 		if "" == updated {
 			// BlockTree 迁移，v2.6.3 之前没有 updated 字段
-			reindexTree(rootID, i, size)
+			reindexTree(rootID, i, size, luteEngine)
 			continue
 		}
 
@@ -190,7 +194,7 @@ func reindexTreeByUpdated(rootUpdatedMap, dbRootUpdatedMap map[string]string) {
 		dbUpdated, _ := time.Parse("20060102150405", rootUpdated)
 		if dbUpdated.Before(btUpdated.Add(-10 * time.Minute)) {
 			logging.LogWarnf("tree [%s] is not up to date, reindex it", rootID)
-			reindexTree(rootID, i, size)
+			reindexTree(rootID, i, size, luteEngine)
 			continue
 		}
 
@@ -231,8 +235,8 @@ func reindexTreeByUpdated(rootUpdatedMap, dbRootUpdatedMap map[string]string) {
 	sql.BatchRemoveTreeQueue(toRemoveRootIDs)
 }
 
-func reindexTreeByPath(box, p string, i, size int) {
+func reindexTreeByPath(box, p string, i, size int, luteEngine *lute.Lute) {
-	tree, err := LoadTree(box, p)
+	tree, err := filesys.LoadTree(box, p, luteEngine)
 	if nil != err {
 		return
 	}
@@ -240,14 +244,14 @@ func reindexTreeByPath(box, p string, i, size int) {
 	reindexTree0(tree, i, size)
 }
 
-func reindexTree(rootID string, i, size int) {
+func reindexTree(rootID string, i, size int, luteEngine *lute.Lute) {
 	root := treenode.GetBlockTree(rootID)
 	if nil == root {
 		logging.LogWarnf("root block [%s] not found", rootID)
 		return
 	}
 
-	tree, err := LoadTree(root.BoxID, root.Path)
+	tree, err := filesys.LoadTree(root.BoxID, root.Path, luteEngine)
 	if nil != err {
 		if os.IsNotExist(err) {
 			// 文件系统上没有找到该 .sy 文件，则订正块树

kernel/model/listitem.go

@@ -76,7 +76,7 @@ func ListItem2Doc(srcListItemID, targetBoxID, targetPath string) (srcRootBlockID
 		children = append(children, newNode)
 	}
 
-	luteEngine := NewLute()
+	luteEngine := util.NewLute()
 	newTree := &parse.Tree{Root: &ast.Node{Type: ast.NodeDocument, ID: srcListItemID}, Context: &parse.Context{ParseOption: luteEngine.ParseOptions}}
 	for _, c := range children {
 		newTree.Root.AppendChild(c)

kernel/model/repository.go

@@ -1075,7 +1075,7 @@ func processSyncMergeResult(exit, byHand bool, start time.Time, mergeResult *dej
 		// 云端同步发生冲突时生成副本 https://github.com/siyuan-note/siyuan/issues/5687
 
 		historyDir := filepath.Join(util.HistoryDir, mergeResult.Time.Format("2006-01-02-150405")+"-sync")
-		luteEngine := NewLute()
+		luteEngine := util.NewLute()
 		for _, file := range mergeResult.Conflicts {
 			if !strings.HasSuffix(file.Path, ".sy") {
 				continue

kernel/model/sync.go

@@ -32,6 +32,7 @@ import (
 	"github.com/siyuan-note/dejavu/cloud"
 	"github.com/siyuan-note/logging"
 	"github.com/siyuan-note/siyuan/kernel/conf"
+	"github.com/siyuan-note/siyuan/kernel/filesys"
 	"github.com/siyuan-note/siyuan/kernel/sql"
 	"github.com/siyuan-note/siyuan/kernel/treenode"
 	"github.com/siyuan-note/siyuan/kernel/util"
@@ -267,6 +268,7 @@ func incReindex(upserts, removes []string) {
 	msg := fmt.Sprintf(Conf.Language(35))
 	util.PushStatusBar(msg)
 
+	luteEngine := util.NewLute()
 	// 先执行 remove，否则移动文档时 upsert 会被忽略，导致未被索引
 	bootProgressPart := 10 / float64(len(removes))
 	for _, removeFile := range removes {
@@ -311,7 +313,7 @@ func incReindex(upserts, removes []string) {
 		util.IncBootProgress(bootProgressPart, msg)
 		util.PushStatusBar(msg)
 
-		tree, err0 := LoadTree(box, p)
+		tree, err0 := filesys.LoadTree(box, p, luteEngine)
 		if nil != err0 {
 			continue
 		}

kernel/model/transaction.go

@@ -26,6 +26,7 @@ import (
 	"time"
 
 	"github.com/88250/gulu"
+	"github.com/88250/lute"
 	"github.com/88250/lute/ast"
 	"github.com/88250/lute/editor"
 	"github.com/88250/lute/lex"
@@ -34,6 +35,7 @@ import (
 	"github.com/siyuan-note/filelock"
 	"github.com/siyuan-note/logging"
 	"github.com/siyuan-note/siyuan/kernel/cache"
+	"github.com/siyuan-note/siyuan/kernel/filesys"
 	"github.com/siyuan-note/siyuan/kernel/sql"
 	"github.com/siyuan-note/siyuan/kernel/treenode"
 	"github.com/siyuan-note/siyuan/kernel/util"
@@ -419,8 +421,7 @@ func (tx *Transaction) doPrependInsert(operation *Operation) (ret *TxErr) {
 	}
 
 	data := strings.ReplaceAll(operation.Data.(string), editor.FrontEndCaret, "")
-	luteEngine := NewLute()
+	subTree := tx.luteEngine.BlockDOM2Tree(data)
-	subTree := luteEngine.BlockDOM2Tree(data)
 	insertedNode := subTree.Root.FirstChild
 	if nil == insertedNode {
 		return &TxErr{code: TxErrCodeBlockNotFound, msg: "invalid data tree", id: block.ID}
@@ -507,8 +508,7 @@ func (tx *Transaction) doAppendInsert(operation *Operation) (ret *TxErr) {
 	}
 
 	data := strings.ReplaceAll(operation.Data.(string), editor.FrontEndCaret, "")
-	luteEngine := NewLute()
+	subTree := tx.luteEngine.BlockDOM2Tree(data)
-	subTree := luteEngine.BlockDOM2Tree(data)
 	insertedNode := subTree.Root.FirstChild
 	if nil == insertedNode {
 		return &TxErr{code: TxErrCodeBlockNotFound, msg: "invalid data tree", id: block.ID}
@@ -720,8 +720,7 @@ func (tx *Transaction) doInsert(operation *Operation) (ret *TxErr) {
 	}
 
 	data := strings.ReplaceAll(operation.Data.(string), editor.FrontEndCaret, "")
-	luteEngine := NewLute()
+	subTree := tx.luteEngine.BlockDOM2Tree(data)
-	subTree := luteEngine.BlockDOM2Tree(data)
 
 	p := block.Path
 	assets := getAssetsDir(filepath.Join(util.DataDir, block.BoxID), filepath.Dir(filepath.Join(util.DataDir, block.BoxID, p)))
@@ -870,8 +869,7 @@ func (tx *Transaction) doUpdate(operation *Operation) (ret *TxErr) {
 		return &TxErr{code: TxErrCodeBlockNotFound, id: id}
 	}
 
-	luteEngine := NewLute()
+	subTree := tx.luteEngine.BlockDOM2Tree(data)
-	subTree := luteEngine.BlockDOM2Tree(data)
 	subTree.ID, subTree.Box, subTree.Path = tree.ID, tree.Box, tree.Path
 	oldNode := treenode.GetNodeInTree(tree, id)
 	if nil == oldNode {
@@ -1034,6 +1032,8 @@ type Transaction struct {
 
 	trees map[string]*parse.Tree
 	nodes map[string]*ast.Node
+
+	luteEngine *lute.Lute
 }
 
 func (tx *Transaction) begin() (err error) {
@@ -1042,6 +1042,7 @@ func (tx *Transaction) begin() (err error) {
 	}
 	tx.trees = map[string]*parse.Tree{}
 	tx.nodes = map[string]*ast.Node{}
+	tx.luteEngine = util.NewLute()
 	return
 }
 
@@ -1077,7 +1078,7 @@ func (tx *Transaction) loadTree(id string) (ret *parse.Tree, err error) {
 		return
 	}
 
-	ret, err = LoadTree(box, p)
+	ret, err = filesys.LoadTree(box, p, tx.luteEngine)
 	if nil != err {
 		return
 	}

kernel/model/tree.go

@@ -164,19 +164,8 @@ func loadTreeByBlockID(id string) (ret *parse.Tree, err error) {
 
 		return nil, ErrBlockNotFound
 	}
-	ret, err = LoadTree(bt.BoxID, bt.Path)
+
-	if nil != err {
+	luteEngine := util.NewLute()
-		return
+	ret, err = filesys.LoadTree(bt.BoxID, bt.Path, luteEngine)
-	}
 	return
 }
-
-func LoadTree(boxID, p string) (*parse.Tree, error) {
-	luteEngine := NewLute()
-	tree, err := filesys.LoadTree(boxID, p, luteEngine)
-	if nil != err {
-		logging.LogErrorf("load tree [%s] failed: %s", boxID+p, err)
-		return nil, err
-	}
-	return tree, nil
-}

kernel/sql/block_query.go

@@ -555,22 +555,6 @@ func GetAllChildBlocks(rootID, condition string) (ret []*Block) {
 	return
 }
 
-func GetRefExistedBlocks() (ret []*Block) {
-	stmt := "SELECT * FROM blocks WHERE markdown LIKE ? OR markdown LIKE ?"
-	rows, err := query(stmt, "%((20%", "%<<20%")
-	if nil != err {
-		logging.LogErrorf("sql query [%s] failed: %s", stmt, err)
-		return
-	}
-	defer rows.Close()
-	for rows.Next() {
-		if block := scanBlockRows(rows); nil != block {
-			ret = append(ret, block)
-		}
-	}
-	return
-}
-
 func GetBlock(id string) (ret *Block) {
 	ret = getBlockCache(id)
 	if nil != ret {