From f5ceb847f42fef4df9034531b51de9dc73f1479a Mon Sep 17 00:00:00 2001
From: Daniel <845765@qq.com>
Date: Thu, 11 Apr 2024 21:54:34 +0800
Subject: [PATCH] :art: Improve data indexing performance when importing
 .sy.zip and markdown https://github.com/siyuan-note/siyuan/issues/10986

---
 kernel/model/assets.go         |  4 ++--
 kernel/model/attribute_view.go |  2 +-
 kernel/model/block.go          |  6 +++---
 kernel/model/blockial.go       |  8 ++++----
 kernel/model/bookmark.go       |  4 ++--
 kernel/model/export.go         |  2 +-
 kernel/model/file.go           | 25 +++++++++++++++++++------
 kernel/model/format.go         |  2 +-
 kernel/model/heading.go        |  6 +++---
 kernel/model/import.go         |  3 +--
 kernel/model/index_fix.go      |  2 +-
 kernel/model/listitem.go       |  4 ++--
 kernel/model/search.go         |  2 +-
 kernel/model/tag.go            |  4 ++--
 kernel/model/transaction.go    |  4 ++--
 15 files changed, 45 insertions(+), 33 deletions(-)

diff --git a/kernel/model/assets.go b/kernel/model/assets.go
index fdf2c51bc..e52a360ae 100644
--- a/kernel/model/assets.go
+++ b/kernel/model/assets.go
@@ -235,7 +235,7 @@ func NetImg2LocalAssets(rootID, originalURL string) (err error) {
 	})
 	if 0 < files {
 		util.PushUpdateMsg(msgId, Conf.Language(113), 7000)
-		if err = writeJSONQueue(tree); nil != err {
+		if err = writeTreeUpsertQueue(tree); nil != err {
 			return
 		}
 		util.PushUpdateMsg(msgId, fmt.Sprintf(Conf.Language(120), files), 5000)
@@ -422,7 +422,7 @@ func NetAssets2LocalAssets(rootID string) (err error) {
 
 	if 0 < files {
 		util.PushUpdateMsg(msgId, Conf.Language(113), 7000)
-		if err = writeJSONQueue(tree); nil != err {
+		if err = writeTreeUpsertQueue(tree); nil != err {
 			return
 		}
 		util.PushUpdateMsg(msgId, fmt.Sprintf(Conf.Language(120), files), 5000)
diff --git a/kernel/model/attribute_view.go b/kernel/model/attribute_view.go
index a0823574d..24a177923 100644
--- a/kernel/model/attribute_view.go
+++ b/kernel/model/attribute_view.go
@@ -1609,7 +1609,7 @@ func (tx *Transaction) doRemoveAttrViewView(operation *Operation) (ret *TxErr) {
 	}
 
 	for _, tree := range trees {
-		if err = indexWriteJSONQueue(tree); nil != err {
+		if err = indexWriteTreeUpsertQueue(tree); nil != err {
 			return
 		}
 	}
diff --git a/kernel/model/block.go b/kernel/model/block.go
index e47ba420a..2fe6e7993 100644
--- a/kernel/model/block.go
+++ b/kernel/model/block.go
@@ -205,7 +205,7 @@ func TransferBlockRef(fromID, toID string, refIDs []string) (err error) {
 			}
 		}
 
-		if err = indexWriteJSONQueue(tree); nil != err {
+		if err = indexWriteTreeUpsertQueue(tree); nil != err {
 			return
 		}
 	}
@@ -317,11 +317,11 @@ func SwapBlockRef(refID, defID string, includeChildren bool) (err error) {
 	}
 	refPivot.Unlink()
 
-	if err = indexWriteJSONQueue(refTree); nil != err {
+	if err = indexWriteTreeUpsertQueue(refTree); nil != err {
 		return
 	}
 	if !sameTree {
-		if err = indexWriteJSONQueue(defTree); nil != err {
+		if err = indexWriteTreeUpsertQueue(defTree); nil != err {
 			return
 		}
 	}
diff --git a/kernel/model/blockial.go b/kernel/model/blockial.go
index 9f93e347d..d9aab5a75 100644
--- a/kernel/model/blockial.go
+++ b/kernel/model/blockial.go
@@ -87,7 +87,7 @@ func SetBlockReminder(id string, timed string) (err error) {
 		node.SetIALAttr(attrName, timed)
 		util.PushMsg(fmt.Sprintf(Conf.Language(101), time.UnixMilli(timedMills).Format("2006-01-02 15:04")), 5000)
 	}
-	if err = indexWriteJSONQueue(tree); nil != err {
+	if err = indexWriteTreeUpsertQueue(tree); nil != err {
 		return
 	}
 	IncSync()
@@ -143,7 +143,7 @@ func BatchSetBlockAttrs(blockAttrs []map[string]interface{}) (err error) {
 	}
 
 	for _, tree := range trees {
-		if err = indexWriteJSONQueue(tree); nil != err {
+		if err = indexWriteTreeUpsertQueue(tree); nil != err {
 			return
 		}
 	}
@@ -180,7 +180,7 @@ func setNodeAttrs(node *ast.Node, tree *parse.Tree, nameValues map[string]string
 		return
 	}
 
-	if err = indexWriteJSONQueue(tree); nil != err {
+	if err = indexWriteTreeUpsertQueue(tree); nil != err {
 		return
 	}
 
@@ -279,7 +279,7 @@ func ResetBlockAttrs(id string, nameValues map[string]string) (err error) {
 		updateRefTextRenameDoc(tree)
 	}
 
-	if err = indexWriteJSONQueue(tree); nil != err {
+	if err = indexWriteTreeUpsertQueue(tree); nil != err {
 		return
 	}
 	IncSync()
diff --git a/kernel/model/bookmark.go b/kernel/model/bookmark.go
index dac5e6768..c2d9dab7c 100644
--- a/kernel/model/bookmark.go
+++ b/kernel/model/bookmark.go
@@ -64,7 +64,7 @@ func RemoveBookmark(bookmark string) (err error) {
 		}
 
 		util.PushEndlessProgress(fmt.Sprintf(Conf.Language(111), util.EscapeHTML(tree.Root.IALAttr("title"))))
-		if err = writeJSONQueue(tree); nil != err {
+		if err = writeTreeUpsertQueue(tree); nil != err {
 			util.ClearPushProgress(100)
 			return
 		}
@@ -122,7 +122,7 @@ func RenameBookmark(oldBookmark, newBookmark string) (err error) {
 		}
 
 		util.PushEndlessProgress(fmt.Sprintf(Conf.Language(111), util.EscapeHTML(tree.Root.IALAttr("title"))))
-		if err = writeJSONQueue(tree); nil != err {
+		if err = writeTreeUpsertQueue(tree); nil != err {
 			util.ClearPushProgress(100)
 			return
 		}
diff --git a/kernel/model/export.go b/kernel/model/export.go
index be8fd4a4a..f3012adfe 100644
--- a/kernel/model/export.go
+++ b/kernel/model/export.go
@@ -295,7 +295,7 @@ func Export2Liandi(id string) (err error) {
 		articleId = result.Data.(string)
 		tree, _ = LoadTreeByBlockID(id) // 这里必须重新加载，因为前面导出时已经修改了树结构
 		tree.Root.SetIALAttr(liandiArticleIdAttrName, articleId)
-		if err = writeJSONQueue(tree); nil != err {
+		if err = writeTreeUpsertQueue(tree); nil != err {
 			return
 		}
 	}
diff --git a/kernel/model/file.go b/kernel/model/file.go
index 6f2457da6..1083cb7b3 100644
--- a/kernel/model/file.go
+++ b/kernel/model/file.go
@@ -1065,7 +1065,7 @@ func loadNodesByMode(node *ast.Node, inputIndex, mode, size int, isDoc, isHeadin
 	return
 }
 
-func writeJSONQueue(tree *parse.Tree) (err error) {
+func writeTreeUpsertQueue(tree *parse.Tree) (err error) {
 	if err = filesys.WriteTree(tree); nil != err {
 		return
 	}
@@ -1073,9 +1073,22 @@ func writeJSONQueue(tree *parse.Tree) (err error) {
 	return
 }
 
-func indexWriteJSONQueue(tree *parse.Tree) (err error) {
+func writeTreeIndexQueue(tree *parse.Tree) (err error) {
+	if err = filesys.WriteTree(tree); nil != err {
+		return
+	}
+	sql.IndexTreeQueue(tree)
+	return
+}
+
+func indexWriteTreeIndexQueue(tree *parse.Tree) (err error) {
 	treenode.IndexBlockTree(tree)
-	return writeJSONQueue(tree)
+	return writeTreeIndexQueue(tree)
+}
+
+func indexWriteTreeUpsertQueue(tree *parse.Tree) (err error) {
+	treenode.IndexBlockTree(tree)
+	return writeTreeUpsertQueue(tree)
 }
 
 func renameWriteJSONQueue(tree *parse.Tree) (err error) {
@@ -1181,7 +1194,7 @@ func CreateDailyNote(boxID string) (p string, existed bool, err error) {
 		date := time.Now().Format("20060102")
 		if tree.Root.IALAttr("custom-dailynote-"+date) == "" {
 			tree.Root.SetIALAttr("custom-dailynote-"+date, date)
-			if err = indexWriteJSONQueue(tree); nil != err {
+			if err = indexWriteTreeUpsertQueue(tree); nil != err {
 				return
 			}
 		}
@@ -1232,7 +1245,7 @@ func CreateDailyNote(boxID string) (p string, existed bool, err error) {
 			}
 
 			tree.Root.SetIALAttr("updated", util.CurrentTimeSecondsStr())
-			if err = indexWriteJSONQueue(tree); nil != err {
+			if err = indexWriteTreeUpsertQueue(tree); nil != err {
 				return
 			}
 		}
@@ -1249,7 +1262,7 @@ func CreateDailyNote(boxID string) (p string, existed bool, err error) {
 	p = tree.Path
 	date := time.Now().Format("20060102")
 	tree.Root.SetIALAttr("custom-dailynote-"+date, date)
-	if err = indexWriteJSONQueue(tree); nil != err {
+	if err = indexWriteTreeUpsertQueue(tree); nil != err {
 		return
 	}
 
diff --git a/kernel/model/format.go b/kernel/model/format.go
index f6457b5fc..de4408ddc 100644
--- a/kernel/model/format.go
+++ b/kernel/model/format.go
@@ -68,7 +68,7 @@ func AutoSpace(rootID string) (err error) {
 	newTree.Path = tree.Path
 	newTree.HPath = tree.HPath
 	newTree.Box = tree.Box
-	err = writeJSONQueue(newTree)
+	err = writeTreeUpsertQueue(newTree)
 	if nil != err {
 		return
 	}
diff --git a/kernel/model/heading.go b/kernel/model/heading.go
index 72fbd8046..c445f974f 100644
--- a/kernel/model/heading.go
+++ b/kernel/model/heading.go
@@ -256,7 +256,7 @@ func Doc2Heading(srcID, targetID string, after bool) (srcTreeBox, srcTreePath st
 	targetTree.Root.SetIALAttr("updated", util.CurrentTimeSecondsStr())
 	treenode.RemoveBlockTreesByRootID(srcTree.ID)
 	treenode.RemoveBlockTreesByRootID(targetTree.ID)
-	err = indexWriteJSONQueue(targetTree)
+	err = indexWriteTreeUpsertQueue(targetTree)
 	IncSync()
 	RefreshBacklink(srcTree.ID)
 	RefreshBacklink(targetTree.ID)
@@ -355,7 +355,7 @@ func Heading2Doc(srcHeadingID, targetBoxID, targetPath string) (srcRootBlockID,
 		srcTree.Root.AppendChild(treenode.NewParagraph())
 	}
 	treenode.RemoveBlockTreesByRootID(srcTree.ID)
-	if err = indexWriteJSONQueue(srcTree); nil != err {
+	if err = indexWriteTreeUpsertQueue(srcTree); nil != err {
 		return "", "", err
 	}
 
@@ -363,7 +363,7 @@ func Heading2Doc(srcHeadingID, targetBoxID, targetPath string) (srcRootBlockID,
 	newTree.Root.SetIALAttr("updated", util.CurrentTimeSecondsStr())
 	newTree.Root.Spec = "1"
 	box.addMinSort(path.Dir(newTargetPath), newTree.ID)
-	if err = indexWriteJSONQueue(newTree); nil != err {
+	if err = indexWriteTreeUpsertQueue(newTree); nil != err {
 		return "", "", err
 	}
 	IncSync()
diff --git a/kernel/model/import.go b/kernel/model/import.go
index 282059c9a..c70967552 100644
--- a/kernel/model/import.go
+++ b/kernel/model/import.go
@@ -894,8 +894,7 @@ func ImportFromLocalPath(boxID, localPath string, toPath string) (err error) {
 		buildBlockRefInText()
 
 		for i, tree := range importTrees {
-			treenode.IndexBlockTree(tree)
-			sql.IndexTreeQueue(tree)
+			indexWriteTreeIndexQueue(tree)
 			if 0 == i%4 {
 				util.PushEndlessProgress(fmt.Sprintf(Conf.Language(66), fmt.Sprintf("%d/%d ", i, len(importTrees))+tree.HPath))
 			}
diff --git a/kernel/model/index_fix.go b/kernel/model/index_fix.go
index 87bc07fa2..ef3a867c1 100644
--- a/kernel/model/index_fix.go
+++ b/kernel/model/index_fix.go
@@ -481,7 +481,7 @@ func reindexTree0(tree *parse.Tree, i, size int) {
 	if "" == updated {
 		updated = util.TimeFromID(tree.Root.ID)
 		tree.Root.SetIALAttr("updated", updated)
-		indexWriteJSONQueue(tree)
+		indexWriteTreeUpsertQueue(tree)
 	} else {
 		treenode.IndexBlockTree(tree)
 		sql.IndexTreeQueue(tree)
diff --git a/kernel/model/listitem.go b/kernel/model/listitem.go
index 68199e2c9..9c8325372 100644
--- a/kernel/model/listitem.go
+++ b/kernel/model/listitem.go
@@ -97,7 +97,7 @@ func ListItem2Doc(srcListItemID, targetBoxID, targetPath string) (srcRootBlockID
 		srcTree.Root.AppendChild(treenode.NewParagraph())
 	}
 	treenode.RemoveBlockTreesByRootID(srcTree.ID)
-	if err = indexWriteJSONQueue(srcTree); nil != err {
+	if err = indexWriteTreeUpsertQueue(srcTree); nil != err {
 		return "", "", err
 	}
 
@@ -105,7 +105,7 @@ func ListItem2Doc(srcListItemID, targetBoxID, targetPath string) (srcRootBlockID
 	newTree.Root.SetIALAttr("updated", util.CurrentTimeSecondsStr())
 	newTree.Root.Spec = "1"
 	box.addMinSort(path.Dir(newTargetPath), newTree.ID)
-	if err = indexWriteJSONQueue(newTree); nil != err {
+	if err = indexWriteTreeUpsertQueue(newTree); nil != err {
 		return "", "", err
 	}
 	IncSync()
diff --git a/kernel/model/search.go b/kernel/model/search.go
index ba519f4e5..a026ec60f 100644
--- a/kernel/model/search.go
+++ b/kernel/model/search.go
@@ -726,7 +726,7 @@ func FindReplace(keyword, replacement string, replaceTypes map[string]bool, ids
 				return ast.WalkContinue
 			})
 
-			if err = writeJSONQueue(tree); nil != err {
+			if err = writeTreeUpsertQueue(tree); nil != err {
 				return
 			}
 		}
diff --git a/kernel/model/tag.go b/kernel/model/tag.go
index 1b93ce2e1..78b1b1151 100644
--- a/kernel/model/tag.go
+++ b/kernel/model/tag.go
@@ -93,7 +93,7 @@ func RemoveTag(label string) (err error) {
 			n.Unlink()
 		}
 		util.PushEndlessProgress(fmt.Sprintf(Conf.Language(111), util.EscapeHTML(tree.Root.IALAttr("title"))))
-		if err = writeJSONQueue(tree); nil != err {
+		if err = writeTreeUpsertQueue(tree); nil != err {
 			util.ClearPushProgress(100)
 			return
 		}
@@ -176,7 +176,7 @@ func RenameTag(oldLabel, newLabel string) (err error) {
 			}
 		}
 		util.PushEndlessProgress(fmt.Sprintf(Conf.Language(111), util.EscapeHTML(tree.Root.IALAttr("title"))))
-		if err = writeJSONQueue(tree); nil != err {
+		if err = writeTreeUpsertQueue(tree); nil != err {
 			util.ClearPushProgress(100)
 			return
 		}
diff --git a/kernel/model/transaction.go b/kernel/model/transaction.go
index c47c066cf..cf7fb0293 100644
--- a/kernel/model/transaction.go
+++ b/kernel/model/transaction.go
@@ -1309,7 +1309,7 @@ func (tx *Transaction) begin() (err error) {
 
 func (tx *Transaction) commit() (err error) {
 	for _, tree := range tx.trees {
-		if err = writeJSONQueue(tree); nil != err {
+		if err = writeTreeUpsertQueue(tree); nil != err {
 			return
 		}
 
@@ -1474,7 +1474,7 @@ func refreshDynamicRefTexts(updatedDefNodes map[string]*ast.Node, updatedTrees m
 
 	// 3. 保存变更
 	for _, tree := range changedRefTree {
-		indexWriteJSONQueue(tree)
+		indexWriteTreeUpsertQueue(tree)
 	}
 }