🎨 改进内核任务调度机制提升稳定性 https://github.com/siyuan-note/siyuan/issues/7113

app/appearance/langs/en_US.json

@@ -983,7 +983,7 @@
     "105": "Corrupted data repo have been automatically reset",
     "106": "Maximum length is limited to 512 characters",
     "107": "Moving document [%s]",
-    "108": "Cleaning obsolete indexes...",
+    "108": "TODO",
     "109": "Remove reminder completed [%s]",
     "110": "Renaming...",
     "111": "Saving document [%s]...",

app/appearance/langs/es_ES.json

@@ -983,7 +983,7 @@
     "105": "El repositorio de datos corruptos se ha restablecido automáticamente",
     "106": "La longitud máxima está limitada a 512 caracteres",
     "107": "Moviendo documento [%s]",
-    "108": "Limpiando índices obsoletos...",
+    "108": "TODO",
     "109": "Eliminación de recordatorios completada [%s]",
     "110": "Renombrar...",
     "111": "Guardando documento [%s]...",

app/appearance/langs/fr_FR.json

@@ -983,7 +983,7 @@
     "105": "Le référentiel de données corrompu a été automatiquement réinitialisé",
     "106": "La longueur maximale est limitée à 512 caractères",
     "107": "Déplacement du document [%s]",
-    "108": "Nettoyage des index obsolètes...",
+    "108": "TODO",
     "109": "Supprimer le rappel terminé [%s]",
     "110": "Renommer...",
     "111": "Enregistrement du document [%s]...",

app/appearance/langs/zh_CHT.json

@@ -983,7 +983,7 @@
     "105": "已經自動重置損壞的數據倉庫",
     "106": "最大長度限制為 512 字元",
     "107": "正在移動文檔 [%s]",
-    "108": "正在清理已過時的索引...",
+    "108": "TODO",
     "109": "移除提醒完畢 [%s]",
     "110": "正在重命名...",
     "111": "正在保存文檔 [%s]...",

app/appearance/langs/zh_CN.json

@@ -983,7 +983,7 @@
     "105": "已经自动重置损坏的数据仓库",
     "106": "最大长度限制为 512 字符",
     "107": "正在移动文档 [%s]",
-    "108": "正在清理已过时的索引...",
+    "108": "TODO",
     "109": "移除提醒完毕 [%s]",
     "110": "正在重命名...",
     "111": "正在保存文档 [%s]...",

kernel/api/transaction.go

@@ -25,9 +25,7 @@ import (
 	"github.com/88250/gulu"
 	"github.com/gin-gonic/gin"
 	"github.com/siyuan-note/filelock"
-	"github.com/siyuan-note/logging"
 	"github.com/siyuan-note/siyuan/kernel/model"
-	"github.com/siyuan-note/siyuan/kernel/sql"
 	"github.com/siyuan-note/siyuan/kernel/util"
 )
 
@@ -67,17 +65,6 @@ func performTransactions(c *gin.Context) {
 		ret.Code = 1
 		return
 	}
-	if nil != err {
-		tx, txErr := sql.BeginTx()
-		if nil != txErr {
-			logging.LogFatalf("transaction failed: %s", txErr)
-			return
-		}
-		sql.ClearBoxHash(tx)
-		sql.CommitTx(tx)
-		logging.LogFatalf("transaction failed: %s", err)
-		return
-	}
 
 	ret.Data = transactions
 

kernel/model/file.go

@@ -912,15 +912,7 @@ func createTreeTx(tree *parse.Tree) {
 	transaction := &Transaction{DoOperations: []*Operation{{Action: "create", Data: tree}}}
 	err := PerformTransactions(&[]*Transaction{transaction})
 	if nil != err {
-		tx, txErr := sql.BeginTx()
-		if nil != txErr {
-			logging.LogFatalf("transaction failed: %s", txErr)
-			return
-		}
-		sql.ClearBoxHash(tx)
-		sql.CommitTx(tx)
 		logging.LogFatalf("transaction failed: %s", err)
-		return
 	}
 }
 
@@ -1472,7 +1464,6 @@ func createDoc(boxID, p, title, dom string) (err error) {
 			logging.LogFatalf("transaction failed: %s", txErr)
 			return
 		}
-		sql.ClearBoxHash(tx)
 		sql.CommitTx(tx)
 		logging.LogFatalf("transaction failed: %s", err)
 		return

kernel/model/index.go

@@ -17,12 +17,8 @@
 package model
 
 import (
-	"bytes"
-	"crypto/sha256"
 	"fmt"
-	"github.com/siyuan-note/siyuan/kernel/task"
 	"runtime/debug"
-	"sort"
 	"strings"
 	"time"
 
@@ -34,6 +30,7 @@ import (
 	"github.com/siyuan-note/siyuan/kernel/cache"
 	"github.com/siyuan-note/siyuan/kernel/filesys"
 	"github.com/siyuan-note/siyuan/kernel/sql"
+	"github.com/siyuan-note/siyuan/kernel/task"
 	"github.com/siyuan-note/siyuan/kernel/treenode"
 	"github.com/siyuan-note/siyuan/kernel/util"
 )
@@ -47,7 +44,6 @@ func unindex(boxID string) {
 	if nil != err {
 		return
 	}
-	sql.RemoveBoxHash(tx, boxID)
 	sql.DeleteByBoxTx(tx, boxID)
 	sql.CommitTx(tx)
 	ids := treenode.RemoveBlockTreesByBoxID(boxID)
@@ -83,8 +79,6 @@ func index(boxID string, fullRebuildIndex bool) {
 	bootProgressPart := 10.0 / float64(boxLen) / float64(len(files))
 
 	luteEngine := NewLute()
-	idTitleMap := map[string]string{}
-	idHashMap := map[string]string{}
 	var treeCount int
 	var treeSize int64
 	util.PushEndlessProgress(fmt.Sprintf("["+box.Name+"] "+Conf.Language(64), len(files)))
@@ -105,7 +99,7 @@ func index(boxID string, fullRebuildIndex bool) {
 		}
 
 		docIAL := parse.IAL2MapUnEsc(tree.Root.KramdownIAL)
-		if "" == docIAL["updated"] {
+		if "" == docIAL["updated"] { // 早期的数据可能没有 updated 属性，这里进行订正
 			updated := util.TimeFromID(tree.Root.ID)
 			tree.Root.SetIALAttr("updated", updated)
 			docIAL["updated"] = updated
@@ -117,12 +111,7 @@ func index(boxID string, fullRebuildIndex bool) {
 		util.IncBootProgress(bootProgressPart, fmt.Sprintf(Conf.Language(92), util.ShortPathForBootingDisplay(tree.Path)))
 		treeSize += file.size
 		treeCount++
-		// 缓存文档标题，后面做 Path -> HPath 路径映射时需要
-		idTitleMap[tree.ID] = tree.Root.IALAttr("title")
-		// 缓存块树
 		treenode.IndexBlockTree(tree)
-		// 缓存 ID-Hash，后面需要用于判断是否要重建库
-		idHashMap[tree.ID] = tree.Hash
 		if 1 < i && 0 == i%64 {
 			util.PushEndlessProgress(fmt.Sprintf(Conf.Language(88), i, len(files)-i))
 		}
@@ -131,25 +120,7 @@ func index(boxID string, fullRebuildIndex bool) {
 
 	box.UpdateHistoryGenerated() // 初始化历史生成时间为当前时间
 
-	// 检查是否需要重新建库
-	util.SetBootDetails("Checking data hashes...")
-	var ids []string
-	for id := range idTitleMap {
-		ids = append(ids, id)
-	}
-	sort.Slice(ids, func(i, j int) bool { return ids[i] >= ids[j] })
-	buf := bytes.Buffer{}
-	for _, id := range ids {
-		hash, _ := idHashMap[id]
-		buf.WriteString(hash)
-		util.SetBootDetails("Checking hash " + hash)
-	}
-	boxHash := fmt.Sprintf("%x", sha256.Sum256(buf.Bytes()))
-
-	dbBoxHash := sql.GetBoxHash(box.ID)
-	if boxHash == dbBoxHash {
-		//logging.LogInfof("use existing database for box [%s]", box.ID)
-		util.SetBootDetails("Use existing database for notebook " + box.ID)
+	if !fullRebuildIndex {
 		return
 	}
 
@@ -159,26 +130,8 @@ func index(boxID string, fullRebuildIndex bool) {
 	defer sql.EnableCache()
 
 	start := time.Now()
-	if !fullRebuildIndex {
-		tx, err := sql.BeginTx()
-		if nil != err {
-			return
-		}
-		sql.PutBoxHash(tx, box.ID, boxHash)
-		util.SetBootDetails("Cleaning obsolete indexes...")
-		util.PushEndlessProgress(Conf.Language(108))
-		sql.DeleteByBoxTx(tx, box.ID)
-		if err = sql.CommitTx(tx); nil != err {
-			return
-		}
-	}
-
 	bootProgressPart = 20.0 / float64(boxLen) / float64(treeCount)
-
-	context := map[string]interface{}{eventbus.CtxPushMsg: eventbus.CtxPushMsgToStatusBarAndProgress}
 	i = 0
-	// 块级行级入库，缓存块
-	// 这里不能并行插入，因为 SQLite 不支持
 	for _, file := range files {
 		if file.isdir || !strings.HasSuffix(file.name, ".sy") {
 			continue
@@ -191,23 +144,15 @@ func index(boxID string, fullRebuildIndex bool) {
 		}
 
 		util.IncBootProgress(bootProgressPart, fmt.Sprintf(Conf.Language(93), util.ShortPathForBootingDisplay(tree.Path)))
-		tx, err := sql.BeginTx()
-		if nil != err {
-			continue
-		}
-		if err = sql.InsertBlocksSpans(tx, tree, context); nil != err {
-			sql.RollbackTx(tx)
-			continue
-		}
-		if err = sql.CommitTx(tx); nil != err {
-			continue
-		}
+		sql.UpsertTreeQueue(tree)
 		if 1 < i && 0 == i%64 {
 			util.PushEndlessProgress(fmt.Sprintf("["+box.Name+"] "+Conf.Language(53), i, treeCount-i))
 		}
 		i++
 	}
 
+	sql.WaitForWritingDatabase()
+
 	end := time.Now()
 	elapsed := end.Sub(start).Seconds()
 	logging.LogInfof("rebuilt database for notebook [%s] in [%.2fs], tree [count=%d, size=%s]", box.ID, elapsed, treeCount, humanize.Bytes(uint64(treeSize)))

kernel/sql/block.go

@@ -65,10 +65,6 @@ func updateRootContent(tx *sql.Tx, content, updated, id string) {
 	cache.RemoveBlockIAL(id)
 }
 
-func InsertBlock(tx *sql.Tx, block *Block, context map[string]interface{}) (err error) {
-	return insertBlocks(tx, []*Block{block}, context)
-}
-
 func UpdateBlockContent(block *Block) {
 	tx, err := BeginTx()
 	if nil != err {

kernel/sql/queue.go

@@ -17,15 +17,10 @@
 package sql
 
 import (
-	"bytes"
-	"crypto/sha256"
-	"database/sql"
-	"fmt"
 	"path"
 	"sync"
 	"time"
 
-	"github.com/88250/lute/ast"
 	"github.com/88250/lute/parse"
 	"github.com/emirpasic/gods/sets/hashset"
 	"github.com/siyuan-note/eventbus"
@@ -136,23 +131,6 @@ func FlushQueue() {
 	if 5000 < elapsed {
 		logging.LogInfof("op tx [%dms]", elapsed)
 	}
-
-	start = time.Now()
-	tx, err = BeginTx()
-	if nil != err {
-		return
-	}
-	for _, box := range boxes.Values() {
-		if !ast.IsNodeIDPattern(box.(string)) {
-			continue
-		}
-		updateBoxHash(tx, box.(string))
-	}
-	CommitTx(tx)
-	elapsed = time.Now().Sub(start).Milliseconds()
-	if 1000 < elapsed {
-		logging.LogInfof("hash tx [%dms]", elapsed)
-	}
 }
 
 func mergeUpsertTrees() (ops []*treeQueueOperation) {
@@ -229,28 +207,3 @@ func RemoveTreePathQueue(treeBox, treePathPrefix string) {
 	newOp := &treeQueueOperation{removeTreeBox: treeBox, removeTreePath: treePathPrefix, inQueueTime: time.Now(), action: "delete"}
 	operationQueue = append(operationQueue, newOp)
 }
-
-func updateBoxHash(tx *sql.Tx, boxID string) {
-	sum := boxChecksum(boxID)
-	PutBoxHash(tx, boxID, sum)
-}
-
-func boxChecksum(box string) (ret string) {
-	rows, err := query("SELECT hash FROM blocks WHERE type = 'd' AND box = ? ORDER BY id DESC", box)
-	if nil != err {
-		logging.LogErrorf("sql query failed: %s", err)
-		return
-	}
-	defer rows.Close()
-	buf := bytes.Buffer{}
-	for rows.Next() {
-		var hash string
-		if err = rows.Scan(&hash); nil != err {
-			logging.LogErrorf("query scan field failed: %s", err)
-			return
-		}
-		buf.WriteString(hash)
-	}
-	ret = fmt.Sprintf("%x", sha256.Sum256(buf.Bytes()))
-	return
-}

kernel/sql/stat.go

@@ -54,27 +54,6 @@ func setDatabaseVer() {
 	CommitTx(tx)
 }
 
-func ClearBoxHash(tx *sql.Tx) {
-	stmt := "DELETE FROM stat WHERE `key` LIKE '%_hash'"
-	execStmtTx(tx, stmt)
-}
-
-func RemoveBoxHash(tx *sql.Tx, box string) {
-	key := box + "_hash"
-	stmt := "DELETE FROM stat WHERE `key` = '" + key + "'"
-	execStmtTx(tx, stmt)
-}
-
-func PutBoxHash(tx *sql.Tx, box, hash string) {
-	key := box + "_hash"
-	putStat(tx, key, hash)
-}
-
-func GetBoxHash(box string) string {
-	key := box + "_hash"
-	return getStat(key)
-}
-
 func putStat(tx *sql.Tx, key, value string) (err error) {
 	stmt := "DELETE FROM stat WHERE `key` = '" + key + "'"
 	if err = execStmtTx(tx, stmt); nil != err {

kernel/sql/upsert.go

@@ -36,13 +36,6 @@ func init() {
 	luteEngine.RenderOptions.KramdownBlockIAL = false // 数据库 markdown 字段为标准 md，但是要保留 span block ial
 }
 
-func InsertBlocksSpans(tx *sql.Tx, tree *parse.Tree, context map[string]interface{}) (err error) {
-	if err = insertBlocksSpans(tx, tree, context); nil != err {
-		logging.LogErrorf("insert tree [%s] into database failed: %s", tree.Box+tree.Path, err)
-	}
-	return
-}
-
 func InsertRefs(tx *sql.Tx, tree *parse.Tree) {
 	if err := insertRef(tx, tree); nil != err {
 		logging.LogErrorf("insert refs tree [%s] into database failed: %s", tree.Box+tree.Path, err)
@@ -395,23 +388,6 @@ func insertFileAnnotationRefs0(tx *sql.Tx, bulk []*FileAnnotationRef) (err error
 	return
 }
 
-func insertBlocksSpans(tx *sql.Tx, tree *parse.Tree, context map[string]interface{}) (err error) {
-	blocks, spans, assets, attributes := fromTree(tree.Root, tree)
-	if err = insertBlocks(tx, blocks, context); nil != err {
-		return
-	}
-	if err = insertSpans(tx, spans); nil != err {
-		return
-	}
-	if err = insertAssets(tx, assets); nil != err {
-		return
-	}
-	if err = insertAttributes(tx, attributes); nil != err {
-		return
-	}
-	return
-}
-
 func insertRef(tx *sql.Tx, tree *parse.Tree) (err error) {
 	refs, fileAnnotationRefs := refsFromTree(tree)
 	if err = insertRefs(tx, refs); nil != err {