Merge remote-tracking branch 'origin/dev' into dev

app/appearance/langs/en_US.json

@@ -876,6 +876,7 @@
     "task.database.index.commit": "Execute database index commit",
     "task.database.index.ref": "Execute database index reference",
     "task.database.index.fix": "Execute database index fix",
+    "task.database.cache": "Execute database cache",
     "task.ocr.image": "Execute image OCR to extract text",
     "task.history.generateDoc": "Execute GenerateDoc History",
     "task.database.index.embedBlock": "Execute database index embed block",

app/appearance/langs/es_ES.json

@@ -876,6 +876,7 @@
     "task.database.index.commit": "Ejecutar la confirmación del índice de la base de datos",
     "task.database.index.ref": "Ejecutar referencia de índice de base de datos",
     "task.database.index.fix": "Ejecutar corrección del índice de la base de datos",
+    "task.database.cache": "Ejecutar caché de base de datos",
     "task.ocr.image": "Ejecutar OCR de imagen para extraer texto",
     "task.history.generateDoc": "Ejecutar Historial GenerateDoc",
     "task.database.index.embedBlock": "Ejecutar bloque de incrustación de índice de base de datos",

app/appearance/langs/fr_FR.json

@@ -876,6 +876,7 @@
     "task.database.index.commit": "Effectuer la validation de l'index de la base de données",
     "task.database.index.ref": "Exécuter la référence d'index de la base de données",
     "task.database.index.fix": "Effectuer la correction de l'index de la base de données",
+    "task.database.cache": "Effectuer le cache de la base de données",
     "task.ocr.image": "Exécute l'OCR d'image pour extraire le texte",
     "task.history.generateDoc": "Exécuter l'historique de GenerateDoc",
     "task.database.index.embedBlock": "Exécuter le bloc d'intégration d'index de base de données",

app/appearance/langs/zh_CHT.json

@@ -876,6 +876,7 @@
     "task.database.index.commit": "執行數據庫索引提交",
     "task.database.index.ref": "執行數據庫索引引用",
     "task.database.index.fix": "執行數據庫索引訂正",
+    "task.database.cache": "執行數據庫緩存",
     "task.ocr.image": "執行圖片 OCR 提取文本",
     "task.history.generateDoc": "執行生成文件歷史",
     "task.database.index.embedBlock": "執行數據庫索引嵌入塊",

app/appearance/langs/zh_CN.json

@@ -876,6 +876,7 @@
     "task.database.index.commit": "执行数据库索引提交",
     "task.database.index.ref": "执行数据库索引引用",
     "task.database.index.fix": "执行数据库索引订正",
+    "task.database.cache": "执行数据库缓存",
     "task.ocr.image": "执行图片 OCR 提取文本",
     "task.history.generateDoc": "执行生成文件历史",
     "task.database.index.embedBlock": "执行数据库索引嵌入块",

kernel/api/block.go

@@ -40,7 +40,14 @@ func transferBlockRef(c *gin.Context) {
 	}
 
 	fromID := arg["fromID"].(string)
+	if util.InvalidIDPattern(fromID, ret) {
+		return
+	}
 	toID := arg["toID"].(string)
+	if util.InvalidIDPattern(toID, ret) {
+		return
+	}
+
 	err := model.TransferBlockRef(fromID, toID)
 	if nil != err {
 		ret.Code = -1

kernel/model/box.go

@@ -505,7 +505,10 @@ func ReloadUI() {
 }
 
 func FullReindex() {
-	task.PrependTask(task.DatabaseIndexFull, fullReindex)
+	task.AppendTask(task.DatabaseIndexFull, fullReindex)
+	task.AppendTask(task.DatabaseCache, sql.EnableCache)
+	task.AppendTask(task.DatabaseIndexRef, IndexRefs)
+	task.AppendTask(task.ReloadUI, util.ReloadUI)
 }
 
 func fullReindex() {
@@ -523,12 +526,9 @@ func fullReindex() {
 	for _, openedBox := range openedBoxes {
 		index(openedBox.ID)
 	}
-	sql.EnableCache()
 	treenode.SaveBlockTree(true)
 	LoadFlashcards()
-	IndexRefs()
 	debug.FreeOSMemory()
-	util.ReloadUI()
 }
 
 func ChangeBoxSort(boxIDs []string) {

kernel/model/file.go

@@ -1256,7 +1256,7 @@ func removeDoc(box *Box, p string) {
 	}
 	util.PushEvent(evt)
 
-	task.PrependTask(task.DatabaseIndex, removeDoc0, box, p, childrenDir)
+	task.AppendTask(task.DatabaseIndex, removeDoc0, box, p, childrenDir)
 }
 
 func removeDoc0(box *Box, p, childrenDir string) {

kernel/model/history.go

@@ -50,7 +50,7 @@ func AutoGenerateDocHistory() {
 	ChangeHistoryTick(Conf.Editor.GenerateHistoryInterval)
 	for {
 		<-historyTicker.C
-		task.PrependTask(task.HistoryGenerateDoc, generateDocHistory)
+		task.AppendTask(task.HistoryGenerateDoc, generateDocHistory)
 	}
 }
 
@@ -244,7 +244,6 @@ func RollbackDocHistory(boxID, historyPath string) (err error) {
 
 	FullReindex()
 	IncSync()
-	ReloadUI()
 	return nil
 }
 

kernel/model/import.go

@@ -441,7 +441,6 @@ func ImportData(zipPath string) (err error) {
 
 	IncSync()
 	FullReindex()
-	ReloadUI()
 	return
 }
 

kernel/model/index.go

@@ -39,7 +39,7 @@ import (
 )
 
 func (box *Box) Unindex() {
-	task.PrependTask(task.DatabaseIndex, unindex, box.ID)
+	task.AppendTask(task.DatabaseIndex, unindex, box.ID)
 }
 
 func unindex(boxID string) {
@@ -49,7 +49,7 @@ func unindex(boxID string) {
 }
 
 func (box *Box) Index() {
-	task.PrependTask(task.DatabaseIndex, index, box.ID)
+	task.AppendTask(task.DatabaseIndex, index, box.ID)
 	task.AppendTask(task.DatabaseIndexRef, IndexRefs)
 }
 

kernel/model/index_fix.go

@@ -39,20 +39,29 @@ import (
 
 // FixIndexJob 自动校验数据库索引 https://github.com/siyuan-note/siyuan/issues/7016
 func FixIndexJob() {
-	task.AppendTask(task.DatabaseIndexFix, autoFixIndex)
+	task.AppendTask(task.DatabaseIndexFix, removeDuplicateDatabaseIndex)
+	sql.WaitForWritingDatabase()
+
+	task.AppendTask(task.DatabaseIndexFix, fixBlockTreeByFileSys)
+	sql.WaitForWritingDatabase()
+
+	task.AppendTask(task.DatabaseIndexFix, fixDatabaseIndexByBlockTree)
+	sql.WaitForWritingDatabase()
+
+	util.PushStatusBar(Conf.Language(185))
+	debug.FreeOSMemory()
 }
 
 var autoFixLock = sync.Mutex{}
 
-func autoFixIndex() {
+// removeDuplicateDatabaseIndex 删除重复的数据库索引。
+func removeDuplicateDatabaseIndex() {
 	defer logging.Recover()
 
 	autoFixLock.Lock()
 	defer autoFixLock.Unlock()
 
 	util.PushStatusBar(Conf.Language(58))
-
-	// 去除重复的数据库块记录
 	duplicatedRootIDs := sql.GetDuplicatedRootIDs("blocks")
 	if 1 > len(duplicatedRootIDs) {
 		duplicatedRootIDs = sql.GetDuplicatedRootIDs("blocks_fts")
@@ -87,11 +96,16 @@ func autoFixIndex() {
 	if 0 < deletes {
 		logging.LogWarnf("exist more than one tree duplicated [%d], reindex it", deletes)
 	}
+}
+
+// fixBlockTreeByFileSys 通过文件系统订正块树。
+func fixBlockTreeByFileSys() {
+	defer logging.Recover()
+
+	autoFixLock.Lock()
+	defer autoFixLock.Unlock()
 
 	util.PushStatusBar(Conf.Language(58))
-	sql.WaitForWritingDatabase()
-	util.PushStatusBar(Conf.Language(58))
-	// 根据文件系统补全块树
 	boxes := Conf.GetOpenedBoxes()
 	for _, box := range boxes {
 		boxPath := filepath.Join(util.DataDir, box.ID)
@@ -130,26 +144,23 @@ func autoFixIndex() {
 		}
 	}
 
-	util.PushStatusBar(Conf.Language(58))
-	sql.WaitForWritingDatabase()
-	util.PushStatusBar(Conf.Language(58))
 	// 清理已关闭的笔记本块树
 	boxes = Conf.GetClosedBoxes()
 	for _, box := range boxes {
 		treenode.RemoveBlockTreesByBoxID(box.ID)
 	}
+}
 
-	// 对比块树和数据库并订正数据库
+// fixDatabaseIndexByBlockTree 通过块树订正数据库索引。
+func fixDatabaseIndexByBlockTree() {
+	defer logging.Recover()
+
+	util.PushStatusBar(Conf.Language(58))
 	rootUpdatedMap := treenode.GetRootUpdated()
 	dbRootUpdatedMap, err := sql.GetRootUpdated()
 	if nil == err {
 		reindexTreeByUpdated(rootUpdatedMap, dbRootUpdatedMap)
 	}
-
-	util.PushStatusBar(Conf.Language(58))
-	sql.WaitForWritingDatabase()
-	util.PushStatusBar(Conf.Language(185))
-	debug.FreeOSMemory()
 }
 
 func reindexTreeByUpdated(rootUpdatedMap, dbRootUpdatedMap map[string]string) {

kernel/model/repository.go

@@ -49,7 +49,6 @@ import (
 	"github.com/siyuan-note/siyuan/kernel/cache"
 	"github.com/siyuan-note/siyuan/kernel/conf"
 	"github.com/siyuan-note/siyuan/kernel/filesys"
-	"github.com/siyuan-note/siyuan/kernel/sql"
 	"github.com/siyuan-note/siyuan/kernel/task"
 	"github.com/siyuan-note/siyuan/kernel/treenode"
 	"github.com/siyuan-note/siyuan/kernel/util"
@@ -504,7 +503,7 @@ func InitRepoKey() (err error) {
 }
 
 func CheckoutRepo(id string) {
-	task.PrependTask(task.RepoCheckout, checkoutRepo, id)
+	task.AppendTask(task.RepoCheckout, checkoutRepo, id)
 }
 
 func checkoutRepo(id string) {
@@ -523,7 +522,6 @@ func checkoutRepo(id string) {
 
 	util.PushEndlessProgress(Conf.Language(63))
 	WaitForWritingFiles()
-	sql.WaitForWritingDatabase()
 	CloseWatchAssets()
 	defer WatchAssets()
 
@@ -541,7 +539,6 @@ func checkoutRepo(id string) {
 	}
 
 	FullReindex()
-	ReloadUI()
 
 	if syncEnabled {
 		func() {
@@ -942,7 +939,6 @@ func syncRepo(exit, byHand bool) (err error) {
 		// 云端同步发生冲突时生成副本 https://github.com/siyuan-note/siyuan/issues/5687
 
 		luteEngine := NewLute()
-		waitTx := false
 		for _, file := range mergeResult.Conflicts {
 			if !strings.HasSuffix(file.Path, ".sy") {
 				continue
@@ -965,10 +961,6 @@ func syncRepo(exit, byHand bool) (err error) {
 
 			resetTree(tree, "Conflicted")
 			createTreeTx(tree)
-			waitTx = true
-		}
-		if waitTx {
-			sql.WaitForWritingDatabase()
 		}
 	}
 
@@ -1028,9 +1020,6 @@ func syncRepo(exit, byHand bool) (err error) {
 	cache.ClearDocsIAL()              // 同步后文档树文档图标没有更新 https://github.com/siyuan-note/siyuan/issues/4939
 	if needFullReindex(upsertTrees) { // 改进同步后全量重建索引判断 https://github.com/siyuan-note/siyuan/issues/5764
 		FullReindex()
-		if !exit {
-			ReloadUI()
-		}
 		return
 	}
 

kernel/model/sync.go

@@ -97,7 +97,9 @@ func SyncData(boot, exit, byHand bool) {
 func syncData(boot, exit, byHand bool) {
 	defer logging.Recover()
 
+	util.BroadcastByType("main", "syncing", 0, Conf.Language(81), nil)
 	if !checkSync(boot, exit, byHand) {
+		util.BroadcastByType("main", "syncing", 1, "", nil)
 		return
 	}
 
@@ -118,7 +120,6 @@ func syncData(boot, exit, byHand bool) {
 	now := util.CurrentTimeMillis()
 	Conf.Sync.Synced = now
 
-	util.BroadcastByType("main", "syncing", 0, Conf.Language(81), nil)
 	err := syncRepo(exit, byHand)
 	synced := util.Millisecond2Time(Conf.Sync.Synced).Format("2006-01-02 15:04:05") + "\n\n"
 	if nil == err {

kernel/sql/queue.go

@@ -118,8 +118,9 @@ func FlushQueue() {
 		context["current"] = i
 		context["total"] = total
 		if err = execOp(op, tx, context); nil != err {
+			tx.Rollback()
 			logging.LogErrorf("queue operation failed: %s", err)
-			return
+			continue
 		}
 
 		if err = commitTx(tx); nil != err {

kernel/task/queue.go

@@ -39,27 +39,20 @@ type Task struct {
 	Created time.Time
 }
 
-func PrependTask(action string, handler interface{}, args ...interface{}) {
-	queueLock.Lock()
-	defer queueLock.Unlock()
-
-	if util.IsExiting {
-		//logging.LogWarnf("task queue is paused, action [%s] will be ignored", action)
-		return
-	}
-
-	taskQueue = append([]*Task{newTask(action, handler, args...)}, taskQueue...)
-}
-
 func AppendTask(action string, handler interface{}, args ...interface{}) {
-	queueLock.Lock()
-	defer queueLock.Unlock()
-
 	if util.IsExiting {
 		//logging.LogWarnf("task queue is paused, action [%s] will be ignored", action)
 		return
 	}
 
+	currentActions := getCurrentActions()
+	if gulu.Str.Contains(action, currentActions) && gulu.Str.Contains(action, uniqueActions) {
+		//logging.LogWarnf("task [%s] is already in queue, will be ignored", action)
+		return
+	}
+
+	queueLock.Lock()
+	defer queueLock.Unlock()
 	taskQueue = append(taskQueue, newTask(action, handler, args...))
 }
 
@@ -72,6 +65,20 @@ func newTask(action string, handler interface{}, args ...interface{}) *Task {
 	}
 }
 
+func getCurrentActions() (ret []string) {
+	queueLock.Lock()
+	defer queueLock.Unlock()
+
+	if "" != currentTaskAction {
+		ret = append(ret, currentTaskAction)
+	}
+
+	for _, task := range taskQueue {
+		ret = append(ret, task.Action)
+	}
+	return
+}
+
 const (
 	RepoCheckout            = "task.repo.checkout"             // 从快照中检出
 	DatabaseIndexFull       = "task.database.index.full"       // 重建索引
@@ -79,12 +86,23 @@ const (
 	DatabaseIndexCommit     = "task.database.index.commit"     // 数据库索引提交
 	DatabaseIndexRef        = "task.database.index.ref"        // 数据库索引引用
 	DatabaseIndexFix        = "task.database.index.fix"        // 数据库索引订正
+	DatabaseCache           = "task.database.cache"            // 数据库缓存
 	OCRImage                = "task.ocr.image"                 // 图片 OCR 提取文本
 	HistoryGenerateDoc      = "task.history.generateDoc"       // 生成文件历史
 	DatabaseIndexEmbedBlock = "task.database.index.embedBlock" // 数据库索引嵌入块
 	ReloadUI                = "task.reload.ui"                 // 重载 UI
 )
 
+// uniqueActions 描述了唯一的任务，即队列中只能存在一个在执行的任务。
+var uniqueActions = []string{
+	RepoCheckout,
+	DatabaseIndexFull,
+	DatabaseIndexCommit,
+	OCRImage,
+	HistoryGenerateDoc,
+	DatabaseIndexEmbedBlock,
+}
+
 func Contain(action string, moreActions ...string) bool {
 	actions := append(moreActions, action)
 	actions = gulu.Str.RemoveDuplicatedElem(actions)
@@ -99,13 +117,12 @@ func Contain(action string, moreActions ...string) bool {
 
 func StatusJob() {
 	tasks := taskQueue
-	data := map[string]interface{}{}
 	var items []map[string]interface{}
 	count := map[string]int{}
+	actionLangs := util.TaskActionLangs[util.Lang]
 	for _, task := range tasks {
-		actionLangs := util.TaskActionLangs[util.Lang]
 		action := task.Action
-		if c := count[action]; 3 < c {
+		if c := count[action]; 2 < c {
 			logging.LogWarnf("too many tasks [%s], ignore show its status", action)
 			continue
 		}
@@ -120,9 +137,19 @@ func StatusJob() {
 		item := map[string]interface{}{"action": action}
 		items = append(items, item)
 	}
+
+	if "" != currentTaskAction {
+		if nil != actionLangs {
+			if label := actionLangs[currentTaskAction]; nil != label {
+				items = append([]map[string]interface{}{map[string]interface{}{"action": label.(string)}}, items...)
+			}
+		}
+	}
+
 	if 1 > len(items) {
 		items = []map[string]interface{}{}
 	}
+	data := map[string]interface{}{}
 	data["tasks"] = items
 	util.PushBackgroundTask(data)
 }
@@ -140,6 +167,8 @@ func ExecTaskJob() {
 	execTask(task)
 }
 
+var currentTaskAction string
+
 func popTask() (ret *Task) {
 	queueLock.Lock()
 	defer queueLock.Unlock()
@@ -165,7 +194,9 @@ func execTask(task *Task) {
 		}
 	}
 
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second*7)
+	currentTaskAction = task.Action
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
 	defer cancel()
 	ch := make(chan bool, 1)
 	go func() {
@@ -175,8 +206,10 @@ func execTask(task *Task) {
 
 	select {
 	case <-ctx.Done():
-		//logging.LogWarnf("task [%s] timeout", task.Action)
+		logging.LogWarnf("task [%s] timeout", task.Action)
 	case <-ch:
 		//logging.LogInfof("task [%s] done", task.Action)
 	}
+
+	currentTaskAction = ""
 }

kernel/treenode/tree.go

@@ -54,7 +54,7 @@ func NodeHash(node *ast.Node, tree *parse.Tree, luteEngine *lute.Lute) string {
 		md = FormatNode(node, luteEngine)
 	}
 	hpath := tree.HPath
-	data := tree.Path + hpath + string(ial) + md
+	data := tree.Box + tree.Path + hpath + string(ial) + md
 	return fmt.Sprintf("%x", sha256.Sum256(gulu.Str.ToBytes(data)))[:7]
 }