From b9f44309cb183a0fb7075f830921842688d6506b Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Mon, 6 Feb 2023 22:05:45 +0800 Subject: [PATCH 01/11] =?UTF-8?q?:art:=20=E9=99=8D=E4=BD=8E=E9=87=8D?= =?UTF-8?q?=E5=BB=BA=E7=B4=A2=E5=BC=95=E5=86=85=E5=AD=98=E5=8D=A0=E7=94=A8?= =?UTF-8?q?=20Fix=20https://github.com/siyuan-note/siyuan/issues/7268?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/appearance/langs/en_US.json | 1 + app/appearance/langs/es_ES.json | 1 + app/appearance/langs/fr_FR.json | 1 + app/appearance/langs/zh_CHT.json | 1 + app/appearance/langs/zh_CN.json | 1 + kernel/model/box.go | 6 +++--- kernel/model/history.go | 1 - kernel/model/import.go | 1 - kernel/model/repository.go | 4 ---- kernel/model/sync.go | 3 ++- kernel/task/queue.go | 9 +++++---- 11 files changed, 15 insertions(+), 14 deletions(-) diff --git a/app/appearance/langs/en_US.json b/app/appearance/langs/en_US.json index 9fdd439f7..215269044 100644 --- a/app/appearance/langs/en_US.json +++ b/app/appearance/langs/en_US.json @@ -876,6 +876,7 @@ "task.database.index.commit": "Execute database index commit", "task.database.index.ref": "Execute database index reference", "task.database.index.fix": "Execute database index fix", + "task.database.cache": "Execute database cache", "task.ocr.image": "Execute image OCR to extract text", "task.history.generateDoc": "Execute GenerateDoc History", "task.database.index.embedBlock": "Execute database index embed block", diff --git a/app/appearance/langs/es_ES.json b/app/appearance/langs/es_ES.json index 2883639ad..6bd897aed 100644 --- a/app/appearance/langs/es_ES.json +++ b/app/appearance/langs/es_ES.json @@ -876,6 +876,7 @@ "task.database.index.commit": "Ejecutar la confirmación del índice de la base de datos", "task.database.index.ref": "Ejecutar referencia de índice de base de datos", "task.database.index.fix": "Ejecutar corrección del índice de la base de datos", + "task.database.cache": "Ejecutar caché de base de datos", "task.ocr.image": "Ejecutar OCR de imagen para extraer texto", "task.history.generateDoc": "Ejecutar Historial GenerateDoc", "task.database.index.embedBlock": "Ejecutar bloque de incrustación de índice de base de datos", diff --git a/app/appearance/langs/fr_FR.json b/app/appearance/langs/fr_FR.json index d14cd6722..a8086a4c1 100644 --- a/app/appearance/langs/fr_FR.json +++ b/app/appearance/langs/fr_FR.json @@ -876,6 +876,7 @@ "task.database.index.commit": "Effectuer la validation de l'index de la base de données", "task.database.index.ref": "Exécuter la référence d'index de la base de données", "task.database.index.fix": "Effectuer la correction de l'index de la base de données", + "task.database.cache": "Effectuer le cache de la base de données", "task.ocr.image": "Exécute l'OCR d'image pour extraire le texte", "task.history.generateDoc": "Exécuter l'historique de GenerateDoc", "task.database.index.embedBlock": "Exécuter le bloc d'intégration d'index de base de données", diff --git a/app/appearance/langs/zh_CHT.json b/app/appearance/langs/zh_CHT.json index 3cea716ad..9b437d3c0 100644 --- a/app/appearance/langs/zh_CHT.json +++ b/app/appearance/langs/zh_CHT.json @@ -876,6 +876,7 @@ "task.database.index.commit": "執行數據庫索引提交", "task.database.index.ref": "執行數據庫索引引用", "task.database.index.fix": "執行數據庫索引訂正", + "task.database.cache": "執行數據庫緩存", "task.ocr.image": "執行圖片 OCR 提取文本", "task.history.generateDoc": "執行生成文件歷史", "task.database.index.embedBlock": "執行數據庫索引嵌入塊", diff --git a/app/appearance/langs/zh_CN.json b/app/appearance/langs/zh_CN.json index a61cad21c..e03dae7c2 100644 --- a/app/appearance/langs/zh_CN.json +++ b/app/appearance/langs/zh_CN.json @@ -876,6 +876,7 @@ "task.database.index.commit": "执行数据库索引提交", "task.database.index.ref": "执行数据库索引引用", "task.database.index.fix": "执行数据库索引订正", + "task.database.cache": "执行数据库缓存", "task.ocr.image": "执行图片 OCR 提取文本", "task.history.generateDoc": "执行生成文件历史", "task.database.index.embedBlock": "执行数据库索引嵌入块", diff --git a/kernel/model/box.go b/kernel/model/box.go index b9bb46da4..1cad8fa81 100644 --- a/kernel/model/box.go +++ b/kernel/model/box.go @@ -506,6 +506,9 @@ func ReloadUI() { func FullReindex() { task.PrependTask(task.DatabaseIndexFull, fullReindex) + task.AppendTask(task.DatabaseCache, sql.EnableCache) + task.AppendTask(task.DatabaseIndexRef, IndexRefs) + task.AppendTask(task.ReloadUI, util.ReloadUI) } func fullReindex() { @@ -523,12 +526,9 @@ func fullReindex() { for _, openedBox := range openedBoxes { index(openedBox.ID) } - sql.EnableCache() treenode.SaveBlockTree(true) LoadFlashcards() - IndexRefs() debug.FreeOSMemory() - util.ReloadUI() } func ChangeBoxSort(boxIDs []string) { diff --git a/kernel/model/history.go b/kernel/model/history.go index 1032b94fb..05ec3ed88 100644 --- a/kernel/model/history.go +++ b/kernel/model/history.go @@ -244,7 +244,6 @@ func RollbackDocHistory(boxID, historyPath string) (err error) { FullReindex() IncSync() - ReloadUI() return nil } diff --git a/kernel/model/import.go b/kernel/model/import.go index 88beaf952..bdb35caad 100644 --- a/kernel/model/import.go +++ b/kernel/model/import.go @@ -441,7 +441,6 @@ func ImportData(zipPath string) (err error) { IncSync() FullReindex() - ReloadUI() return } diff --git a/kernel/model/repository.go b/kernel/model/repository.go index 3a8f09424..6b42965e6 100644 --- a/kernel/model/repository.go +++ b/kernel/model/repository.go @@ -541,7 +541,6 @@ func checkoutRepo(id string) { } FullReindex() - ReloadUI() if syncEnabled { func() { @@ -1028,9 +1027,6 @@ func syncRepo(exit, byHand bool) (err error) { cache.ClearDocsIAL() // 同步后文档树文档图标没有更新 https://github.com/siyuan-note/siyuan/issues/4939 if needFullReindex(upsertTrees) { // 改进同步后全量重建索引判断 https://github.com/siyuan-note/siyuan/issues/5764 FullReindex() - if !exit { - ReloadUI() - } return } diff --git a/kernel/model/sync.go b/kernel/model/sync.go index e8462d68f..f4376d63b 100644 --- a/kernel/model/sync.go +++ b/kernel/model/sync.go @@ -97,7 +97,9 @@ func SyncData(boot, exit, byHand bool) { func syncData(boot, exit, byHand bool) { defer logging.Recover() + util.BroadcastByType("main", "syncing", 0, Conf.Language(81), nil) if !checkSync(boot, exit, byHand) { + util.BroadcastByType("main", "syncing", 1, "", nil) return } @@ -118,7 +120,6 @@ func syncData(boot, exit, byHand bool) { now := util.CurrentTimeMillis() Conf.Sync.Synced = now - util.BroadcastByType("main", "syncing", 0, Conf.Language(81), nil) err := syncRepo(exit, byHand) synced := util.Millisecond2Time(Conf.Sync.Synced).Format("2006-01-02 15:04:05") + "\n\n" if nil == err { diff --git a/kernel/task/queue.go b/kernel/task/queue.go index 32d8dec4a..45c7b2935 100644 --- a/kernel/task/queue.go +++ b/kernel/task/queue.go @@ -79,6 +79,7 @@ const ( DatabaseIndexCommit = "task.database.index.commit" // 数据库索引提交 DatabaseIndexRef = "task.database.index.ref" // 数据库索引引用 DatabaseIndexFix = "task.database.index.fix" // 数据库索引订正 + DatabaseCache = "task.database.cache" // 数据库缓存 OCRImage = "task.ocr.image" // 图片 OCR 提取文本 HistoryGenerateDoc = "task.history.generateDoc" // 生成文件历史 DatabaseIndexEmbedBlock = "task.database.index.embedBlock" // 数据库索引嵌入块 @@ -105,8 +106,8 @@ func StatusJob() { for _, task := range tasks { actionLangs := util.TaskActionLangs[util.Lang] action := task.Action - if c := count[action]; 3 < c { - logging.LogWarnf("too many tasks [%s], ignore show its status", action) + if c := count[action]; 2 < c { + //logging.LogWarnf("too many tasks [%s], ignore show its status", action) continue } count[action]++ @@ -165,7 +166,7 @@ func execTask(task *Task) { } } - ctx, cancel := context.WithTimeout(context.Background(), time.Second*7) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) defer cancel() ch := make(chan bool, 1) go func() { @@ -175,7 +176,7 @@ func execTask(task *Task) { select { case <-ctx.Done(): - //logging.LogWarnf("task [%s] timeout", task.Action) + logging.LogWarnf("task [%s] timeout", task.Action) case <-ch: //logging.LogInfof("task [%s] done", task.Action) } From 9a7a155ac710b9b3904082c9d0909156f0eeb8d0 Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Mon, 6 Feb 2023 22:17:39 +0800 Subject: [PATCH 02/11] =?UTF-8?q?:art:=20=E5=90=8E=E5=8F=B0=E4=BB=BB?= =?UTF-8?q?=E5=8A=A1=E6=98=BE=E7=A4=BA=E5=BD=93=E5=89=8D=E6=AD=A3=E5=9C=A8?= =?UTF-8?q?=E6=89=A7=E8=A1=8C=E7=9A=84=E4=BB=BB=E5=8A=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/task/queue.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/kernel/task/queue.go b/kernel/task/queue.go index 45c7b2935..b73601226 100644 --- a/kernel/task/queue.go +++ b/kernel/task/queue.go @@ -121,6 +121,11 @@ func StatusJob() { item := map[string]interface{}{"action": action} items = append(items, item) } + + if "" != currentTaskAction { + items = append([]map[string]interface{}{map[string]interface{}{"action": currentTaskAction}}, items...) + } + if 1 > len(items) { items = []map[string]interface{}{} } @@ -141,6 +146,8 @@ func ExecTaskJob() { execTask(task) } +var currentTaskAction string + func popTask() (ret *Task) { queueLock.Lock() defer queueLock.Unlock() @@ -166,6 +173,8 @@ func execTask(task *Task) { } } + currentTaskAction = task.Action + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) defer cancel() ch := make(chan bool, 1) From 9f491713ad105fa4a9fbee38d45e74fa785f6d81 Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Mon, 6 Feb 2023 22:24:48 +0800 Subject: [PATCH 03/11] =?UTF-8?q?:art:=20=E5=90=8E=E5=8F=B0=E4=BB=BB?= =?UTF-8?q?=E5=8A=A1=E6=98=BE=E7=A4=BA=E5=BD=93=E5=89=8D=E6=AD=A3=E5=9C=A8?= =?UTF-8?q?=E6=89=A7=E8=A1=8C=E7=9A=84=E4=BB=BB=E5=8A=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/task/queue.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/kernel/task/queue.go b/kernel/task/queue.go index b73601226..ad74551f8 100644 --- a/kernel/task/queue.go +++ b/kernel/task/queue.go @@ -100,11 +100,10 @@ func Contain(action string, moreActions ...string) bool { func StatusJob() { tasks := taskQueue - data := map[string]interface{}{} var items []map[string]interface{} count := map[string]int{} + actionLangs := util.TaskActionLangs[util.Lang] for _, task := range tasks { - actionLangs := util.TaskActionLangs[util.Lang] action := task.Action if c := count[action]; 2 < c { //logging.LogWarnf("too many tasks [%s], ignore show its status", action) @@ -123,12 +122,17 @@ func StatusJob() { } if "" != currentTaskAction { - items = append([]map[string]interface{}{map[string]interface{}{"action": currentTaskAction}}, items...) + if nil != actionLangs { + if label := actionLangs[currentTaskAction]; nil != label { + items = append([]map[string]interface{}{map[string]interface{}{"action": label.(string)}}, items...) + } + } } if 1 > len(items) { items = []map[string]interface{}{} } + data := map[string]interface{}{} data["tasks"] = items util.PushBackgroundTask(data) } From 8a7df8af788e8646c15105a553c8ef62cffa120b Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Tue, 7 Feb 2023 09:20:32 +0800 Subject: [PATCH 04/11] =?UTF-8?q?:art:=20=E5=90=8E=E5=8F=B0=E4=BB=BB?= =?UTF-8?q?=E5=8A=A1=E6=8C=89=E4=BB=BB=E5=8A=A1=E5=8A=A0=E5=85=A5=E5=85=88?= =?UTF-8?q?=E5=90=8E=E9=A1=BA=E5=BA=8F=E5=8E=BB=E9=87=8D=E6=89=A7=E8=A1=8C?= =?UTF-8?q?=20Fix=20https://github.com/siyuan-note/siyuan/issues/7270?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/model/box.go | 2 +- kernel/model/file.go | 2 +- kernel/model/history.go | 2 +- kernel/model/index.go | 4 +-- kernel/model/repository.go | 2 +- kernel/task/queue.go | 50 ++++++++++++++++++++++++++------------ 6 files changed, 40 insertions(+), 22 deletions(-) diff --git a/kernel/model/box.go b/kernel/model/box.go index 1cad8fa81..20e7c7710 100644 --- a/kernel/model/box.go +++ b/kernel/model/box.go @@ -505,7 +505,7 @@ func ReloadUI() { } func FullReindex() { - task.PrependTask(task.DatabaseIndexFull, fullReindex) + task.AppendTask(task.DatabaseIndexFull, fullReindex) task.AppendTask(task.DatabaseCache, sql.EnableCache) task.AppendTask(task.DatabaseIndexRef, IndexRefs) task.AppendTask(task.ReloadUI, util.ReloadUI) diff --git a/kernel/model/file.go b/kernel/model/file.go index 1483d25d9..a9fec8522 100644 --- a/kernel/model/file.go +++ b/kernel/model/file.go @@ -1256,7 +1256,7 @@ func removeDoc(box *Box, p string) { } util.PushEvent(evt) - task.PrependTask(task.DatabaseIndex, removeDoc0, box, p, childrenDir) + task.AppendTask(task.DatabaseIndex, removeDoc0, box, p, childrenDir) } func removeDoc0(box *Box, p, childrenDir string) { diff --git a/kernel/model/history.go b/kernel/model/history.go index 05ec3ed88..046cea6df 100644 --- a/kernel/model/history.go +++ b/kernel/model/history.go @@ -50,7 +50,7 @@ func AutoGenerateDocHistory() { ChangeHistoryTick(Conf.Editor.GenerateHistoryInterval) for { <-historyTicker.C - task.PrependTask(task.HistoryGenerateDoc, generateDocHistory) + task.AppendTask(task.HistoryGenerateDoc, generateDocHistory) } } diff --git a/kernel/model/index.go b/kernel/model/index.go index 533648040..2d321826b 100644 --- a/kernel/model/index.go +++ b/kernel/model/index.go @@ -39,7 +39,7 @@ import ( ) func (box *Box) Unindex() { - task.PrependTask(task.DatabaseIndex, unindex, box.ID) + task.AppendTask(task.DatabaseIndex, unindex, box.ID) } func unindex(boxID string) { @@ -49,7 +49,7 @@ func unindex(boxID string) { } func (box *Box) Index() { - task.PrependTask(task.DatabaseIndex, index, box.ID) + task.AppendTask(task.DatabaseIndex, index, box.ID) task.AppendTask(task.DatabaseIndexRef, IndexRefs) } diff --git a/kernel/model/repository.go b/kernel/model/repository.go index 6b42965e6..5238b6f5b 100644 --- a/kernel/model/repository.go +++ b/kernel/model/repository.go @@ -504,7 +504,7 @@ func InitRepoKey() (err error) { } func CheckoutRepo(id string) { - task.PrependTask(task.RepoCheckout, checkoutRepo, id) + task.AppendTask(task.RepoCheckout, checkoutRepo, id) } func checkoutRepo(id string) { diff --git a/kernel/task/queue.go b/kernel/task/queue.go index ad74551f8..99c3324a3 100644 --- a/kernel/task/queue.go +++ b/kernel/task/queue.go @@ -39,27 +39,20 @@ type Task struct { Created time.Time } -func PrependTask(action string, handler interface{}, args ...interface{}) { - queueLock.Lock() - defer queueLock.Unlock() - - if util.IsExiting { - //logging.LogWarnf("task queue is paused, action [%s] will be ignored", action) - return - } - - taskQueue = append([]*Task{newTask(action, handler, args...)}, taskQueue...) -} - func AppendTask(action string, handler interface{}, args ...interface{}) { - queueLock.Lock() - defer queueLock.Unlock() - if util.IsExiting { //logging.LogWarnf("task queue is paused, action [%s] will be ignored", action) return } + currentActions := getCurrentActions() + if gulu.Str.Contains(action, currentActions) && gulu.Str.Contains(action, uniqueActions) { + //logging.LogWarnf("task [%s] is already in queue, will be ignored", action) + return + } + + queueLock.Lock() + defer queueLock.Unlock() taskQueue = append(taskQueue, newTask(action, handler, args...)) } @@ -72,6 +65,20 @@ func newTask(action string, handler interface{}, args ...interface{}) *Task { } } +func getCurrentActions() (ret []string) { + queueLock.Lock() + defer queueLock.Unlock() + + if "" != currentTaskAction { + ret = append(ret, currentTaskAction) + } + + for _, task := range taskQueue { + ret = append(ret, task.Action) + } + return +} + const ( RepoCheckout = "task.repo.checkout" // 从快照中检出 DatabaseIndexFull = "task.database.index.full" // 重建索引 @@ -86,6 +93,17 @@ const ( ReloadUI = "task.reload.ui" // 重载 UI ) +// uniqueActions 描述了唯一的任务，即队列中只能存在一个在执行的任务。 +var uniqueActions = []string{ + RepoCheckout, + DatabaseIndexFull, + DatabaseIndexCommit, + DatabaseIndexFix, + OCRImage, + HistoryGenerateDoc, + DatabaseIndexEmbedBlock, +} + func Contain(action string, moreActions ...string) bool { actions := append(moreActions, action) actions = gulu.Str.RemoveDuplicatedElem(actions) @@ -106,7 +124,7 @@ func StatusJob() { for _, task := range tasks { action := task.Action if c := count[action]; 2 < c { - //logging.LogWarnf("too many tasks [%s], ignore show its status", action) + logging.LogWarnf("too many tasks [%s], ignore show its status", action) continue } count[action]++ From bf31293395a45219f92645975fe3e41136b3a8d7 Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Tue, 7 Feb 2023 09:32:07 +0800 Subject: [PATCH 05/11] =?UTF-8?q?:art:=20=E8=BF=81=E5=87=BA=E5=BF=AB?= =?UTF-8?q?=E7=85=A7=E5=92=8C=E5=90=8C=E6=AD=A5=E6=95=B0=E6=8D=AE=E7=94=9F?= =?UTF-8?q?=E6=88=90=E5=86=B2=E7=AA=81=E5=89=AF=E6=9C=AC=E4=B8=8D=E5=86=8D?= =?UTF-8?q?=E7=AD=89=E5=BE=85=E6=95=B0=E6=8D=AE=E5=BA=93=E4=BA=8B=E5=8A=A1?= =?UTF-8?q?=E6=89=A7=E8=A1=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/model/repository.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/kernel/model/repository.go b/kernel/model/repository.go index 5238b6f5b..39c1ab21b 100644 --- a/kernel/model/repository.go +++ b/kernel/model/repository.go @@ -49,7 +49,6 @@ import ( "github.com/siyuan-note/siyuan/kernel/cache" "github.com/siyuan-note/siyuan/kernel/conf" "github.com/siyuan-note/siyuan/kernel/filesys" - "github.com/siyuan-note/siyuan/kernel/sql" "github.com/siyuan-note/siyuan/kernel/task" "github.com/siyuan-note/siyuan/kernel/treenode" "github.com/siyuan-note/siyuan/kernel/util" @@ -523,7 +522,6 @@ func checkoutRepo(id string) { util.PushEndlessProgress(Conf.Language(63)) WaitForWritingFiles() - sql.WaitForWritingDatabase() CloseWatchAssets() defer WatchAssets() @@ -941,7 +939,6 @@ func syncRepo(exit, byHand bool) (err error) { // 云端同步发生冲突时生成副本 https://github.com/siyuan-note/siyuan/issues/5687 luteEngine := NewLute() - waitTx := false for _, file := range mergeResult.Conflicts { if !strings.HasSuffix(file.Path, ".sy") { continue @@ -964,10 +961,6 @@ func syncRepo(exit, byHand bool) (err error) { resetTree(tree, "Conflicted") createTreeTx(tree) - waitTx = true - } - if waitTx { - sql.WaitForWritingDatabase() } } From daaa9e67fd25c5514abaac5c813fd3d7495efd04 Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Tue, 7 Feb 2023 09:58:13 +0800 Subject: [PATCH 06/11] =?UTF-8?q?:art:=20=E6=95=B0=E6=8D=AE=E8=AE=A2?= =?UTF-8?q?=E6=AD=A3=E5=88=86=E6=AD=A5=E6=89=A7=E8=A1=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/model/index_fix.go | 41 ++++++++++++++++++++++++--------------- kernel/task/queue.go | 3 +-- 2 files changed, 26 insertions(+), 18 deletions(-) diff --git a/kernel/model/index_fix.go b/kernel/model/index_fix.go index a63cd8d53..65b2fc920 100644 --- a/kernel/model/index_fix.go +++ b/kernel/model/index_fix.go @@ -39,20 +39,27 @@ import ( // FixIndexJob 自动校验数据库索引 https://github.com/siyuan-note/siyuan/issues/7016 func FixIndexJob() { - task.AppendTask(task.DatabaseIndexFix, autoFixIndex) + task.AppendTask(task.DatabaseIndexFix, removeDuplicateDatabaseIndex) + sql.WaitForWritingDatabase() + task.AppendTask(task.DatabaseIndexFix, fixBlockTreeByFileSys) + sql.WaitForWritingDatabase() + task.AppendTask(task.DatabaseIndexFix, fixDatabaseIndexByBlockTree) + + sql.WaitForWritingDatabase() + util.PushStatusBar(Conf.Language(185)) + debug.FreeOSMemory() } var autoFixLock = sync.Mutex{} -func autoFixIndex() { +// removeDuplicateDatabaseIndex 删除重复的数据库索引。 +func removeDuplicateDatabaseIndex() { defer logging.Recover() autoFixLock.Lock() defer autoFixLock.Unlock() util.PushStatusBar(Conf.Language(58)) - - // 去除重复的数据库块记录 duplicatedRootIDs := sql.GetDuplicatedRootIDs("blocks") if 1 > len(duplicatedRootIDs) { duplicatedRootIDs = sql.GetDuplicatedRootIDs("blocks_fts") @@ -87,11 +94,16 @@ func autoFixIndex() { if 0 < deletes { logging.LogWarnf("exist more than one tree duplicated [%d], reindex it", deletes) } +} + +// fixBlockTreeByFileSys 通过文件系统订正块树。 +func fixBlockTreeByFileSys() { + defer logging.Recover() + + autoFixLock.Lock() + defer autoFixLock.Unlock() util.PushStatusBar(Conf.Language(58)) - sql.WaitForWritingDatabase() - util.PushStatusBar(Conf.Language(58)) - // 根据文件系统补全块树 boxes := Conf.GetOpenedBoxes() for _, box := range boxes { boxPath := filepath.Join(util.DataDir, box.ID) @@ -130,26 +142,23 @@ func autoFixIndex() { } } - util.PushStatusBar(Conf.Language(58)) - sql.WaitForWritingDatabase() - util.PushStatusBar(Conf.Language(58)) // 清理已关闭的笔记本块树 boxes = Conf.GetClosedBoxes() for _, box := range boxes { treenode.RemoveBlockTreesByBoxID(box.ID) } +} - // 对比块树和数据库并订正数据库 +// fixDatabaseIndexByBlockTree 通过块树订正数据库索引。 +func fixDatabaseIndexByBlockTree() { + defer logging.Recover() + + util.PushStatusBar(Conf.Language(58)) rootUpdatedMap := treenode.GetRootUpdated() dbRootUpdatedMap, err := sql.GetRootUpdated() if nil == err { reindexTreeByUpdated(rootUpdatedMap, dbRootUpdatedMap) } - - util.PushStatusBar(Conf.Language(58)) - sql.WaitForWritingDatabase() - util.PushStatusBar(Conf.Language(185)) - debug.FreeOSMemory() } func reindexTreeByUpdated(rootUpdatedMap, dbRootUpdatedMap map[string]string) { diff --git a/kernel/task/queue.go b/kernel/task/queue.go index 99c3324a3..d17391d05 100644 --- a/kernel/task/queue.go +++ b/kernel/task/queue.go @@ -98,7 +98,6 @@ var uniqueActions = []string{ RepoCheckout, DatabaseIndexFull, DatabaseIndexCommit, - DatabaseIndexFix, OCRImage, HistoryGenerateDoc, DatabaseIndexEmbedBlock, @@ -197,7 +196,7 @@ func execTask(task *Task) { currentTaskAction = task.Action - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() ch := make(chan bool, 1) go func() { From 8aaf5567673ba0f572ddc33b2df7b0a8ffe5b11a Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Tue, 7 Feb 2023 10:00:15 +0800 Subject: [PATCH 07/11] =?UTF-8?q?:art:=20=E6=95=B0=E6=8D=AE=E8=AE=A2?= =?UTF-8?q?=E6=AD=A3=E5=88=86=E6=AD=A5=E6=89=A7=E8=A1=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/task/queue.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/kernel/task/queue.go b/kernel/task/queue.go index d17391d05..ac178affb 100644 --- a/kernel/task/queue.go +++ b/kernel/task/queue.go @@ -210,4 +210,6 @@ func execTask(task *Task) { case <-ch: //logging.LogInfof("task [%s] done", task.Action) } + + currentTaskAction = "" } From 4041af465b783ea07c460298e1b01dec56981958 Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Tue, 7 Feb 2023 10:05:20 +0800 Subject: [PATCH 08/11] =?UTF-8?q?:art:=20=E6=94=AF=E6=8C=81=E8=BD=AC?= =?UTF-8?q?=E7=A7=BB=E5=BC=95=E7=94=A8=20https://github.com/siyuan-note/si?= =?UTF-8?q?yuan/issues/4025?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/api/block.go | 7 +++++++ kernel/model/index_fix.go | 4 +++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/kernel/api/block.go b/kernel/api/block.go index 3e5ac4a37..5301270c2 100644 --- a/kernel/api/block.go +++ b/kernel/api/block.go @@ -40,7 +40,14 @@ func transferBlockRef(c *gin.Context) { } fromID := arg["fromID"].(string) + if util.InvalidIDPattern(fromID, ret) { + return + } toID := arg["toID"].(string) + if util.InvalidIDPattern(toID, ret) { + return + } + err := model.TransferBlockRef(fromID, toID) if nil != err { ret.Code = -1 diff --git a/kernel/model/index_fix.go b/kernel/model/index_fix.go index 65b2fc920..67a77f0fa 100644 --- a/kernel/model/index_fix.go +++ b/kernel/model/index_fix.go @@ -41,11 +41,13 @@ import ( func FixIndexJob() { task.AppendTask(task.DatabaseIndexFix, removeDuplicateDatabaseIndex) sql.WaitForWritingDatabase() + task.AppendTask(task.DatabaseIndexFix, fixBlockTreeByFileSys) sql.WaitForWritingDatabase() - task.AppendTask(task.DatabaseIndexFix, fixDatabaseIndexByBlockTree) + task.AppendTask(task.DatabaseIndexFix, fixDatabaseIndexByBlockTree) sql.WaitForWritingDatabase() + util.PushStatusBar(Conf.Language(185)) debug.FreeOSMemory() } From 209a853040bfadb1348507882f3841536e6254c5 Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Tue, 7 Feb 2023 10:11:38 +0800 Subject: [PATCH 09/11] =?UTF-8?q?:art:=20=E6=95=B0=E6=8D=AE=E5=BA=93?= =?UTF-8?q?=E4=BA=8B=E5=8A=A1=E9=98=9F=E5=88=97=E6=89=A7=E8=A1=8C=E6=8A=A5?= =?UTF-8?q?=E9=94=99=E5=9B=9E=E6=BB=9A=E9=87=8A=E6=94=BE=E8=BF=9E=E6=8E=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/sql/queue.go | 1 + 1 file changed, 1 insertion(+) diff --git a/kernel/sql/queue.go b/kernel/sql/queue.go index 8ab363c01..25447d197 100644 --- a/kernel/sql/queue.go +++ b/kernel/sql/queue.go @@ -118,6 +118,7 @@ func FlushQueue() { context["current"] = i context["total"] = total if err = execOp(op, tx, context); nil != err { + tx.Rollback() logging.LogErrorf("queue operation failed: %s", err) return } From c5c9884d3881db02758f078e4f15c7942c3a60d8 Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Tue, 7 Feb 2023 10:40:24 +0800 Subject: [PATCH 10/11] =?UTF-8?q?:art:=20=E6=95=B0=E6=8D=AE=E5=BA=93?= =?UTF-8?q?=E4=BA=8B=E5=8A=A1=E9=98=9F=E5=88=97=E6=89=A7=E8=A1=8C=E6=8A=A5?= =?UTF-8?q?=E9=94=99=E5=9B=9E=E6=BB=9A=E9=87=8A=E6=94=BE=E8=BF=9E=E6=8E=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/sql/queue.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/sql/queue.go b/kernel/sql/queue.go index 25447d197..7ac078ae2 100644 --- a/kernel/sql/queue.go +++ b/kernel/sql/queue.go @@ -120,7 +120,7 @@ func FlushQueue() { if err = execOp(op, tx, context); nil != err { tx.Rollback() logging.LogErrorf("queue operation failed: %s", err) - return + continue } if err = commitTx(tx); nil != err { From 63303cec81a7abd1d35599261bcf01549f525ffd Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Tue, 7 Feb 2023 10:58:39 +0800 Subject: [PATCH 11/11] =?UTF-8?q?:bug:=20=E7=A7=BB=E5=8A=A8=E6=96=87?= =?UTF-8?q?=E6=A1=A3=E6=97=B6=20hpath=20=E6=9C=AA=E8=B7=9F=E9=9A=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/treenode/tree.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/treenode/tree.go b/kernel/treenode/tree.go index 9274069a4..27c9b2bf1 100644 --- a/kernel/treenode/tree.go +++ b/kernel/treenode/tree.go @@ -54,7 +54,7 @@ func NodeHash(node *ast.Node, tree *parse.Tree, luteEngine *lute.Lute) string { md = FormatNode(node, luteEngine) } hpath := tree.HPath - data := tree.Path + hpath + string(ial) + md + data := tree.Box + tree.Path + hpath + string(ial) + md return fmt.Sprintf("%x", sha256.Sum256(gulu.Str.ToBytes(data)))[:7] }