From 6a442ad85d42d2ead32909c8038cf2187c2bcbd6 Mon Sep 17 00:00:00 2001 From: Daniel <845765@qq.com> Date: Sun, 17 Nov 2024 01:33:21 +0800 Subject: [PATCH] :art: Automatic purge for local data repo https://github.com/siyuan-note/siyuan/issues/13091 --- app/appearance/langs/de_DE.json | 2 + app/appearance/langs/en_US.json | 2 + app/appearance/langs/es_ES.json | 2 + app/appearance/langs/fr_FR.json | 2 + app/appearance/langs/he_IL.json | 2 + app/appearance/langs/it_IT.json | 2 + app/appearance/langs/ja_JP.json | 2 + app/appearance/langs/pl_PL.json | 2 + app/appearance/langs/ru_RU.json | 2 + app/appearance/langs/zh_CHT.json | 2 + app/appearance/langs/zh_CN.json | 2 + app/src/config/about.ts | 43 +++++++++-- app/src/mobile/settings/about.ts | 18 +++++ app/src/types/config.d.ts | 10 ++- kernel/api/repo.go | 34 +++++++++ kernel/api/router.go | 2 + kernel/conf/repo.go | 8 +- kernel/go.mod | 2 +- kernel/go.sum | 4 +- kernel/job/cron.go | 1 + kernel/model/conf.go | 6 ++ kernel/model/repository.go | 122 ++++++++++++++++++++++++++++++- kernel/task/queue.go | 2 + 23 files changed, 259 insertions(+), 15 deletions(-) diff --git a/app/appearance/langs/de_DE.json b/app/appearance/langs/de_DE.json index 0f423d316..a320762c7 100644 --- a/app/appearance/langs/de_DE.json +++ b/app/appearance/langs/de_DE.json @@ -1,4 +1,6 @@ { + "dataRepoAutoPurgeIndexRetentionDays": "Daten-Snapshot-Aufbewahrungstage", + "dataRepoAutoPurgeRetentionIndexesDaily": "Daten-Snapshots pro Tag", "fields": "Attribut", "dynamicEmoji": "Dynamisches Icon", "backlinkContainChildren": "Enthalten Rückverweise untergeordnete Blöcke", diff --git a/app/appearance/langs/en_US.json b/app/appearance/langs/en_US.json index a0d6698ac..4407659dc 100644 --- a/app/appearance/langs/en_US.json +++ b/app/appearance/langs/en_US.json @@ -1,4 +1,6 @@ { + "dataRepoAutoPurgeIndexRetentionDays": "Data snapshot retention days", + "dataRepoAutoPurgeRetentionIndexesDaily": "Data snapshots per day", "fields": "Fields", "dynamicEmoji": "Dynamic icon", "backlinkContainChildren": "Do backlinks contain child blocks", diff --git a/app/appearance/langs/es_ES.json b/app/appearance/langs/es_ES.json index 9de756a24..c3f922b5f 100644 --- a/app/appearance/langs/es_ES.json +++ b/app/appearance/langs/es_ES.json @@ -1,4 +1,6 @@ { + "dataRepoAutoPurgeIndexRetentionDays": "Días de retención de instantáneas de datos", + "dataRepoAutoPurgeRetentionIndexesDaily": "Número de instantáneas de datos por día", "fields": "Atributo", "dynamicEmoji": "Icono dinámico", "backlinkContainChildren": "¿Los enlaces inversos contienen bloques secundarios?", diff --git a/app/appearance/langs/fr_FR.json b/app/appearance/langs/fr_FR.json index 8a66aebc7..6f00edb46 100644 --- a/app/appearance/langs/fr_FR.json +++ b/app/appearance/langs/fr_FR.json @@ -1,4 +1,6 @@ { + "dataRepoAutoPurgeIndexRetentionDays": "Jours de rétention des instantanés de données", + "dataRepoAutoPurgeRetentionIndexesDaily": "Nombre d'instantanés de données par jour", "fields": "Attribut", "dynamicEmoji": "Icône dynamique", "backlinkContainChildren": "Les liens retour contiennent-ils des sous-blocs", diff --git a/app/appearance/langs/he_IL.json b/app/appearance/langs/he_IL.json index 1f9dcbfa3..65ac52aa9 100644 --- a/app/appearance/langs/he_IL.json +++ b/app/appearance/langs/he_IL.json @@ -1,4 +1,6 @@ { + "dataRepoAutoPurgeIndexRetentionDays": "ימי שמירת תמונות נתונים", + "dataRepoAutoPurgeRetentionIndexesDaily": "מספר תמונות נתונים ביום", "fields": "מאפיין", "dynamicEmoji": "אייקון דינמי", "backlinkContainChildren": "האם קישורים חוזרים כוללים בלוקים משניים", diff --git a/app/appearance/langs/it_IT.json b/app/appearance/langs/it_IT.json index 1bdacf124..e03d0f7ba 100644 --- a/app/appearance/langs/it_IT.json +++ b/app/appearance/langs/it_IT.json @@ -1,4 +1,6 @@ { + "dataRepoAutoPurgeIndexRetentionDays": "Giorni di conservazione degli snapshot dei dati", + "dataRepoAutoPurgeRetentionIndexesDaily": "Numero di snapshot dei dati al giorno", "fields": "Campi", "dynamicEmoji": "Emoji dinamica", "backlinkContainChildren": "I backlink contengono blocchi figli", diff --git a/app/appearance/langs/ja_JP.json b/app/appearance/langs/ja_JP.json index 1384f2598..b3d94e9af 100644 --- a/app/appearance/langs/ja_JP.json +++ b/app/appearance/langs/ja_JP.json @@ -1,4 +1,6 @@ { + "dataRepoAutoPurgeIndexRetentionDays": "データスナップショットの保持日数", + "dataRepoAutoPurgeRetentionIndexesDaily": "データスナップショットの毎日の保持数", "fields": "属性", "dynamicEmoji": "動的アイコン", "backlinkContainChildren": "バックリンクに子ブロックを含めるかどうか", diff --git a/app/appearance/langs/pl_PL.json b/app/appearance/langs/pl_PL.json index aab82c0ec..478b1f1a1 100644 --- a/app/appearance/langs/pl_PL.json +++ b/app/appearance/langs/pl_PL.json @@ -1,4 +1,6 @@ { + "dataRepoAutoPurgeIndexRetentionDays": "Dni przechowywania migawek danych", + "dataRepoAutoPurgeRetentionIndexesDaily": "Liczba migawek danych dziennie", "fields": "Atrybut", "dynamicEmoji": "Ikona dynamiczna", "backlinkContainChildren": "Czy linki zwrotne zawierają bloki podrzędne", diff --git a/app/appearance/langs/ru_RU.json b/app/appearance/langs/ru_RU.json index 1c2b24acd..650f6b19f 100644 --- a/app/appearance/langs/ru_RU.json +++ b/app/appearance/langs/ru_RU.json @@ -1,4 +1,6 @@ { + "dataRepoAutoPurgeIndexRetentionDays": "Срок хранения снимков данных", + "dataRepoAutoPurgeRetentionIndexesDaily": "Количество снимков данных в день", "fields": "Атрибут", "dynamicEmoji": "Динамическая иконка", "backlinkContainChildren": "Включать ли дочерние блоки в обратные ссылки", diff --git a/app/appearance/langs/zh_CHT.json b/app/appearance/langs/zh_CHT.json index 3bbe9fe3c..3f373e320 100644 --- a/app/appearance/langs/zh_CHT.json +++ b/app/appearance/langs/zh_CHT.json @@ -1,4 +1,6 @@ { + "dataRepoAutoPurgeIndexRetentionDays": "數據快照保留天數", + "dataRepoAutoPurgeRetentionIndexesDaily": "數據快照每天保留個數", "fields": "欄位", "dynamicEmoji": "動態圖標", "backlinkContainChildren": "反向鏈接包含子塊", diff --git a/app/appearance/langs/zh_CN.json b/app/appearance/langs/zh_CN.json index 5511c0693..fb2deba66 100644 --- a/app/appearance/langs/zh_CN.json +++ b/app/appearance/langs/zh_CN.json @@ -1,4 +1,6 @@ { + "dataRepoAutoPurgeIndexRetentionDays": "数据快照保留天数", + "dataRepoAutoPurgeRetentionIndexesDaily": "数据快照每天保留个数", "fields": "字段", "dynamicEmoji": "动态图标", "backlinkContainChildren": "反向链接包含子块", diff --git a/app/src/config/about.ts b/app/src/config/about.ts index 767b0b6c3..6f741abee 100644 --- a/app/src/config/about.ts +++ b/app/src/config/about.ts @@ -142,16 +142,31 @@ export const about = { -
-
+
+
${window.siyuan.languages.dataRepoPurge} -
${window.siyuan.languages.dataRepoPurgeTip}
-
- -
+
+
+
${window.siyuan.languages.dataRepoPurgeTip}
+ + +
+
+
+
${window.siyuan.languages.dataRepoAutoPurgeIndexRetentionDays}
+ + +
+
+
+
${window.siyuan.languages.dataRepoAutoPurgeRetentionIndexesDaily}
+ + +
+
${window.siyuan.languages.systemLog} @@ -208,6 +223,18 @@ ${checkUpdateHTML} if (window.siyuan.config.system.isInsider) { about.element.querySelector("#isInsider").innerHTML = "Insider Preview"; } + const indexRetentionDaysElement = about.element.querySelector("#indexRetentionDays") as HTMLInputElement; + indexRetentionDaysElement.addEventListener("change", () => { + fetchPost("/api/repo/setRepoIndexRetentionDays", {days: parseInt(indexRetentionDaysElement.value)}, () => { + window.siyuan.config.repo.indexRetentionDays = parseInt(indexRetentionDaysElement.value); + }); + }); + const retentionIndexesDailyElement = about.element.querySelector("#retentionIndexesDaily") as HTMLInputElement; + retentionIndexesDailyElement.addEventListener("change", () => { + fetchPost("/api/repo/setRetentionIndexesDaily", {indexes: parseInt(retentionIndexesDailyElement.value)}, () => { + window.siyuan.config.repo.retentionIndexesDaily = parseInt(retentionIndexesDailyElement.value); + }); + }); const tokenElement = about.element.querySelector("#token") as HTMLInputElement; tokenElement.addEventListener("click", () => { tokenElement.select(); diff --git a/app/src/mobile/settings/about.ts b/app/src/mobile/settings/about.ts index 6a97b02a2..21f749c27 100644 --- a/app/src/mobile/settings/about.ts +++ b/app/src/mobile/settings/about.ts @@ -83,6 +83,12 @@ export const initAbout = () => { ${window.siyuan.languages.purge}
${window.siyuan.languages.dataRepoPurgeTip}
+
+ +
${window.siyuan.languages.dataRepoAutoPurgeIndexRetentionDays}
+
+ +
${window.siyuan.languages.dataRepoAutoPurgeRetentionIndexesDaily}
${window.siyuan.languages.systemLog} @@ -410,6 +416,18 @@ export const initAbout = () => { modelMainElement.querySelector("#tokenTip").innerHTML = window.siyuan.languages.about14.replace("${token}", window.siyuan.config.api.token); }); }); + const indexRetentionDaysElement = modelMainElement.querySelector("#indexRetentionDays") as HTMLInputElement; + indexRetentionDaysElement.addEventListener("change", () => { + fetchPost("/api/repo/setRepoIndexRetentionDays", {days: parseInt(indexRetentionDaysElement.value)}, () => { + window.siyuan.config.repo.indexRetentionDays = parseInt(indexRetentionDaysElement.value); + }); + }); + const retentionIndexesDailyElement = modelMainElement.querySelector("#retentionIndexesDaily") as HTMLInputElement; + retentionIndexesDailyElement.addEventListener("change", () => { + fetchPost("/api/repo/setRetentionIndexesDaily", {indexes: parseInt(retentionIndexesDailyElement.value)}, () => { + window.siyuan.config.repo.retentionIndexesDaily = parseInt(retentionIndexesDailyElement.value); + }); + }); } }); }; diff --git a/app/src/types/config.d.ts b/app/src/types/config.d.ts index 5a1a9362d..421be973c 100644 --- a/app/src/types/config.d.ts +++ b/app/src/types/config.d.ts @@ -1110,10 +1110,18 @@ declare namespace Config { */ key: string; /** - * Synchronous index timing, if it exceeds this time, the user is prompted that the index + * Sync index timing, if it exceeds this time, the user is prompted that the index * performance is degraded (unit: milliseconds) */ syncIndexTiming: number; + /** + * Automatic purge for local data repo index retention days + */ + indexRetentionDays: number; + /** + * Automatic purge for local data repo indexes retention daily + */ + retentionIndexesDaily: number; } /** diff --git a/kernel/api/repo.go b/kernel/api/repo.go index cea78bff8..a9befb1ad 100644 --- a/kernel/api/repo.go +++ b/kernel/api/repo.go @@ -29,6 +29,40 @@ import ( "github.com/siyuan-note/siyuan/kernel/util" ) +func setRepoIndexRetentionDays(c *gin.Context) { + ret := gulu.Ret.NewResult() + defer c.JSON(http.StatusOK, ret) + + arg, ok := util.JsonArg(c, ret) + if !ok { + return + } + days := int(arg["days"].(float64)) + if 1 > days { + days = 180 + } + + model.Conf.Repo.IndexRetentionDays = days + model.Conf.Save() +} + +func setRetentionIndexesDaily(c *gin.Context) { + ret := gulu.Ret.NewResult() + defer c.JSON(http.StatusOK, ret) + + arg, ok := util.JsonArg(c, ret) + if !ok { + return + } + indexes := int(arg["indexes"].(float64)) + if 1 > indexes { + indexes = 180 + } + + model.Conf.Repo.RetentionIndexesDaily = indexes + model.Conf.Save() +} + func getRepoFile(c *gin.Context) { // Add internal kernel API `/api/repo/getRepoFile` https://github.com/siyuan-note/siyuan/issues/10101 diff --git a/kernel/api/router.go b/kernel/api/router.go index b3bf8edda..66af996f3 100644 --- a/kernel/api/router.go +++ b/kernel/api/router.go @@ -390,6 +390,8 @@ func ServeAPI(ginServer *gin.Engine) { ginServer.Handle("POST", "/api/repo/diffRepoSnapshots", model.CheckAuth, model.CheckAdminRole, diffRepoSnapshots) ginServer.Handle("POST", "/api/repo/openRepoSnapshotDoc", model.CheckAuth, model.CheckAdminRole, openRepoSnapshotDoc) ginServer.Handle("POST", "/api/repo/getRepoFile", model.CheckAuth, model.CheckAdminRole, getRepoFile) + ginServer.Handle("POST", "/api/repo/setRepoIndexRetentionDays", model.CheckAuth, model.CheckAdminRole, setRepoIndexRetentionDays) + ginServer.Handle("POST", "/api/repo/setRetentionIndexesDaily", model.CheckAuth, model.CheckAdminRole, setRetentionIndexesDaily) ginServer.Handle("POST", "/api/riff/createRiffDeck", model.CheckAuth, model.CheckAdminRole, model.CheckReadonly, createRiffDeck) ginServer.Handle("POST", "/api/riff/renameRiffDeck", model.CheckAuth, model.CheckAdminRole, model.CheckReadonly, renameRiffDeck) diff --git a/kernel/conf/repo.go b/kernel/conf/repo.go index a335a2905..0ca43131a 100644 --- a/kernel/conf/repo.go +++ b/kernel/conf/repo.go @@ -29,11 +29,17 @@ type Repo struct { // If the data repo indexing time is greater than 12s, prompt user to purge the data repo https://github.com/siyuan-note/siyuan/issues/9613 // Supports configuring data sync index time-consuming prompts https://github.com/siyuan-note/siyuan/issues/9698 SyncIndexTiming int64 `json:"syncIndexTiming"` + + // 自动清理数据仓库 Automatic purge for local data repo https://github.com/siyuan-note/siyuan/issues/13091 + IndexRetentionDays int `json:"indexRetentionDays"` // 索引保留天数 + RetentionIndexesDaily int `json:"retentionIndexesDaily"` // 每日保留索引数 } func NewRepo() *Repo { return &Repo{ - SyncIndexTiming: 12 * 1000, + SyncIndexTiming: 12 * 1000, + IndexRetentionDays: 180, + RetentionIndexesDaily: 2, } } diff --git a/kernel/go.mod b/kernel/go.mod index f7b2135d9..07888ae52 100644 --- a/kernel/go.mod +++ b/kernel/go.mod @@ -56,7 +56,7 @@ require ( github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 github.com/sashabaranov/go-openai v1.29.1 github.com/shirou/gopsutil/v3 v3.24.5 - github.com/siyuan-note/dejavu v0.0.0-20241116085410-899893f6ae97 + github.com/siyuan-note/dejavu v0.0.0-20241116172754-506198a10836 github.com/siyuan-note/encryption v0.0.0-20231219001248-1e028a4d13b4 github.com/siyuan-note/eventbus v0.0.0-20240627125516-396fdb0f0f97 github.com/siyuan-note/filelock v0.0.0-20240724034355-d1ed7bf21d04 diff --git a/kernel/go.sum b/kernel/go.sum index 41ab0a60d..a1be5dc40 100644 --- a/kernel/go.sum +++ b/kernel/go.sum @@ -340,8 +340,8 @@ github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+D github.com/shurcooL/gofontwoff v0.0.0-20181114050219-180f79e6909d h1:lvCTyBbr36+tqMccdGMwuEU+hjux/zL6xSmf5S9ITaA= github.com/shurcooL/gofontwoff v0.0.0-20181114050219-180f79e6909d/go.mod h1:05UtEgK5zq39gLST6uB0cf3NEHjETfB4Fgr3Gx5R9Vw= github.com/simplereach/timeutils v1.2.0/go.mod h1:VVbQDfN/FHRZa1LSqcwo4kNZ62OOyqLLGQKYB3pB0Q8= -github.com/siyuan-note/dejavu v0.0.0-20241116085410-899893f6ae97 h1:JkJG+dYDHSPCrzKXfi/yczlobfgiDSDHoS6ozRMNRNI= -github.com/siyuan-note/dejavu v0.0.0-20241116085410-899893f6ae97/go.mod h1:sVINGoilQS1l5ZQJJBHzUwKmyHhx8qdQps7gKqZVbgU= +github.com/siyuan-note/dejavu v0.0.0-20241116172754-506198a10836 h1:+hHN29Ifk2KqdXFgFIj++3bgqEOv8SjHClmFcSvj0eE= +github.com/siyuan-note/dejavu v0.0.0-20241116172754-506198a10836/go.mod h1:sVINGoilQS1l5ZQJJBHzUwKmyHhx8qdQps7gKqZVbgU= github.com/siyuan-note/encryption v0.0.0-20231219001248-1e028a4d13b4 h1:kJaw5L/evyW6LcB9IQT8PR4ppx8JVqOFP9Ix3rfwSrc= github.com/siyuan-note/encryption v0.0.0-20231219001248-1e028a4d13b4/go.mod h1:UYcCCY+0wh+GmUoDOaO63j1sV5lgy7laLAk1XhEiUis= github.com/siyuan-note/eventbus v0.0.0-20240627125516-396fdb0f0f97 h1:lM5v8BfNtbOL5jYwhCdMYBcYtr06IYBKjjSLAPMKTM8= diff --git a/kernel/job/cron.go b/kernel/job/cron.go index 5db915460..511933f4f 100644 --- a/kernel/job/cron.go +++ b/kernel/job/cron.go @@ -42,6 +42,7 @@ func StartCron() { go every(30*time.Second, model.OCRAssetsJob) go every(30*time.Second, model.FlushAssetsTextsJob) go every(30*time.Second, model.HookDesktopUIProcJob) + go every(24*time.Hour, model.AutoPurgeRepoJob) } func every(interval time.Duration, f func()) { diff --git a/kernel/model/conf.go b/kernel/model/conf.go index 79f48fa23..d50c9f743 100644 --- a/kernel/model/conf.go +++ b/kernel/model/conf.go @@ -385,6 +385,12 @@ func InitConf() { if 12000 > Conf.Repo.SyncIndexTiming { Conf.Repo.SyncIndexTiming = 12 * 1000 } + if 1 > Conf.Repo.IndexRetentionDays { + Conf.Repo.IndexRetentionDays = 180 + } + if 1 > Conf.Repo.RetentionIndexesDaily { + Conf.Repo.RetentionIndexesDaily = 2 + } if nil == Conf.Search { Conf.Search = conf.NewSearch() diff --git a/kernel/model/repository.go b/kernel/model/repository.go index e85af5597..0e054d814 100644 --- a/kernel/model/repository.go +++ b/kernel/model/repository.go @@ -24,6 +24,7 @@ import ( "errors" "fmt" "math" + mathRand "math/rand" "mime" "net/http" "os" @@ -59,6 +60,116 @@ import ( "github.com/studio-b12/gowebdav" ) +// AutoPurgeRepoJob 自动清理数据仓库 https://github.com/siyuan-note/siyuan/issues/13091 +func AutoPurgeRepoJob() { + task.AppendTaskWithTimeout(task.RepoAutoPurge, 12*time.Hour, autoPurgeRepo, true) +} + +var ( + autoPurgeRepoAfterFirstSync = false + lastAutoPurgeRepo = time.Time{} +) + +func autoPurgeRepo(cron bool) { + if cron && !autoPurgeRepoAfterFirstSync { + return + } + if time.Since(lastAutoPurgeRepo) < 6*time.Hour { + return + } + + autoPurgeRepoAfterFirstSync = true + defer func() { + lastAutoPurgeRepo = time.Now() + }() + + if 1 > len(Conf.Repo.Key) { + return + } + + repo, err := newRepository() + if err != nil { + return + } + + now := time.Now() + + dateGroupedIndexes := map[string][]*dejavu.Log{} // 按照日期分组 + // 收集指定日期内需要保留的索引 + var date string + page := 1 + for { + indexLogs, pageCount, _, err := repo.GetIndexLogs(page, 512) + if nil != err { + logging.LogErrorf("get data repo index logs failed: %s", err) + return + } + if 1 > len(indexLogs) { + break + } + + tooOld := false + for _, index := range indexLogs { + if now.UnixMilli()-index.Created <= int64(Conf.Repo.IndexRetentionDays)*24*60*60*1000 { + date = time.UnixMilli(index.Created).Format("2006-01-02") + if _, ok := dateGroupedIndexes[date]; !ok { + dateGroupedIndexes[date] = []*dejavu.Log{} + } + dateGroupedIndexes[date] = append(dateGroupedIndexes[date], index) + } else { + tooOld = true + break + } + } + if tooOld { + break + } + page++ + if page > pageCount { + break + } + } + + // 筛选出每日需要保留的索引 + var retentionIndexIDs []string + for _, indexes := range dateGroupedIndexes { + if len(indexes) <= Conf.Repo.RetentionIndexesDaily { + continue + } + + keepIndexes := hashset.New() + keepIndexes.Add(indexes[0]) // 每天最后一个固定保留 + // 随机保留指定数量的索引 + for i := 0; i < Conf.Repo.RetentionIndexesDaily*7; i++ { + keepIndexes.Add(indexes[mathRand.Intn(len(indexes)-1)]) + if keepIndexes.Size() >= Conf.Repo.RetentionIndexesDaily { + break + } + } + + for _, keepIndex := range keepIndexes.Values() { + retentionIndexIDs = append(retentionIndexIDs, keepIndex.(*dejavu.Log).ID) + } + } + + retentionIndexIDs = gulu.Str.RemoveDuplicatedElem(retentionIndexIDs) + if 1 > len(retentionIndexIDs) { + logging.LogInfof("no index to purge [ellapsed=%.2fs]", time.Since(now).Seconds()) + return + } + + stat, err := repo.Purge(retentionIndexIDs...) + if err != nil { + return + } + + deletedIndexes := stat.Indexes + deletedObjects := stat.Objects + deletedSize := humanize.BytesCustomCeil(uint64(stat.Size), 2) + logging.LogInfof("purge data repo completed [ellapsed=%.2fs, indexes=%d, objects=%d, size=%s]", + time.Since(now).Seconds(), deletedIndexes, deletedObjects, deletedSize) +} + func GetRepoFile(fileID string) (ret []byte, p string, err error) { if 1 > len(Conf.Repo.Key) { err = errors.New(Conf.Language(26)) @@ -518,6 +629,7 @@ func PurgeRepo() (err error) { return } + now := time.Now() stat, err := repo.Purge() if err != nil { return @@ -526,6 +638,8 @@ func PurgeRepo() (err error) { deletedIndexes := stat.Indexes deletedObjects := stat.Objects deletedSize := humanize.BytesCustomCeil(uint64(stat.Size), 2) + logging.LogInfof("purge data repo completed [ellapsed=%.2fs, indexes=%d, objects=%d, size=%s]", + time.Since(now).Seconds(), deletedIndexes, deletedObjects, deletedSize) msg = fmt.Sprintf(Conf.Language(203), deletedIndexes, deletedObjects, deletedSize) util.PushMsg(msg, 7000) return @@ -1348,8 +1462,12 @@ func syncRepo(exit, byHand bool) (dataChanged bool, err error) { processSyncMergeResult(exit, byHand, mergeResult, trafficStat, "a", elapsed) if !exit { - // 首次数据同步执行完成后再执行索引订正 Index fixing should not be performed before data synchronization https://github.com/siyuan-note/siyuan/issues/10761 - go checkIndex() + go func() { + // 首次数据同步执行完成后再执行索引订正 Index fixing should not be performed before data synchronization https://github.com/siyuan-note/siyuan/issues/10761 + checkIndex() + // 索引订正结束后执行数据仓库清理 Automatic purge for local data repo https://github.com/siyuan-note/siyuan/issues/13091 + autoPurgeRepo(false) + }() } return } diff --git a/kernel/task/queue.go b/kernel/task/queue.go index a1cbdf52c..30f47ca59 100644 --- a/kernel/task/queue.go +++ b/kernel/task/queue.go @@ -119,6 +119,7 @@ func getCurrentTasks() (ret []*Task) { const ( RepoCheckout = "task.repo.checkout" // 从快照中检出 + RepoAutoPurge = "task.repo.autoPurge" // 自动清理数据仓库 DatabaseIndexFull = "task.database.index.full" // 重建索引 DatabaseIndex = "task.database.index" // 数据库索引 DatabaseIndexCommit = "task.database.index.commit" // 数据库索引提交 @@ -143,6 +144,7 @@ const ( // uniqueActions 描述了唯一的任务，即队列中只能存在一个在执行的任务。 var uniqueActions = []string{ RepoCheckout, + RepoAutoPurge, DatabaseIndexFull, DatabaseIndexCommit, OCRImage,