mirror of
https://github.com/siyuan-note/siyuan.git
synced 2025-05-03 00:00:38 +08:00
🎨 Automatic purge for local data repo https://github.com/siyuan-note/siyuan/issues/13091
This commit is contained in:
parent
93b687a407
commit
6a442ad85d
@ -1,4 +1,6 @@
|
|||||||
{
|
{
|
||||||
|
"dataRepoAutoPurgeIndexRetentionDays": "Daten-Snapshot-Aufbewahrungstage",
|
||||||
|
"dataRepoAutoPurgeRetentionIndexesDaily": "Daten-Snapshots pro Tag",
|
||||||
"fields": "Attribut",
|
"fields": "Attribut",
|
||||||
"dynamicEmoji": "Dynamisches Icon",
|
"dynamicEmoji": "Dynamisches Icon",
|
||||||
"backlinkContainChildren": "Enthalten Rückverweise untergeordnete Blöcke",
|
"backlinkContainChildren": "Enthalten Rückverweise untergeordnete Blöcke",
|
||||||
|
@ -1,4 +1,6 @@
|
|||||||
{
|
{
|
||||||
|
"dataRepoAutoPurgeIndexRetentionDays": "Data snapshot retention days",
|
||||||
|
"dataRepoAutoPurgeRetentionIndexesDaily": "Data snapshots per day",
|
||||||
"fields": "Fields",
|
"fields": "Fields",
|
||||||
"dynamicEmoji": "Dynamic icon",
|
"dynamicEmoji": "Dynamic icon",
|
||||||
"backlinkContainChildren": "Do backlinks contain child blocks",
|
"backlinkContainChildren": "Do backlinks contain child blocks",
|
||||||
|
@ -1,4 +1,6 @@
|
|||||||
{
|
{
|
||||||
|
"dataRepoAutoPurgeIndexRetentionDays": "Días de retención de instantáneas de datos",
|
||||||
|
"dataRepoAutoPurgeRetentionIndexesDaily": "Número de instantáneas de datos por día",
|
||||||
"fields": "Atributo",
|
"fields": "Atributo",
|
||||||
"dynamicEmoji": "Icono dinámico",
|
"dynamicEmoji": "Icono dinámico",
|
||||||
"backlinkContainChildren": "¿Los enlaces inversos contienen bloques secundarios?",
|
"backlinkContainChildren": "¿Los enlaces inversos contienen bloques secundarios?",
|
||||||
|
@ -1,4 +1,6 @@
|
|||||||
{
|
{
|
||||||
|
"dataRepoAutoPurgeIndexRetentionDays": "Jours de rétention des instantanés de données",
|
||||||
|
"dataRepoAutoPurgeRetentionIndexesDaily": "Nombre d'instantanés de données par jour",
|
||||||
"fields": "Attribut",
|
"fields": "Attribut",
|
||||||
"dynamicEmoji": "Icône dynamique",
|
"dynamicEmoji": "Icône dynamique",
|
||||||
"backlinkContainChildren": "Les liens retour contiennent-ils des sous-blocs",
|
"backlinkContainChildren": "Les liens retour contiennent-ils des sous-blocs",
|
||||||
|
@ -1,4 +1,6 @@
|
|||||||
{
|
{
|
||||||
|
"dataRepoAutoPurgeIndexRetentionDays": "ימי שמירת תמונות נתונים",
|
||||||
|
"dataRepoAutoPurgeRetentionIndexesDaily": "מספר תמונות נתונים ביום",
|
||||||
"fields": "מאפיין",
|
"fields": "מאפיין",
|
||||||
"dynamicEmoji": "אייקון דינמי",
|
"dynamicEmoji": "אייקון דינמי",
|
||||||
"backlinkContainChildren": "האם קישורים חוזרים כוללים בלוקים משניים",
|
"backlinkContainChildren": "האם קישורים חוזרים כוללים בלוקים משניים",
|
||||||
|
@ -1,4 +1,6 @@
|
|||||||
{
|
{
|
||||||
|
"dataRepoAutoPurgeIndexRetentionDays": "Giorni di conservazione degli snapshot dei dati",
|
||||||
|
"dataRepoAutoPurgeRetentionIndexesDaily": "Numero di snapshot dei dati al giorno",
|
||||||
"fields": "Campi",
|
"fields": "Campi",
|
||||||
"dynamicEmoji": "Emoji dinamica",
|
"dynamicEmoji": "Emoji dinamica",
|
||||||
"backlinkContainChildren": "I backlink contengono blocchi figli",
|
"backlinkContainChildren": "I backlink contengono blocchi figli",
|
||||||
|
@ -1,4 +1,6 @@
|
|||||||
{
|
{
|
||||||
|
"dataRepoAutoPurgeIndexRetentionDays": "データスナップショットの保持日数",
|
||||||
|
"dataRepoAutoPurgeRetentionIndexesDaily": "データスナップショットの毎日の保持数",
|
||||||
"fields": "属性",
|
"fields": "属性",
|
||||||
"dynamicEmoji": "動的アイコン",
|
"dynamicEmoji": "動的アイコン",
|
||||||
"backlinkContainChildren": "バックリンクに子ブロックを含めるかどうか",
|
"backlinkContainChildren": "バックリンクに子ブロックを含めるかどうか",
|
||||||
|
@ -1,4 +1,6 @@
|
|||||||
{
|
{
|
||||||
|
"dataRepoAutoPurgeIndexRetentionDays": "Dni przechowywania migawek danych",
|
||||||
|
"dataRepoAutoPurgeRetentionIndexesDaily": "Liczba migawek danych dziennie",
|
||||||
"fields": "Atrybut",
|
"fields": "Atrybut",
|
||||||
"dynamicEmoji": "Ikona dynamiczna",
|
"dynamicEmoji": "Ikona dynamiczna",
|
||||||
"backlinkContainChildren": "Czy linki zwrotne zawierają bloki podrzędne",
|
"backlinkContainChildren": "Czy linki zwrotne zawierają bloki podrzędne",
|
||||||
|
@ -1,4 +1,6 @@
|
|||||||
{
|
{
|
||||||
|
"dataRepoAutoPurgeIndexRetentionDays": "Срок хранения снимков данных",
|
||||||
|
"dataRepoAutoPurgeRetentionIndexesDaily": "Количество снимков данных в день",
|
||||||
"fields": "Атрибут",
|
"fields": "Атрибут",
|
||||||
"dynamicEmoji": "Динамическая иконка",
|
"dynamicEmoji": "Динамическая иконка",
|
||||||
"backlinkContainChildren": "Включать ли дочерние блоки в обратные ссылки",
|
"backlinkContainChildren": "Включать ли дочерние блоки в обратные ссылки",
|
||||||
|
@ -1,4 +1,6 @@
|
|||||||
{
|
{
|
||||||
|
"dataRepoAutoPurgeIndexRetentionDays": "數據快照保留天數",
|
||||||
|
"dataRepoAutoPurgeRetentionIndexesDaily": "數據快照每天保留個數",
|
||||||
"fields": "欄位",
|
"fields": "欄位",
|
||||||
"dynamicEmoji": "動態圖標",
|
"dynamicEmoji": "動態圖標",
|
||||||
"backlinkContainChildren": "反向鏈接包含子塊",
|
"backlinkContainChildren": "反向鏈接包含子塊",
|
||||||
|
@ -1,4 +1,6 @@
|
|||||||
{
|
{
|
||||||
|
"dataRepoAutoPurgeIndexRetentionDays": "数据快照保留天数",
|
||||||
|
"dataRepoAutoPurgeRetentionIndexesDaily": "数据快照每天保留个数",
|
||||||
"fields": "字段",
|
"fields": "字段",
|
||||||
"dynamicEmoji": "动态图标",
|
"dynamicEmoji": "动态图标",
|
||||||
"backlinkContainChildren": "反向链接包含子块",
|
"backlinkContainChildren": "反向链接包含子块",
|
||||||
|
@ -142,16 +142,31 @@ export const about = {
|
|||||||
</button>
|
</button>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
<div class="fn__flex b3-label config__item">
|
<div class="b3-label fn__flex"><div class="fn__block">
|
||||||
<div class="fn__flex-1">
|
<div>
|
||||||
${window.siyuan.languages.dataRepoPurge}
|
${window.siyuan.languages.dataRepoPurge}
|
||||||
<div class="b3-label__text">${window.siyuan.languages.dataRepoPurgeTip}</div>
|
|
||||||
</div>
|
</div>
|
||||||
<div class="fn__space"></div>
|
<div class="fn__hr"></div>
|
||||||
<button id="purgeRepo" class="b3-button b3-button--outline fn__size200 fn__flex-center">
|
<div class="fn__flex config__item">
|
||||||
<svg><use xlink:href="#iconTrashcan"></use></svg>${window.siyuan.languages.purge}
|
<div class="fn__flex-center fn__flex-1 ft__on-surface">${window.siyuan.languages.dataRepoPurgeTip}</div>
|
||||||
</button>
|
<span class="fn__space"></span>
|
||||||
</div>
|
<button id="purgeRepo" class="b3-button b3-button--outline fn__size200 fn__flex-center">
|
||||||
|
<svg><use xlink:href="#iconTrashcan"></use></svg>${window.siyuan.languages.purge}
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
<div class="fn__hr"></div>
|
||||||
|
<div class="fn__flex config__item">
|
||||||
|
<div class="fn__flex-center fn__flex-1 ft__on-surface">${window.siyuan.languages.dataRepoAutoPurgeIndexRetentionDays}</div>
|
||||||
|
<span class="fn__space"></span>
|
||||||
|
<input class="b3-text-field fn__flex-center fn__size200" min="1" type="number" id="indexRetentionDays" value="${window.siyuan.config.repo.indexRetentionDays}">
|
||||||
|
</div>
|
||||||
|
<div class="fn__hr"></div>
|
||||||
|
<div class="fn__flex config__item">
|
||||||
|
<div class="fn__flex-center fn__flex-1 ft__on-surface">${window.siyuan.languages.dataRepoAutoPurgeRetentionIndexesDaily}</div>
|
||||||
|
<span class="fn__space"></span>
|
||||||
|
<input class="b3-text-field fn__flex-center fn__size200" min="1" type="number" id="retentionIndexesDaily" value="${window.siyuan.config.repo.retentionIndexesDaily}">
|
||||||
|
</div>
|
||||||
|
</div></div>
|
||||||
<div class="fn__flex b3-label config__item">
|
<div class="fn__flex b3-label config__item">
|
||||||
<div class="fn__flex-1">
|
<div class="fn__flex-1">
|
||||||
${window.siyuan.languages.systemLog}
|
${window.siyuan.languages.systemLog}
|
||||||
@ -208,6 +223,18 @@ ${checkUpdateHTML}
|
|||||||
if (window.siyuan.config.system.isInsider) {
|
if (window.siyuan.config.system.isInsider) {
|
||||||
about.element.querySelector("#isInsider").innerHTML = "<span class='ft__secondary'>Insider Preview</span>";
|
about.element.querySelector("#isInsider").innerHTML = "<span class='ft__secondary'>Insider Preview</span>";
|
||||||
}
|
}
|
||||||
|
const indexRetentionDaysElement = about.element.querySelector("#indexRetentionDays") as HTMLInputElement;
|
||||||
|
indexRetentionDaysElement.addEventListener("change", () => {
|
||||||
|
fetchPost("/api/repo/setRepoIndexRetentionDays", {days: parseInt(indexRetentionDaysElement.value)}, () => {
|
||||||
|
window.siyuan.config.repo.indexRetentionDays = parseInt(indexRetentionDaysElement.value);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
const retentionIndexesDailyElement = about.element.querySelector("#retentionIndexesDaily") as HTMLInputElement;
|
||||||
|
retentionIndexesDailyElement.addEventListener("change", () => {
|
||||||
|
fetchPost("/api/repo/setRetentionIndexesDaily", {indexes: parseInt(retentionIndexesDailyElement.value)}, () => {
|
||||||
|
window.siyuan.config.repo.retentionIndexesDaily = parseInt(retentionIndexesDailyElement.value);
|
||||||
|
});
|
||||||
|
});
|
||||||
const tokenElement = about.element.querySelector("#token") as HTMLInputElement;
|
const tokenElement = about.element.querySelector("#token") as HTMLInputElement;
|
||||||
tokenElement.addEventListener("click", () => {
|
tokenElement.addEventListener("click", () => {
|
||||||
tokenElement.select();
|
tokenElement.select();
|
||||||
|
@ -83,6 +83,12 @@ export const initAbout = () => {
|
|||||||
<svg><use xlink:href="#iconTrashcan"></use></svg>${window.siyuan.languages.purge}
|
<svg><use xlink:href="#iconTrashcan"></use></svg>${window.siyuan.languages.purge}
|
||||||
</button>
|
</button>
|
||||||
<div class="b3-label__text">${window.siyuan.languages.dataRepoPurgeTip}</div>
|
<div class="b3-label__text">${window.siyuan.languages.dataRepoPurgeTip}</div>
|
||||||
|
<div class="fn__hr"></div>
|
||||||
|
<input class="b3-text-field fn__block" style="padding-right: 64px;" id="indexRetentionDays" min="1" type="number" class="b3-text-field" value="${window.siyuan.config.repo.indexRetentionDays}">
|
||||||
|
<div class="b3-label__text">${window.siyuan.languages.dataRepoAutoPurgeIndexRetentionDays}</div>
|
||||||
|
<div class="fn__hr"></div>
|
||||||
|
<input class="b3-text-field fn__block" style="padding-right: 64px;" id="retentionIndexesDaily" min="1" type="number" class="b3-text-field" value="${window.siyuan.config.repo.retentionIndexesDaily}">
|
||||||
|
<div class="b3-label__text">${window.siyuan.languages.dataRepoAutoPurgeRetentionIndexesDaily}</div>
|
||||||
</div>
|
</div>
|
||||||
<div class="b3-label">
|
<div class="b3-label">
|
||||||
${window.siyuan.languages.systemLog}
|
${window.siyuan.languages.systemLog}
|
||||||
@ -410,6 +416,18 @@ export const initAbout = () => {
|
|||||||
modelMainElement.querySelector("#tokenTip").innerHTML = window.siyuan.languages.about14.replace("${token}", window.siyuan.config.api.token);
|
modelMainElement.querySelector("#tokenTip").innerHTML = window.siyuan.languages.about14.replace("${token}", window.siyuan.config.api.token);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
const indexRetentionDaysElement = modelMainElement.querySelector("#indexRetentionDays") as HTMLInputElement;
|
||||||
|
indexRetentionDaysElement.addEventListener("change", () => {
|
||||||
|
fetchPost("/api/repo/setRepoIndexRetentionDays", {days: parseInt(indexRetentionDaysElement.value)}, () => {
|
||||||
|
window.siyuan.config.repo.indexRetentionDays = parseInt(indexRetentionDaysElement.value);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
const retentionIndexesDailyElement = modelMainElement.querySelector("#retentionIndexesDaily") as HTMLInputElement;
|
||||||
|
retentionIndexesDailyElement.addEventListener("change", () => {
|
||||||
|
fetchPost("/api/repo/setRetentionIndexesDaily", {indexes: parseInt(retentionIndexesDailyElement.value)}, () => {
|
||||||
|
window.siyuan.config.repo.retentionIndexesDaily = parseInt(retentionIndexesDailyElement.value);
|
||||||
|
});
|
||||||
|
});
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
};
|
};
|
||||||
|
10
app/src/types/config.d.ts
vendored
10
app/src/types/config.d.ts
vendored
@ -1110,10 +1110,18 @@ declare namespace Config {
|
|||||||
*/
|
*/
|
||||||
key: string;
|
key: string;
|
||||||
/**
|
/**
|
||||||
* Synchronous index timing, if it exceeds this time, the user is prompted that the index
|
* Sync index timing, if it exceeds this time, the user is prompted that the index
|
||||||
* performance is degraded (unit: milliseconds)
|
* performance is degraded (unit: milliseconds)
|
||||||
*/
|
*/
|
||||||
syncIndexTiming: number;
|
syncIndexTiming: number;
|
||||||
|
/**
|
||||||
|
* Automatic purge for local data repo index retention days
|
||||||
|
*/
|
||||||
|
indexRetentionDays: number;
|
||||||
|
/**
|
||||||
|
* Automatic purge for local data repo indexes retention daily
|
||||||
|
*/
|
||||||
|
retentionIndexesDaily: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -29,6 +29,40 @@ import (
|
|||||||
"github.com/siyuan-note/siyuan/kernel/util"
|
"github.com/siyuan-note/siyuan/kernel/util"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
func setRepoIndexRetentionDays(c *gin.Context) {
|
||||||
|
ret := gulu.Ret.NewResult()
|
||||||
|
defer c.JSON(http.StatusOK, ret)
|
||||||
|
|
||||||
|
arg, ok := util.JsonArg(c, ret)
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
days := int(arg["days"].(float64))
|
||||||
|
if 1 > days {
|
||||||
|
days = 180
|
||||||
|
}
|
||||||
|
|
||||||
|
model.Conf.Repo.IndexRetentionDays = days
|
||||||
|
model.Conf.Save()
|
||||||
|
}
|
||||||
|
|
||||||
|
func setRetentionIndexesDaily(c *gin.Context) {
|
||||||
|
ret := gulu.Ret.NewResult()
|
||||||
|
defer c.JSON(http.StatusOK, ret)
|
||||||
|
|
||||||
|
arg, ok := util.JsonArg(c, ret)
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
indexes := int(arg["indexes"].(float64))
|
||||||
|
if 1 > indexes {
|
||||||
|
indexes = 180
|
||||||
|
}
|
||||||
|
|
||||||
|
model.Conf.Repo.RetentionIndexesDaily = indexes
|
||||||
|
model.Conf.Save()
|
||||||
|
}
|
||||||
|
|
||||||
func getRepoFile(c *gin.Context) {
|
func getRepoFile(c *gin.Context) {
|
||||||
// Add internal kernel API `/api/repo/getRepoFile` https://github.com/siyuan-note/siyuan/issues/10101
|
// Add internal kernel API `/api/repo/getRepoFile` https://github.com/siyuan-note/siyuan/issues/10101
|
||||||
|
|
||||||
|
@ -390,6 +390,8 @@ func ServeAPI(ginServer *gin.Engine) {
|
|||||||
ginServer.Handle("POST", "/api/repo/diffRepoSnapshots", model.CheckAuth, model.CheckAdminRole, diffRepoSnapshots)
|
ginServer.Handle("POST", "/api/repo/diffRepoSnapshots", model.CheckAuth, model.CheckAdminRole, diffRepoSnapshots)
|
||||||
ginServer.Handle("POST", "/api/repo/openRepoSnapshotDoc", model.CheckAuth, model.CheckAdminRole, openRepoSnapshotDoc)
|
ginServer.Handle("POST", "/api/repo/openRepoSnapshotDoc", model.CheckAuth, model.CheckAdminRole, openRepoSnapshotDoc)
|
||||||
ginServer.Handle("POST", "/api/repo/getRepoFile", model.CheckAuth, model.CheckAdminRole, getRepoFile)
|
ginServer.Handle("POST", "/api/repo/getRepoFile", model.CheckAuth, model.CheckAdminRole, getRepoFile)
|
||||||
|
ginServer.Handle("POST", "/api/repo/setRepoIndexRetentionDays", model.CheckAuth, model.CheckAdminRole, setRepoIndexRetentionDays)
|
||||||
|
ginServer.Handle("POST", "/api/repo/setRetentionIndexesDaily", model.CheckAuth, model.CheckAdminRole, setRetentionIndexesDaily)
|
||||||
|
|
||||||
ginServer.Handle("POST", "/api/riff/createRiffDeck", model.CheckAuth, model.CheckAdminRole, model.CheckReadonly, createRiffDeck)
|
ginServer.Handle("POST", "/api/riff/createRiffDeck", model.CheckAuth, model.CheckAdminRole, model.CheckReadonly, createRiffDeck)
|
||||||
ginServer.Handle("POST", "/api/riff/renameRiffDeck", model.CheckAuth, model.CheckAdminRole, model.CheckReadonly, renameRiffDeck)
|
ginServer.Handle("POST", "/api/riff/renameRiffDeck", model.CheckAuth, model.CheckAdminRole, model.CheckReadonly, renameRiffDeck)
|
||||||
|
@ -29,11 +29,17 @@ type Repo struct {
|
|||||||
// If the data repo indexing time is greater than 12s, prompt user to purge the data repo https://github.com/siyuan-note/siyuan/issues/9613
|
// If the data repo indexing time is greater than 12s, prompt user to purge the data repo https://github.com/siyuan-note/siyuan/issues/9613
|
||||||
// Supports configuring data sync index time-consuming prompts https://github.com/siyuan-note/siyuan/issues/9698
|
// Supports configuring data sync index time-consuming prompts https://github.com/siyuan-note/siyuan/issues/9698
|
||||||
SyncIndexTiming int64 `json:"syncIndexTiming"`
|
SyncIndexTiming int64 `json:"syncIndexTiming"`
|
||||||
|
|
||||||
|
// 自动清理数据仓库 Automatic purge for local data repo https://github.com/siyuan-note/siyuan/issues/13091
|
||||||
|
IndexRetentionDays int `json:"indexRetentionDays"` // 索引保留天数
|
||||||
|
RetentionIndexesDaily int `json:"retentionIndexesDaily"` // 每日保留索引数
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewRepo() *Repo {
|
func NewRepo() *Repo {
|
||||||
return &Repo{
|
return &Repo{
|
||||||
SyncIndexTiming: 12 * 1000,
|
SyncIndexTiming: 12 * 1000,
|
||||||
|
IndexRetentionDays: 180,
|
||||||
|
RetentionIndexesDaily: 2,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -56,7 +56,7 @@ require (
|
|||||||
github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06
|
github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06
|
||||||
github.com/sashabaranov/go-openai v1.29.1
|
github.com/sashabaranov/go-openai v1.29.1
|
||||||
github.com/shirou/gopsutil/v3 v3.24.5
|
github.com/shirou/gopsutil/v3 v3.24.5
|
||||||
github.com/siyuan-note/dejavu v0.0.0-20241116085410-899893f6ae97
|
github.com/siyuan-note/dejavu v0.0.0-20241116172754-506198a10836
|
||||||
github.com/siyuan-note/encryption v0.0.0-20231219001248-1e028a4d13b4
|
github.com/siyuan-note/encryption v0.0.0-20231219001248-1e028a4d13b4
|
||||||
github.com/siyuan-note/eventbus v0.0.0-20240627125516-396fdb0f0f97
|
github.com/siyuan-note/eventbus v0.0.0-20240627125516-396fdb0f0f97
|
||||||
github.com/siyuan-note/filelock v0.0.0-20240724034355-d1ed7bf21d04
|
github.com/siyuan-note/filelock v0.0.0-20240724034355-d1ed7bf21d04
|
||||||
|
@ -340,8 +340,8 @@ github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+D
|
|||||||
github.com/shurcooL/gofontwoff v0.0.0-20181114050219-180f79e6909d h1:lvCTyBbr36+tqMccdGMwuEU+hjux/zL6xSmf5S9ITaA=
|
github.com/shurcooL/gofontwoff v0.0.0-20181114050219-180f79e6909d h1:lvCTyBbr36+tqMccdGMwuEU+hjux/zL6xSmf5S9ITaA=
|
||||||
github.com/shurcooL/gofontwoff v0.0.0-20181114050219-180f79e6909d/go.mod h1:05UtEgK5zq39gLST6uB0cf3NEHjETfB4Fgr3Gx5R9Vw=
|
github.com/shurcooL/gofontwoff v0.0.0-20181114050219-180f79e6909d/go.mod h1:05UtEgK5zq39gLST6uB0cf3NEHjETfB4Fgr3Gx5R9Vw=
|
||||||
github.com/simplereach/timeutils v1.2.0/go.mod h1:VVbQDfN/FHRZa1LSqcwo4kNZ62OOyqLLGQKYB3pB0Q8=
|
github.com/simplereach/timeutils v1.2.0/go.mod h1:VVbQDfN/FHRZa1LSqcwo4kNZ62OOyqLLGQKYB3pB0Q8=
|
||||||
github.com/siyuan-note/dejavu v0.0.0-20241116085410-899893f6ae97 h1:JkJG+dYDHSPCrzKXfi/yczlobfgiDSDHoS6ozRMNRNI=
|
github.com/siyuan-note/dejavu v0.0.0-20241116172754-506198a10836 h1:+hHN29Ifk2KqdXFgFIj++3bgqEOv8SjHClmFcSvj0eE=
|
||||||
github.com/siyuan-note/dejavu v0.0.0-20241116085410-899893f6ae97/go.mod h1:sVINGoilQS1l5ZQJJBHzUwKmyHhx8qdQps7gKqZVbgU=
|
github.com/siyuan-note/dejavu v0.0.0-20241116172754-506198a10836/go.mod h1:sVINGoilQS1l5ZQJJBHzUwKmyHhx8qdQps7gKqZVbgU=
|
||||||
github.com/siyuan-note/encryption v0.0.0-20231219001248-1e028a4d13b4 h1:kJaw5L/evyW6LcB9IQT8PR4ppx8JVqOFP9Ix3rfwSrc=
|
github.com/siyuan-note/encryption v0.0.0-20231219001248-1e028a4d13b4 h1:kJaw5L/evyW6LcB9IQT8PR4ppx8JVqOFP9Ix3rfwSrc=
|
||||||
github.com/siyuan-note/encryption v0.0.0-20231219001248-1e028a4d13b4/go.mod h1:UYcCCY+0wh+GmUoDOaO63j1sV5lgy7laLAk1XhEiUis=
|
github.com/siyuan-note/encryption v0.0.0-20231219001248-1e028a4d13b4/go.mod h1:UYcCCY+0wh+GmUoDOaO63j1sV5lgy7laLAk1XhEiUis=
|
||||||
github.com/siyuan-note/eventbus v0.0.0-20240627125516-396fdb0f0f97 h1:lM5v8BfNtbOL5jYwhCdMYBcYtr06IYBKjjSLAPMKTM8=
|
github.com/siyuan-note/eventbus v0.0.0-20240627125516-396fdb0f0f97 h1:lM5v8BfNtbOL5jYwhCdMYBcYtr06IYBKjjSLAPMKTM8=
|
||||||
|
@ -42,6 +42,7 @@ func StartCron() {
|
|||||||
go every(30*time.Second, model.OCRAssetsJob)
|
go every(30*time.Second, model.OCRAssetsJob)
|
||||||
go every(30*time.Second, model.FlushAssetsTextsJob)
|
go every(30*time.Second, model.FlushAssetsTextsJob)
|
||||||
go every(30*time.Second, model.HookDesktopUIProcJob)
|
go every(30*time.Second, model.HookDesktopUIProcJob)
|
||||||
|
go every(24*time.Hour, model.AutoPurgeRepoJob)
|
||||||
}
|
}
|
||||||
|
|
||||||
func every(interval time.Duration, f func()) {
|
func every(interval time.Duration, f func()) {
|
||||||
|
@ -385,6 +385,12 @@ func InitConf() {
|
|||||||
if 12000 > Conf.Repo.SyncIndexTiming {
|
if 12000 > Conf.Repo.SyncIndexTiming {
|
||||||
Conf.Repo.SyncIndexTiming = 12 * 1000
|
Conf.Repo.SyncIndexTiming = 12 * 1000
|
||||||
}
|
}
|
||||||
|
if 1 > Conf.Repo.IndexRetentionDays {
|
||||||
|
Conf.Repo.IndexRetentionDays = 180
|
||||||
|
}
|
||||||
|
if 1 > Conf.Repo.RetentionIndexesDaily {
|
||||||
|
Conf.Repo.RetentionIndexesDaily = 2
|
||||||
|
}
|
||||||
|
|
||||||
if nil == Conf.Search {
|
if nil == Conf.Search {
|
||||||
Conf.Search = conf.NewSearch()
|
Conf.Search = conf.NewSearch()
|
||||||
|
@ -24,6 +24,7 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"math"
|
"math"
|
||||||
|
mathRand "math/rand"
|
||||||
"mime"
|
"mime"
|
||||||
"net/http"
|
"net/http"
|
||||||
"os"
|
"os"
|
||||||
@ -59,6 +60,116 @@ import (
|
|||||||
"github.com/studio-b12/gowebdav"
|
"github.com/studio-b12/gowebdav"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// AutoPurgeRepoJob 自动清理数据仓库 https://github.com/siyuan-note/siyuan/issues/13091
|
||||||
|
func AutoPurgeRepoJob() {
|
||||||
|
task.AppendTaskWithTimeout(task.RepoAutoPurge, 12*time.Hour, autoPurgeRepo, true)
|
||||||
|
}
|
||||||
|
|
||||||
|
var (
|
||||||
|
autoPurgeRepoAfterFirstSync = false
|
||||||
|
lastAutoPurgeRepo = time.Time{}
|
||||||
|
)
|
||||||
|
|
||||||
|
func autoPurgeRepo(cron bool) {
|
||||||
|
if cron && !autoPurgeRepoAfterFirstSync {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if time.Since(lastAutoPurgeRepo) < 6*time.Hour {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
autoPurgeRepoAfterFirstSync = true
|
||||||
|
defer func() {
|
||||||
|
lastAutoPurgeRepo = time.Now()
|
||||||
|
}()
|
||||||
|
|
||||||
|
if 1 > len(Conf.Repo.Key) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
repo, err := newRepository()
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
now := time.Now()
|
||||||
|
|
||||||
|
dateGroupedIndexes := map[string][]*dejavu.Log{} // 按照日期分组
|
||||||
|
// 收集指定日期内需要保留的索引
|
||||||
|
var date string
|
||||||
|
page := 1
|
||||||
|
for {
|
||||||
|
indexLogs, pageCount, _, err := repo.GetIndexLogs(page, 512)
|
||||||
|
if nil != err {
|
||||||
|
logging.LogErrorf("get data repo index logs failed: %s", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if 1 > len(indexLogs) {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
|
||||||
|
tooOld := false
|
||||||
|
for _, index := range indexLogs {
|
||||||
|
if now.UnixMilli()-index.Created <= int64(Conf.Repo.IndexRetentionDays)*24*60*60*1000 {
|
||||||
|
date = time.UnixMilli(index.Created).Format("2006-01-02")
|
||||||
|
if _, ok := dateGroupedIndexes[date]; !ok {
|
||||||
|
dateGroupedIndexes[date] = []*dejavu.Log{}
|
||||||
|
}
|
||||||
|
dateGroupedIndexes[date] = append(dateGroupedIndexes[date], index)
|
||||||
|
} else {
|
||||||
|
tooOld = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if tooOld {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
page++
|
||||||
|
if page > pageCount {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 筛选出每日需要保留的索引
|
||||||
|
var retentionIndexIDs []string
|
||||||
|
for _, indexes := range dateGroupedIndexes {
|
||||||
|
if len(indexes) <= Conf.Repo.RetentionIndexesDaily {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
keepIndexes := hashset.New()
|
||||||
|
keepIndexes.Add(indexes[0]) // 每天最后一个固定保留
|
||||||
|
// 随机保留指定数量的索引
|
||||||
|
for i := 0; i < Conf.Repo.RetentionIndexesDaily*7; i++ {
|
||||||
|
keepIndexes.Add(indexes[mathRand.Intn(len(indexes)-1)])
|
||||||
|
if keepIndexes.Size() >= Conf.Repo.RetentionIndexesDaily {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, keepIndex := range keepIndexes.Values() {
|
||||||
|
retentionIndexIDs = append(retentionIndexIDs, keepIndex.(*dejavu.Log).ID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
retentionIndexIDs = gulu.Str.RemoveDuplicatedElem(retentionIndexIDs)
|
||||||
|
if 1 > len(retentionIndexIDs) {
|
||||||
|
logging.LogInfof("no index to purge [ellapsed=%.2fs]", time.Since(now).Seconds())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
stat, err := repo.Purge(retentionIndexIDs...)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
deletedIndexes := stat.Indexes
|
||||||
|
deletedObjects := stat.Objects
|
||||||
|
deletedSize := humanize.BytesCustomCeil(uint64(stat.Size), 2)
|
||||||
|
logging.LogInfof("purge data repo completed [ellapsed=%.2fs, indexes=%d, objects=%d, size=%s]",
|
||||||
|
time.Since(now).Seconds(), deletedIndexes, deletedObjects, deletedSize)
|
||||||
|
}
|
||||||
|
|
||||||
func GetRepoFile(fileID string) (ret []byte, p string, err error) {
|
func GetRepoFile(fileID string) (ret []byte, p string, err error) {
|
||||||
if 1 > len(Conf.Repo.Key) {
|
if 1 > len(Conf.Repo.Key) {
|
||||||
err = errors.New(Conf.Language(26))
|
err = errors.New(Conf.Language(26))
|
||||||
@ -518,6 +629,7 @@ func PurgeRepo() (err error) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
now := time.Now()
|
||||||
stat, err := repo.Purge()
|
stat, err := repo.Purge()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return
|
return
|
||||||
@ -526,6 +638,8 @@ func PurgeRepo() (err error) {
|
|||||||
deletedIndexes := stat.Indexes
|
deletedIndexes := stat.Indexes
|
||||||
deletedObjects := stat.Objects
|
deletedObjects := stat.Objects
|
||||||
deletedSize := humanize.BytesCustomCeil(uint64(stat.Size), 2)
|
deletedSize := humanize.BytesCustomCeil(uint64(stat.Size), 2)
|
||||||
|
logging.LogInfof("purge data repo completed [ellapsed=%.2fs, indexes=%d, objects=%d, size=%s]",
|
||||||
|
time.Since(now).Seconds(), deletedIndexes, deletedObjects, deletedSize)
|
||||||
msg = fmt.Sprintf(Conf.Language(203), deletedIndexes, deletedObjects, deletedSize)
|
msg = fmt.Sprintf(Conf.Language(203), deletedIndexes, deletedObjects, deletedSize)
|
||||||
util.PushMsg(msg, 7000)
|
util.PushMsg(msg, 7000)
|
||||||
return
|
return
|
||||||
@ -1348,8 +1462,12 @@ func syncRepo(exit, byHand bool) (dataChanged bool, err error) {
|
|||||||
processSyncMergeResult(exit, byHand, mergeResult, trafficStat, "a", elapsed)
|
processSyncMergeResult(exit, byHand, mergeResult, trafficStat, "a", elapsed)
|
||||||
|
|
||||||
if !exit {
|
if !exit {
|
||||||
// 首次数据同步执行完成后再执行索引订正 Index fixing should not be performed before data synchronization https://github.com/siyuan-note/siyuan/issues/10761
|
go func() {
|
||||||
go checkIndex()
|
// 首次数据同步执行完成后再执行索引订正 Index fixing should not be performed before data synchronization https://github.com/siyuan-note/siyuan/issues/10761
|
||||||
|
checkIndex()
|
||||||
|
// 索引订正结束后执行数据仓库清理 Automatic purge for local data repo https://github.com/siyuan-note/siyuan/issues/13091
|
||||||
|
autoPurgeRepo(false)
|
||||||
|
}()
|
||||||
}
|
}
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
@ -119,6 +119,7 @@ func getCurrentTasks() (ret []*Task) {
|
|||||||
|
|
||||||
const (
|
const (
|
||||||
RepoCheckout = "task.repo.checkout" // 从快照中检出
|
RepoCheckout = "task.repo.checkout" // 从快照中检出
|
||||||
|
RepoAutoPurge = "task.repo.autoPurge" // 自动清理数据仓库
|
||||||
DatabaseIndexFull = "task.database.index.full" // 重建索引
|
DatabaseIndexFull = "task.database.index.full" // 重建索引
|
||||||
DatabaseIndex = "task.database.index" // 数据库索引
|
DatabaseIndex = "task.database.index" // 数据库索引
|
||||||
DatabaseIndexCommit = "task.database.index.commit" // 数据库索引提交
|
DatabaseIndexCommit = "task.database.index.commit" // 数据库索引提交
|
||||||
@ -143,6 +144,7 @@ const (
|
|||||||
// uniqueActions 描述了唯一的任务,即队列中只能存在一个在执行的任务。
|
// uniqueActions 描述了唯一的任务,即队列中只能存在一个在执行的任务。
|
||||||
var uniqueActions = []string{
|
var uniqueActions = []string{
|
||||||
RepoCheckout,
|
RepoCheckout,
|
||||||
|
RepoAutoPurge,
|
||||||
DatabaseIndexFull,
|
DatabaseIndexFull,
|
||||||
DatabaseIndexCommit,
|
DatabaseIndexCommit,
|
||||||
OCRImage,
|
OCRImage,
|
||||||
|
Loading…
Reference in New Issue
Block a user