diff --git a/app/appearance/langs/en_US.json b/app/appearance/langs/en_US.json index 891fffbb8..03aaf167c 100644 --- a/app/appearance/langs/en_US.json +++ b/app/appearance/langs/en_US.json @@ -983,7 +983,7 @@ "105": "Corrupted data repo have been automatically reset", "106": "Maximum length is limited to 512 characters", "107": "Moving document [%s]", - "108": "Cleaning obsolete indexes...", + "108": "TODO", "109": "Remove reminder completed [%s]", "110": "Renaming...", "111": "Saving document [%s]...", diff --git a/app/appearance/langs/es_ES.json b/app/appearance/langs/es_ES.json index cb1ff166a..ee55b5907 100644 --- a/app/appearance/langs/es_ES.json +++ b/app/appearance/langs/es_ES.json @@ -983,7 +983,7 @@ "105": "El repositorio de datos corruptos se ha restablecido automáticamente", "106": "La longitud máxima está limitada a 512 caracteres", "107": "Moviendo documento [%s]", - "108": "Limpiando índices obsoletos...", + "108": "TODO", "109": "Eliminación de recordatorios completada [%s]", "110": "Renombrar...", "111": "Guardando documento [%s]...", diff --git a/app/appearance/langs/fr_FR.json b/app/appearance/langs/fr_FR.json index 982c8bbae..6243e7af2 100644 --- a/app/appearance/langs/fr_FR.json +++ b/app/appearance/langs/fr_FR.json @@ -983,7 +983,7 @@ "105": "Le référentiel de données corrompu a été automatiquement réinitialisé", "106": "La longueur maximale est limitée à 512 caractères", "107": "Déplacement du document [%s]", - "108": "Nettoyage des index obsolètes...", + "108": "TODO", "109": "Supprimer le rappel terminé [%s]", "110": "Renommer...", "111": "Enregistrement du document [%s]...", diff --git a/app/appearance/langs/zh_CHT.json b/app/appearance/langs/zh_CHT.json index b152fdc43..a9862e96b 100644 --- a/app/appearance/langs/zh_CHT.json +++ b/app/appearance/langs/zh_CHT.json @@ -983,7 +983,7 @@ "105": "已經自動重置損壞的數據倉庫", "106": "最大長度限制為 512 字元", "107": "正在移動文檔 [%s]", - "108": "正在清理已過時的索引...", + "108": "TODO", "109": "移除提醒完畢 [%s]", "110": "正在重命名...", "111": "正在保存文檔 [%s]...", diff --git a/app/appearance/langs/zh_CN.json b/app/appearance/langs/zh_CN.json index 2434bb55c..00923c276 100644 --- a/app/appearance/langs/zh_CN.json +++ b/app/appearance/langs/zh_CN.json @@ -983,7 +983,7 @@ "105": "已经自动重置损坏的数据仓库", "106": "最大长度限制为 512 字符", "107": "正在移动文档 [%s]", - "108": "正在清理已过时的索引...", + "108": "TODO", "109": "移除提醒完毕 [%s]", "110": "正在重命名...", "111": "正在保存文档 [%s]...", diff --git a/kernel/api/transaction.go b/kernel/api/transaction.go index d50a73fdd..a7f3c21d0 100644 --- a/kernel/api/transaction.go +++ b/kernel/api/transaction.go @@ -25,9 +25,7 @@ import ( "github.com/88250/gulu" "github.com/gin-gonic/gin" "github.com/siyuan-note/filelock" - "github.com/siyuan-note/logging" "github.com/siyuan-note/siyuan/kernel/model" - "github.com/siyuan-note/siyuan/kernel/sql" "github.com/siyuan-note/siyuan/kernel/util" ) @@ -67,17 +65,6 @@ func performTransactions(c *gin.Context) { ret.Code = 1 return } - if nil != err { - tx, txErr := sql.BeginTx() - if nil != txErr { - logging.LogFatalf("transaction failed: %s", txErr) - return - } - sql.ClearBoxHash(tx) - sql.CommitTx(tx) - logging.LogFatalf("transaction failed: %s", err) - return - } ret.Data = transactions diff --git a/kernel/model/file.go b/kernel/model/file.go index b00162de3..2ca36ee88 100644 --- a/kernel/model/file.go +++ b/kernel/model/file.go @@ -912,15 +912,7 @@ func createTreeTx(tree *parse.Tree) { transaction := &Transaction{DoOperations: []*Operation{{Action: "create", Data: tree}}} err := PerformTransactions(&[]*Transaction{transaction}) if nil != err { - tx, txErr := sql.BeginTx() - if nil != txErr { - logging.LogFatalf("transaction failed: %s", txErr) - return - } - sql.ClearBoxHash(tx) - sql.CommitTx(tx) logging.LogFatalf("transaction failed: %s", err) - return } } @@ -1472,7 +1464,6 @@ func createDoc(boxID, p, title, dom string) (err error) { logging.LogFatalf("transaction failed: %s", txErr) return } - sql.ClearBoxHash(tx) sql.CommitTx(tx) logging.LogFatalf("transaction failed: %s", err) return diff --git a/kernel/model/index.go b/kernel/model/index.go index cb7cb5387..cc9810a36 100644 --- a/kernel/model/index.go +++ b/kernel/model/index.go @@ -17,12 +17,8 @@ package model import ( - "bytes" - "crypto/sha256" "fmt" - "github.com/siyuan-note/siyuan/kernel/task" "runtime/debug" - "sort" "strings" "time" @@ -34,6 +30,7 @@ import ( "github.com/siyuan-note/siyuan/kernel/cache" "github.com/siyuan-note/siyuan/kernel/filesys" "github.com/siyuan-note/siyuan/kernel/sql" + "github.com/siyuan-note/siyuan/kernel/task" "github.com/siyuan-note/siyuan/kernel/treenode" "github.com/siyuan-note/siyuan/kernel/util" ) @@ -47,7 +44,6 @@ func unindex(boxID string) { if nil != err { return } - sql.RemoveBoxHash(tx, boxID) sql.DeleteByBoxTx(tx, boxID) sql.CommitTx(tx) ids := treenode.RemoveBlockTreesByBoxID(boxID) @@ -83,8 +79,6 @@ func index(boxID string, fullRebuildIndex bool) { bootProgressPart := 10.0 / float64(boxLen) / float64(len(files)) luteEngine := NewLute() - idTitleMap := map[string]string{} - idHashMap := map[string]string{} var treeCount int var treeSize int64 util.PushEndlessProgress(fmt.Sprintf("["+box.Name+"] "+Conf.Language(64), len(files))) @@ -105,7 +99,7 @@ func index(boxID string, fullRebuildIndex bool) { } docIAL := parse.IAL2MapUnEsc(tree.Root.KramdownIAL) - if "" == docIAL["updated"] { + if "" == docIAL["updated"] { // 早期的数据可能没有 updated 属性,这里进行订正 updated := util.TimeFromID(tree.Root.ID) tree.Root.SetIALAttr("updated", updated) docIAL["updated"] = updated @@ -117,12 +111,7 @@ func index(boxID string, fullRebuildIndex bool) { util.IncBootProgress(bootProgressPart, fmt.Sprintf(Conf.Language(92), util.ShortPathForBootingDisplay(tree.Path))) treeSize += file.size treeCount++ - // 缓存文档标题,后面做 Path -> HPath 路径映射时需要 - idTitleMap[tree.ID] = tree.Root.IALAttr("title") - // 缓存块树 treenode.IndexBlockTree(tree) - // 缓存 ID-Hash,后面需要用于判断是否要重建库 - idHashMap[tree.ID] = tree.Hash if 1 < i && 0 == i%64 { util.PushEndlessProgress(fmt.Sprintf(Conf.Language(88), i, len(files)-i)) } @@ -131,25 +120,7 @@ func index(boxID string, fullRebuildIndex bool) { box.UpdateHistoryGenerated() // 初始化历史生成时间为当前时间 - // 检查是否需要重新建库 - util.SetBootDetails("Checking data hashes...") - var ids []string - for id := range idTitleMap { - ids = append(ids, id) - } - sort.Slice(ids, func(i, j int) bool { return ids[i] >= ids[j] }) - buf := bytes.Buffer{} - for _, id := range ids { - hash, _ := idHashMap[id] - buf.WriteString(hash) - util.SetBootDetails("Checking hash " + hash) - } - boxHash := fmt.Sprintf("%x", sha256.Sum256(buf.Bytes())) - - dbBoxHash := sql.GetBoxHash(box.ID) - if boxHash == dbBoxHash { - //logging.LogInfof("use existing database for box [%s]", box.ID) - util.SetBootDetails("Use existing database for notebook " + box.ID) + if !fullRebuildIndex { return } @@ -159,26 +130,8 @@ func index(boxID string, fullRebuildIndex bool) { defer sql.EnableCache() start := time.Now() - if !fullRebuildIndex { - tx, err := sql.BeginTx() - if nil != err { - return - } - sql.PutBoxHash(tx, box.ID, boxHash) - util.SetBootDetails("Cleaning obsolete indexes...") - util.PushEndlessProgress(Conf.Language(108)) - sql.DeleteByBoxTx(tx, box.ID) - if err = sql.CommitTx(tx); nil != err { - return - } - } - bootProgressPart = 20.0 / float64(boxLen) / float64(treeCount) - - context := map[string]interface{}{eventbus.CtxPushMsg: eventbus.CtxPushMsgToStatusBarAndProgress} i = 0 - // 块级行级入库,缓存块 - // 这里不能并行插入,因为 SQLite 不支持 for _, file := range files { if file.isdir || !strings.HasSuffix(file.name, ".sy") { continue @@ -191,23 +144,15 @@ func index(boxID string, fullRebuildIndex bool) { } util.IncBootProgress(bootProgressPart, fmt.Sprintf(Conf.Language(93), util.ShortPathForBootingDisplay(tree.Path))) - tx, err := sql.BeginTx() - if nil != err { - continue - } - if err = sql.InsertBlocksSpans(tx, tree, context); nil != err { - sql.RollbackTx(tx) - continue - } - if err = sql.CommitTx(tx); nil != err { - continue - } + sql.UpsertTreeQueue(tree) if 1 < i && 0 == i%64 { util.PushEndlessProgress(fmt.Sprintf("["+box.Name+"] "+Conf.Language(53), i, treeCount-i)) } i++ } + sql.WaitForWritingDatabase() + end := time.Now() elapsed := end.Sub(start).Seconds() logging.LogInfof("rebuilt database for notebook [%s] in [%.2fs], tree [count=%d, size=%s]", box.ID, elapsed, treeCount, humanize.Bytes(uint64(treeSize))) diff --git a/kernel/sql/block.go b/kernel/sql/block.go index eb0d11db3..fbd7392b6 100644 --- a/kernel/sql/block.go +++ b/kernel/sql/block.go @@ -65,10 +65,6 @@ func updateRootContent(tx *sql.Tx, content, updated, id string) { cache.RemoveBlockIAL(id) } -func InsertBlock(tx *sql.Tx, block *Block, context map[string]interface{}) (err error) { - return insertBlocks(tx, []*Block{block}, context) -} - func UpdateBlockContent(block *Block) { tx, err := BeginTx() if nil != err { diff --git a/kernel/sql/queue.go b/kernel/sql/queue.go index 2dd7349c8..30e672505 100644 --- a/kernel/sql/queue.go +++ b/kernel/sql/queue.go @@ -17,15 +17,10 @@ package sql import ( - "bytes" - "crypto/sha256" - "database/sql" - "fmt" "path" "sync" "time" - "github.com/88250/lute/ast" "github.com/88250/lute/parse" "github.com/emirpasic/gods/sets/hashset" "github.com/siyuan-note/eventbus" @@ -136,23 +131,6 @@ func FlushQueue() { if 5000 < elapsed { logging.LogInfof("op tx [%dms]", elapsed) } - - start = time.Now() - tx, err = BeginTx() - if nil != err { - return - } - for _, box := range boxes.Values() { - if !ast.IsNodeIDPattern(box.(string)) { - continue - } - updateBoxHash(tx, box.(string)) - } - CommitTx(tx) - elapsed = time.Now().Sub(start).Milliseconds() - if 1000 < elapsed { - logging.LogInfof("hash tx [%dms]", elapsed) - } } func mergeUpsertTrees() (ops []*treeQueueOperation) { @@ -229,28 +207,3 @@ func RemoveTreePathQueue(treeBox, treePathPrefix string) { newOp := &treeQueueOperation{removeTreeBox: treeBox, removeTreePath: treePathPrefix, inQueueTime: time.Now(), action: "delete"} operationQueue = append(operationQueue, newOp) } - -func updateBoxHash(tx *sql.Tx, boxID string) { - sum := boxChecksum(boxID) - PutBoxHash(tx, boxID, sum) -} - -func boxChecksum(box string) (ret string) { - rows, err := query("SELECT hash FROM blocks WHERE type = 'd' AND box = ? ORDER BY id DESC", box) - if nil != err { - logging.LogErrorf("sql query failed: %s", err) - return - } - defer rows.Close() - buf := bytes.Buffer{} - for rows.Next() { - var hash string - if err = rows.Scan(&hash); nil != err { - logging.LogErrorf("query scan field failed: %s", err) - return - } - buf.WriteString(hash) - } - ret = fmt.Sprintf("%x", sha256.Sum256(buf.Bytes())) - return -} diff --git a/kernel/sql/stat.go b/kernel/sql/stat.go index e8222d8d5..1b1baa5ec 100644 --- a/kernel/sql/stat.go +++ b/kernel/sql/stat.go @@ -54,27 +54,6 @@ func setDatabaseVer() { CommitTx(tx) } -func ClearBoxHash(tx *sql.Tx) { - stmt := "DELETE FROM stat WHERE `key` LIKE '%_hash'" - execStmtTx(tx, stmt) -} - -func RemoveBoxHash(tx *sql.Tx, box string) { - key := box + "_hash" - stmt := "DELETE FROM stat WHERE `key` = '" + key + "'" - execStmtTx(tx, stmt) -} - -func PutBoxHash(tx *sql.Tx, box, hash string) { - key := box + "_hash" - putStat(tx, key, hash) -} - -func GetBoxHash(box string) string { - key := box + "_hash" - return getStat(key) -} - func putStat(tx *sql.Tx, key, value string) (err error) { stmt := "DELETE FROM stat WHERE `key` = '" + key + "'" if err = execStmtTx(tx, stmt); nil != err { diff --git a/kernel/sql/upsert.go b/kernel/sql/upsert.go index 24d2c7049..380a52475 100644 --- a/kernel/sql/upsert.go +++ b/kernel/sql/upsert.go @@ -36,13 +36,6 @@ func init() { luteEngine.RenderOptions.KramdownBlockIAL = false // 数据库 markdown 字段为标准 md,但是要保留 span block ial } -func InsertBlocksSpans(tx *sql.Tx, tree *parse.Tree, context map[string]interface{}) (err error) { - if err = insertBlocksSpans(tx, tree, context); nil != err { - logging.LogErrorf("insert tree [%s] into database failed: %s", tree.Box+tree.Path, err) - } - return -} - func InsertRefs(tx *sql.Tx, tree *parse.Tree) { if err := insertRef(tx, tree); nil != err { logging.LogErrorf("insert refs tree [%s] into database failed: %s", tree.Box+tree.Path, err) @@ -395,23 +388,6 @@ func insertFileAnnotationRefs0(tx *sql.Tx, bulk []*FileAnnotationRef) (err error return } -func insertBlocksSpans(tx *sql.Tx, tree *parse.Tree, context map[string]interface{}) (err error) { - blocks, spans, assets, attributes := fromTree(tree.Root, tree) - if err = insertBlocks(tx, blocks, context); nil != err { - return - } - if err = insertSpans(tx, spans); nil != err { - return - } - if err = insertAssets(tx, assets); nil != err { - return - } - if err = insertAttributes(tx, attributes); nil != err { - return - } - return -} - func insertRef(tx *sql.Tx, tree *parse.Tree) (err error) { refs, fileAnnotationRefs := refsFromTree(tree) if err = insertRefs(tx, refs); nil != err {