diff --git a/kernel/model/box.go b/kernel/model/box.go index 176658520..fe46e0748 100644 --- a/kernel/model/box.go +++ b/kernel/model/box.go @@ -25,6 +25,7 @@ import ( "os" "path" "path/filepath" + "runtime" "sort" "strings" "sync" @@ -516,12 +517,15 @@ func fullReindex() { } treenode.InitBlockTree(true) + sql.DisableCache() openedBoxes := Conf.GetOpenedBoxes() for _, openedBox := range openedBoxes { index(openedBox.ID) } + sql.EnableCache() treenode.SaveBlockTree(true) LoadFlashcards() + runtime.GC() } func ChangeBoxSort(boxIDs []string) { diff --git a/kernel/model/index.go b/kernel/model/index.go index 424eedfee..d252b9a42 100644 --- a/kernel/model/index.go +++ b/kernel/model/index.go @@ -125,6 +125,7 @@ func index(boxID string) { end := time.Now() elapsed := end.Sub(start).Seconds() logging.LogInfof("rebuilt database for notebook [%s] in [%.2fs], tree [count=%d, size=%s]", box.ID, elapsed, treeCount, humanize.Bytes(uint64(treeSize))) + runtime.GC() return } @@ -186,6 +187,49 @@ func IndexRefs() { util.PushStatusBar(fmt.Sprintf(Conf.Language(55), i)) } +// AutoIndexEmbedBlock 嵌入块支持搜索 https://github.com/siyuan-note/siyuan/issues/7112 +func AutoIndexEmbedBlock() { + for { + embedBlocks := sql.QueryEmptyContentEmbedBlocks() + task.AppendTask(task.DatabaseIndexEmbedBlock, autoIndexEmbedBlock, embedBlocks) + time.Sleep(10 * time.Minute) + } +} + +func autoIndexEmbedBlock(embedBlocks []*sql.Block) { + for i, embedBlock := range embedBlocks { + stmt := strings.TrimPrefix(embedBlock.Markdown, "{{") + stmt = strings.TrimSuffix(stmt, "}}") + queryResultBlocks := sql.SelectBlocksRawStmtNoParse(stmt, 102400) + for _, block := range queryResultBlocks { + embedBlock.Content += block.Content + } + if "" == embedBlock.Content { + embedBlock.Content = "no query result" + } + sql.UpdateBlockContent(embedBlock) + + if 63 <= i { // 一次任务中最多处理 64 个嵌入块，防止卡顿 + break + } + } +} + +func updateEmbedBlockContent(embedBlockID string, queryResultBlocks []*EmbedBlock) { + embedBlock := sql.GetBlock(embedBlockID) + if nil == embedBlock { + return + } + + for _, block := range queryResultBlocks { + embedBlock.Content += block.Block.Markdown + } + if "" == embedBlock.Content { + embedBlock.Content = "no query result" + } + sql.UpdateBlockContent(embedBlock) +} + func init() { //eventbus.Subscribe(eventbus.EvtSQLInsertBlocks, func(context map[string]interface{}, current, total, blockCount int, hash string) { // if util.ContainerAndroid == util.Container || util.ContainerIOS == util.Container { diff --git a/kernel/model/index_fix.go b/kernel/model/index_fix.go new file mode 100644 index 000000000..ea8850cb7 --- /dev/null +++ b/kernel/model/index_fix.go @@ -0,0 +1,231 @@ +// SiYuan - Build Your Eternal Digital Garden +// Copyright (c) 2020-present, b3log.org +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package model + +import ( + "fmt" + "os" + "path" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/88250/lute/ast" + "github.com/88250/lute/html" + "github.com/88250/lute/parse" + "github.com/siyuan-note/logging" + "github.com/siyuan-note/siyuan/kernel/sql" + "github.com/siyuan-note/siyuan/kernel/task" + "github.com/siyuan-note/siyuan/kernel/treenode" + "github.com/siyuan-note/siyuan/kernel/util" +) + +// AutoFixIndex 自动校验数据库索引 https://github.com/siyuan-note/siyuan/issues/7016 +func AutoFixIndex() { + for { + task.AppendTask(task.DatabaseIndexFix, autoFixIndex) + time.Sleep(10 * time.Minute) + } +} + +var autoFixLock = sync.Mutex{} + +func autoFixIndex() { + defer logging.Recover() + + // 根据文件系统补全块树 + boxes := Conf.GetOpenedBoxes() + for _, box := range boxes { + boxPath := filepath.Join(util.DataDir, box.ID) + var paths []string + filepath.Walk(boxPath, func(path string, info os.FileInfo, err error) error { + if !info.IsDir() && filepath.Ext(path) == ".sy" { + p := path[len(boxPath):] + p = filepath.ToSlash(p) + paths = append(paths, p) + } + return nil + }) + + size := len(paths) + + redundantPaths := treenode.GetRedundantPaths(box.ID, paths) + for _, p := range redundantPaths { + treenode.RemoveBlockTreesByPath(p) + } + + missingPaths := treenode.GetNotExistPaths(box.ID, paths) + for i, p := range missingPaths { + id := path.Base(p) + id = strings.TrimSuffix(id, ".sy") + if !ast.IsNodeIDPattern(id) { + continue + } + + reindexTreeByPath(box.ID, p, i, size) + if util.IsExiting { + break + } + } + + if util.IsExiting { + break + } + } + + // 清理已关闭的笔记本块树 + boxes = Conf.GetClosedBoxes() + for _, box := range boxes { + treenode.RemoveBlockTreesByBoxID(box.ID) + } + + // 对比块树和数据库并订正数据库 + rootUpdatedMap := treenode.GetRootUpdated() + dbRootUpdatedMap, err := sql.GetRootUpdated("blocks") + if nil == err { + reindexTreeByUpdated(rootUpdatedMap, dbRootUpdatedMap, "blocks") + } + dbFtsRootUpdatedMap, err := sql.GetRootUpdated("blocks_fts") + if nil == err { + reindexTreeByUpdated(rootUpdatedMap, dbFtsRootUpdatedMap, "blocks_fts") + } + if !Conf.Search.CaseSensitive { + dbFtsRootUpdatedMap, err = sql.GetRootUpdated("blocks_fts_case_insensitive") + if nil == err { + reindexTreeByUpdated(rootUpdatedMap, dbFtsRootUpdatedMap, "blocks_fts_case_insensitive") + } + } + + // 去除重复的数据库块记录 + duplicatedRootIDs := sql.GetDuplicatedRootIDs("blocks") + if 1 > len(duplicatedRootIDs) { + duplicatedRootIDs = sql.GetDuplicatedRootIDs("blocks_fts") + if 1 > len(duplicatedRootIDs) && !Conf.Search.CaseSensitive { + duplicatedRootIDs = sql.GetDuplicatedRootIDs("blocks_fts_case_insensitive") + } + } + size := len(duplicatedRootIDs) + for i, rootID := range duplicatedRootIDs { + root := sql.GetBlock(rootID) + if nil == root { + continue + } + + logging.LogWarnf("exist more than one tree [%s], reindex it", rootID) + sql.RemoveTreeQueue(root.Box, rootID) + reindexTree(rootID, i, size) + + if util.IsExiting { + break + } + } + + util.PushStatusBar(Conf.Language(185)) +} + +func reindexTreeByUpdated(rootUpdatedMap, dbRootUpdatedMap map[string]string, blocksTable string) { + i := -1 + size := len(rootUpdatedMap) + for rootID, updated := range rootUpdatedMap { + i++ + + if util.IsExiting { + break + } + + rootUpdated := dbRootUpdatedMap[rootID] + if "" == rootUpdated { + //logging.LogWarnf("not found tree [%s] in database, reindex it", rootID) + reindexTree(rootID, i, size) + continue + } + + if "" == updated { + // BlockTree 迁移，v2.6.3 之前没有 updated 字段 + reindexTree(rootID, i, size) + continue + } + + btUpdated, _ := time.Parse("20060102150405", updated) + dbUpdated, _ := time.Parse("20060102150405", rootUpdated) + if dbUpdated.Before(btUpdated.Add(-10 * time.Minute)) { + logging.LogWarnf("tree [%s] is not up to date, reindex it", rootID) + reindexTree(rootID, i, size) + continue + } + + if util.IsExiting { + break + } + } + + for rootID, _ := range dbRootUpdatedMap { + if _, ok := rootUpdatedMap[rootID]; !ok { + logging.LogWarnf("tree [%s] is not in block tree, remove it from [%s]", rootID, blocksTable) + sql.DeleteTree(blocksTable, rootID) + } + + if util.IsExiting { + break + } + } +} + +func reindexTreeByPath(box, p string, i, size int) { + tree, err := LoadTree(box, p) + if nil != err { + return + } + + reindexTree0(tree, i, size) +} + +func reindexTree(rootID string, i, size int) { + root := treenode.GetBlockTree(rootID) + if nil == root { + logging.LogWarnf("root block not found", rootID) + return + } + + tree, err := LoadTree(root.BoxID, root.Path) + if nil != err { + if os.IsNotExist(err) { + // 文件系统上没有找到该 .sy 文件，则订正块树 + treenode.RemoveBlockTreesByRootID(rootID) + } + return + } + + reindexTree0(tree, i, size) +} + +func reindexTree0(tree *parse.Tree, i, size int) { + updated := tree.Root.IALAttr("updated") + if "" == updated { + updated = util.TimeFromID(tree.Root.ID) + tree.Root.SetIALAttr("updated", updated) + indexWriteJSONQueue(tree) + } else { + treenode.IndexBlockTree(tree) + sql.IndexTreeQueue(tree.Box, tree.Path) + } + + if 0 == i%64 { + util.PushStatusBar(fmt.Sprintf(Conf.Language(183), i, size, html.EscapeHTMLStr(path.Base(tree.HPath)))) + } +} diff --git a/kernel/model/transaction.go b/kernel/model/transaction.go index 9eaceedea..a075d115c 100644 --- a/kernel/model/transaction.go +++ b/kernel/model/transaction.go @@ -20,9 +20,6 @@ import ( "bytes" "errors" "fmt" - "github.com/siyuan-note/siyuan/kernel/task" - "os" - "path" "path/filepath" "strings" "sync" @@ -31,7 +28,6 @@ import ( "github.com/88250/gulu" "github.com/88250/lute/ast" "github.com/88250/lute/editor" - "github.com/88250/lute/html" "github.com/88250/lute/lex" "github.com/88250/lute/parse" "github.com/emirpasic/gods/sets/hashset" @@ -1235,241 +1231,3 @@ func updateRefText(refNode *ast.Node, changedDefNodes map[string]*ast.Node) (cha }) return } - -// AutoIndexEmbedBlock 嵌入块支持搜索 https://github.com/siyuan-note/siyuan/issues/7112 -func AutoIndexEmbedBlock() { - for { - embedBlocks := sql.QueryEmptyContentEmbedBlocks() - task.AppendTask(task.DatabaseIndexEmbedBlock, autoIndexEmbedBlock, embedBlocks) - time.Sleep(10 * time.Minute) - } -} - -func autoIndexEmbedBlock(embedBlocks []*sql.Block) { - for i, embedBlock := range embedBlocks { - stmt := strings.TrimPrefix(embedBlock.Markdown, "{{") - stmt = strings.TrimSuffix(stmt, "}}") - queryResultBlocks := sql.SelectBlocksRawStmtNoParse(stmt, 102400) - for _, block := range queryResultBlocks { - embedBlock.Content += block.Content - } - if "" == embedBlock.Content { - embedBlock.Content = "no query result" - } - sql.UpdateBlockContent(embedBlock) - - if 63 <= i { // 一次任务中最多处理 64 个嵌入块，防止卡顿 - break - } - } -} - -func updateEmbedBlockContent(embedBlockID string, queryResultBlocks []*EmbedBlock) { - embedBlock := sql.GetBlock(embedBlockID) - if nil == embedBlock { - return - } - - for _, block := range queryResultBlocks { - embedBlock.Content += block.Block.Markdown - } - if "" == embedBlock.Content { - embedBlock.Content = "no query result" - } - sql.UpdateBlockContent(embedBlock) -} - -// AutoFixIndex 自动校验数据库索引 https://github.com/siyuan-note/siyuan/issues/7016 -func AutoFixIndex() { - for { - task.AppendTask(task.DatabaseIndexFix, autoFixIndex) - time.Sleep(10 * time.Minute) - } -} - -var autoFixLock = sync.Mutex{} - -func autoFixIndex() { - defer logging.Recover() - - // 根据文件系统补全块树 - boxes := Conf.GetOpenedBoxes() - for _, box := range boxes { - boxPath := filepath.Join(util.DataDir, box.ID) - var paths []string - filepath.Walk(boxPath, func(path string, info os.FileInfo, err error) error { - if !info.IsDir() && filepath.Ext(path) == ".sy" { - p := path[len(boxPath):] - p = filepath.ToSlash(p) - paths = append(paths, p) - } - return nil - }) - - size := len(paths) - - redundantPaths := treenode.GetRedundantPaths(box.ID, paths) - for _, p := range redundantPaths { - treenode.RemoveBlockTreesByPath(p) - } - - missingPaths := treenode.GetNotExistPaths(box.ID, paths) - for i, p := range missingPaths { - id := path.Base(p) - id = strings.TrimSuffix(id, ".sy") - if !ast.IsNodeIDPattern(id) { - continue - } - - reindexTreeByPath(box.ID, p, i, size) - if util.IsExiting { - break - } - } - - if util.IsExiting { - break - } - } - - // 清理已关闭的笔记本块树 - boxes = Conf.GetClosedBoxes() - for _, box := range boxes { - treenode.RemoveBlockTreesByBoxID(box.ID) - } - - // 对比块树和数据库并订正数据库 - rootUpdatedMap := treenode.GetRootUpdated() - dbRootUpdatedMap, err := sql.GetRootUpdated("blocks") - if nil == err { - reindexTreeByUpdated(rootUpdatedMap, dbRootUpdatedMap, "blocks") - } - dbFtsRootUpdatedMap, err := sql.GetRootUpdated("blocks_fts") - if nil == err { - reindexTreeByUpdated(rootUpdatedMap, dbFtsRootUpdatedMap, "blocks_fts") - } - if !Conf.Search.CaseSensitive { - dbFtsRootUpdatedMap, err = sql.GetRootUpdated("blocks_fts_case_insensitive") - if nil == err { - reindexTreeByUpdated(rootUpdatedMap, dbFtsRootUpdatedMap, "blocks_fts_case_insensitive") - } - } - - // 去除重复的数据库块记录 - duplicatedRootIDs := sql.GetDuplicatedRootIDs("blocks") - if 1 > len(duplicatedRootIDs) { - duplicatedRootIDs = sql.GetDuplicatedRootIDs("blocks_fts") - if 1 > len(duplicatedRootIDs) && !Conf.Search.CaseSensitive { - duplicatedRootIDs = sql.GetDuplicatedRootIDs("blocks_fts_case_insensitive") - } - } - size := len(duplicatedRootIDs) - for i, rootID := range duplicatedRootIDs { - root := sql.GetBlock(rootID) - if nil == root { - continue - } - - logging.LogWarnf("exist more than one tree [%s], reindex it", rootID) - sql.RemoveTreeQueue(root.Box, rootID) - reindexTree(rootID, i, size) - - if util.IsExiting { - break - } - } - - util.PushStatusBar(Conf.Language(185)) -} - -func reindexTreeByUpdated(rootUpdatedMap, dbRootUpdatedMap map[string]string, blocksTable string) { - i := -1 - size := len(rootUpdatedMap) - for rootID, updated := range rootUpdatedMap { - i++ - - if util.IsExiting { - break - } - - rootUpdated := dbRootUpdatedMap[rootID] - if "" == rootUpdated { - logging.LogWarnf("not found tree [%s] in database, reindex it", rootID) - reindexTree(rootID, i, size) - continue - } - - if "" == updated { - // BlockTree 迁移，v2.6.3 之前没有 updated 字段 - reindexTree(rootID, i, size) - continue - } - - btUpdated, _ := time.Parse("20060102150405", updated) - dbUpdated, _ := time.Parse("20060102150405", rootUpdated) - if dbUpdated.Before(btUpdated.Add(-10 * time.Minute)) { - logging.LogWarnf("tree [%s] is not up to date, reindex it", rootID) - reindexTree(rootID, i, size) - continue - } - - if util.IsExiting { - break - } - } - - for rootID, _ := range dbRootUpdatedMap { - if _, ok := rootUpdatedMap[rootID]; !ok { - logging.LogWarnf("tree [%s] is not in block tree, remove it from [%s]", rootID, blocksTable) - sql.DeleteTree(blocksTable, rootID) - } - - if util.IsExiting { - break - } - } -} - -func reindexTreeByPath(box, p string, i, size int) { - tree, err := LoadTree(box, p) - if nil != err { - return - } - - reindexTree0(tree, i, size) -} - -func reindexTree(rootID string, i, size int) { - root := treenode.GetBlockTree(rootID) - if nil == root { - logging.LogWarnf("root block not found", rootID) - return - } - - tree, err := LoadTree(root.BoxID, root.Path) - if nil != err { - if os.IsNotExist(err) { - // 文件系统上没有找到该 .sy 文件，则订正块树 - treenode.RemoveBlockTreesByRootID(rootID) - } - return - } - - reindexTree0(tree, i, size) -} - -func reindexTree0(tree *parse.Tree, i, size int) { - updated := tree.Root.IALAttr("updated") - if "" == updated { - updated = util.TimeFromID(tree.Root.ID) - tree.Root.SetIALAttr("updated", updated) - indexWriteJSONQueue(tree) - } else { - treenode.IndexBlockTree(tree) - sql.IndexTreeQueue(tree.Box, tree.Path) - } - - if 0 == i%64 { - util.PushStatusBar(fmt.Sprintf(Conf.Language(183), i, size, html.EscapeHTMLStr(path.Base(tree.HPath)))) - } -}