🎨 改进内核任务调度机制提升稳定性 https://github.com/siyuan-note/siyuan/issues/7113

kernel/model/box.go

@@ -25,6 +25,7 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+	"runtime"
 	"sort"
 	"strings"
 	"sync"
@@ -516,12 +517,15 @@ func fullReindex() {
 	}
 	treenode.InitBlockTree(true)
 
+	sql.DisableCache()
 	openedBoxes := Conf.GetOpenedBoxes()
 	for _, openedBox := range openedBoxes {
 		index(openedBox.ID)
 	}
+	sql.EnableCache()
 	treenode.SaveBlockTree(true)
 	LoadFlashcards()
+	runtime.GC()
 }
 
 func ChangeBoxSort(boxIDs []string) {

kernel/model/index.go

@@ -125,6 +125,7 @@ func index(boxID string) {
 	end := time.Now()
 	elapsed := end.Sub(start).Seconds()
 	logging.LogInfof("rebuilt database for notebook [%s] in [%.2fs], tree [count=%d, size=%s]", box.ID, elapsed, treeCount, humanize.Bytes(uint64(treeSize)))
+	runtime.GC()
 	return
 }
 
@@ -186,6 +187,49 @@ func IndexRefs() {
 	util.PushStatusBar(fmt.Sprintf(Conf.Language(55), i))
 }
 
+// AutoIndexEmbedBlock 嵌入块支持搜索 https://github.com/siyuan-note/siyuan/issues/7112
+func AutoIndexEmbedBlock() {
+	for {
+		embedBlocks := sql.QueryEmptyContentEmbedBlocks()
+		task.AppendTask(task.DatabaseIndexEmbedBlock, autoIndexEmbedBlock, embedBlocks)
+		time.Sleep(10 * time.Minute)
+	}
+}
+
+func autoIndexEmbedBlock(embedBlocks []*sql.Block) {
+	for i, embedBlock := range embedBlocks {
+		stmt := strings.TrimPrefix(embedBlock.Markdown, "{{")
+		stmt = strings.TrimSuffix(stmt, "}}")
+		queryResultBlocks := sql.SelectBlocksRawStmtNoParse(stmt, 102400)
+		for _, block := range queryResultBlocks {
+			embedBlock.Content += block.Content
+		}
+		if "" == embedBlock.Content {
+			embedBlock.Content = "no query result"
+		}
+		sql.UpdateBlockContent(embedBlock)
+
+		if 63 <= i { // 一次任务中最多处理 64 个嵌入块，防止卡顿
+			break
+		}
+	}
+}
+
+func updateEmbedBlockContent(embedBlockID string, queryResultBlocks []*EmbedBlock) {
+	embedBlock := sql.GetBlock(embedBlockID)
+	if nil == embedBlock {
+		return
+	}
+
+	for _, block := range queryResultBlocks {
+		embedBlock.Content += block.Block.Markdown
+	}
+	if "" == embedBlock.Content {
+		embedBlock.Content = "no query result"
+	}
+	sql.UpdateBlockContent(embedBlock)
+}
+
 func init() {
 	//eventbus.Subscribe(eventbus.EvtSQLInsertBlocks, func(context map[string]interface{}, current, total, blockCount int, hash string) {
 	//	if util.ContainerAndroid == util.Container || util.ContainerIOS == util.Container {

kernel/model/index_fix.go

@@ -0,0 +1,231 @@
+// SiYuan - Build Your Eternal Digital Garden
+// Copyright (c) 2020-present, b3log.org
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+package model
+
+import (
+	"fmt"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/88250/lute/ast"
+	"github.com/88250/lute/html"
+	"github.com/88250/lute/parse"
+	"github.com/siyuan-note/logging"
+	"github.com/siyuan-note/siyuan/kernel/sql"
+	"github.com/siyuan-note/siyuan/kernel/task"
+	"github.com/siyuan-note/siyuan/kernel/treenode"
+	"github.com/siyuan-note/siyuan/kernel/util"
+)
+
+// AutoFixIndex 自动校验数据库索引 https://github.com/siyuan-note/siyuan/issues/7016
+func AutoFixIndex() {
+	for {
+		task.AppendTask(task.DatabaseIndexFix, autoFixIndex)
+		time.Sleep(10 * time.Minute)
+	}
+}
+
+var autoFixLock = sync.Mutex{}
+
+func autoFixIndex() {
+	defer logging.Recover()
+
+	// 根据文件系统补全块树
+	boxes := Conf.GetOpenedBoxes()
+	for _, box := range boxes {
+		boxPath := filepath.Join(util.DataDir, box.ID)
+		var paths []string
+		filepath.Walk(boxPath, func(path string, info os.FileInfo, err error) error {
+			if !info.IsDir() && filepath.Ext(path) == ".sy" {
+				p := path[len(boxPath):]
+				p = filepath.ToSlash(p)
+				paths = append(paths, p)
+			}
+			return nil
+		})
+
+		size := len(paths)
+
+		redundantPaths := treenode.GetRedundantPaths(box.ID, paths)
+		for _, p := range redundantPaths {
+			treenode.RemoveBlockTreesByPath(p)
+		}
+
+		missingPaths := treenode.GetNotExistPaths(box.ID, paths)
+		for i, p := range missingPaths {
+			id := path.Base(p)
+			id = strings.TrimSuffix(id, ".sy")
+			if !ast.IsNodeIDPattern(id) {
+				continue
+			}
+
+			reindexTreeByPath(box.ID, p, i, size)
+			if util.IsExiting {
+				break
+			}
+		}
+
+		if util.IsExiting {
+			break
+		}
+	}
+
+	// 清理已关闭的笔记本块树
+	boxes = Conf.GetClosedBoxes()
+	for _, box := range boxes {
+		treenode.RemoveBlockTreesByBoxID(box.ID)
+	}
+
+	// 对比块树和数据库并订正数据库
+	rootUpdatedMap := treenode.GetRootUpdated()
+	dbRootUpdatedMap, err := sql.GetRootUpdated("blocks")
+	if nil == err {
+		reindexTreeByUpdated(rootUpdatedMap, dbRootUpdatedMap, "blocks")
+	}
+	dbFtsRootUpdatedMap, err := sql.GetRootUpdated("blocks_fts")
+	if nil == err {
+		reindexTreeByUpdated(rootUpdatedMap, dbFtsRootUpdatedMap, "blocks_fts")
+	}
+	if !Conf.Search.CaseSensitive {
+		dbFtsRootUpdatedMap, err = sql.GetRootUpdated("blocks_fts_case_insensitive")
+		if nil == err {
+			reindexTreeByUpdated(rootUpdatedMap, dbFtsRootUpdatedMap, "blocks_fts_case_insensitive")
+		}
+	}
+
+	// 去除重复的数据库块记录
+	duplicatedRootIDs := sql.GetDuplicatedRootIDs("blocks")
+	if 1 > len(duplicatedRootIDs) {
+		duplicatedRootIDs = sql.GetDuplicatedRootIDs("blocks_fts")
+		if 1 > len(duplicatedRootIDs) && !Conf.Search.CaseSensitive {
+			duplicatedRootIDs = sql.GetDuplicatedRootIDs("blocks_fts_case_insensitive")
+		}
+	}
+	size := len(duplicatedRootIDs)
+	for i, rootID := range duplicatedRootIDs {
+		root := sql.GetBlock(rootID)
+		if nil == root {
+			continue
+		}
+
+		logging.LogWarnf("exist more than one tree [%s], reindex it", rootID)
+		sql.RemoveTreeQueue(root.Box, rootID)
+		reindexTree(rootID, i, size)
+
+		if util.IsExiting {
+			break
+		}
+	}
+
+	util.PushStatusBar(Conf.Language(185))
+}
+
+func reindexTreeByUpdated(rootUpdatedMap, dbRootUpdatedMap map[string]string, blocksTable string) {
+	i := -1
+	size := len(rootUpdatedMap)
+	for rootID, updated := range rootUpdatedMap {
+		i++
+
+		if util.IsExiting {
+			break
+		}
+
+		rootUpdated := dbRootUpdatedMap[rootID]
+		if "" == rootUpdated {
+			//logging.LogWarnf("not found tree [%s] in database, reindex it", rootID)
+			reindexTree(rootID, i, size)
+			continue
+		}
+
+		if "" == updated {
+			// BlockTree 迁移，v2.6.3 之前没有 updated 字段
+			reindexTree(rootID, i, size)
+			continue
+		}
+
+		btUpdated, _ := time.Parse("20060102150405", updated)
+		dbUpdated, _ := time.Parse("20060102150405", rootUpdated)
+		if dbUpdated.Before(btUpdated.Add(-10 * time.Minute)) {
+			logging.LogWarnf("tree [%s] is not up to date, reindex it", rootID)
+			reindexTree(rootID, i, size)
+			continue
+		}
+
+		if util.IsExiting {
+			break
+		}
+	}
+
+	for rootID, _ := range dbRootUpdatedMap {
+		if _, ok := rootUpdatedMap[rootID]; !ok {
+			logging.LogWarnf("tree [%s] is not in block tree, remove it from [%s]", rootID, blocksTable)
+			sql.DeleteTree(blocksTable, rootID)
+		}
+
+		if util.IsExiting {
+			break
+		}
+	}
+}
+
+func reindexTreeByPath(box, p string, i, size int) {
+	tree, err := LoadTree(box, p)
+	if nil != err {
+		return
+	}
+
+	reindexTree0(tree, i, size)
+}
+
+func reindexTree(rootID string, i, size int) {
+	root := treenode.GetBlockTree(rootID)
+	if nil == root {
+		logging.LogWarnf("root block not found", rootID)
+		return
+	}
+
+	tree, err := LoadTree(root.BoxID, root.Path)
+	if nil != err {
+		if os.IsNotExist(err) {
+			// 文件系统上没有找到该 .sy 文件，则订正块树
+			treenode.RemoveBlockTreesByRootID(rootID)
+		}
+		return
+	}
+
+	reindexTree0(tree, i, size)
+}
+
+func reindexTree0(tree *parse.Tree, i, size int) {
+	updated := tree.Root.IALAttr("updated")
+	if "" == updated {
+		updated = util.TimeFromID(tree.Root.ID)
+		tree.Root.SetIALAttr("updated", updated)
+		indexWriteJSONQueue(tree)
+	} else {
+		treenode.IndexBlockTree(tree)
+		sql.IndexTreeQueue(tree.Box, tree.Path)
+	}
+
+	if 0 == i%64 {
+		util.PushStatusBar(fmt.Sprintf(Conf.Language(183), i, size, html.EscapeHTMLStr(path.Base(tree.HPath))))
+	}
+}

kernel/model/transaction.go

@@ -20,9 +20,6 @@ import (
 	"bytes"
 	"errors"
 	"fmt"
-	"github.com/siyuan-note/siyuan/kernel/task"
-	"os"
-	"path"
 	"path/filepath"
 	"strings"
 	"sync"
@@ -31,7 +28,6 @@ import (
 	"github.com/88250/gulu"
 	"github.com/88250/lute/ast"
 	"github.com/88250/lute/editor"
-	"github.com/88250/lute/html"
 	"github.com/88250/lute/lex"
 	"github.com/88250/lute/parse"
 	"github.com/emirpasic/gods/sets/hashset"
@@ -1235,241 +1231,3 @@ func updateRefText(refNode *ast.Node, changedDefNodes map[string]*ast.Node) (cha
 	})
 	return
 }
-
-// AutoIndexEmbedBlock 嵌入块支持搜索 https://github.com/siyuan-note/siyuan/issues/7112
-func AutoIndexEmbedBlock() {
-	for {
-		embedBlocks := sql.QueryEmptyContentEmbedBlocks()
-		task.AppendTask(task.DatabaseIndexEmbedBlock, autoIndexEmbedBlock, embedBlocks)
-		time.Sleep(10 * time.Minute)
-	}
-}
-
-func autoIndexEmbedBlock(embedBlocks []*sql.Block) {
-	for i, embedBlock := range embedBlocks {
-		stmt := strings.TrimPrefix(embedBlock.Markdown, "{{")
-		stmt = strings.TrimSuffix(stmt, "}}")
-		queryResultBlocks := sql.SelectBlocksRawStmtNoParse(stmt, 102400)
-		for _, block := range queryResultBlocks {
-			embedBlock.Content += block.Content
-		}
-		if "" == embedBlock.Content {
-			embedBlock.Content = "no query result"
-		}
-		sql.UpdateBlockContent(embedBlock)
-
-		if 63 <= i { // 一次任务中最多处理 64 个嵌入块，防止卡顿
-			break
-		}
-	}
-}
-
-func updateEmbedBlockContent(embedBlockID string, queryResultBlocks []*EmbedBlock) {
-	embedBlock := sql.GetBlock(embedBlockID)
-	if nil == embedBlock {
-		return
-	}
-
-	for _, block := range queryResultBlocks {
-		embedBlock.Content += block.Block.Markdown
-	}
-	if "" == embedBlock.Content {
-		embedBlock.Content = "no query result"
-	}
-	sql.UpdateBlockContent(embedBlock)
-}
-
-// AutoFixIndex 自动校验数据库索引 https://github.com/siyuan-note/siyuan/issues/7016
-func AutoFixIndex() {
-	for {
-		task.AppendTask(task.DatabaseIndexFix, autoFixIndex)
-		time.Sleep(10 * time.Minute)
-	}
-}
-
-var autoFixLock = sync.Mutex{}
-
-func autoFixIndex() {
-	defer logging.Recover()
-
-	// 根据文件系统补全块树
-	boxes := Conf.GetOpenedBoxes()
-	for _, box := range boxes {
-		boxPath := filepath.Join(util.DataDir, box.ID)
-		var paths []string
-		filepath.Walk(boxPath, func(path string, info os.FileInfo, err error) error {
-			if !info.IsDir() && filepath.Ext(path) == ".sy" {
-				p := path[len(boxPath):]
-				p = filepath.ToSlash(p)
-				paths = append(paths, p)
-			}
-			return nil
-		})
-
-		size := len(paths)
-
-		redundantPaths := treenode.GetRedundantPaths(box.ID, paths)
-		for _, p := range redundantPaths {
-			treenode.RemoveBlockTreesByPath(p)
-		}
-
-		missingPaths := treenode.GetNotExistPaths(box.ID, paths)
-		for i, p := range missingPaths {
-			id := path.Base(p)
-			id = strings.TrimSuffix(id, ".sy")
-			if !ast.IsNodeIDPattern(id) {
-				continue
-			}
-
-			reindexTreeByPath(box.ID, p, i, size)
-			if util.IsExiting {
-				break
-			}
-		}
-
-		if util.IsExiting {
-			break
-		}
-	}
-
-	// 清理已关闭的笔记本块树
-	boxes = Conf.GetClosedBoxes()
-	for _, box := range boxes {
-		treenode.RemoveBlockTreesByBoxID(box.ID)
-	}
-
-	// 对比块树和数据库并订正数据库
-	rootUpdatedMap := treenode.GetRootUpdated()
-	dbRootUpdatedMap, err := sql.GetRootUpdated("blocks")
-	if nil == err {
-		reindexTreeByUpdated(rootUpdatedMap, dbRootUpdatedMap, "blocks")
-	}
-	dbFtsRootUpdatedMap, err := sql.GetRootUpdated("blocks_fts")
-	if nil == err {
-		reindexTreeByUpdated(rootUpdatedMap, dbFtsRootUpdatedMap, "blocks_fts")
-	}
-	if !Conf.Search.CaseSensitive {
-		dbFtsRootUpdatedMap, err = sql.GetRootUpdated("blocks_fts_case_insensitive")
-		if nil == err {
-			reindexTreeByUpdated(rootUpdatedMap, dbFtsRootUpdatedMap, "blocks_fts_case_insensitive")
-		}
-	}
-
-	// 去除重复的数据库块记录
-	duplicatedRootIDs := sql.GetDuplicatedRootIDs("blocks")
-	if 1 > len(duplicatedRootIDs) {
-		duplicatedRootIDs = sql.GetDuplicatedRootIDs("blocks_fts")
-		if 1 > len(duplicatedRootIDs) && !Conf.Search.CaseSensitive {
-			duplicatedRootIDs = sql.GetDuplicatedRootIDs("blocks_fts_case_insensitive")
-		}
-	}
-	size := len(duplicatedRootIDs)
-	for i, rootID := range duplicatedRootIDs {
-		root := sql.GetBlock(rootID)
-		if nil == root {
-			continue
-		}
-
-		logging.LogWarnf("exist more than one tree [%s], reindex it", rootID)
-		sql.RemoveTreeQueue(root.Box, rootID)
-		reindexTree(rootID, i, size)
-
-		if util.IsExiting {
-			break
-		}
-	}
-
-	util.PushStatusBar(Conf.Language(185))
-}
-
-func reindexTreeByUpdated(rootUpdatedMap, dbRootUpdatedMap map[string]string, blocksTable string) {
-	i := -1
-	size := len(rootUpdatedMap)
-	for rootID, updated := range rootUpdatedMap {
-		i++
-
-		if util.IsExiting {
-			break
-		}
-
-		rootUpdated := dbRootUpdatedMap[rootID]
-		if "" == rootUpdated {
-			logging.LogWarnf("not found tree [%s] in database, reindex it", rootID)
-			reindexTree(rootID, i, size)
-			continue
-		}
-
-		if "" == updated {
-			// BlockTree 迁移，v2.6.3 之前没有 updated 字段
-			reindexTree(rootID, i, size)
-			continue
-		}
-
-		btUpdated, _ := time.Parse("20060102150405", updated)
-		dbUpdated, _ := time.Parse("20060102150405", rootUpdated)
-		if dbUpdated.Before(btUpdated.Add(-10 * time.Minute)) {
-			logging.LogWarnf("tree [%s] is not up to date, reindex it", rootID)
-			reindexTree(rootID, i, size)
-			continue
-		}
-
-		if util.IsExiting {
-			break
-		}
-	}
-
-	for rootID, _ := range dbRootUpdatedMap {
-		if _, ok := rootUpdatedMap[rootID]; !ok {
-			logging.LogWarnf("tree [%s] is not in block tree, remove it from [%s]", rootID, blocksTable)
-			sql.DeleteTree(blocksTable, rootID)
-		}
-
-		if util.IsExiting {
-			break
-		}
-	}
-}
-
-func reindexTreeByPath(box, p string, i, size int) {
-	tree, err := LoadTree(box, p)
-	if nil != err {
-		return
-	}
-
-	reindexTree0(tree, i, size)
-}
-
-func reindexTree(rootID string, i, size int) {
-	root := treenode.GetBlockTree(rootID)
-	if nil == root {
-		logging.LogWarnf("root block not found", rootID)
-		return
-	}
-
-	tree, err := LoadTree(root.BoxID, root.Path)
-	if nil != err {
-		if os.IsNotExist(err) {
-			// 文件系统上没有找到该 .sy 文件，则订正块树
-			treenode.RemoveBlockTreesByRootID(rootID)
-		}
-		return
-	}
-
-	reindexTree0(tree, i, size)
-}
-
-func reindexTree0(tree *parse.Tree, i, size int) {
-	updated := tree.Root.IALAttr("updated")
-	if "" == updated {
-		updated = util.TimeFromID(tree.Root.ID)
-		tree.Root.SetIALAttr("updated", updated)
-		indexWriteJSONQueue(tree)
-	} else {
-		treenode.IndexBlockTree(tree)
-		sql.IndexTreeQueue(tree.Box, tree.Path)
-	}
-
-	if 0 == i%64 {
-		util.PushStatusBar(fmt.Sprintf(Conf.Language(183), i, size, html.EscapeHTMLStr(path.Base(tree.HPath))))
-	}
-}