♻️ Improve database loading performance https://github.com/siyuan-note/siyuan/issues/12818

kernel/filesys/tree.go

@@ -25,11 +25,13 @@ import (
 	"path/filepath"
 	"strings"
 	"sync"
+	"time"
 
 	"github.com/88250/lute"
 	"github.com/88250/lute/parse"
 	"github.com/88250/lute/render"
 	jsoniter "github.com/json-iterator/go"
+	"github.com/panjf2000/ants/v2"
 	"github.com/siyuan-note/filelock"
 	"github.com/siyuan-note/logging"
 	"github.com/siyuan-note/siyuan/kernel/cache"
@@ -53,7 +55,7 @@ func LoadTrees(ids []string) (ret map[string]*parse.Tree) {
 		blockIDs[bt.RootID] = append(blockIDs[bt.RootID], bt.ID)
 	}
 
-	trees, errs := BatchLoadTrees(boxIDs, paths, luteEngine)
+	trees, errs := batchLoadTrees(boxIDs, paths, luteEngine)
 	for i := range trees {
 		tree := trees[i]
 		err := errs[i]
@@ -82,21 +84,16 @@ func LoadTree(boxID, p string, luteEngine *lute.Lute) (ret *parse.Tree, err erro
 	return
 }
 
-func BatchLoadTrees(boxIDs, paths []string, luteEngine *lute.Lute) (ret []*parse.Tree, errs []error) {
+func batchLoadTrees(boxIDs, paths []string, luteEngine *lute.Lute) (ret []*parse.Tree, errs []error) {
-	var wg sync.WaitGroup
+	waitGroup := sync.WaitGroup{}
 	lock := sync.Mutex{}
 	loaded := map[string]bool{}
-	for i := range paths {
-		if loaded[boxIDs[i]+paths[i]] {
-			continue
-		}
 
-		loaded[boxIDs[i]+paths[i]] = true
+	start := time.Now()
-		wg.Add(1)
+	p, _ := ants.NewPoolWithFunc(8, func(arg interface{}) {
-
+		defer waitGroup.Done()
-		go func(i int) {
-			defer wg.Done()
 
+		i := arg.(int)
 		boxID := boxIDs[i]
 		path := paths[i]
 		tree, err := LoadTree(boxID, path, luteEngine)
@@ -104,9 +101,20 @@ func BatchLoadTrees(boxIDs, paths []string, luteEngine *lute.Lute) (ret []*parse
 		ret = append(ret, tree)
 		errs = append(errs, err)
 		lock.Unlock()
-		}(i)
+	})
+	for i := range paths {
+		if loaded[boxIDs[i]+paths[i]] {
+			continue
 		}
-	wg.Wait()
+
+		loaded[boxIDs[i]+paths[i]] = true
+
+		waitGroup.Add(1)
+		p.Invoke(i)
+	}
+	waitGroup.Wait()
+	p.Release()
+	logging.LogInfof("batch load trees [%d] cost [%s]", len(paths), time.Since(start))
 	return
 }
 

kernel/sql/av.go

@@ -176,6 +176,16 @@ func RenderAttributeViewTable(attrView *av.AttributeView, view *av.View, query s
 		ret.Rows = append(ret.Rows, &tableRow)
 	}
 
+	// 批量获取块属性以提升性能
+	var ialIDs []string
+	for _, row := range ret.Rows {
+		block := row.GetBlockValue()
+		if nil != block && !block.IsDetached {
+			ialIDs = append(ialIDs, row.ID)
+		}
+	}
+	ials := BatchGetBlockAttrs(ialIDs)
+
 	// 渲染自动生成的列值，比如关联列、汇总列、创建时间列和更新时间列
 	for _, row := range ret.Rows {
 		for _, cell := range row.Cells {
@@ -266,11 +276,11 @@ func RenderAttributeViewTable(attrView *av.AttributeView, view *av.View, query s
 				keyValues = append(keyValues, &av.KeyValues{Key: createdKey, Values: []*av.Value{{ID: cell.Value.ID, KeyID: createdKey.ID, BlockID: row.ID, Type: av.KeyTypeCreated, Created: cell.Value.Created}}})
 				rows[row.ID] = keyValues
 			case av.KeyTypeUpdated: // 渲染更新时间
-				ial := map[string]string{}
+				ial := ials[row.ID]
-				block := row.GetBlockValue()
+				if nil == ial {
-				if nil != block && !block.IsDetached {
+					ial = map[string]string{}
-					ial = GetBlockAttrs(row.ID)
 				}
+				block := row.GetBlockValue()
 				updatedStr := ial["updated"]
 				if "" == updatedStr && nil != block {
 					cell.Value.Updated = av.NewFormattedValueUpdated(block.Block.Updated, 0, av.UpdatedFormatNone)
@@ -302,10 +312,9 @@ func RenderAttributeViewTable(attrView *av.AttributeView, view *av.View, query s
 			switch cell.ValueType {
 			case av.KeyTypeTemplate: // 渲染模板列
 				keyValues := rows[row.ID]
-				ial := map[string]string{}
+				ial := ials[row.ID]
-				block := row.GetBlockValue()
+				if nil == ial {
-				if nil != block && !block.IsDetached {
+					ial = map[string]string{}
-					ial = GetBlockAttrs(row.ID)
 				}
 				content, renderErr := RenderTemplateCol(ial, keyValues, cell.Value.Template.Content)
 				cell.Value.Template.Content = content