github/siyuan
4
0
Fork 0
mirror of https://github.com/siyuan-note/siyuan.git synced 2025-05-02 22:21:28 +08:00
Code Issues Packages Projects Releases Wiki Activity

⚡ 改进重命名资源文件性能 https://github.com/siyuan-note/siyuan/issues/7322

Browse Source 
改进建立引用索引 https://github.com/siyuan-note/siyuan/issues/7320
This commit is contained in:
Liang Ding 2023-02-10 14:28:10 +08:00
parent 6743b21193
commit b6ea32462e
No known key found for this signature in database
GPG Key ID: 136F30F901A2231D
23 changed files with 155 additions and 132 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
kernel/api/block_op.go
View File

@ -17,14 +17,15 @@
package api
import (
"github.com/siyuan-note/siyuan/kernel/treenode"
"net/http"
"github.com/88250/gulu"
"github.com/88250/lute"
"github.com/88250/lute/ast"
"github.com/gin-gonic/gin"
"github.com/siyuan-note/siyuan/kernel/filesys"
"github.com/siyuan-note/siyuan/kernel/model"
"github.com/siyuan-note/siyuan/kernel/treenode"
"github.com/siyuan-note/siyuan/kernel/util"
)
@ -44,7 +45,7 @@ func appendBlock(c *gin.Context) {
return
}
if "markdown" == dataType {
luteEngine := model.NewLute()
luteEngine := util.NewLute()
data = dataBlockDOM(data, luteEngine)
}
@ -89,7 +90,7 @@ func prependBlock(c *gin.Context) {
return
}
if "markdown" == dataType {
luteEngine := model.NewLute()
luteEngine := util.NewLute()
data = dataBlockDOM(data, luteEngine)
}
@ -150,7 +151,7 @@ func insertBlock(c *gin.Context) {
}
if "markdown" == dataType {
luteEngine := model.NewLute()
luteEngine := util.NewLute()
data = dataBlockDOM(data, luteEngine)
}
@ -197,7 +198,7 @@ func updateBlock(c *gin.Context) {
return
}
luteEngine := model.NewLute()
luteEngine := util.NewLute()
if "markdown" == dataType {
data = dataBlockDOM(data, luteEngine)
}
@ -217,7 +218,7 @@ func updateBlock(c *gin.Context) {
var transactions []*model.Transaction
if "NodeDocument" == block.Type {
oldTree, err := model.LoadTree(block.Box, block.Path)
oldTree, err := filesys.LoadTree(block.Box, block.Path, luteEngine)
if nil != err {
ret.Code = -1
ret.Msg = "load tree failed: " + err.Error()

10
kernel/api/filetree.go
View File

@ -28,6 +28,7 @@ import (
"github.com/88250/gulu"
"github.com/gin-gonic/gin"
"github.com/siyuan-note/filelock"
"github.com/siyuan-note/siyuan/kernel/filesys"
"github.com/siyuan-note/siyuan/kernel/model"
"github.com/siyuan-note/siyuan/kernel/util"
)
@ -86,7 +87,8 @@ func heading2Doc(c *gin.Context) {
}
model.WaitForWritingFiles()
tree, err := model.LoadTree(targetNotebook, targetPath)
luteEngine := util.NewLute()
tree, err := filesys.LoadTree(targetNotebook, targetPath, luteEngine)
if nil != err {
ret.Code = -1
ret.Msg = err.Error()
@ -130,7 +132,8 @@ func li2Doc(c *gin.Context) {
}
model.WaitForWritingFiles()
tree, err := model.LoadTree(targetNotebook, targetPath)
luteEngine := util.NewLute()
tree, err := filesys.LoadTree(targetNotebook, targetPath, luteEngine)
if nil != err {
ret.Code = -1
ret.Msg = err.Error()
@ -424,7 +427,8 @@ func createDailyNote(c *gin.Context) {
box := model.Conf.Box(notebook)
model.WaitForWritingFiles()
tree, err := model.LoadTree(box.ID, p)
luteEngine := util.NewLute()
tree, err := filesys.LoadTree(box.ID, p, luteEngine)
if nil != err {
ret.Code = -1
ret.Msg = err.Error()

2
kernel/api/lute.go
View File

@ -62,7 +62,7 @@ func html2BlockDOM(c *gin.Context) {
return
}
luteEngine := model.NewLute()
luteEngine := util.NewLute()
var unlinks []*ast.Node
tree := parse.Parse("", []byte(markdown), luteEngine.ParseOptions)
ast.Walk(tree.Root, func(n *ast.Node, entering bool) ast.WalkStatus {

5
kernel/filesys/tree.go
View File

@ -40,6 +40,7 @@ func LoadTree(boxID, p string, luteEngine *lute.Lute) (ret *parse.Tree, err erro
filePath := filepath.Join(util.DataDir, boxID, p)
data, err := filelock.ReadFile(filePath)
if nil != err {
logging.LogErrorf("load tree [%s] failed: %s", p, err)
return
}
@ -50,7 +51,9 @@ func LoadTree(boxID, p string, luteEngine *lute.Lute) (ret *parse.Tree, err erro
func LoadTreeByData(data []byte, boxID, p string, luteEngine *lute.Lute) (ret *parse.Tree, err error) {
ret = parseJSON2Tree(boxID, p, data, luteEngine)
if nil == ret {
return nil, errors.New("parse tree failed")
logging.LogErrorf("parse tree [%s] failed", p)
err = errors.New("parse tree failed")
return
}
ret.Path = p
ret.Root.Path = p

8
kernel/model/assets.go
View File

@ -434,7 +434,7 @@ func RemoveUnusedAssets() (ret []string) {
IncSync()
}
indexHistoryDir(filepath.Base(historyDir), NewLute())
indexHistoryDir(filepath.Base(historyDir), util.NewLute())
cache.LoadAssets()
return
}
@ -468,7 +468,7 @@ func RemoveUnusedAsset(p string) (ret string) {
ret = absPath
IncSync()
indexHistoryDir(filepath.Base(historyDir), NewLute())
indexHistoryDir(filepath.Base(historyDir), util.NewLute())
cache.RemoveAsset(p)
return
}
@ -504,7 +504,7 @@ func RenameAsset(oldPath, newName string) (err error) {
return
}
luteEngine := NewLute()
luteEngine := util.NewLute()
for _, notebook := range notebooks {
pages := pagedPaths(filepath.Join(util.DataDir, notebook.ID), 32)
@ -562,7 +562,7 @@ func UnusedAssets() (ret []string) {
if nil != err {
return
}
luteEngine := NewLute()
luteEngine := util.NewLute()
for _, notebook := range notebooks {
dests := map[string]bool{}

2
kernel/model/backlink.go
View File

@ -665,7 +665,7 @@ func searchBackmention(mentionKeywords []string, keyword string, excludeBacklink
}
blocks := fromSQLBlocks(&sqlBlocks, strings.Join(terms, search.TermSep), beforeLen)
luteEngine := NewLute()
luteEngine := util.NewLute()
var tmp []*Block
for _, b := range blocks {
tree := parse.Parse("", gulu.Str.ToBytes(b.Markdown), luteEngine.ParseOptions)

6
kernel/model/block.go
View File

@ -294,7 +294,7 @@ func GetHeadingDeleteTransaction(id string) (transaction *Transaction, err error
nodes = append(nodes, treenode.HeadingChildren(node)...)
transaction = &Transaction{}
luteEngine := NewLute()
luteEngine := util.NewLute()
for _, n := range nodes {
op := &Operation{}
op.ID = n.ID
@ -347,7 +347,7 @@ func GetHeadingChildrenDOM(id string) (ret string) {
nodes := append([]*ast.Node{}, heading)
children := treenode.HeadingChildren(heading)
nodes = append(nodes, children...)
luteEngine := NewLute()
luteEngine := util.NewLute()
ret = renderBlockDOMByNodes(nodes, luteEngine)
return
}
@ -383,7 +383,7 @@ func GetHeadingLevelTransaction(id string, level int) (transaction *Transaction,
}
transaction = &Transaction{}
luteEngine := NewLute()
luteEngine := util.NewLute()
for _, c := range childrenHeadings {
op := &Operation{}
op.ID = c.ID

4
kernel/model/box.go
View File

@ -38,6 +38,7 @@ import (
"github.com/siyuan-note/filelock"
"github.com/siyuan-note/logging"
"github.com/siyuan-note/siyuan/kernel/conf"
"github.com/siyuan-note/siyuan/kernel/filesys"
"github.com/siyuan-note/siyuan/kernel/sql"
"github.com/siyuan-note/siyuan/kernel/task"
"github.com/siyuan-note/siyuan/kernel/treenode"
@ -386,12 +387,13 @@ func (box *Box) renameSubTrees(tree *parse.Tree) {
func (box *Box) moveTrees0(files []*FileInfo) {
totals := len(files) + 5
showProgress := 64 < totals
luteEngine := util.NewLute()
for i, subFile := range files {
if !strings.HasSuffix(subFile.path, ".sy") {
continue
}
subTree, err := LoadTree(box.ID, subFile.path) // LoadTree 会重新构造 HPath
subTree, err := filesys.LoadTree(box.ID, subFile.path, luteEngine) // LoadTree 会重新构造 HPath
if nil != err {
continue
}

4
kernel/model/export_merge.go
View File

@ -19,6 +19,7 @@ package model
import (
"github.com/88250/lute/ast"
"github.com/88250/lute/parse"
"github.com/siyuan-note/siyuan/kernel/filesys"
)
func mergeSubDocs(rootTree *parse.Tree) (ret *parse.Tree, err error) {
@ -74,7 +75,8 @@ func walkBlock(insertPoint *ast.Node, block *Block, level int) (err error) {
}
func loadTreeNodes(box string, p string, level int) (ret []*ast.Node, err error) {
tree, err := LoadTree(box, p)
luteEngine := NewLute()
tree, err := filesys.LoadTree(box, p, luteEngine)
if nil != err {
return
}

49
kernel/model/file.go
View File

@ -19,7 +19,6 @@ package model
import (
"errors"
"fmt"
"github.com/siyuan-note/siyuan/kernel/task"
"math"
"os"
"path"
@ -31,6 +30,7 @@ import (
"unicode/utf8"
"github.com/88250/gulu"
"github.com/88250/lute"
"github.com/88250/lute/ast"
"github.com/88250/lute/html"
"github.com/88250/lute/parse"
@ -44,6 +44,7 @@ import (
"github.com/siyuan-note/siyuan/kernel/filesys"
"github.com/siyuan-note/siyuan/kernel/search"
"github.com/siyuan-note/siyuan/kernel/sql"
"github.com/siyuan-note/siyuan/kernel/task"
"github.com/siyuan-note/siyuan/kernel/treenode"
"github.com/siyuan-note/siyuan/kernel/util"
)
@ -378,7 +379,7 @@ func ListDocTree(boxID, path string, sortMode int) (ret []*File, totals int, err
}
func ContentStat(content string) (ret *util.BlockStatResult) {
luteEngine := NewLute()
luteEngine := util.NewLute()
tree := luteEngine.BlockDOM2Tree(content)
runeCnt, wordCnt, linkCnt, imgCnt, refCnt := tree.Root.Stat()
return &util.BlockStatResult{
@ -393,6 +394,7 @@ func ContentStat(content string) (ret *util.BlockStatResult) {
func BlocksWordCount(ids []string) (ret *util.BlockStatResult) {
ret = &util.BlockStatResult{}
trees := map[string]*parse.Tree{} // 缓存
luteEngine := util.NewLute()
for _, id := range ids {
bt := treenode.GetBlockTree(id)
if nil == bt {
@ -402,7 +404,7 @@ func BlocksWordCount(ids []string) (ret *util.BlockStatResult) {
tree := trees[bt.RootID]
if nil == tree {
tree, _ = LoadTree(bt.BoxID, bt.Path)
tree, _ = filesys.LoadTree(bt.BoxID, bt.Path, luteEngine)
if nil == tree {
continue
}
@ -927,7 +929,7 @@ func CreateDocByMd(boxID, p, title, md string, sorts []string) (tree *parse.Tree
return
}
luteEngine := NewLute()
luteEngine := util.NewLute()
dom := luteEngine.Md2BlockDOM(md, false)
tree, err = createDoc(box.ID, p, title, dom)
if nil != err {
@ -946,7 +948,7 @@ func CreateWithMarkdown(boxID, hPath, md string) (id string, err error) {
}
WaitForWritingFiles()
luteEngine := NewLute()
luteEngine := util.NewLute()
dom := luteEngine.Md2BlockDOM(md, false)
id, _, err = createDocsByHPath(box.ID, hPath, dom)
return
@ -958,7 +960,8 @@ func GetHPathByPath(boxID, p string) (hPath string, err error) {
return
}
tree, err := LoadTree(boxID, p)
luteEngine := util.NewLute()
tree, err := filesys.LoadTree(boxID, p, luteEngine)
if nil != err {
return
}
@ -1038,8 +1041,9 @@ func MoveDocs(fromPaths []string, toBoxID, toPath string) (err error) {
}
WaitForWritingFiles()
luteEngine := util.NewLute()
for fromPath, fromBox := range pathsBoxes {
_, err = moveDoc(fromBox, fromPath, toBox, toPath)
_, err = moveDoc(fromBox, fromPath, toBox, toPath, luteEngine)
if nil != err {
return
}
@ -1055,7 +1059,7 @@ func MoveDocs(fromPaths []string, toBoxID, toPath string) (err error) {
return
}
func moveDoc(fromBox *Box, fromPath string, toBox *Box, toPath string) (newPath string, err error) {
func moveDoc(fromBox *Box, fromPath string, toBox *Box, toPath string, luteEngine *lute.Lute) (newPath string, err error) {
isSameBox := fromBox.ID == toBox.ID
if isSameBox {
@ -1070,7 +1074,7 @@ func moveDoc(fromBox *Box, fromPath string, toBox *Box, toPath string) (newPath
}
}
tree, err := LoadTree(fromBox.ID, fromPath)
tree, err := filesys.LoadTree(fromBox.ID, fromPath, luteEngine)
if nil != err {
err = ErrBlockNotFound
return
@ -1083,9 +1087,9 @@ func moveDoc(fromBox *Box, fromPath string, toBox *Box, toPath string) (newPath
if !moveToRoot {
var toTree *parse.Tree
if isSameBox {
toTree, err = LoadTree(fromBox.ID, toPath)
toTree, err = filesys.LoadTree(fromBox.ID, toPath, luteEngine)
} else {
toTree, err = LoadTree(toBox.ID, toPath)
toTree, err = filesys.LoadTree(toBox.ID, toPath, luteEngine)
}
if nil != err {
err = ErrBlockNotFound
@ -1136,7 +1140,7 @@ func moveDoc(fromBox *Box, fromPath string, toBox *Box, toPath string) (newPath
return
}
tree, err = LoadTree(fromBox.ID, newPath)
tree, err = filesys.LoadTree(fromBox.ID, newPath, luteEngine)
if nil != err {
return
}
@ -1152,7 +1156,7 @@ func moveDoc(fromBox *Box, fromPath string, toBox *Box, toPath string) (newPath
return
}
tree, err = LoadTree(toBox.ID, newPath)
tree, err = filesys.LoadTree(toBox.ID, newPath, luteEngine)
if nil != err {
return
}
@ -1180,7 +1184,8 @@ func RemoveDoc(boxID, p string) {
}
WaitForWritingFiles()
removeDoc(box, p)
luteEngine := util.NewLute()
removeDoc(box, p, luteEngine)
IncSync()
return
}
@ -1192,14 +1197,15 @@ func RemoveDocs(paths []string) {
paths = util.FilterSelfChildDocs(paths)
pathsBoxes := getBoxesByPaths(paths)
WaitForWritingFiles()
luteEngine := util.NewLute()
for p, box := range pathsBoxes {
removeDoc(box, p)
removeDoc(box, p, luteEngine)
}
return
}
func removeDoc(box *Box, p string) {
tree, _ := LoadTree(box.ID, p)
func removeDoc(box *Box, p string, luteEngine *lute.Lute) {
tree, _ := filesys.LoadTree(box.ID, p, luteEngine)
if nil == tree {
return
}
@ -1231,7 +1237,7 @@ func removeDoc(box *Box, p string) {
return
}
}
indexHistoryDir(filepath.Base(historyDir), NewLute())
indexHistoryDir(filepath.Base(historyDir), util.NewLute())
if existChildren {
if err = box.Remove(childrenDir); nil != err {
@ -1274,7 +1280,8 @@ func RenameDoc(boxID, p, title string) (err error) {
}
WaitForWritingFiles()
tree, err := LoadTree(box.ID, p)
luteEngine := util.NewLute()
tree, err := filesys.LoadTree(box.ID, p, luteEngine)
if nil != err {
return
}
@ -1370,7 +1377,7 @@ func CreateDailyNote(boxID string) (p string, existed bool, err error) {
if nil == err {
tree.Root.FirstChild.Unlink()
luteEngine := NewLute()
luteEngine := util.NewLute()
newTree := luteEngine.BlockDOM2Tree(dom)
var children []*ast.Node
for c := newTree.Root.FirstChild; nil != c; c = c.Next {
@ -1450,7 +1457,7 @@ func createDoc(boxID, p, title, dom string) (tree *parse.Tree, err error) {
return
}
luteEngine := NewLute()
luteEngine := util.NewLute()
tree = luteEngine.BlockDOM2Tree(dom)
tree.Box = boxID
tree.Path = p

2
kernel/model/format.go
View File

@ -105,5 +105,5 @@ func generateFormatHistory(tree *parse.Tree) {
return
}
indexHistoryDir(filepath.Base(historyDir), NewLute())
indexHistoryDir(filepath.Base(historyDir), util.NewLute())
}

3
kernel/model/graph.go
View File

@ -18,6 +18,7 @@ package model
import (
"bytes"
"github.com/siyuan-note/siyuan/kernel/util"
"math"
"strings"
"unicode/utf8"
@ -693,7 +694,7 @@ func query2Stmt(queryStr string) (ret string) {
buf.WriteString("id = '" + queryStr + "'")
} else {
var tags []string
luteEngine := NewLute()
luteEngine := util.NewLute()
t := parse.Inline("", []byte(queryStr), luteEngine.ParseOptions)
ast.Walk(t.Root, func(n *ast.Node, entering bool) ast.WalkStatus {
if !entering {

2
kernel/model/heading.go
View File

@ -290,7 +290,7 @@ func Heading2Doc(srcHeadingID, targetBoxID, targetPath string) (srcRootBlockID,
}
headingNode.RemoveIALAttr("fold")
luteEngine := NewLute()
luteEngine := util.NewLute()
newTree := &parse.Tree{Root: &ast.Node{Type: ast.NodeDocument, ID: srcHeadingID}, Context: &parse.Context{ParseOption: luteEngine.ParseOptions}}
children = treenode.HeadingChildren(headingNode)
for _, c := range children {

4
kernel/model/history.go
View File

@ -482,7 +482,7 @@ func (box *Box) generateDocHistory0() {
}
}
indexHistoryDir(filepath.Base(historyDir), NewLute())
indexHistoryDir(filepath.Base(historyDir), util.NewLute())
return
}
@ -593,7 +593,7 @@ func ReindexHistory() (err error) {
defer util.PushClearProgress()
sql.InitHistoryDatabase(true)
lutEngine := NewLute()
lutEngine := util.NewLute()
for _, historyDir := range historyDirs {
if !historyDir.IsDir() {
continue

4
kernel/model/import.go
View File

@ -53,7 +53,7 @@ import (
func HTML2Markdown(htmlStr string) (markdown string, err error) {
assetDirPath := filepath.Join(util.DataDir, "assets")
luteEngine := NewLute()
luteEngine := util.NewLute()
luteEngine.SetProtyleWYSIWYG(false)
tree := luteEngine.HTML2Tree(htmlStr)
ast.Walk(tree.Root, func(n *ast.Node, entering bool) ast.WalkStatus {
@ -387,7 +387,7 @@ func ImportSY(zipPath, boxID, toPath string) (err error) {
absPath := filepath.Join(targetDir, treePath)
p := strings.TrimPrefix(absPath, boxAbsPath)
p = filepath.ToSlash(p)
tree, err := LoadTree(boxID, p)
tree, err := filesys.LoadTree(boxID, p, luteEngine)
if nil != err {
logging.LogErrorf("load tree [%s] failed: %s", treePath, err)
continue

75
kernel/model/index.go
View File

@ -17,18 +17,22 @@
package model
import (
"bytes"
"fmt"
"path/filepath"
"runtime"
"runtime/debug"
"strings"
"sync"
"time"
"github.com/88250/gulu"
"github.com/88250/lute/ast"
"github.com/88250/lute/parse"
"github.com/dustin/go-humanize"
"github.com/emirpasic/gods/sets/hashset"
"github.com/panjf2000/ants/v2"
"github.com/siyuan-note/eventbus"
"github.com/siyuan-note/filelock"
"github.com/siyuan-note/logging"
"github.com/siyuan-note/siyuan/kernel/cache"
"github.com/siyuan-note/siyuan/kernel/filesys"
@ -68,7 +72,7 @@ func index(boxID string) {
bootProgressPart := 30.0 / float64(boxLen) / float64(len(files))
start := time.Now()
luteEngine := NewLute()
luteEngine := util.NewLute()
var treeCount int
var treeSize int64
i := 0
@ -135,53 +139,72 @@ func IndexRefs() {
util.SetBootDetails("Resolving refs...")
util.PushStatusBar(Conf.Language(54))
// 引用入库
util.SetBootDetails("Indexing refs...")
refBlocks := sql.GetRefExistedBlocks()
refTreeIDs := hashset.New()
for _, refBlock := range refBlocks {
refTreeIDs.Add(refBlock.RootID)
}
i := 0
if 0 < refTreeIDs.Size() {
luteEngine := NewLute()
bootProgressPart := 10.0 / float64(refTreeIDs.Size())
for _, box := range Conf.GetOpenedBoxes() {
var defBlockIDs []string
luteEngine := util.NewLute()
boxes := Conf.GetOpenedBoxes()
for _, box := range boxes {
sql.DeleteBoxRefsQueue(box.ID)
files := box.ListFiles("/")
for _, file := range files {
if file.isdir || !strings.HasSuffix(file.name, ".sy") {
pages := pagedPaths(filepath.Join(util.DataDir, box.ID), 32)
for _, paths := range pages {
for _, treeAbsPath := range paths {
data, readErr := filelock.ReadFile(treeAbsPath)
if nil != readErr {
logging.LogWarnf("get data [path=%s] failed: %s", treeAbsPath, readErr)
continue
}
if file.isdir || !strings.HasSuffix(file.name, ".sy") {
if !bytes.Contains(data, []byte("TextMarkBlockRefID")) && !bytes.Contains(data, []byte("TextMarkFileAnnotationRefID")) {
continue
}
id := strings.TrimSuffix(file.name, ".sy")
if !refTreeIDs.Contains(id) {
p := filepath.ToSlash(strings.TrimPrefix(treeAbsPath, filepath.Join(util.DataDir, box.ID)))
tree, parseErr := filesys.LoadTreeByData(data, box.ID, p, luteEngine)
if nil != parseErr {
logging.LogWarnf("parse json to tree [%s] failed: %s", treeAbsPath, parseErr)
continue
}
util.IncBootProgress(bootProgressPart, "Indexing ref "+util.ShortPathForBootingDisplay(file.path))
ast.Walk(tree.Root, func(n *ast.Node, entering bool) ast.WalkStatus {
if !entering {
return ast.WalkContinue
}
tree, err := filesys.LoadTree(box.ID, file.path, luteEngine)
if nil != err {
logging.LogErrorf("parse box [%s] tree [%s] failed", box.ID, file.path)
if n.IsTextMarkType("block-ref") {
defBlockIDs = append(defBlockIDs, n.TextMarkBlockRefID)
} else if n.IsTextMarkType("file-annotation-ref") {
defBlockIDs = append(defBlockIDs, n.TextMarkFileAnnotationRefID)
}
return ast.WalkContinue
})
}
}
}
defBlockIDs = gulu.Str.RemoveDuplicatedElem(defBlockIDs)
i := 0
size := len(defBlockIDs)
if 0 < size {
bootProgressPart := 10.0 / float64(size)
for _, defBlockID := range defBlockIDs {
defTree, loadErr := LoadTreeByID(defBlockID)
if nil != loadErr {
continue
}
sql.InsertRefsTreeQueue(tree)
util.IncBootProgress(bootProgressPart, "Indexing ref "+defTree.ID)
sql.InsertRefsTreeQueue(defTree)
if 1 < i && 0 == i%64 {
util.PushStatusBar(fmt.Sprintf(Conf.Language(55), i))
}
i++
}
}
}
logging.LogInfof("resolved refs [%d] in [%dms]", len(refBlocks), time.Now().Sub(start).Milliseconds())
logging.LogInfof("resolved refs [%d] in [%dms]", size, time.Now().Sub(start).Milliseconds())
util.PushStatusBar(fmt.Sprintf(Conf.Language(55), i))
}

20
kernel/model/index_fix.go
View File

@ -27,10 +27,12 @@ import (
"time"
"github.com/88250/gulu"
"github.com/88250/lute"
"github.com/88250/lute/ast"
"github.com/88250/lute/html"
"github.com/88250/lute/parse"
"github.com/siyuan-note/logging"
"github.com/siyuan-note/siyuan/kernel/filesys"
"github.com/siyuan-note/siyuan/kernel/sql"
"github.com/siyuan-note/siyuan/kernel/task"
"github.com/siyuan-note/siyuan/kernel/treenode"
@ -107,6 +109,7 @@ func fixBlockTreeByFileSys() {
util.PushStatusBar(Conf.Language(58))
boxes := Conf.GetOpenedBoxes()
luteEngine := lute.New()
for _, box := range boxes {
boxPath := filepath.Join(util.DataDir, box.ID)
var paths []string
@ -133,7 +136,7 @@ func fixBlockTreeByFileSys() {
continue
}
reindexTreeByPath(box.ID, p, i, size)
reindexTreeByPath(box.ID, p, i, size, luteEngine)
if util.IsExiting {
break
}
@ -166,6 +169,7 @@ func fixDatabaseIndexByBlockTree() {
func reindexTreeByUpdated(rootUpdatedMap, dbRootUpdatedMap map[string]string) {
i := -1
size := len(rootUpdatedMap)
luteEngine := util.NewLute()
for rootID, updated := range rootUpdatedMap {
i++
@ -176,13 +180,13 @@ func reindexTreeByUpdated(rootUpdatedMap, dbRootUpdatedMap map[string]string) {
rootUpdated := dbRootUpdatedMap[rootID]
if "" == rootUpdated {
//logging.LogWarnf("not found tree [%s] in database, reindex it", rootID)
reindexTree(rootID, i, size)
reindexTree(rootID, i, size, luteEngine)
continue
}
if "" == updated {
// BlockTree 迁移，v2.6.3 之前没有 updated 字段
reindexTree(rootID, i, size)
reindexTree(rootID, i, size, luteEngine)
continue
}
@ -190,7 +194,7 @@ func reindexTreeByUpdated(rootUpdatedMap, dbRootUpdatedMap map[string]string) {
dbUpdated, _ := time.Parse("20060102150405", rootUpdated)
if dbUpdated.Before(btUpdated.Add(-10 * time.Minute)) {
logging.LogWarnf("tree [%s] is not up to date, reindex it", rootID)
reindexTree(rootID, i, size)
reindexTree(rootID, i, size, luteEngine)
continue
}
@ -231,8 +235,8 @@ func reindexTreeByUpdated(rootUpdatedMap, dbRootUpdatedMap map[string]string) {
sql.BatchRemoveTreeQueue(toRemoveRootIDs)
}
func reindexTreeByPath(box, p string, i, size int) {
tree, err := LoadTree(box, p)
func reindexTreeByPath(box, p string, i, size int, luteEngine *lute.Lute) {
tree, err := filesys.LoadTree(box, p, luteEngine)
if nil != err {
return
}
@ -240,14 +244,14 @@ func reindexTreeByPath(box, p string, i, size int) {
reindexTree0(tree, i, size)
}
func reindexTree(rootID string, i, size int) {
func reindexTree(rootID string, i, size int, luteEngine *lute.Lute) {
root := treenode.GetBlockTree(rootID)
if nil == root {
logging.LogWarnf("root block [%s] not found", rootID)
return
}
tree, err := LoadTree(root.BoxID, root.Path)
tree, err := filesys.LoadTree(root.BoxID, root.Path, luteEngine)
if nil != err {
if os.IsNotExist(err) {
// 文件系统上没有找到该 .sy 文件，则订正块树

2
kernel/model/listitem.go
View File

@ -76,7 +76,7 @@ func ListItem2Doc(srcListItemID, targetBoxID, targetPath string) (srcRootBlockID
children = append(children, newNode)
}
luteEngine := NewLute()
luteEngine := util.NewLute()
newTree := &parse.Tree{Root: &ast.Node{Type: ast.NodeDocument, ID: srcListItemID}, Context: &parse.Context{ParseOption: luteEngine.ParseOptions}}
for _, c := range children {
newTree.Root.AppendChild(c)

2
kernel/model/repository.go
View File

@ -1075,7 +1075,7 @@ func processSyncMergeResult(exit, byHand bool, start time.Time, mergeResult *dej
// 云端同步发生冲突时生成副本 https://github.com/siyuan-note/siyuan/issues/5687
historyDir := filepath.Join(util.HistoryDir, mergeResult.Time.Format("2006-01-02-150405")+"-sync")
luteEngine := NewLute()
luteEngine := util.NewLute()
for _, file := range mergeResult.Conflicts {
if !strings.HasSuffix(file.Path, ".sy") {
continue

4
kernel/model/sync.go
View File

@ -32,6 +32,7 @@ import (
"github.com/siyuan-note/dejavu/cloud"
"github.com/siyuan-note/logging"
"github.com/siyuan-note/siyuan/kernel/conf"
"github.com/siyuan-note/siyuan/kernel/filesys"
"github.com/siyuan-note/siyuan/kernel/sql"
"github.com/siyuan-note/siyuan/kernel/treenode"
"github.com/siyuan-note/siyuan/kernel/util"
@ -267,6 +268,7 @@ func incReindex(upserts, removes []string) {
msg := fmt.Sprintf(Conf.Language(35))
util.PushStatusBar(msg)
luteEngine := util.NewLute()
// 先执行 remove，否则移动文档时 upsert 会被忽略，导致未被索引
bootProgressPart := 10 / float64(len(removes))
for _, removeFile := range removes {
@ -311,7 +313,7 @@ func incReindex(upserts, removes []string) {
util.IncBootProgress(bootProgressPart, msg)
util.PushStatusBar(msg)
tree, err0 := LoadTree(box, p)
tree, err0 := filesys.LoadTree(box, p, luteEngine)
if nil != err0 {
continue
}

19
kernel/model/transaction.go
View File

@ -26,6 +26,7 @@ import (
"time"
"github.com/88250/gulu"
"github.com/88250/lute"
"github.com/88250/lute/ast"
"github.com/88250/lute/editor"
"github.com/88250/lute/lex"
@ -34,6 +35,7 @@ import (
"github.com/siyuan-note/filelock"
"github.com/siyuan-note/logging"
"github.com/siyuan-note/siyuan/kernel/cache"
"github.com/siyuan-note/siyuan/kernel/filesys"
"github.com/siyuan-note/siyuan/kernel/sql"
"github.com/siyuan-note/siyuan/kernel/treenode"
"github.com/siyuan-note/siyuan/kernel/util"
@ -419,8 +421,7 @@ func (tx *Transaction) doPrependInsert(operation *Operation) (ret *TxErr) {
}
data := strings.ReplaceAll(operation.Data.(string), editor.FrontEndCaret, "")
luteEngine := NewLute()
subTree := luteEngine.BlockDOM2Tree(data)
subTree := tx.luteEngine.BlockDOM2Tree(data)
insertedNode := subTree.Root.FirstChild
if nil == insertedNode {
return &TxErr{code: TxErrCodeBlockNotFound, msg: "invalid data tree", id: block.ID}
@ -507,8 +508,7 @@ func (tx *Transaction) doAppendInsert(operation *Operation) (ret *TxErr) {
}
data := strings.ReplaceAll(operation.Data.(string), editor.FrontEndCaret, "")
luteEngine := NewLute()
subTree := luteEngine.BlockDOM2Tree(data)
subTree := tx.luteEngine.BlockDOM2Tree(data)
insertedNode := subTree.Root.FirstChild
if nil == insertedNode {
return &TxErr{code: TxErrCodeBlockNotFound, msg: "invalid data tree", id: block.ID}
@ -720,8 +720,7 @@ func (tx *Transaction) doInsert(operation *Operation) (ret *TxErr) {
}
data := strings.ReplaceAll(operation.Data.(string), editor.FrontEndCaret, "")
luteEngine := NewLute()
subTree := luteEngine.BlockDOM2Tree(data)
subTree := tx.luteEngine.BlockDOM2Tree(data)
p := block.Path
assets := getAssetsDir(filepath.Join(util.DataDir, block.BoxID), filepath.Dir(filepath.Join(util.DataDir, block.BoxID, p)))
@ -870,8 +869,7 @@ func (tx *Transaction) doUpdate(operation *Operation) (ret *TxErr) {
return &TxErr{code: TxErrCodeBlockNotFound, id: id}
}
luteEngine := NewLute()
subTree := luteEngine.BlockDOM2Tree(data)
subTree := tx.luteEngine.BlockDOM2Tree(data)
subTree.ID, subTree.Box, subTree.Path = tree.ID, tree.Box, tree.Path
oldNode := treenode.GetNodeInTree(tree, id)
if nil == oldNode {
@ -1034,6 +1032,8 @@ type Transaction struct {
trees map[string]*parse.Tree
nodes map[string]*ast.Node
luteEngine *lute.Lute
}
func (tx *Transaction) begin() (err error) {
@ -1042,6 +1042,7 @@ func (tx *Transaction) begin() (err error) {
}
tx.trees = map[string]*parse.Tree{}
tx.nodes = map[string]*ast.Node{}
tx.luteEngine = util.NewLute()
return
}
@ -1077,7 +1078,7 @@ func (tx *Transaction) loadTree(id string) (ret *parse.Tree, err error) {
return
}
ret, err = LoadTree(box, p)
ret, err = filesys.LoadTree(box, p, tx.luteEngine)
if nil != err {
return
}

17
kernel/model/tree.go
View File

@ -164,19 +164,8 @@ func loadTreeByBlockID(id string) (ret *parse.Tree, err error) {
return nil, ErrBlockNotFound
}
ret, err = LoadTree(bt.BoxID, bt.Path)
if nil != err {
return
}
return
}
func LoadTree(boxID, p string) (*parse.Tree, error) {
luteEngine := NewLute()
tree, err := filesys.LoadTree(boxID, p, luteEngine)
if nil != err {
logging.LogErrorf("load tree [%s] failed: %s", boxID+p, err)
return nil, err
}
return tree, nil
luteEngine := util.NewLute()
ret, err = filesys.LoadTree(bt.BoxID, bt.Path, luteEngine)
return
}

16
kernel/sql/block_query.go
View File

@ -555,22 +555,6 @@ func GetAllChildBlocks(rootID, condition string) (ret []*Block) {
return
}
func GetRefExistedBlocks() (ret []*Block) {
stmt := "SELECT * FROM blocks WHERE markdown LIKE ? OR markdown LIKE ?"
rows, err := query(stmt, "%((20%", "%<<20%")
if nil != err {
logging.LogErrorf("sql query [%s] failed: %s", stmt, err)
return
}
defer rows.Close()
for rows.Next() {
if block := scanBlockRows(rows); nil != block {
ret = append(ret, block)
}
}
return
}
func GetBlock(id string) (ret *Block) {
ret = getBlockCache(id)
if nil != ret {