From b5fc879628809657c2fd8a40cf8d24fa5ed6dc97 Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Wed, 25 Jan 2023 12:33:38 +0800 Subject: [PATCH 1/3] =?UTF-8?q?:art:=20=E6=94=B9=E8=BF=9B=E5=86=85?= =?UTF-8?q?=E6=A0=B8=E4=BB=BB=E5=8A=A1=E8=B0=83=E5=BA=A6=E6=9C=BA=E5=88=B6?= =?UTF-8?q?=E6=8F=90=E5=8D=87=E7=A8=B3=E5=AE=9A=E6=80=A7=20https://github.?= =?UTF-8?q?com/siyuan-note/siyuan/issues/7113?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/model/index.go | 2 +- kernel/model/transaction.go | 2 +- kernel/sql/queue.go | 21 +++++++++++++++++++-- kernel/sql/upsert.go | 11 +++++++++++ 4 files changed, 32 insertions(+), 4 deletions(-) diff --git a/kernel/model/index.go b/kernel/model/index.go index 662167154..080a7a25b 100644 --- a/kernel/model/index.go +++ b/kernel/model/index.go @@ -105,7 +105,7 @@ func index(boxID string) { cache.PutDocIAL(file.path, docIAL) treenode.ReindexBlockTree(tree) - sql.UpsertTreeQueue(tree) + sql.IndexTreeQueue(box.ID, file.path) util.IncBootProgress(bootProgressPart, fmt.Sprintf(Conf.Language(92), util.ShortPathForBootingDisplay(tree.Path))) treeSize += file.size diff --git a/kernel/model/transaction.go b/kernel/model/transaction.go index 1ab9c7980..71dd9d2f7 100644 --- a/kernel/model/transaction.go +++ b/kernel/model/transaction.go @@ -1447,7 +1447,7 @@ func reindexTree0(tree *parse.Tree, i, size int) { indexWriteJSONQueue(tree) } else { treenode.ReindexBlockTree(tree) - sql.UpsertTreeQueue(tree) + sql.IndexTreeQueue(tree.Box, tree.Path) } util.PushStatusBar(fmt.Sprintf(Conf.Language(183), i, size, html.EscapeHTMLStr(path.Base(tree.HPath)))) } diff --git a/kernel/sql/queue.go b/kernel/sql/queue.go index f326188ed..e3e361f10 100644 --- a/kernel/sql/queue.go +++ b/kernel/sql/queue.go @@ -37,12 +37,13 @@ var ( type dbQueueOperation struct { inQueueTime time.Time - action string // upsert/delete/delete_id/rename/delete_box/delete_box_refs/insert_refs + action string // upsert/delete/delete_id/rename/delete_box/delete_box_refs/insert_refs/index + indexPath string // index upsertTree *parse.Tree // upsert/insert_refs removeTreeBox, removeTreePath string // delete removeTreeIDBox, removeTreeID string // delete_id - box string // delete_box/delete_box_refs + box string // delete_box/delete_box_refs/index renameTree *parse.Tree // rename renameTreeOldHPath string // rename } @@ -110,6 +111,8 @@ func FlushQueue() { } switch op.action { + case "index": + err = indexTree(tx, op.box, op.indexPath, context) case "upsert": err = upsertTree(tx, op.upsertTree, context) case "delete": @@ -231,6 +234,20 @@ func DeleteBoxQueue(boxID string) { operationQueue = append(operationQueue, newOp) } +func IndexTreeQueue(box, p string) { + dbQueueLock.Lock() + defer dbQueueLock.Unlock() + + newOp := &dbQueueOperation{indexPath: p, box: box, inQueueTime: time.Now(), action: "index"} + for i, op := range operationQueue { + if "index" == op.action && op.indexPath == p && op.box == box { // 相同树则覆盖 + operationQueue[i] = newOp + return + } + } + operationQueue = append(operationQueue, newOp) +} + func UpsertTreeQueue(tree *parse.Tree) { dbQueueLock.Lock() defer dbQueueLock.Unlock() diff --git a/kernel/sql/upsert.go b/kernel/sql/upsert.go index 184c748c8..3e7516467 100644 --- a/kernel/sql/upsert.go +++ b/kernel/sql/upsert.go @@ -21,6 +21,7 @@ import ( "crypto/sha256" "database/sql" "fmt" + "github.com/siyuan-note/siyuan/kernel/filesys" "strings" "github.com/88250/lute/parse" @@ -392,6 +393,16 @@ func insertRefs(tx *sql.Tx, tree *parse.Tree) (err error) { return err } +func indexTree(tx *sql.Tx, box, p string, context map[string]interface{}) (err error) { + tree, err := filesys.LoadTree(box, p, luteEngine) + if nil != err { + return + } + + err = upsertTree(tx, tree, context) + return +} + func upsertTree(tx *sql.Tx, tree *parse.Tree, context map[string]interface{}) (err error) { oldBlockHashes := queryBlockHashes(tree.ID) blocks, spans, assets, attributes := fromTree(tree.Root, tree) From 430c8cbf6a59db9aab32e169b351ec2069d4d881 Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Wed, 25 Jan 2023 12:54:25 +0800 Subject: [PATCH 2/3] =?UTF-8?q?:art:=20=E6=94=B9=E8=BF=9B=E5=86=85?= =?UTF-8?q?=E6=A0=B8=E4=BB=BB=E5=8A=A1=E8=B0=83=E5=BA=A6=E6=9C=BA=E5=88=B6?= =?UTF-8?q?=E6=8F=90=E5=8D=87=E7=A8=B3=E5=AE=9A=E6=80=A7=20https://github.?= =?UTF-8?q?com/siyuan-note/siyuan/issues/7113?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/sql/queue.go | 77 +++++++++++++++++++++++++-------------------- 1 file changed, 43 insertions(+), 34 deletions(-) diff --git a/kernel/sql/queue.go b/kernel/sql/queue.go index e3e361f10..e1f613eb3 100644 --- a/kernel/sql/queue.go +++ b/kernel/sql/queue.go @@ -17,6 +17,9 @@ package sql import ( + "database/sql" + "errors" + "fmt" "path" "sync" "time" @@ -103,57 +106,31 @@ func FlushQueue() { return } + var execOps int context := map[string]interface{}{eventbus.CtxPushMsg: eventbus.CtxPushMsgToStatusBar} - execOps := 0 for i, op := range ops { if util.IsExiting { - break - } - - switch op.action { - case "index": - err = indexTree(tx, op.box, op.indexPath, context) - case "upsert": - err = upsertTree(tx, op.upsertTree, context) - case "delete": - err = batchDeleteByPathPrefix(tx, op.removeTreeBox, op.removeTreePath) - case "delete_id": - err = deleteByRootID(tx, op.removeTreeID) - case "rename": - err = batchUpdateHPath(tx, op.renameTree.Box, op.renameTree.ID, op.renameTreeOldHPath, op.renameTree.HPath) - if nil != err { - break - } - err = updateRootContent(tx, path.Base(op.renameTree.HPath), op.renameTree.Root.IALAttr("updated"), op.renameTree.ID) - case "delete_box": - err = deleteByBoxTx(tx, op.box) - case "delete_box_refs": - err = deleteRefsByBoxTx(tx, op.box) - case "insert_refs": - err = insertRefs(tx, op.upsertTree) - case "update_refs": - err = upsertRefs(tx, op.upsertTree) - default: - logging.LogErrorf("unknown operation [%s]", op.action) - break + return } + err = execOp(op, tx, context) execOps++ + if nil != err { logging.LogErrorf("queue operation failed: %s", err) - break + return } - if 0 < i && 0 == i%64 { + if 0 < i && 0 == execOps%64 { if err = commitTx(tx); nil != err { logging.LogErrorf("commit tx failed: %s", err) - break + return } execOps = 0 tx, err = beginTx() if nil != err { - break + return } } } @@ -169,6 +146,38 @@ func FlushQueue() { } } +func execOp(op *dbQueueOperation, tx *sql.Tx, context map[string]interface{}) (err error) { + switch op.action { + case "index": + err = indexTree(tx, op.box, op.indexPath, context) + case "upsert": + err = upsertTree(tx, op.upsertTree, context) + case "delete": + err = batchDeleteByPathPrefix(tx, op.removeTreeBox, op.removeTreePath) + case "delete_id": + err = deleteByRootID(tx, op.removeTreeID) + case "rename": + err = batchUpdateHPath(tx, op.renameTree.Box, op.renameTree.ID, op.renameTreeOldHPath, op.renameTree.HPath) + if nil != err { + break + } + err = updateRootContent(tx, path.Base(op.renameTree.HPath), op.renameTree.Root.IALAttr("updated"), op.renameTree.ID) + case "delete_box": + err = deleteByBoxTx(tx, op.box) + case "delete_box_refs": + err = deleteRefsByBoxTx(tx, op.box) + case "insert_refs": + err = insertRefs(tx, op.upsertTree) + case "update_refs": + err = upsertRefs(tx, op.upsertTree) + default: + msg := fmt.Sprint("unknown operation [%s]", op.action) + logging.LogErrorf(msg) + err = errors.New(msg) + } + return +} + func mergeUpsertTrees() (ops []*dbQueueOperation) { dbQueueLock.Lock() defer dbQueueLock.Unlock() From 9d009fe5f92d1f724b4a748c3f5419ef1c0bb6ff Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Wed, 25 Jan 2023 13:09:10 +0800 Subject: [PATCH 3/3] =?UTF-8?q?:art:=20=E6=94=B9=E8=BF=9B=E5=86=85?= =?UTF-8?q?=E6=A0=B8=E4=BB=BB=E5=8A=A1=E8=B0=83=E5=BA=A6=E6=9C=BA=E5=88=B6?= =?UTF-8?q?=E6=8F=90=E5=8D=87=E7=A8=B3=E5=AE=9A=E6=80=A7=20https://github.?= =?UTF-8?q?com/siyuan-note/siyuan/issues/7113?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/sql/upsert.go | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/kernel/sql/upsert.go b/kernel/sql/upsert.go index 3e7516467..f0d9875a7 100644 --- a/kernel/sql/upsert.go +++ b/kernel/sql/upsert.go @@ -399,7 +399,14 @@ func indexTree(tx *sql.Tx, box, p string, context map[string]interface{}) (err e return } - err = upsertTree(tx, tree, context) + err = insertTree(tx, tree, context) + return +} + +func insertTree(tx *sql.Tx, tree *parse.Tree, context map[string]interface{}) (err error) { + blocks, spans, assets, attributes := fromTree(tree.Root, tree) + refs, fileAnnotationRefs := refsFromTree(tree) + err = insertTree0(tx, tree, context, blocks, spans, assets, attributes, refs, fileAnnotationRefs) return } @@ -431,7 +438,10 @@ func upsertTree(tx *sql.Tx, tree *parse.Tree, context map[string]interface{}) (e for _, b := range blocks { toRemoves = append(toRemoves, b.ID) } - deleteBlocksByIDs(tx, toRemoves) + + if err = deleteBlocksByIDs(tx, toRemoves); nil != err { + return + } if err = deleteSpansByPathTx(tx, tree.Box, tree.Path); nil != err { return @@ -449,11 +459,20 @@ func upsertTree(tx *sql.Tx, tree *parse.Tree, context map[string]interface{}) (e return } + refs, fileAnnotationRefs := refsFromTree(tree) + if err = insertTree0(tx, tree, context, blocks, spans, assets, attributes, refs, fileAnnotationRefs); nil != err { + return + } + return err +} + +func insertTree0(tx *sql.Tx, tree *parse.Tree, context map[string]interface{}, + blocks []*Block, spans []*Span, assets []*Asset, attributes []*Attribute, + refs []*Ref, fileAnnotationRefs []*FileAnnotationRef) (err error) { if err = insertBlocks(tx, blocks, context); nil != err { return } - refs, fileAnnotationRefs := refsFromTree(tree) if err = insertBlockRefs(tx, refs); nil != err { return } @@ -476,5 +495,5 @@ func upsertTree(tx *sql.Tx, tree *parse.Tree, context map[string]interface{}) (e if err = insertAttributes(tx, attributes); nil != err { return } - return err + return }