github/siyuan
4
0
Fork 0
mirror of https://github.com/siyuan-note/siyuan.git synced 2025-05-16 17:10:53 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/dev' into dev

Browse Source
This commit is contained in:
Vanessa 2023-02-07 11:41:39 +08:00
commit 27f9c943fe
17 changed files with 106 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
app/appearance/langs/en_US.json
View File

@ -876,6 +876,7 @@
"task.database.index.commit": "Execute database index commit",
"task.database.index.ref": "Execute database index reference",
"task.database.index.fix": "Execute database index fix",
"task.database.cache": "Execute database cache",
"task.ocr.image": "Execute image OCR to extract text",
"task.history.generateDoc": "Execute GenerateDoc History",
"task.database.index.embedBlock": "Execute database index embed block",

1
app/appearance/langs/es_ES.json
View File

@ -876,6 +876,7 @@
"task.database.index.commit": "Ejecutar la confirmación del índice de la base de datos",
"task.database.index.ref": "Ejecutar referencia de índice de base de datos",
"task.database.index.fix": "Ejecutar corrección del índice de la base de datos",
"task.database.cache": "Ejecutar caché de base de datos",
"task.ocr.image": "Ejecutar OCR de imagen para extraer texto",
"task.history.generateDoc": "Ejecutar Historial GenerateDoc",
"task.database.index.embedBlock": "Ejecutar bloque de incrustación de índice de base de datos",

1
app/appearance/langs/fr_FR.json
View File

@ -876,6 +876,7 @@
"task.database.index.commit": "Effectuer la validation de l'index de la base de données",
"task.database.index.ref": "Exécuter la référence d'index de la base de données",
"task.database.index.fix": "Effectuer la correction de l'index de la base de données",
"task.database.cache": "Effectuer le cache de la base de données",
"task.ocr.image": "Exécute l'OCR d'image pour extraire le texte",
"task.history.generateDoc": "Exécuter l'historique de GenerateDoc",
"task.database.index.embedBlock": "Exécuter le bloc d'intégration d'index de base de données",

1
app/appearance/langs/zh_CHT.json
View File

@ -876,6 +876,7 @@
"task.database.index.commit": "執行數據庫索引提交",
"task.database.index.ref": "執行數據庫索引引用",
"task.database.index.fix": "執行數據庫索引訂正",
"task.database.cache": "執行數據庫緩存",
"task.ocr.image": "執行圖片 OCR 提取文本",
"task.history.generateDoc": "執行生成文件歷史",
"task.database.index.embedBlock": "執行數據庫索引嵌入塊",

1
app/appearance/langs/zh_CN.json
View File

@ -876,6 +876,7 @@
"task.database.index.commit": "执行数据库索引提交",
"task.database.index.ref": "执行数据库索引引用",
"task.database.index.fix": "执行数据库索引订正",
"task.database.cache": "执行数据库缓存",
"task.ocr.image": "执行图片 OCR 提取文本",
"task.history.generateDoc": "执行生成文件历史",
"task.database.index.embedBlock": "执行数据库索引嵌入块",

7
kernel/api/block.go
View File

@ -40,7 +40,14 @@ func transferBlockRef(c *gin.Context) {
}
fromID := arg["fromID"].(string)
if util.InvalidIDPattern(fromID, ret) {
return
}
toID := arg["toID"].(string)
if util.InvalidIDPattern(toID, ret) {
return
}
err := model.TransferBlockRef(fromID, toID)
if nil != err {
ret.Code = -1

8
kernel/model/box.go
View File

@ -505,7 +505,10 @@ func ReloadUI() {
}
func FullReindex() {
task.PrependTask(task.DatabaseIndexFull, fullReindex)
task.AppendTask(task.DatabaseIndexFull, fullReindex)
task.AppendTask(task.DatabaseCache, sql.EnableCache)
task.AppendTask(task.DatabaseIndexRef, IndexRefs)
task.AppendTask(task.ReloadUI, util.ReloadUI)
}
func fullReindex() {
@ -523,12 +526,9 @@ func fullReindex() {
for _, openedBox := range openedBoxes {
index(openedBox.ID)
}
sql.EnableCache()
treenode.SaveBlockTree(true)
LoadFlashcards()
IndexRefs()
debug.FreeOSMemory()
util.ReloadUI()
}
func ChangeBoxSort(boxIDs []string) {

2
kernel/model/file.go
View File

@ -1256,7 +1256,7 @@ func removeDoc(box *Box, p string) {
}
util.PushEvent(evt)
task.PrependTask(task.DatabaseIndex, removeDoc0, box, p, childrenDir)
task.AppendTask(task.DatabaseIndex, removeDoc0, box, p, childrenDir)
}
func removeDoc0(box *Box, p, childrenDir string) {

3
kernel/model/history.go
View File

@ -50,7 +50,7 @@ func AutoGenerateDocHistory() {
ChangeHistoryTick(Conf.Editor.GenerateHistoryInterval)
for {
<-historyTicker.C
task.PrependTask(task.HistoryGenerateDoc, generateDocHistory)
task.AppendTask(task.HistoryGenerateDoc, generateDocHistory)
}
}
@ -244,7 +244,6 @@ func RollbackDocHistory(boxID, historyPath string) (err error) {
FullReindex()
IncSync()
ReloadUI()
return nil
}

1
kernel/model/import.go
View File

@ -441,7 +441,6 @@ func ImportData(zipPath string) (err error) {
IncSync()
FullReindex()
ReloadUI()
return
}

4
kernel/model/index.go
View File

@ -39,7 +39,7 @@ import (
)
func (box *Box) Unindex() {
task.PrependTask(task.DatabaseIndex, unindex, box.ID)
task.AppendTask(task.DatabaseIndex, unindex, box.ID)
}
func unindex(boxID string) {
@ -49,7 +49,7 @@ func unindex(boxID string) {
}
func (box *Box) Index() {
task.PrependTask(task.DatabaseIndex, index, box.ID)
task.AppendTask(task.DatabaseIndex, index, box.ID)
task.AppendTask(task.DatabaseIndexRef, IndexRefs)
}

43
kernel/model/index_fix.go
View File

@ -39,20 +39,29 @@ import (
// FixIndexJob 自动校验数据库索引 https://github.com/siyuan-note/siyuan/issues/7016
func FixIndexJob() {
task.AppendTask(task.DatabaseIndexFix, autoFixIndex)
task.AppendTask(task.DatabaseIndexFix, removeDuplicateDatabaseIndex)
sql.WaitForWritingDatabase()
task.AppendTask(task.DatabaseIndexFix, fixBlockTreeByFileSys)
sql.WaitForWritingDatabase()
task.AppendTask(task.DatabaseIndexFix, fixDatabaseIndexByBlockTree)
sql.WaitForWritingDatabase()
util.PushStatusBar(Conf.Language(185))
debug.FreeOSMemory()
}
var autoFixLock = sync.Mutex{}
func autoFixIndex() {
// removeDuplicateDatabaseIndex 删除重复的数据库索引。
func removeDuplicateDatabaseIndex() {
defer logging.Recover()
autoFixLock.Lock()
defer autoFixLock.Unlock()
util.PushStatusBar(Conf.Language(58))
// 去除重复的数据库块记录
duplicatedRootIDs := sql.GetDuplicatedRootIDs("blocks")
if 1 > len(duplicatedRootIDs) {
duplicatedRootIDs = sql.GetDuplicatedRootIDs("blocks_fts")
@ -87,11 +96,16 @@ func autoFixIndex() {
if 0 < deletes {
logging.LogWarnf("exist more than one tree duplicated [%d], reindex it", deletes)
}
}
// fixBlockTreeByFileSys 通过文件系统订正块树。
func fixBlockTreeByFileSys() {
defer logging.Recover()
autoFixLock.Lock()
defer autoFixLock.Unlock()
util.PushStatusBar(Conf.Language(58))
sql.WaitForWritingDatabase()
util.PushStatusBar(Conf.Language(58))
// 根据文件系统补全块树
boxes := Conf.GetOpenedBoxes()
for _, box := range boxes {
boxPath := filepath.Join(util.DataDir, box.ID)
@ -130,26 +144,23 @@ func autoFixIndex() {
}
}
util.PushStatusBar(Conf.Language(58))
sql.WaitForWritingDatabase()
util.PushStatusBar(Conf.Language(58))
// 清理已关闭的笔记本块树
boxes = Conf.GetClosedBoxes()
for _, box := range boxes {
treenode.RemoveBlockTreesByBoxID(box.ID)
}
}
// 对比块树和数据库并订正数据库
// fixDatabaseIndexByBlockTree 通过块树订正数据库索引。
func fixDatabaseIndexByBlockTree() {
defer logging.Recover()
util.PushStatusBar(Conf.Language(58))
rootUpdatedMap := treenode.GetRootUpdated()
dbRootUpdatedMap, err := sql.GetRootUpdated()
if nil == err {
reindexTreeByUpdated(rootUpdatedMap, dbRootUpdatedMap)
}
util.PushStatusBar(Conf.Language(58))
sql.WaitForWritingDatabase()
util.PushStatusBar(Conf.Language(185))
debug.FreeOSMemory()
}
func reindexTreeByUpdated(rootUpdatedMap, dbRootUpdatedMap map[string]string) {

13
kernel/model/repository.go
View File

@ -49,7 +49,6 @@ import (
"github.com/siyuan-note/siyuan/kernel/cache"
"github.com/siyuan-note/siyuan/kernel/conf"
"github.com/siyuan-note/siyuan/kernel/filesys"
"github.com/siyuan-note/siyuan/kernel/sql"
"github.com/siyuan-note/siyuan/kernel/task"
"github.com/siyuan-note/siyuan/kernel/treenode"
"github.com/siyuan-note/siyuan/kernel/util"
@ -504,7 +503,7 @@ func InitRepoKey() (err error) {
}
func CheckoutRepo(id string) {
task.PrependTask(task.RepoCheckout, checkoutRepo, id)
task.AppendTask(task.RepoCheckout, checkoutRepo, id)
}
func checkoutRepo(id string) {
@ -523,7 +522,6 @@ func checkoutRepo(id string) {
util.PushEndlessProgress(Conf.Language(63))
WaitForWritingFiles()
sql.WaitForWritingDatabase()
CloseWatchAssets()
defer WatchAssets()
@ -541,7 +539,6 @@ func checkoutRepo(id string) {
}
FullReindex()
ReloadUI()
if syncEnabled {
func() {
@ -942,7 +939,6 @@ func syncRepo(exit, byHand bool) (err error) {
// 云端同步发生冲突时生成副本 https://github.com/siyuan-note/siyuan/issues/5687
luteEngine := NewLute()
waitTx := false
for _, file := range mergeResult.Conflicts {
if !strings.HasSuffix(file.Path, ".sy") {
continue
@ -965,10 +961,6 @@ func syncRepo(exit, byHand bool) (err error) {
resetTree(tree, "Conflicted")
createTreeTx(tree)
waitTx = true
}
if waitTx {
sql.WaitForWritingDatabase()
}
}
@ -1028,9 +1020,6 @@ func syncRepo(exit, byHand bool) (err error) {
cache.ClearDocsIAL() // 同步后文档树文档图标没有更新 https://github.com/siyuan-note/siyuan/issues/4939
if needFullReindex(upsertTrees) { // 改进同步后全量重建索引判断 https://github.com/siyuan-note/siyuan/issues/5764
FullReindex()
if !exit {
ReloadUI()
}
return
}

3
kernel/model/sync.go
View File

@ -97,7 +97,9 @@ func SyncData(boot, exit, byHand bool) {
func syncData(boot, exit, byHand bool) {
defer logging.Recover()
util.BroadcastByType("main", "syncing", 0, Conf.Language(81), nil)
if !checkSync(boot, exit, byHand) {
util.BroadcastByType("main", "syncing", 1, "", nil)
return
}
@ -118,7 +120,6 @@ func syncData(boot, exit, byHand bool) {
now := util.CurrentTimeMillis()
Conf.Sync.Synced = now
util.BroadcastByType("main", "syncing", 0, Conf.Language(81), nil)
err := syncRepo(exit, byHand)
synced := util.Millisecond2Time(Conf.Sync.Synced).Format("2006-01-02 15:04:05") + "\n\n"
if nil == err {

3
kernel/sql/queue.go
View File

@ -118,8 +118,9 @@ func FlushQueue() {
context["current"] = i
context["total"] = total
if err = execOp(op, tx, context); nil != err {
tx.Rollback()
logging.LogErrorf("queue operation failed: %s", err)
return
continue
}
if err = commitTx(tx); nil != err {

73
kernel/task/queue.go
View File

@ -39,27 +39,20 @@ type Task struct {
Created time.Time
}
func PrependTask(action string, handler interface{}, args ...interface{}) {
queueLock.Lock()
defer queueLock.Unlock()
if util.IsExiting {
//logging.LogWarnf("task queue is paused, action [%s] will be ignored", action)
return
}
taskQueue = append([]*Task{newTask(action, handler, args...)}, taskQueue...)
}
func AppendTask(action string, handler interface{}, args ...interface{}) {
queueLock.Lock()
defer queueLock.Unlock()
if util.IsExiting {
//logging.LogWarnf("task queue is paused, action [%s] will be ignored", action)
return
}
currentActions := getCurrentActions()
if gulu.Str.Contains(action, currentActions) && gulu.Str.Contains(action, uniqueActions) {
//logging.LogWarnf("task [%s] is already in queue, will be ignored", action)
return
}
queueLock.Lock()
defer queueLock.Unlock()
taskQueue = append(taskQueue, newTask(action, handler, args...))
}
@ -72,6 +65,20 @@ func newTask(action string, handler interface{}, args ...interface{}) *Task {
}
}
func getCurrentActions() (ret []string) {
queueLock.Lock()
defer queueLock.Unlock()
if "" != currentTaskAction {
ret = append(ret, currentTaskAction)
}
for _, task := range taskQueue {
ret = append(ret, task.Action)
}
return
}
const (
RepoCheckout = "task.repo.checkout" // 从快照中检出
DatabaseIndexFull = "task.database.index.full" // 重建索引
@ -79,12 +86,23 @@ const (
DatabaseIndexCommit = "task.database.index.commit" // 数据库索引提交
DatabaseIndexRef = "task.database.index.ref" // 数据库索引引用
DatabaseIndexFix = "task.database.index.fix" // 数据库索引订正
DatabaseCache = "task.database.cache" // 数据库缓存
OCRImage = "task.ocr.image" // 图片 OCR 提取文本
HistoryGenerateDoc = "task.history.generateDoc" // 生成文件历史
DatabaseIndexEmbedBlock = "task.database.index.embedBlock" // 数据库索引嵌入块
ReloadUI = "task.reload.ui" // 重载 UI
)
// uniqueActions 描述了唯一的任务，即队列中只能存在一个在执行的任务。
var uniqueActions = []string{
RepoCheckout,
DatabaseIndexFull,
DatabaseIndexCommit,
OCRImage,
HistoryGenerateDoc,
DatabaseIndexEmbedBlock,
}
func Contain(action string, moreActions ...string) bool {
actions := append(moreActions, action)
actions = gulu.Str.RemoveDuplicatedElem(actions)
@ -99,13 +117,12 @@ func Contain(action string, moreActions ...string) bool {
func StatusJob() {
tasks := taskQueue
data := map[string]interface{}{}
var items []map[string]interface{}
count := map[string]int{}
actionLangs := util.TaskActionLangs[util.Lang]
for _, task := range tasks {
actionLangs := util.TaskActionLangs[util.Lang]
action := task.Action
if c := count[action]; 3 < c {
if c := count[action]; 2 < c {
logging.LogWarnf("too many tasks [%s], ignore show its status", action)
continue
}
@ -120,9 +137,19 @@ func StatusJob() {
item := map[string]interface{}{"action": action}
items = append(items, item)
}
if "" != currentTaskAction {
if nil != actionLangs {
if label := actionLangs[currentTaskAction]; nil != label {
items = append([]map[string]interface{}{map[string]interface{}{"action": label.(string)}}, items...)
}
}
}
if 1 > len(items) {
items = []map[string]interface{}{}
}
data := map[string]interface{}{}
data["tasks"] = items
util.PushBackgroundTask(data)
}
@ -140,6 +167,8 @@ func ExecTaskJob() {
execTask(task)
}
var currentTaskAction string
func popTask() (ret *Task) {
queueLock.Lock()
defer queueLock.Unlock()
@ -165,7 +194,9 @@ func execTask(task *Task) {
}
}
ctx, cancel := context.WithTimeout(context.Background(), time.Second*7)
currentTaskAction = task.Action
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
ch := make(chan bool, 1)
go func() {
@ -175,8 +206,10 @@ func execTask(task *Task) {
select {
case <-ctx.Done():
//logging.LogWarnf("task [%s] timeout", task.Action)
logging.LogWarnf("task [%s] timeout", task.Action)
case <-ch:
//logging.LogInfof("task [%s] done", task.Action)
}
currentTaskAction = ""
}

2
kernel/treenode/tree.go
View File

@ -54,7 +54,7 @@ func NodeHash(node *ast.Node, tree *parse.Tree, luteEngine *lute.Lute) string {
md = FormatNode(node, luteEngine)
}
hpath := tree.HPath
data := tree.Path + hpath + string(ial) + md
data := tree.Box + tree.Path + hpath + string(ial) + md
return fmt.Sprintf("%x", sha256.Sum256(gulu.Str.ToBytes(data)))[:7]
}