From 514c747fe3e9fa15240cd2e08a40b3699765f0aa Mon Sep 17 00:00:00 2001 From: Daniel <845765@qq.com> Date: Fri, 10 Nov 2023 20:25:29 +0800 Subject: [PATCH] :art: Improve data sync --- kernel/go.mod | 16 ++++++++-------- kernel/go.sum | 30 ++++++++++++++++-------------- kernel/model/repository.go | 26 ++++++++++++-------------- 3 files changed, 36 insertions(+), 36 deletions(-) diff --git a/kernel/go.mod b/kernel/go.mod index 3fdafe6e6..a12fb2d81 100644 --- a/kernel/go.mod +++ b/kernel/go.mod @@ -47,9 +47,9 @@ require ( github.com/panjf2000/ants/v2 v2.8.2 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/radovskyb/watcher v1.0.7 - github.com/sashabaranov/go-openai v1.16.1 + github.com/sashabaranov/go-openai v1.17.5 github.com/shirou/gopsutil/v3 v3.23.10 - github.com/siyuan-note/dejavu v0.0.0-20231108034856-670a6dc62c56 + github.com/siyuan-note/dejavu v0.0.0-20231110121748-867038071de4 github.com/siyuan-note/encryption v0.0.0-20220713091850-5ecd92177b75 github.com/siyuan-note/eventbus v0.0.0-20230804030110-cf250f838c80 github.com/siyuan-note/filelock v0.0.0-20231107122348-6ed75b0b525a @@ -61,7 +61,7 @@ require ( github.com/vmihailenco/msgpack/v5 v5.4.1 github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 github.com/xuri/excelize/v2 v2.8.0 - golang.org/x/image v0.13.0 + golang.org/x/image v0.14.0 golang.org/x/mobile v0.0.0-20230901161150-52620a4a7557 golang.org/x/mod v0.14.0 golang.org/x/text v0.14.0 @@ -77,7 +77,7 @@ require ( github.com/andybalholm/brotli v1.0.6 // indirect github.com/andybalholm/cascadia v1.3.2 // indirect github.com/asaskevich/EventBus v0.0.0-20200907212545-49d423059eef // indirect - github.com/aws/aws-sdk-go v1.47.5 // indirect + github.com/aws/aws-sdk-go v1.47.8 // indirect github.com/bytedance/sonic v1.10.2 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect @@ -156,12 +156,12 @@ require ( github.com/yusufpapurcu/wmi v1.2.3 // indirect go.uber.org/mock v0.3.0 // indirect golang.org/x/arch v0.6.0 // indirect - golang.org/x/crypto v0.14.0 // indirect - golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect - golang.org/x/net v0.17.0 // indirect + golang.org/x/crypto v0.15.0 // indirect + golang.org/x/exp v0.0.0-20231108232855-2478ac86f678 // indirect + golang.org/x/net v0.18.0 // indirect golang.org/x/sync v0.5.0 // indirect golang.org/x/sys v0.14.0 // indirect - golang.org/x/tools v0.14.0 // indirect + golang.org/x/tools v0.15.0 // indirect google.golang.org/protobuf v1.31.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/kernel/go.sum b/kernel/go.sum index 70aa57882..d665a7b3c 100644 --- a/kernel/go.sum +++ b/kernel/go.sum @@ -49,8 +49,8 @@ github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de h1:FxWPpzIjnTlhP github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de/go.mod h1:DCaWoUhZrYW9p1lxo/cm8EmUOOzAPSEZNGF2DK1dJgw= github.com/asaskevich/EventBus v0.0.0-20200907212545-49d423059eef h1:2JGTg6JapxP9/R33ZaagQtAM4EkkSYnIAlOG5EI8gkM= github.com/asaskevich/EventBus v0.0.0-20200907212545-49d423059eef/go.mod h1:JS7hed4L1fj0hXcyEejnW57/7LCetXggd+vwrRnYeII= -github.com/aws/aws-sdk-go v1.47.5 h1:U2JlfPmrUoz5p+2X/XwKxmaJFo2oV+LbJqx8jyEvyAY= -github.com/aws/aws-sdk-go v1.47.5/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= +github.com/aws/aws-sdk-go v1.47.8 h1:VCFyO5UTREnhR0HRf9roqFfJeeRVin58zUy+pBMhwjY= +github.com/aws/aws-sdk-go v1.47.8/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= github.com/bytedance/sonic v1.10.0-rc/go.mod h1:ElCzW+ufi8qKqNW0FY314xriJhyJhuoJ3gFZdAHF7NM= github.com/bytedance/sonic v1.10.2 h1:GQebETVBxYB7JGWJtLBi07OVzWwt+8dWA00gEVW2ZFE= @@ -337,8 +337,8 @@ github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZV github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 h1:OkMGxebDjyw0ULyrTYWeN0UNCCkmCWfjPnIA2W6oviI= github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06/go.mod h1:+ePHsJ1keEjQtpvf9HHw0f4ZeJ0TLRsxhunSI2hYJSs= -github.com/sashabaranov/go-openai v1.16.1 h1:R7c8jhXeQEtHkD6MoXQfkgzz/7JI6jjFKl+DXawaejQ= -github.com/sashabaranov/go-openai v1.16.1/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg= +github.com/sashabaranov/go-openai v1.17.5 h1:ItBzlrrfTtkFWOFlgfOhk3y/xRBC4PJol4gdbiK7hgg= +github.com/sashabaranov/go-openai v1.17.5/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg= github.com/scylladb/termtables v0.0.0-20191203121021-c4c0b6d42ff4/go.mod h1:C1a7PQSMz9NShzorzCiG2fk9+xuCgLkPeCvMHYR2OWg= github.com/shirou/gopsutil/v3 v3.23.10 h1:/N42opWlYzegYaVkWejXWJpbzKv2JDy3mrgGzKsh9hM= github.com/shirou/gopsutil/v3 v3.23.10/go.mod h1:JIE26kpucQi+innVlAUnIEOSBhBUkirr5b44yr55+WE= @@ -352,8 +352,8 @@ github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFR github.com/shurcooL/gofontwoff v0.0.0-20181114050219-180f79e6909d h1:lvCTyBbr36+tqMccdGMwuEU+hjux/zL6xSmf5S9ITaA= github.com/shurcooL/gofontwoff v0.0.0-20181114050219-180f79e6909d/go.mod h1:05UtEgK5zq39gLST6uB0cf3NEHjETfB4Fgr3Gx5R9Vw= github.com/simplereach/timeutils v1.2.0/go.mod h1:VVbQDfN/FHRZa1LSqcwo4kNZ62OOyqLLGQKYB3pB0Q8= -github.com/siyuan-note/dejavu v0.0.0-20231108034856-670a6dc62c56 h1:qNNxfRgvt7oYCb9zkJuqKoEICfqyJ4ta4SZ4NqMTiis= -github.com/siyuan-note/dejavu v0.0.0-20231108034856-670a6dc62c56/go.mod h1:QJg/fY8KE4eeEOa0Mu2Mp8aBWpshm0qgLaHxNBCc/OM= +github.com/siyuan-note/dejavu v0.0.0-20231110121748-867038071de4 h1:370R/Uis+AY9ivXg671KgKA6iSksvylz9sfhiA2QVl4= +github.com/siyuan-note/dejavu v0.0.0-20231110121748-867038071de4/go.mod h1:6RdIofCDZP8vL0qYnVZfB12NhK7cUgh1ooUxRvRdlmI= github.com/siyuan-note/encryption v0.0.0-20220713091850-5ecd92177b75 h1:Bi7/7f29LW+Fm0cHc0J1NO1cZqyJwljSWVmfOqVZgaE= github.com/siyuan-note/encryption v0.0.0-20220713091850-5ecd92177b75/go.mod h1:H8fyqqAbp9XreANjeSbc72zEdFfKTXYN34tc1TjZwtw= github.com/siyuan-note/eventbus v0.0.0-20230804030110-cf250f838c80 h1:XghjHKJd+SiL0DkGYFVC+UGUDFtnR4v9gkAbPeh9Eq8= @@ -438,14 +438,15 @@ golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= -golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= -golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= -golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= +golang.org/x/crypto v0.15.0 h1:frVn1TEaCEaZcn3Tmd7Y2b5KKPaZ+I32Q2OA3kYp5TA= +golang.org/x/crypto v0.15.0/go.mod h1:4ChreQoLWfG3xLDer1WdlH5NdlQ3+mwnQq1YTKY+72g= +golang.org/x/exp v0.0.0-20231108232855-2478ac86f678 h1:mchzmB1XO2pMaKFRqk/+MV3mgGG96aqaPXaMifQU47w= +golang.org/x/exp v0.0.0-20231108232855-2478ac86f678/go.mod h1:zk2irFbV9DP96SEBUUAy67IdHUaZuSnrz1n472HUCLE= golang.org/x/image v0.0.0-20210628002857-a66eb6448b8d/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= golang.org/x/image v0.11.0/go.mod h1:bglhjqbqVuEb9e9+eNR45Jfu7D+T4Qan+NhQk8Ck2P8= -golang.org/x/image v0.13.0 h1:3cge/F/QTkNLauhf2QoE9zp+7sr+ZcL4HnoZmdwg9sg= -golang.org/x/image v0.13.0/go.mod h1:6mmbMOeV28HuMTgA6OSRkdXKYw/t5W9Uwn2Yv1r3Yxk= +golang.org/x/image v0.14.0 h1:tNgSxAFe3jC4uYqvZdTr84SZoM1KfwdC9SKIFrLjFn4= +golang.org/x/image v0.14.0/go.mod h1:HUYqC05R2ZcZ3ejNQsIHQDQiwWM4JBqmm6MKANTp4LE= golang.org/x/mobile v0.0.0-20230901161150-52620a4a7557 h1:mLrcd+qwh23kzD7ej1VxCa+A23UNr+BCjSj2tNX8/NM= golang.org/x/mobile v0.0.0-20230901161150-52620a4a7557/go.mod h1:f0gjFM6UTH7y1WEZBm/kquBYsogL+NQtllKFy4Rdulc= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= @@ -468,8 +469,9 @@ golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg= +golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -528,8 +530,8 @@ golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc= -golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg= +golang.org/x/tools v0.15.0 h1:zdAyfUGbYmuVokhzVmghFl2ZJh5QhcfebBgmVPFYA+8= +golang.org/x/tools v0.15.0/go.mod h1:hpksKq4dtpQWS1uQ61JkdqWM3LscIS6Slf+VVkm+wQk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= diff --git a/kernel/model/repository.go b/kernel/model/repository.go index f46e1c82e..94332e97e 100644 --- a/kernel/model/repository.go +++ b/kernel/model/repository.go @@ -967,7 +967,7 @@ func syncRepoDownload() (err error) { } start := time.Now() - err = indexRepoBeforeCloudSync(repo) + _, _, err = indexRepoBeforeCloudSync(repo) if nil != err { planSyncAfter(fixSyncInterval) return @@ -1035,7 +1035,7 @@ func syncRepoUpload() (err error) { } start := time.Now() - err = indexRepoBeforeCloudSync(repo) + _, _, err = indexRepoBeforeCloudSync(repo) if nil != err { planSyncAfter(fixSyncInterval) return @@ -1105,7 +1105,7 @@ func bootSyncRepo() (err error) { } start := time.Now() - err = indexRepoBeforeCloudSync(repo) + _, _, err = indexRepoBeforeCloudSync(repo) if nil != err { autoSyncErrCount++ planSyncAfter(fixSyncInterval) @@ -1193,9 +1193,8 @@ func syncRepo(exit, byHand bool) (dataChanged bool, err error) { return } - latest, _ := repo.Latest() start := time.Now() - err = indexRepoBeforeCloudSync(repo) + beforeIndex, afterIndex, err := indexRepoBeforeCloudSync(repo) if nil != err { autoSyncErrCount++ planSyncAfter(fixSyncInterval) @@ -1235,8 +1234,7 @@ func syncRepo(exit, byHand bool) (dataChanged bool, err error) { return } - syncedLatest, _ := repo.Latest() - dataChanged = nil == latest || latest.ID != syncedLatest.ID + dataChanged = nil == beforeIndex || beforeIndex.ID != afterIndex.ID util.PushStatusBar(fmt.Sprintf(Conf.Language(149), elapsed.Seconds())) Conf.Sync.Synced = util.CurrentTimeMillis() @@ -1447,15 +1445,15 @@ func needFullReindex(upsertTrees int) bool { var promotedPurgeDataRepo bool -func indexRepoBeforeCloudSync(repo *dejavu.Repo) (err error) { +func indexRepoBeforeCloudSync(repo *dejavu.Repo) (beforeIndex, afterIndex *entity.Index, err error) { start := time.Now() - latest, _ := repo.Latest() - index, err := repo.Index("[Sync] Cloud sync", map[string]interface{}{ + beforeIndex, _ = repo.Latest() + afterIndex, err = repo.Index("[Sync] Cloud sync", map[string]interface{}{ eventbus.CtxPushMsg: eventbus.CtxPushMsgToStatusBar, }) if errors.Is(err, dejavu.ErrNotFoundObject) { var resetErr error - index, resetErr = resetRepository(repo) + afterIndex, resetErr = resetRepository(repo) if nil != resetErr { return } @@ -1471,10 +1469,10 @@ func indexRepoBeforeCloudSync(repo *dejavu.Repo) (err error) { } elapsed := time.Since(start) - if nil == latest || latest.ID != index.ID { + if nil == beforeIndex || beforeIndex.ID != afterIndex.ID { // 对新创建的快照需要更新备注，加入耗时统计 - index.Memo = fmt.Sprintf("[Sync] Cloud sync, completed in %.2fs", elapsed.Seconds()) - if err = repo.PutIndex(index); nil != err { + afterIndex.Memo = fmt.Sprintf("[Sync] Cloud sync, completed in %.2fs", elapsed.Seconds()) + if err = repo.PutIndex(afterIndex); nil != err { util.PushStatusBar("Save data snapshot for cloud sync failed") logging.LogErrorf("put index into data repo before cloud sync failed: %s", err) return