🎨 支持多设备操作不同文档后云端同步合并 https://github.com/siyuan-note/siyuan/issues/5092

kernel/model/osssync.go

@@ -33,6 +33,8 @@ import (
 	"github.com/88250/gulu"
 	"github.com/panjf2000/ants/v2"
 	"github.com/qiniu/go-sdk/v7/storage"
+	"github.com/siyuan-note/siyuan/kernel/sql"
+	"github.com/siyuan-note/siyuan/kernel/treenode"
 	"github.com/siyuan-note/siyuan/kernel/util"
 )
 
@@ -161,7 +163,7 @@ func listCloudSyncDirOSS() (dirs []map[string]interface{}, size int64, err error
 	return
 }
 
-func ossDownload(localDirPath, cloudDirPath string, bootOrExit bool) (fetchedFilesCount int, transferSize uint64, downloadedFiles []string, err error) {
+func ossDownload(localDirPath, cloudDirPath string, bootOrExit bool) (fetchedFilesCount int, transferSize uint64, downloadedFiles map[string]bool, err error) {
 	if !gulu.File.IsExist(localDirPath) {
 		return
 	}
@@ -186,6 +188,7 @@ func ossDownload(localDirPath, cloudDirPath string, bootOrExit bool) (fetchedFil
 	needPushProgress := 32 < len(cloudFetches)
 	waitGroup := &sync.WaitGroup{}
 	var downloadErr error
+	downloadedFiles = map[string]bool{}
 	poolSize := 4
 	if poolSize > len(cloudFetches)-1 /* 不计入 /.siyuan/conf.json，配置文件最后单独下载 */ {
 		poolSize = len(cloudFetches)
@@ -202,7 +205,7 @@ func ossDownload(localDirPath, cloudDirPath string, bootOrExit bool) (fetchedFil
 			downloadErr = err // 仅记录最后一次错误
 			return
 		}
-		downloadedFiles = append(downloadedFiles, fetch)
+		downloadedFiles[fetch] = true // FIXME: 并发修改 map
 
 		if needPushProgress {
 			msg := fmt.Sprintf(Conf.Language(103), fetchedFilesCount, len(cloudFetches)-fetchedFilesCount)
@@ -335,6 +338,7 @@ func ossUpload(localDirPath, cloudDirPath, cloudDevice string, boot bool, remove
 
 	var localUpserts, cloudRemoves []string
 	var cloudFileList map[string]*CloudIndex
+	var downloadList map[string]bool
 	if "" != localDevice && localDevice == cloudDevice {
 		//util.LogInfof("cloud device is the same as local device, get index from local")
 		localUpserts, cloudRemoves, err = cloudUpsertRemoveLocalListOSS(localDirPath, removeList, upsertList, excludes)
@@ -346,10 +350,47 @@ func ossUpload(localDirPath, cloudDirPath, cloudDevice string, boot bool, remove
 	}
 
 	if 0 < len(cloudFileList) {
-		localUpserts, cloudRemoves, err = cloudUpsertRemoveListOSS(localDirPath, cloudFileList, localFileList, excludes)
+		localUpserts, cloudRemoves, downloadList, err = cloudUpsertRemoveListOSS(localDirPath, cloudFileList, localFileList, removeList, upsertList, excludes)
 		if nil != err {
 			return
 		}
+		if 0 < len(downloadList) {
+			// 下载合并云端变更
+			var tmpFetchedFiles int
+			var tmpTransferSize uint64
+			err = ossDownload0(util.TempDir+"/sync", "sync/"+Conf.Sync.CloudName, "/"+pathJSON, &tmpFetchedFiles, &tmpTransferSize, false)
+			if nil != err {
+				util.LogErrorf("download merge cloud file failed: %s", err)
+			}
+
+			metaPath := filepath.Join(util.TempDir, "/sync/"+pathJSON)
+			mergeErr := syncDirUpsertWorkspaceData(metaPath, downloadList)
+			if nil != mergeErr {
+				util.LogErrorf("download merge cloud file failed: %s", mergeErr)
+			} else {
+				// 增量索引
+				for upsertFile, _ := range downloadList {
+					if !strings.HasSuffix(upsertFile, ".sy") {
+						continue
+					}
+
+					upsertFile = filepath.ToSlash(upsertFile)
+					box := upsertFile[:strings.Index(upsertFile, "/")]
+					p := strings.TrimPrefix(upsertFile, box)
+					tree, err0 := LoadTree(box, p)
+					if nil != err0 {
+						continue
+					}
+					treenode.ReindexBlockTree(tree)
+					sql.UpsertTreeQueue(tree)
+				}
+			}
+
+			// 重新生成云端索引
+			if _, idxErr := genCloudIndex(localDirPath, excludes); nil != idxErr {
+				return
+			}
+		}
 	}
 
 	err = ossRemove0(cloudDirPath, cloudRemoves)
@@ -713,9 +754,10 @@ func cloudUpsertRemoveLocalListOSS(localDirPath string, removedSyncList, upserte
 	return
 }
 
-func cloudUpsertRemoveListOSS(localDirPath string, cloudFileList, localFileList map[string]*CloudIndex, excludes map[string]bool) (localUpserts, cloudRemoves []string, err error) {
+func cloudUpsertRemoveListOSS(localDirPath string, cloudFileList, localFileList map[string]*CloudIndex, removeList, upsertList, excludes map[string]bool) (localUpserts, cloudRemoves []string, downloadList map[string]bool, err error) {
 	localUpserts, cloudRemoves = []string{}, []string{}
 
+	cloudChangedList := map[string]bool{}
 	unchanged := map[string]bool{}
 	for cloudFile, cloudIdx := range cloudFileList {
 		localIdx := localFileList[cloudFile]
@@ -725,7 +767,10 @@ func cloudUpsertRemoveListOSS(localDirPath string, cloudFileList, localFileList
 		}
 		if localIdx.Updated == cloudIdx.Updated {
 			unchanged[filepath.Join(localDirPath, cloudFile)] = true
+			continue
 		}
+
+		cloudChangedList[cloudFile] = true
 	}
 
 	delete(unchanged, filepath.Join(localDirPath, "index.json")) // 同步偶尔报错 `The system cannot find the path specified.` https://github.com/siyuan-note/siyuan/issues/4942
@@ -747,6 +792,14 @@ func cloudUpsertRemoveListOSS(localDirPath string, cloudFileList, localFileList
 		}
 		return nil
 	})
+
+	downloadList = map[string]bool{}
+	for cloudChanged, _ := range cloudChangedList {
+		if upsertList[cloudChanged] || removeList[cloudChanged] || excludes[cloudChanged] {
+			continue
+		}
+		downloadList[cloudChanged] = true
+	}
 	return
 }
 

kernel/model/sync.go

@@ -352,7 +352,8 @@ func SyncData(boot, exit, byHand bool) {
 		Conf.Sync.Stat = msg
 		util.PushErrMsg(msg, 7000)
 
-		err = syncDirUpsertWorkspaceData(downloadedFiles)
+		metaPath := filepath.Join(Conf.Sync.GetSaveDir(), pathJSON)
+		err = syncDirUpsertWorkspaceData(metaPath, downloadedFiles)
 		if nil != err {
 			util.LogErrorf("upsert partially downloaded files to workspace data failed: %s", err)
 		}
@@ -529,17 +530,17 @@ func SetSyncMode(mode int) (err error) {
 
 var syncLock = sync.Mutex{}
 
-func syncDirUpsertWorkspaceData(downloadedFiles []string) (err error) {
+func syncDirUpsertWorkspaceData(metaPath string, downloadedFiles map[string]bool) (err error) {
 	start := time.Now()
 
 	modified := map[string]bool{}
 	syncDir := Conf.Sync.GetSaveDir()
-	for _, file := range downloadedFiles {
+	for file, _ := range downloadedFiles {
 		file = filepath.Join(syncDir, file)
 		modified[file] = true
 	}
 
-	decryptedDataDir, _, err := recoverSyncData(modified)
+	decryptedDataDir, _, err := recoverSyncData(metaPath, modified)
 	if nil != err {
 		util.LogErrorf("decrypt data dir failed: %s", err)
 		return
@@ -567,7 +568,8 @@ func syncDir2WorkspaceData(boot bool) (upsertFiles, removeFiles []string, err er
 	}
 
 	modified := modifiedSyncList(unchanged)
-	decryptedDataDir, upsertFiles, err := recoverSyncData(modified)
+	metaPath := filepath.Join(Conf.Sync.GetSaveDir(), pathJSON)
+	decryptedDataDir, upsertFiles, err := recoverSyncData(metaPath, modified)
 	if nil != err {
 		util.LogErrorf("decrypt data dir failed: %s", err)
 		return
@@ -659,7 +661,7 @@ func genCloudIndex(localDirPath string, excludes map[string]bool) (cloudIndex ma
 	return
 }
 
-func recoverSyncData(modified map[string]bool) (decryptedDataDir string, upsertFiles []string, err error) {
+func recoverSyncData(metaPath string, modified map[string]bool) (decryptedDataDir string, upsertFiles []string, err error) {
 	passwd := Conf.E2EEPasswd
 	decryptedDataDir = filepath.Join(util.WorkspaceDir, "incremental", "sync-decrypt")
 	if err = os.RemoveAll(decryptedDataDir); nil != err {
@@ -670,8 +672,7 @@ func recoverSyncData(modified map[string]bool) (decryptedDataDir string, upsertF
 	}
 
 	syncDir := Conf.Sync.GetSaveDir()
-	meta := filepath.Join(syncDir, pathJSON)
-	data, err := os.ReadFile(meta)
+	data, err := os.ReadFile(metaPath)
 	if nil != err {
 		return
 	}