From 488e87b70bc152ddaacdfb9f1daa1ebfeda2daed Mon Sep 17 00:00:00 2001
From: Daniel <845765@qq.com>
Date: Wed, 16 Oct 2024 19:29:11 +0800
Subject: [PATCH] :art: WebDAV/S3 data sync and backup support configurable
 concurrent requests https://github.com/siyuan-note/siyuan/issues/12798

---
 app/src/config/repos.ts    | 26 ++++++++++++++++++++++++++
 app/src/types/config.d.ts  |  8 ++++++++
 kernel/conf/sync.go        | 28 +++++++++++++++-------------
 kernel/go.mod              |  2 +-
 kernel/go.sum              |  4 ++--
 kernel/model/conf.go       |  6 ++++--
 kernel/model/repository.go |  2 ++
 kernel/model/sync.go       |  2 ++
 kernel/util/path.go        | 15 +++++++++++++++
 9 files changed, 75 insertions(+), 18 deletions(-)

diff --git a/app/src/config/repos.ts b/app/src/config/repos.ts
index 0a9066b84..6aa6496c1 100644
--- a/app/src/config/repos.ts
+++ b/app/src/config/repos.ts
@@ -99,6 +99,11 @@ const renderProvider = (provider: number) => {
         <option ${window.siyuan.config.sync.s3.skipTlsVerify ? "selected" : ""} value="true">Skip</option>
     </select>
 </div>
+<div class="b3-label b3-label--inner fn__flex">
+    <div class="fn__flex-center fn__size200">Concurrent Reqs</div>
+    <div class="fn__space"></div>
+    <input id="s3ConcurrentReqs" class="b3-text-field fn__block" type="number" min="1" max="16" value="${window.siyuan.config.sync.s3.concurrentReqs}">
+</div>
 <div class="b3-label b3-label--inner fn__flex">
     <div class="fn__flex-1"></div>
     <button class="b3-button b3-button--outline fn__size200" data-action="purgeData">
@@ -153,6 +158,11 @@ const renderProvider = (provider: number) => {
         <option ${window.siyuan.config.sync.webdav.skipTlsVerify ? "selected" : ""} value="true">Skip</option>
     </select>
 </div>
+<div class="b3-label b3-label--inner fn__flex">
+    <div class="fn__flex-center fn__size200">Concurrent Reqs</div>
+    <div class="fn__space"></div>
+    <input id="webdavConcurrentReqs" class="b3-text-field fn__block" type="number" min="1" max="16" value="${window.siyuan.config.sync.webdav.concurrentReqs}">
+</div>
 <div class="b3-label b3-label--inner fn__flex">
     <div class="fn__flex-1"></div>
     <button class="b3-button b3-button--outline fn__size200" data-action="purgeData">
@@ -267,6 +277,13 @@ const bindProviderEvent = () => {
                 if (300 < timeout) {
                     timeout = 300;
                 }
+                let concurrentReqs = parseInt((providerPanelElement.querySelector("#s3ConcurrentReqs") as HTMLInputElement).value, 10);
+                if (1 > concurrentReqs) {
+                    concurrentReqs = 1;
+                }
+                if (16 < concurrentReqs) {
+                    concurrentReqs = 16;
+                }
                 (providerPanelElement.querySelector("#timeout") as HTMLInputElement).value = timeout.toString();
                 const s3 = {
                     endpoint: (providerPanelElement.querySelector("#endpoint") as HTMLInputElement).value,
@@ -277,6 +294,7 @@ const bindProviderEvent = () => {
                     region: (providerPanelElement.querySelector("#region") as HTMLInputElement).value,
                     skipTlsVerify: (providerPanelElement.querySelector("#s3SkipTlsVerify") as HTMLInputElement).value === "true",
                     timeout: timeout,
+                    concurrentReqs: concurrentReqs,
                 };
                 fetchPost("/api/sync/setSyncProviderS3", {s3}, () => {
                     window.siyuan.config.sync.s3 = s3;
@@ -289,6 +307,13 @@ const bindProviderEvent = () => {
                 if (300 < timeout) {
                     timeout = 300;
                 }
+                let concurrentReqs = parseInt((providerPanelElement.querySelector("#webdavConcurrentReqs") as HTMLInputElement).value, 10);
+                if (1 > concurrentReqs) {
+                    concurrentReqs = 1;
+                }
+                if (16 < concurrentReqs) {
+                    concurrentReqs = 16;
+                }
                 (providerPanelElement.querySelector("#timeout") as HTMLInputElement).value = timeout.toString();
                 const webdav = {
                     endpoint: (providerPanelElement.querySelector("#endpoint") as HTMLInputElement).value,
@@ -296,6 +321,7 @@ const bindProviderEvent = () => {
                     password: (providerPanelElement.querySelector("#password") as HTMLInputElement).value,
                     skipTlsVerify: (providerPanelElement.querySelector("#webdavSkipTlsVerify") as HTMLInputElement).value === "true",
                     timeout: timeout,
+                    concurrentReqs: concurrentReqs,
                 };
                 fetchPost("/api/sync/setSyncProviderWebDAV", {webdav}, () => {
                     window.siyuan.config.sync.webdav = webdav;
diff --git a/app/src/types/config.d.ts b/app/src/types/config.d.ts
index d4a7232ff..20521e1d8 100644
--- a/app/src/types/config.d.ts
+++ b/app/src/types/config.d.ts
@@ -1381,6 +1381,10 @@ declare namespace Config {
          * Timeout (unit: seconds)
          */
         timeout: number;
+        /**
+         * Concurrent requests.
+         */
+        concurrentReqs: number;
     }
 
     /**
@@ -1403,6 +1407,10 @@ declare namespace Config {
          * Timeout (unit: seconds)
          */
         timeout: number;
+        /**
+         * Concurrent requests.
+         */
+        concurrentReqs: number;
         /**
          * Username
          */
diff --git a/kernel/conf/sync.go b/kernel/conf/sync.go
index 31a7bb2d2..ac37900cc 100644
--- a/kernel/conf/sync.go
+++ b/kernel/conf/sync.go
@@ -41,22 +41,24 @@ func NewSync() *Sync {
 }
 
 type S3 struct {
-	Endpoint      string `json:"endpoint"`      // 服务端点
-	AccessKey     string `json:"accessKey"`     // Access Key
-	SecretKey     string `json:"secretKey"`     // Secret Key
-	Bucket        string `json:"bucket"`        // 存储空间
-	Region        string `json:"region"`        // 存储区域
-	PathStyle     bool   `json:"pathStyle"`     // 是否使用路径风格
-	SkipTlsVerify bool   `json:"skipTlsVerify"` // 是否跳过 TLS 验证
-	Timeout       int    `json:"timeout"`       // 超时时间，单位：秒
+	Endpoint       string `json:"endpoint"`       // 服务端点
+	AccessKey      string `json:"accessKey"`      // Access Key
+	SecretKey      string `json:"secretKey"`      // Secret Key
+	Bucket         string `json:"bucket"`         // 存储空间
+	Region         string `json:"region"`         // 存储区域
+	PathStyle      bool   `json:"pathStyle"`      // 是否使用路径风格
+	SkipTlsVerify  bool   `json:"skipTlsVerify"`  // 是否跳过 TLS 验证
+	Timeout        int    `json:"timeout"`        // 超时时间，单位：秒
+	ConcurrentReqs int    `json:"concurrentReqs"` // 并发请求数
 }
 
 type WebDAV struct {
-	Endpoint      string `json:"endpoint"`      // 服务端点
-	Username      string `json:"username"`      // 用户名
-	Password      string `json:"password"`      // 密码
-	SkipTlsVerify bool   `json:"skipTlsVerify"` // 是否跳过 TLS 验证
-	Timeout       int    `json:"timeout"`       // 超时时间，单位：秒
+	Endpoint       string `json:"endpoint"`       // 服务端点
+	Username       string `json:"username"`       // 用户名
+	Password       string `json:"password"`       // 密码
+	SkipTlsVerify  bool   `json:"skipTlsVerify"`  // 是否跳过 TLS 验证
+	Timeout        int    `json:"timeout"`        // 超时时间，单位：秒
+	ConcurrentReqs int    `json:"concurrentReqs"` // 并发请求数
 }
 
 const (
diff --git a/kernel/go.mod b/kernel/go.mod
index ccb263bab..0bdb0c6e5 100644
--- a/kernel/go.mod
+++ b/kernel/go.mod
@@ -54,7 +54,7 @@ require (
 	github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06
 	github.com/sashabaranov/go-openai v1.29.1
 	github.com/shirou/gopsutil/v3 v3.24.5
-	github.com/siyuan-note/dejavu v0.0.0-20241014034432-7d61fd4d1fe0
+	github.com/siyuan-note/dejavu v0.0.0-20241016112457-5fc03573be87
 	github.com/siyuan-note/encryption v0.0.0-20231219001248-1e028a4d13b4
 	github.com/siyuan-note/eventbus v0.0.0-20240627125516-396fdb0f0f97
 	github.com/siyuan-note/filelock v0.0.0-20240724034355-d1ed7bf21d04
diff --git a/kernel/go.sum b/kernel/go.sum
index 29f49ff17..62d7430b0 100644
--- a/kernel/go.sum
+++ b/kernel/go.sum
@@ -332,8 +332,8 @@ github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+D
 github.com/shurcooL/gofontwoff v0.0.0-20181114050219-180f79e6909d h1:lvCTyBbr36+tqMccdGMwuEU+hjux/zL6xSmf5S9ITaA=
 github.com/shurcooL/gofontwoff v0.0.0-20181114050219-180f79e6909d/go.mod h1:05UtEgK5zq39gLST6uB0cf3NEHjETfB4Fgr3Gx5R9Vw=
 github.com/simplereach/timeutils v1.2.0/go.mod h1:VVbQDfN/FHRZa1LSqcwo4kNZ62OOyqLLGQKYB3pB0Q8=
-github.com/siyuan-note/dejavu v0.0.0-20241014034432-7d61fd4d1fe0 h1:r2XSfIkA6rF0Cj4RqFlJzw9Am7x5xOlA6WcClV7vgm8=
-github.com/siyuan-note/dejavu v0.0.0-20241014034432-7d61fd4d1fe0/go.mod h1:7E+jOWm75Me1ss7Sc5UM6O5xMiATAyCbN3g7MP6Cgxw=
+github.com/siyuan-note/dejavu v0.0.0-20241016112457-5fc03573be87 h1:EG8h35jYH8/iOukQlhB2OE7S35PwOBBn+XMOK1rtgtg=
+github.com/siyuan-note/dejavu v0.0.0-20241016112457-5fc03573be87/go.mod h1:7E+jOWm75Me1ss7Sc5UM6O5xMiATAyCbN3g7MP6Cgxw=
 github.com/siyuan-note/encryption v0.0.0-20231219001248-1e028a4d13b4 h1:kJaw5L/evyW6LcB9IQT8PR4ppx8JVqOFP9Ix3rfwSrc=
 github.com/siyuan-note/encryption v0.0.0-20231219001248-1e028a4d13b4/go.mod h1:UYcCCY+0wh+GmUoDOaO63j1sV5lgy7laLAk1XhEiUis=
 github.com/siyuan-note/eventbus v0.0.0-20240627125516-396fdb0f0f97 h1:lM5v8BfNtbOL5jYwhCdMYBcYtr06IYBKjjSLAPMKTM8=
diff --git a/kernel/model/conf.go b/kernel/model/conf.go
index a8971a5c7..c592e23b8 100644
--- a/kernel/model/conf.go
+++ b/kernel/model/conf.go
@@ -339,15 +339,17 @@ func InitConf() {
 		Conf.Sync.Mode = 1
 	}
 	if nil == Conf.Sync.S3 {
-		Conf.Sync.S3 = &conf.S3{}
+		Conf.Sync.S3 = &conf.S3{SkipTlsVerify: true}
 	}
 	Conf.Sync.S3.Endpoint = util.NormalizeEndpoint(Conf.Sync.S3.Endpoint)
 	Conf.Sync.S3.Timeout = util.NormalizeTimeout(Conf.Sync.S3.Timeout)
+	Conf.Sync.S3.ConcurrentReqs = util.NormalizeConcurrentReqs(Conf.Sync.S3.ConcurrentReqs, conf.ProviderS3)
 	if nil == Conf.Sync.WebDAV {
-		Conf.Sync.WebDAV = &conf.WebDAV{}
+		Conf.Sync.WebDAV = &conf.WebDAV{SkipTlsVerify: true}
 	}
 	Conf.Sync.WebDAV.Endpoint = util.NormalizeEndpoint(Conf.Sync.WebDAV.Endpoint)
 	Conf.Sync.WebDAV.Timeout = util.NormalizeTimeout(Conf.Sync.WebDAV.Timeout)
+	Conf.Sync.WebDAV.ConcurrentReqs = util.NormalizeConcurrentReqs(Conf.Sync.WebDAV.ConcurrentReqs, conf.ProviderWebDAV)
 	if util.ContainerDocker == util.Container {
 		Conf.Sync.Perception = false
 	}
diff --git a/kernel/model/repository.go b/kernel/model/repository.go
index 0a692eaa8..bd94f781c 100644
--- a/kernel/model/repository.go
+++ b/kernel/model/repository.go
@@ -1959,6 +1959,7 @@ func buildCloudConf() (ret *cloud.Conf, err error) {
 			PathStyle:     Conf.Sync.S3.PathStyle,
 			SkipTlsVerify: Conf.Sync.S3.SkipTlsVerify,
 			Timeout:       Conf.Sync.S3.Timeout,
+			ConcurrentReqs: Conf.Sync.S3.ConcurrentReqs,
 		}
 	case conf.ProviderWebDAV:
 		ret.WebDAV = &cloud.ConfWebDAV{
@@ -1967,6 +1968,7 @@ func buildCloudConf() (ret *cloud.Conf, err error) {
 			Password:      Conf.Sync.WebDAV.Password,
 			SkipTlsVerify: Conf.Sync.WebDAV.SkipTlsVerify,
 			Timeout:       Conf.Sync.WebDAV.Timeout,
+			ConcurrentReqs: Conf.Sync.WebDAV.ConcurrentReqs,
 		}
 	default:
 		err = fmt.Errorf("invalid provider [%d]", Conf.Sync.Provider)
diff --git a/kernel/model/sync.go b/kernel/model/sync.go
index b657497f1..dae96728d 100644
--- a/kernel/model/sync.go
+++ b/kernel/model/sync.go
@@ -426,6 +426,7 @@ func SetSyncProviderS3(s3 *conf.S3) (err error) {
 	s3.Bucket = strings.TrimSpace(s3.Bucket)
 	s3.Region = strings.TrimSpace(s3.Region)
 	s3.Timeout = util.NormalizeTimeout(s3.Timeout)
+	s3.ConcurrentReqs = util.NormalizeConcurrentReqs(s3.ConcurrentReqs, conf.ProviderS3)
 
 	if !cloud.IsValidCloudDirName(s3.Bucket) {
 		util.PushErrMsg(Conf.Language(37), 5000)
@@ -450,6 +451,7 @@ func SetSyncProviderWebDAV(webdav *conf.WebDAV) (err error) {
 	webdav.Username = strings.TrimSpace(webdav.Username)
 	webdav.Password = strings.TrimSpace(webdav.Password)
 	webdav.Timeout = util.NormalizeTimeout(webdav.Timeout)
+	webdav.ConcurrentReqs = util.NormalizeConcurrentReqs(webdav.ConcurrentReqs, conf.ProviderWebDAV)
 
 	Conf.Sync.WebDAV = webdav
 	Conf.Save()
diff --git a/kernel/util/path.go b/kernel/util/path.go
index ecee45d9c..a3c63ddb6 100644
--- a/kernel/util/path.go
+++ b/kernel/util/path.go
@@ -179,6 +179,21 @@ func GetChildDocDepth(treeAbsPath string) (ret int) {
 	return
 }
 
+func NormalizeConcurrentReqs(concurrentReqs int, provider int) int {
+	if 1 > concurrentReqs {
+		if 2 == provider { // S3
+			return 8
+		} else if 3 == provider { // WebDAV
+			return 1
+		}
+		return 8
+	}
+	if 16 < concurrentReqs {
+		return 16
+	}
+	return concurrentReqs
+}
+
 func NormalizeTimeout(timeout int) int {
 	if 7 > timeout {
 		if 1 > timeout {