🎨 WebDAV/S3 data sync and backup support configurable concurrent requests https://github.com/siyuan-note/siyuan/issues/12798

app/src/config/repos.ts

@@ -99,6 +99,11 @@ const renderProvider = (provider: number) => {
         <option ${window.siyuan.config.sync.s3.skipTlsVerify ? "selected" : ""} value="true">Skip</option>
     </select>
 </div>
+<div class="b3-label b3-label--inner fn__flex">
+    <div class="fn__flex-center fn__size200">Concurrent Reqs</div>
+    <div class="fn__space"></div>
+    <input id="s3ConcurrentReqs" class="b3-text-field fn__block" type="number" min="1" max="16" value="${window.siyuan.config.sync.s3.concurrentReqs}">
+</div>
 <div class="b3-label b3-label--inner fn__flex">
     <div class="fn__flex-1"></div>
     <button class="b3-button b3-button--outline fn__size200" data-action="purgeData">
@@ -153,6 +158,11 @@ const renderProvider = (provider: number) => {
         <option ${window.siyuan.config.sync.webdav.skipTlsVerify ? "selected" : ""} value="true">Skip</option>
     </select>
 </div>
+<div class="b3-label b3-label--inner fn__flex">
+    <div class="fn__flex-center fn__size200">Concurrent Reqs</div>
+    <div class="fn__space"></div>
+    <input id="webdavConcurrentReqs" class="b3-text-field fn__block" type="number" min="1" max="16" value="${window.siyuan.config.sync.webdav.concurrentReqs}">
+</div>
 <div class="b3-label b3-label--inner fn__flex">
     <div class="fn__flex-1"></div>
     <button class="b3-button b3-button--outline fn__size200" data-action="purgeData">
@@ -267,6 +277,13 @@ const bindProviderEvent = () => {
                 if (300 < timeout) {
                     timeout = 300;
                 }
+                let concurrentReqs = parseInt((providerPanelElement.querySelector("#s3ConcurrentReqs") as HTMLInputElement).value, 10);
+                if (1 > concurrentReqs) {
+                    concurrentReqs = 1;
+                }
+                if (16 < concurrentReqs) {
+                    concurrentReqs = 16;
+                }
                 (providerPanelElement.querySelector("#timeout") as HTMLInputElement).value = timeout.toString();
                 const s3 = {
                     endpoint: (providerPanelElement.querySelector("#endpoint") as HTMLInputElement).value,
@@ -277,6 +294,7 @@ const bindProviderEvent = () => {
                     region: (providerPanelElement.querySelector("#region") as HTMLInputElement).value,
                     skipTlsVerify: (providerPanelElement.querySelector("#s3SkipTlsVerify") as HTMLInputElement).value === "true",
                     timeout: timeout,
+                    concurrentReqs: concurrentReqs,
                 };
                 fetchPost("/api/sync/setSyncProviderS3", {s3}, () => {
                     window.siyuan.config.sync.s3 = s3;
@@ -289,6 +307,13 @@ const bindProviderEvent = () => {
                 if (300 < timeout) {
                     timeout = 300;
                 }
+                let concurrentReqs = parseInt((providerPanelElement.querySelector("#webdavConcurrentReqs") as HTMLInputElement).value, 10);
+                if (1 > concurrentReqs) {
+                    concurrentReqs = 1;
+                }
+                if (16 < concurrentReqs) {
+                    concurrentReqs = 16;
+                }
                 (providerPanelElement.querySelector("#timeout") as HTMLInputElement).value = timeout.toString();
                 const webdav = {
                     endpoint: (providerPanelElement.querySelector("#endpoint") as HTMLInputElement).value,
@@ -296,6 +321,7 @@ const bindProviderEvent = () => {
                     password: (providerPanelElement.querySelector("#password") as HTMLInputElement).value,
                     skipTlsVerify: (providerPanelElement.querySelector("#webdavSkipTlsVerify") as HTMLInputElement).value === "true",
                     timeout: timeout,
+                    concurrentReqs: concurrentReqs,
                 };
                 fetchPost("/api/sync/setSyncProviderWebDAV", {webdav}, () => {
                     window.siyuan.config.sync.webdav = webdav;

app/src/types/config.d.ts

@@ -1381,6 +1381,10 @@ declare namespace Config {
          * Timeout (unit: seconds)
          */
         timeout: number;
+        /**
+         * Concurrent requests.
+         */
+        concurrentReqs: number;
     }
 
     /**
@@ -1403,6 +1407,10 @@ declare namespace Config {
          * Timeout (unit: seconds)
          */
         timeout: number;
+        /**
+         * Concurrent requests.
+         */
+        concurrentReqs: number;
         /**
          * Username
          */

kernel/conf/sync.go

@@ -49,6 +49,7 @@ type S3 struct {
 	PathStyle      bool   `json:"pathStyle"`      // 是否使用路径风格
 	SkipTlsVerify  bool   `json:"skipTlsVerify"`  // 是否跳过 TLS 验证
 	Timeout        int    `json:"timeout"`        // 超时时间，单位：秒
+	ConcurrentReqs int    `json:"concurrentReqs"` // 并发请求数
 }
 
 type WebDAV struct {
@@ -57,6 +58,7 @@ type WebDAV struct {
 	Password       string `json:"password"`       // 密码
 	SkipTlsVerify  bool   `json:"skipTlsVerify"`  // 是否跳过 TLS 验证
 	Timeout        int    `json:"timeout"`        // 超时时间，单位：秒
+	ConcurrentReqs int    `json:"concurrentReqs"` // 并发请求数
 }
 
 const (

kernel/go.mod

@@ -54,7 +54,7 @@ require (
 	github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06
 	github.com/sashabaranov/go-openai v1.29.1
 	github.com/shirou/gopsutil/v3 v3.24.5
-	github.com/siyuan-note/dejavu v0.0.0-20241014034432-7d61fd4d1fe0
+	github.com/siyuan-note/dejavu v0.0.0-20241016112457-5fc03573be87
 	github.com/siyuan-note/encryption v0.0.0-20231219001248-1e028a4d13b4
 	github.com/siyuan-note/eventbus v0.0.0-20240627125516-396fdb0f0f97
 	github.com/siyuan-note/filelock v0.0.0-20240724034355-d1ed7bf21d04

kernel/go.sum

@@ -332,8 +332,8 @@ github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+D
 github.com/shurcooL/gofontwoff v0.0.0-20181114050219-180f79e6909d h1:lvCTyBbr36+tqMccdGMwuEU+hjux/zL6xSmf5S9ITaA=
 github.com/shurcooL/gofontwoff v0.0.0-20181114050219-180f79e6909d/go.mod h1:05UtEgK5zq39gLST6uB0cf3NEHjETfB4Fgr3Gx5R9Vw=
 github.com/simplereach/timeutils v1.2.0/go.mod h1:VVbQDfN/FHRZa1LSqcwo4kNZ62OOyqLLGQKYB3pB0Q8=
-github.com/siyuan-note/dejavu v0.0.0-20241014034432-7d61fd4d1fe0 h1:r2XSfIkA6rF0Cj4RqFlJzw9Am7x5xOlA6WcClV7vgm8=
+github.com/siyuan-note/dejavu v0.0.0-20241016112457-5fc03573be87 h1:EG8h35jYH8/iOukQlhB2OE7S35PwOBBn+XMOK1rtgtg=
-github.com/siyuan-note/dejavu v0.0.0-20241014034432-7d61fd4d1fe0/go.mod h1:7E+jOWm75Me1ss7Sc5UM6O5xMiATAyCbN3g7MP6Cgxw=
+github.com/siyuan-note/dejavu v0.0.0-20241016112457-5fc03573be87/go.mod h1:7E+jOWm75Me1ss7Sc5UM6O5xMiATAyCbN3g7MP6Cgxw=
 github.com/siyuan-note/encryption v0.0.0-20231219001248-1e028a4d13b4 h1:kJaw5L/evyW6LcB9IQT8PR4ppx8JVqOFP9Ix3rfwSrc=
 github.com/siyuan-note/encryption v0.0.0-20231219001248-1e028a4d13b4/go.mod h1:UYcCCY+0wh+GmUoDOaO63j1sV5lgy7laLAk1XhEiUis=
 github.com/siyuan-note/eventbus v0.0.0-20240627125516-396fdb0f0f97 h1:lM5v8BfNtbOL5jYwhCdMYBcYtr06IYBKjjSLAPMKTM8=

kernel/model/conf.go

@@ -339,15 +339,17 @@ func InitConf() {
 		Conf.Sync.Mode = 1
 	}
 	if nil == Conf.Sync.S3 {
-		Conf.Sync.S3 = &conf.S3{}
+		Conf.Sync.S3 = &conf.S3{SkipTlsVerify: true}
 	}
 	Conf.Sync.S3.Endpoint = util.NormalizeEndpoint(Conf.Sync.S3.Endpoint)
 	Conf.Sync.S3.Timeout = util.NormalizeTimeout(Conf.Sync.S3.Timeout)
+	Conf.Sync.S3.ConcurrentReqs = util.NormalizeConcurrentReqs(Conf.Sync.S3.ConcurrentReqs, conf.ProviderS3)
 	if nil == Conf.Sync.WebDAV {
-		Conf.Sync.WebDAV = &conf.WebDAV{}
+		Conf.Sync.WebDAV = &conf.WebDAV{SkipTlsVerify: true}
 	}
 	Conf.Sync.WebDAV.Endpoint = util.NormalizeEndpoint(Conf.Sync.WebDAV.Endpoint)
 	Conf.Sync.WebDAV.Timeout = util.NormalizeTimeout(Conf.Sync.WebDAV.Timeout)
+	Conf.Sync.WebDAV.ConcurrentReqs = util.NormalizeConcurrentReqs(Conf.Sync.WebDAV.ConcurrentReqs, conf.ProviderWebDAV)
 	if util.ContainerDocker == util.Container {
 		Conf.Sync.Perception = false
 	}

kernel/model/repository.go

@@ -1959,6 +1959,7 @@ func buildCloudConf() (ret *cloud.Conf, err error) {
 			PathStyle:     Conf.Sync.S3.PathStyle,
 			SkipTlsVerify: Conf.Sync.S3.SkipTlsVerify,
 			Timeout:       Conf.Sync.S3.Timeout,
+			ConcurrentReqs: Conf.Sync.S3.ConcurrentReqs,
 		}
 	case conf.ProviderWebDAV:
 		ret.WebDAV = &cloud.ConfWebDAV{
@@ -1967,6 +1968,7 @@ func buildCloudConf() (ret *cloud.Conf, err error) {
 			Password:      Conf.Sync.WebDAV.Password,
 			SkipTlsVerify: Conf.Sync.WebDAV.SkipTlsVerify,
 			Timeout:       Conf.Sync.WebDAV.Timeout,
+			ConcurrentReqs: Conf.Sync.WebDAV.ConcurrentReqs,
 		}
 	default:
 		err = fmt.Errorf("invalid provider [%d]", Conf.Sync.Provider)

kernel/model/sync.go

@@ -426,6 +426,7 @@ func SetSyncProviderS3(s3 *conf.S3) (err error) {
 	s3.Bucket = strings.TrimSpace(s3.Bucket)
 	s3.Region = strings.TrimSpace(s3.Region)
 	s3.Timeout = util.NormalizeTimeout(s3.Timeout)
+	s3.ConcurrentReqs = util.NormalizeConcurrentReqs(s3.ConcurrentReqs, conf.ProviderS3)
 
 	if !cloud.IsValidCloudDirName(s3.Bucket) {
 		util.PushErrMsg(Conf.Language(37), 5000)
@@ -450,6 +451,7 @@ func SetSyncProviderWebDAV(webdav *conf.WebDAV) (err error) {
 	webdav.Username = strings.TrimSpace(webdav.Username)
 	webdav.Password = strings.TrimSpace(webdav.Password)
 	webdav.Timeout = util.NormalizeTimeout(webdav.Timeout)
+	webdav.ConcurrentReqs = util.NormalizeConcurrentReqs(webdav.ConcurrentReqs, conf.ProviderWebDAV)
 
 	Conf.Sync.WebDAV = webdav
 	Conf.Save()

kernel/util/path.go

@@ -179,6 +179,21 @@ func GetChildDocDepth(treeAbsPath string) (ret int) {
 	return
 }
 
+func NormalizeConcurrentReqs(concurrentReqs int, provider int) int {
+	if 1 > concurrentReqs {
+		if 2 == provider { // S3
+			return 8
+		} else if 3 == provider { // WebDAV
+			return 1
+		}
+		return 8
+	}
+	if 16 < concurrentReqs {
+		return 16
+	}
+	return concurrentReqs
+}
+
 func NormalizeTimeout(timeout int) int {
 	if 7 > timeout {
 		if 1 > timeout {