🎨 Remove Unicode PUA characters https://github.com/siyuan-note/siyuan/issues/13291

kernel/api/workspace.go

@@ -94,7 +94,7 @@ func createWorkspaceDir(c *gin.Context) {
 	}
 
 	absPath := arg["path"].(string)
-	absPath = gulu.Str.RemoveInvisible(absPath)
+	absPath = util.RemoveInvalid(absPath)
 	absPath = strings.TrimSpace(absPath)
 	if isInvalidWorkspacePath(absPath) {
 		ret.Code = -1

kernel/go.mod

@@ -9,7 +9,7 @@ require (
 	github.com/88250/clipboard v0.1.5
 	github.com/88250/epub v0.0.0-20230830085737-c19055cd1f48
 	github.com/88250/go-humanize v0.0.0-20240424102817-4f78fac47ea7
-	github.com/88250/gulu v1.2.3-0.20240612035750-c9cf5f7a4d02
+	github.com/88250/gulu v1.2.3-0.20241127120230-1ae6a9868a2d
 	github.com/88250/lute v1.7.7-0.20241127031345-f772b0ee2be8
 	github.com/88250/pdfcpu v0.3.14-0.20230401044135-c7369a99720c
 	github.com/88250/vitess-sqlparser v0.0.0-20210205111146-56a2ded2aba1

kernel/go.sum

@@ -12,8 +12,8 @@ github.com/88250/go-humanize v0.0.0-20240424102817-4f78fac47ea7 h1:MafIFwSS0x6A4
 github.com/88250/go-humanize v0.0.0-20240424102817-4f78fac47ea7/go.mod h1:HrKCCTin3YNDSLBD02K0AOljjV6eNwc3/zyEI+xyV1I=
 github.com/88250/go-sqlite3 v1.14.13-0.20231214121541-e7f54c482950 h1:Pa5hMiBceTVVqrYaDlLio2QSKbXMUmAZPbzCwT5eNCw=
 github.com/88250/go-sqlite3 v1.14.13-0.20231214121541-e7f54c482950/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
-github.com/88250/gulu v1.2.3-0.20240612035750-c9cf5f7a4d02 h1:3e5+yobj655pTeKOYMbJrnc1mE51ZkbXIxquTYZuYCY=
+github.com/88250/gulu v1.2.3-0.20241127120230-1ae6a9868a2d h1:dexFyk3UkR4c2xpyrC4Zk4L28xFbfLYAeowIW/7QYEA=
-github.com/88250/gulu v1.2.3-0.20240612035750-c9cf5f7a4d02/go.mod h1:MUfzyfmbPrRDZLqxc7aPrVYveatTHRfoUa5TynPS0i8=
+github.com/88250/gulu v1.2.3-0.20241127120230-1ae6a9868a2d/go.mod h1:MUfzyfmbPrRDZLqxc7aPrVYveatTHRfoUa5TynPS0i8=
 github.com/88250/lute v1.7.7-0.20241127031345-f772b0ee2be8 h1:1gWOQT9m2o3E6X//wvI+bexbXgdHheN6YUZcABHMm4k=
 github.com/88250/lute v1.7.7-0.20241127031345-f772b0ee2be8/go.mod h1:VDAzL8b+oCh+e3NAlmwwLzC53ten0rZlS8NboB7ljtk=
 github.com/88250/pdfcpu v0.3.14-0.20230401044135-c7369a99720c h1:Dl/8S9iLyPMTElnWIBxmjaLiWrkI5P4a21ivwAn5pU0=

kernel/model/assets.go

@@ -607,7 +607,7 @@ func RenameAsset(oldPath, newName string) (newPath string, err error) {
 	defer util.PushClearProgress()
 
 	newName = strings.TrimSpace(newName)
-	newName = gulu.Str.RemoveInvisible(newName)
+	newName = util.RemoveInvalid(newName)
 	if path.Base(oldPath) == newName {
 		return
 	}

kernel/model/cloud_service.go

@@ -566,7 +566,7 @@ func getUser(token string) (*conf.User, error) {
 
 func UseActivationcode(code string) (err error) {
 	code = strings.TrimSpace(code)
-	code = gulu.Str.RemoveInvisible(code)
+	code = util.RemoveInvalid(code)
 	requestResult := gulu.Ret.NewResult()
 	request := httpclient.NewCloudRequest30s()
 	resp, err := request.
@@ -590,7 +590,7 @@ func UseActivationcode(code string) (err error) {
 
 func CheckActivationcode(code string) (retCode int, msg string) {
 	code = strings.TrimSpace(code)
-	code = gulu.Str.RemoveInvisible(code)
+	code = util.RemoveInvalid(code)
 	retCode = 1
 	requestResult := gulu.Ret.NewResult()
 	request := httpclient.NewCloudRequest30s()

kernel/model/export.go

@@ -609,7 +609,7 @@ func ExportDocx(id, savePath string, removeAssets, merge bool) (fullPath string,
 	}
 
 	// Pandoc template for exporting docx https://github.com/siyuan-note/siyuan/issues/8740
-	docxTemplate := gulu.Str.RemoveInvisible(Conf.Export.DocxTemplate)
+	docxTemplate := util.RemoveInvalid(Conf.Export.DocxTemplate)
 	docxTemplate = strings.TrimSpace(docxTemplate)
 	if "" != docxTemplate {
 		if !gulu.File.IsExist(docxTemplate) {

kernel/model/file.go

@@ -1936,7 +1936,7 @@ func createDoc(boxID, p, title, dom string) (tree *parse.Tree, err error) {
 func removeInvisibleCharsInTitle(title string) string {
 	// 不要踢掉 零宽连字符，否则有的 Emoji 会变形 https://github.com/siyuan-note/siyuan/issues/11480
 	title = strings.ReplaceAll(title, string(gulu.ZWJ), "__@ZWJ@__")
-	title = gulu.Str.RemoveInvisible(title)
+	title = util.RemoveInvalid(title)
 	title = strings.ReplaceAll(title, "__@ZWJ@__", string(gulu.ZWJ))
 	return title
 }

kernel/model/history.go

@@ -405,7 +405,7 @@ type HistoryItem struct {
 const fileHistoryPageSize = 32
 
 func FullTextSearchHistory(query, box, op string, typ, page int) (ret []string, pageCount, totalCount int) {
-	query = gulu.Str.RemoveInvisible(query)
+	query = util.RemoveInvalid(query)
 	if "" != query && HistoryTypeDocID != typ {
 		query = stringQuery(query)
 	}
@@ -440,7 +440,7 @@ func FullTextSearchHistory(query, box, op string, typ, page int) (ret []string,
 }
 
 func FullTextSearchHistoryItems(created, query, box, op string, typ int) (ret []*HistoryItem) {
-	query = gulu.Str.RemoveInvisible(query)
+	query = util.RemoveInvalid(query)
 	if "" != query && HistoryTypeDocID != typ {
 		query = stringQuery(query)
 	}

kernel/model/import.go

@@ -67,6 +67,7 @@ func HTML2Markdown(htmlStr string, luteEngine *lute.Lute) (markdown string, with
 }
 
 func HTML2Tree(htmlStr string, luteEngine *lute.Lute) (tree *parse.Tree, withMath bool) {
+	htmlStr = util.RemoveInvalid(htmlStr)
 	assetDirPath := filepath.Join(util.DataDir, "assets")
 	tree = luteEngine.HTML2Tree(htmlStr)
 	ast.Walk(tree.Root, func(n *ast.Node, entering bool) ast.WalkStatus {

kernel/model/mount.go

@@ -35,7 +35,7 @@ import (
 )
 
 func CreateBox(name string) (id string, err error) {
-	name = gulu.Str.RemoveInvisible(name)
+	name = util.RemoveInvalid(name)
 	if 512 < utf8.RuneCountInString(name) {
 		// 限制笔记本名和文档名最大长度为 `512` https://github.com/siyuan-note/siyuan/issues/6299
 		err = errors.New(Conf.Language(106))

kernel/model/repository.go

@@ -1022,7 +1022,7 @@ func TagSnapshot(id, name string) (err error) {
 	}
 
 	name = strings.TrimSpace(name)
-	name = gulu.Str.RemoveInvisible(name)
+	name = util.RemoveInvalid(name)
 	if "" == name {
 		err = errors.New(Conf.Language(142))
 		return

kernel/model/search.go

@@ -1993,7 +1993,7 @@ func getRefSearchIgnoreLines() (ret []string) {
 
 func filterQueryInvisibleChars(query string) string {
 	query = strings.ReplaceAll(query, "　", "_@full_width_space@_")
-	query = gulu.Str.RemoveInvisible(query)
+	query = util.RemoveInvalid(query)
 	query = strings.ReplaceAll(query, "_@full_width_space@_", "　")
 	return query
 }

kernel/model/sync.go

@@ -470,7 +470,7 @@ func CreateCloudSyncDir(name string) (err error) {
 	}
 
 	name = strings.TrimSpace(name)
-	name = gulu.Str.RemoveInvisible(name)
+	name = util.RemoveInvalid(name)
 	if !cloud.IsValidCloudDirName(name) {
 		return errors.New(Conf.Language(37))
 	}

kernel/model/virutalref.go

@@ -33,6 +33,7 @@ import (
 	"github.com/siyuan-note/siyuan/kernel/sql"
 	"github.com/siyuan-note/siyuan/kernel/task"
 	"github.com/siyuan-note/siyuan/kernel/treenode"
+	"github.com/siyuan-note/siyuan/kernel/util"
 )
 
 // virtualBlockRefCache 用于保存块关联的虚拟引用关键字。
@@ -170,7 +171,7 @@ func processVirtualRef(n *ast.Node, unlinks *[]*ast.Node, virtualBlockRefKeyword
 	}
 
 	content := string(n.Tokens)
-	tmp := gulu.Str.RemoveInvisible(content)
+	tmp := util.RemoveInvalid(content)
 	tmp = strings.TrimSpace(tmp)
 	if "" == tmp {
 		return false

kernel/util/file.go

@@ -242,7 +242,7 @@ func FilterFileName(name string) string {
 	name = strings.ReplaceAll(name, ">", "_")
 	name = strings.ReplaceAll(name, "|", "_")
 	name = strings.TrimSpace(name)
-	name = gulu.Str.RemoveInvisible(name) // Remove invisible characters from file names when uploading assets https://github.com/siyuan-note/siyuan/issues/11683
+	name = RemoveInvalid(name) // Remove invisible characters from file names when uploading assets https://github.com/siyuan-note/siyuan/issues/11683
 	return name
 }
 

kernel/util/misc.go

@@ -26,7 +26,6 @@ import (
 	"time"
 	"unicode"
 
-	"github.com/88250/gulu"
 	"github.com/88250/lute/html"
 )
 
@@ -133,7 +132,7 @@ func RemoveRedundantSpace(str string) string {
 }
 
 func Convert2Float(s string) (float64, bool) {
-	s = gulu.Str.RemoveInvisible(s)
+	s = RemoveInvalid(s)
 	s = strings.ReplaceAll(s, " ", "")
 	s = strings.ReplaceAll(s, ",", "")
 	buf := bytes.Buffer{}

kernel/util/ocr.go

@@ -248,7 +248,7 @@ func Tesseract(imgAbsPath string) (ret []map[string]interface{}) {
 		ret = append(ret, dataMap)
 	}
 
-	tsv = gulu.Str.RemoveInvisible(tsv)
+	tsv = RemoveInvalid(tsv)
 	tsv = RemoveRedundantSpace(tsv)
 	msg := fmt.Sprintf("OCR [%s] [%s]", html.EscapeString(info.Name()), html.EscapeString(GetOcrJsonText(ret)))
 	PushStatusBar(msg)
@@ -266,7 +266,7 @@ func GetOcrJsonText(jsonData []map[string]interface{}) (ret string) {
 			}
 		}
 	}
-	ret = gulu.Str.RemoveInvisible(ret)
+	ret = RemoveInvalid(ret)
 	ret = RemoveRedundantSpace(ret)
 	return ret
 }

kernel/util/rune.go

@@ -56,6 +56,12 @@ var emojiRegex = regexp.MustCompile(`/([0-9#][\x{20E3}])|` +
 
 func RemoveEmojiInvisible(text string) (ret string) {
 	ret = emojiRegex.ReplaceAllString(text, "")
-	ret = gulu.Str.RemoveInvisible(ret)
+	ret = RemoveInvalid(ret)
+	return
+}
+
+func RemoveInvalid(text string) (ret string) {
+	ret = gulu.Str.RemoveInvisible(text)
+	ret = gulu.Str.RemovePUA(text)
 	return
 }