mirror of
https://github.com/siyuan-note/siyuan.git
synced 2025-05-17 01:21:14 +08:00
🎨 Supports converting network assets in the database to local https://github.com/siyuan-note/siyuan/issues/12096
This commit is contained in:
Daniel
parent
df8d70995c
commit
b4ece87cc0
@ -100,131 +100,138 @@ func NetAssets2LocalAssets(rootID string, onlyImg bool, originalURL string) (err
|
|||||||
|
|
||||||
destNodes := getRemoteAssetsLinkDestsInTree(tree, onlyImg)
|
destNodes := getRemoteAssetsLinkDestsInTree(tree, onlyImg)
|
||||||
for _, destNode := range destNodes {
|
for _, destNode := range destNodes {
|
||||||
dest := getRemoteAssetsLinkDest(destNode, onlyImg)
|
dests := getRemoteAssetsLinkDests(destNode, onlyImg)
|
||||||
if "" == dest {
|
if 1 > len(dests) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
if strings.HasPrefix(strings.ToLower(dest), "file://") { // 处理本地文件链接
|
for _, dest := range dests {
|
||||||
u := dest[7:]
|
if strings.HasPrefix(strings.ToLower(dest), "file://") { // 处理本地文件链接
|
||||||
unescaped, _ := url.PathUnescape(u)
|
u := dest[7:]
|
||||||
if unescaped != u {
|
unescaped, _ := url.PathUnescape(u)
|
||||||
// `Convert network images/assets to local` supports URL-encoded local file names https://github.com/siyuan-note/siyuan/issues/9929
|
if unescaped != u {
|
||||||
u = unescaped
|
// `Convert network images/assets to local` supports URL-encoded local file names https://github.com/siyuan-note/siyuan/issues/9929
|
||||||
}
|
u = unescaped
|
||||||
if strings.Contains(u, ":") {
|
}
|
||||||
u = strings.TrimPrefix(u, "/")
|
if strings.Contains(u, ":") {
|
||||||
}
|
u = strings.TrimPrefix(u, "/")
|
||||||
|
|
||||||
if !gulu.File.IsExist(u) || gulu.File.IsDir(u) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
name := filepath.Base(u)
|
|
||||||
name = util.FilterUploadFileName(name)
|
|
||||||
name = util.TruncateLenFileName(name)
|
|
||||||
name = "network-asset-" + name
|
|
||||||
name = util.AssetName(name)
|
|
||||||
writePath := filepath.Join(assetsDirPath, name)
|
|
||||||
if err = filelock.Copy(u, writePath); nil != err {
|
|
||||||
logging.LogErrorf("copy [%s] to [%s] failed: %s", u, writePath, err)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
setAssetsLinkDest(destNode, dest, "assets/"+name)
|
|
||||||
files++
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
if strings.HasPrefix(strings.ToLower(dest), "https://") || strings.HasPrefix(strings.ToLower(dest), "http://") || strings.HasPrefix(dest, "//") {
|
|
||||||
if strings.HasPrefix(dest, "//") {
|
|
||||||
// `Convert network images to local` supports `//` https://github.com/siyuan-note/siyuan/issues/10598
|
|
||||||
dest = "https:" + dest
|
|
||||||
}
|
|
||||||
|
|
||||||
u := dest
|
|
||||||
if strings.Contains(u, "qpic.cn") {
|
|
||||||
// 改进 `网络图片转换为本地图片` 微信图片拉取 https://github.com/siyuan-note/siyuan/issues/5052
|
|
||||||
if strings.Contains(u, "http://") {
|
|
||||||
u = strings.Replace(u, "http://", "https://", 1)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// 改进 `网络图片转换为本地图片` 微信图片拉取 https://github.com/siyuan-note/siyuan/issues/6431
|
if !gulu.File.IsExist(u) || gulu.File.IsDir(u) {
|
||||||
// 下面这部分需要注释掉，否则会导致响应 400
|
continue
|
||||||
//if strings.HasSuffix(u, "/0") {
|
|
||||||
// u = strings.Replace(u, "/0", "/640", 1)
|
|
||||||
//} else if strings.Contains(u, "/0?") {
|
|
||||||
// u = strings.Replace(u, "/0?", "/640?", 1)
|
|
||||||
//}
|
|
||||||
}
|
|
||||||
util.PushUpdateMsg(msgId, fmt.Sprintf(Conf.Language(119), u), 15000)
|
|
||||||
request := browserClient.R()
|
|
||||||
request.SetRetryCount(1).SetRetryFixedInterval(3 * time.Second)
|
|
||||||
if "" != originalURL {
|
|
||||||
request.SetHeader("Referer", originalURL) // 改进浏览器剪藏扩展转换本地图片成功率 https://github.com/siyuan-note/siyuan/issues/7464
|
|
||||||
}
|
|
||||||
resp, reqErr := request.Get(u)
|
|
||||||
if strings.Contains(strings.ToLower(resp.GetContentType()), "text/html") {
|
|
||||||
// 忽略超链接网页 `Convert network assets to local` no longer process webpage https://github.com/siyuan-note/siyuan/issues/9965
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
if nil != reqErr {
|
|
||||||
logging.LogErrorf("download network asset [%s] failed: %s", u, reqErr)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if 200 != resp.StatusCode {
|
|
||||||
logging.LogErrorf("download network asset [%s] failed: %d", u, resp.StatusCode)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
if 1024*1024*96 < resp.ContentLength {
|
|
||||||
logging.LogWarnf("network asset [%s]' size [%s] is large then [96 MB], ignore it", u, humanize.IBytes(uint64(resp.ContentLength)))
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
data, repErr := resp.ToBytes()
|
|
||||||
if nil != repErr {
|
|
||||||
logging.LogErrorf("download network asset [%s] failed: %s", u, repErr)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
var name string
|
|
||||||
if strings.Contains(u, "?") {
|
|
||||||
name = u[:strings.Index(u, "?")]
|
|
||||||
name = path.Base(name)
|
|
||||||
} else {
|
|
||||||
name = path.Base(u)
|
|
||||||
}
|
|
||||||
if strings.Contains(name, "#") {
|
|
||||||
name = name[:strings.Index(name, "#")]
|
|
||||||
}
|
|
||||||
name, _ = url.PathUnescape(name)
|
|
||||||
ext := path.Ext(name)
|
|
||||||
if "" == ext {
|
|
||||||
if mtype := mimetype.Detect(data); nil != mtype {
|
|
||||||
ext = mtype.Extension()
|
|
||||||
}
|
}
|
||||||
}
|
|
||||||
if "" == ext {
|
name := filepath.Base(u)
|
||||||
contentType := resp.Header.Get("Content-Type")
|
name = util.FilterUploadFileName(name)
|
||||||
exts, _ := mime.ExtensionsByType(contentType)
|
name = util.TruncateLenFileName(name)
|
||||||
if 0 < len(exts) {
|
name = "network-asset-" + name
|
||||||
ext = exts[0]
|
name = util.AssetName(name)
|
||||||
|
writePath := filepath.Join(assetsDirPath, name)
|
||||||
|
if err = filelock.Copy(u, writePath); nil != err {
|
||||||
|
logging.LogErrorf("copy [%s] to [%s] failed: %s", u, writePath, err)
|
||||||
|
continue
|
||||||
}
|
}
|
||||||
}
|
|
||||||
name = strings.TrimSuffix(name, ext)
|
setAssetsLinkDest(destNode, dest, "assets/"+name)
|
||||||
name = util.FilterUploadFileName(name)
|
files++
|
||||||
name = util.TruncateLenFileName(name)
|
|
||||||
name = "network-asset-" + name + "-" + ast.NewNodeID() + ext
|
|
||||||
writePath := filepath.Join(assetsDirPath, name)
|
|
||||||
if err = filelock.WriteFile(writePath, data); nil != err {
|
|
||||||
logging.LogErrorf("write downloaded network asset [%s] to local asset [%s] failed: %s", u, writePath, err)
|
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
setAssetsLinkDest(destNode, dest, "assets/"+name)
|
if strings.HasPrefix(strings.ToLower(dest), "https://") || strings.HasPrefix(strings.ToLower(dest), "http://") || strings.HasPrefix(dest, "//") {
|
||||||
files++
|
if strings.HasPrefix(dest, "//") {
|
||||||
continue
|
// `Convert network images to local` supports `//` https://github.com/siyuan-note/siyuan/issues/10598
|
||||||
|
dest = "https:" + dest
|
||||||
|
}
|
||||||
|
|
||||||
|
u := dest
|
||||||
|
if strings.Contains(u, "qpic.cn") {
|
||||||
|
// 改进 `网络图片转换为本地图片` 微信图片拉取 https://github.com/siyuan-note/siyuan/issues/5052
|
||||||
|
if strings.Contains(u, "http://") {
|
||||||
|
u = strings.Replace(u, "http://", "https://", 1)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 改进 `网络图片转换为本地图片` 微信图片拉取 https://github.com/siyuan-note/siyuan/issues/6431
|
||||||
|
// 下面这部分需要注释掉，否则会导致响应 400
|
||||||
|
//if strings.HasSuffix(u, "/0") {
|
||||||
|
// u = strings.Replace(u, "/0", "/640", 1)
|
||||||
|
//} else if strings.Contains(u, "/0?") {
|
||||||
|
// u = strings.Replace(u, "/0?", "/640?", 1)
|
||||||
|
//}
|
||||||
|
}
|
||||||
|
|
||||||
|
displayU := u
|
||||||
|
if 64 < len(displayU) {
|
||||||
|
displayU = displayU[:64] + "..."
|
||||||
|
}
|
||||||
|
util.PushUpdateMsg(msgId, fmt.Sprintf(Conf.Language(119), displayU), 15000)
|
||||||
|
request := browserClient.R()
|
||||||
|
request.SetRetryCount(1).SetRetryFixedInterval(3 * time.Second)
|
||||||
|
if "" != originalURL {
|
||||||
|
request.SetHeader("Referer", originalURL) // 改进浏览器剪藏扩展转换本地图片成功率 https://github.com/siyuan-note/siyuan/issues/7464
|
||||||
|
}
|
||||||
|
resp, reqErr := request.Get(u)
|
||||||
|
if strings.Contains(strings.ToLower(resp.GetContentType()), "text/html") {
|
||||||
|
// 忽略超链接网页 `Convert network assets to local` no longer process webpage https://github.com/siyuan-note/siyuan/issues/9965
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if nil != reqErr {
|
||||||
|
logging.LogErrorf("download network asset [%s] failed: %s", u, reqErr)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if 200 != resp.StatusCode {
|
||||||
|
logging.LogErrorf("download network asset [%s] failed: %d", u, resp.StatusCode)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if 1024*1024*96 < resp.ContentLength {
|
||||||
|
logging.LogWarnf("network asset [%s]' size [%s] is large then [96 MB], ignore it", u, humanize.IBytes(uint64(resp.ContentLength)))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
data, repErr := resp.ToBytes()
|
||||||
|
if nil != repErr {
|
||||||
|
logging.LogErrorf("download network asset [%s] failed: %s", u, repErr)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
var name string
|
||||||
|
if strings.Contains(u, "?") {
|
||||||
|
name = u[:strings.Index(u, "?")]
|
||||||
|
name = path.Base(name)
|
||||||
|
} else {
|
||||||
|
name = path.Base(u)
|
||||||
|
}
|
||||||
|
if strings.Contains(name, "#") {
|
||||||
|
name = name[:strings.Index(name, "#")]
|
||||||
|
}
|
||||||
|
name, _ = url.PathUnescape(name)
|
||||||
|
ext := path.Ext(name)
|
||||||
|
if "" == ext {
|
||||||
|
if mtype := mimetype.Detect(data); nil != mtype {
|
||||||
|
ext = mtype.Extension()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if "" == ext {
|
||||||
|
contentType := resp.Header.Get("Content-Type")
|
||||||
|
exts, _ := mime.ExtensionsByType(contentType)
|
||||||
|
if 0 < len(exts) {
|
||||||
|
ext = exts[0]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
name = strings.TrimSuffix(name, ext)
|
||||||
|
name = util.FilterUploadFileName(name)
|
||||||
|
name = util.TruncateLenFileName(name)
|
||||||
|
name = "network-asset-" + name + "-" + ast.NewNodeID() + ext
|
||||||
|
writePath := filepath.Join(assetsDirPath, name)
|
||||||
|
if err = filelock.WriteFile(writePath, data); nil != err {
|
||||||
|
logging.LogErrorf("write downloaded network asset [%s] to local asset [%s] failed: %s", u, writePath, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
setAssetsLinkDest(destNode, dest, "assets/"+name)
|
||||||
|
files++
|
||||||
|
continue
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1051,23 +1058,83 @@ func setAssetsLinkDest(node *ast.Node, oldDest, dest string) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func getRemoteAssetsLinkDest(node *ast.Node, onlyImg bool) (ret string) {
|
func getRemoteAssetsLinkDests(node *ast.Node, onlyImg bool) (ret []string) {
|
||||||
if onlyImg {
|
if onlyImg {
|
||||||
if ast.NodeLinkDest == node.Type && node.ParentIs(ast.NodeImage) {
|
if ast.NodeLinkDest == node.Type {
|
||||||
ret = string(node.Tokens)
|
if node.ParentIs(ast.NodeImage) {
|
||||||
|
if !util.IsAssetLinkDest(node.Tokens) {
|
||||||
|
ret = append(ret, string(node.Tokens))
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
} else if ast.NodeAttributeView == node.Type {
|
||||||
|
attrView, _ := av.ParseAttributeView(node.AttributeViewID)
|
||||||
|
if nil == attrView {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, keyValues := range attrView.KeyValues {
|
||||||
|
if av.KeyTypeMAsset != keyValues.Key.Type {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, value := range keyValues.Values {
|
||||||
|
if 1 > len(value.MAsset) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, asset := range value.MAsset {
|
||||||
|
if av.AssetTypeImage != asset.Type {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
dest := asset.Content
|
||||||
|
if !util.IsAssetLinkDest([]byte(dest)) {
|
||||||
|
ret = append(ret, strings.TrimSpace(dest))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
if ast.NodeLinkDest == node.Type {
|
if ast.NodeLinkDest == node.Type {
|
||||||
ret = string(node.Tokens)
|
if !util.IsAssetLinkDest(node.Tokens) {
|
||||||
|
ret = append(ret, string(node.Tokens))
|
||||||
|
}
|
||||||
} else if node.IsTextMarkType("a") {
|
} else if node.IsTextMarkType("a") {
|
||||||
ret = node.TextMarkAHref
|
if !util.IsAssetLinkDest([]byte(node.TextMarkAHref)) {
|
||||||
|
ret = append(ret, node.TextMarkAHref)
|
||||||
|
}
|
||||||
} else if ast.NodeAudio == node.Type || ast.NodeVideo == node.Type {
|
} else if ast.NodeAudio == node.Type || ast.NodeVideo == node.Type {
|
||||||
ret = treenode.GetNodeSrcTokens(node)
|
src := treenode.GetNodeSrcTokens(node)
|
||||||
}
|
if !util.IsAssetLinkDest([]byte(src)) {
|
||||||
}
|
ret = append(ret, src)
|
||||||
|
}
|
||||||
|
} else if ast.NodeAttributeView == node.Type {
|
||||||
|
attrView, _ := av.ParseAttributeView(node.AttributeViewID)
|
||||||
|
if nil == attrView {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
if util.IsAssetLinkDest([]byte(ret)) {
|
for _, keyValues := range attrView.KeyValues {
|
||||||
ret = ""
|
if av.KeyTypeMAsset != keyValues.Key.Type {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, value := range keyValues.Values {
|
||||||
|
if 1 > len(value.MAsset) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, asset := range value.MAsset {
|
||||||
|
dest := asset.Content
|
||||||
|
if !util.IsAssetLinkDest([]byte(dest)) {
|
||||||
|
ret = append(ret, strings.TrimSpace(dest))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@ -1078,8 +1145,8 @@ func getRemoteAssetsLinkDestsInTree(tree *parse.Tree, onlyImg bool) (nodes []*as
|
|||||||
return ast.WalkContinue
|
return ast.WalkContinue
|
||||||
}
|
}
|
||||||
|
|
||||||
dest := getRemoteAssetsLinkDest(n, onlyImg)
|
dests := getRemoteAssetsLinkDests(n, onlyImg)
|
||||||
if "" == dest {
|
if 1 > len(dests) {
|
||||||
return ast.WalkContinue
|
return ast.WalkContinue
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user