fix(update):

修复腾讯云图床/Github图床上传问题图片上传失败时，取消停止进程新增图片上传时的并发数concurrency，使用Github图床时，并发问题严重，建议设置为1
x-cold · Oct 18, 2022 · 0244f4e · 0244f4e
1 parent c0ebf1e
commit 0244f4e
Show file tree

Hide file tree

Showing 10 changed files with 95 additions and 58 deletions.
diff --git a/README.md b/README.md
@@ -80,6 +80,7 @@ A downloader for articles from yuque（语雀知识库同步工具）
     "lastGeneratePath": "lastGeneratePath.log",
     "imgCdn": {
       "enabled": false,
+      "concurrency": 0,
       "imageBed": "qiniu",
       "host": "",
       "bucket": "",
@@ -113,6 +114,7 @@ imgCdn 语雀图片转图床配置说明
 | 参数名       | 含义                                                                                      | 默认值     |
 |-----------|-----------------------------------------------------------------------------------------|---------|
 | enabled   | 是否开启                                                                                    | false   |
+| concurrency | 上传图片并发数, 0代表无限制，使用github图床时，并发问题严重，建议设置为1                                               | 0       |
 | imageBed  | 选择将图片上传的图床<br/>目前支持腾讯云(cos)、阿里云(oss)和七牛云(qiniu)，又拍云(upyun)，Github图床(github)<br/>默认使用七牛云 | 'qiniu' |
 | host      | 使用七牛云/又拍云图床时，需要指定CDN域名前缀                                                                |         |
 | bucket    | 图床的bucket名称                                                                             | -       |
@@ -233,6 +235,11 @@ DEBUG=yuque-hexo.* yuque-hexo sync
 
 # Changelog
 
+### v1.9.5
+- 修复腾讯云图床/Github图床上传问题
+- 图片上传失败时，取消停止进程
+- 新增图片上传时的并发数concurrency，使用Github图床时，并发问题严重，建议设置为1
+
 ### v1.9.4
 - 🔥 新增GitHub图床和又拍云图床
 

diff --git a/config.js b/config.js
@@ -20,6 +20,7 @@ const defaultConfig = {
   onlyPublished: false,
   onlyPublic: false,
   imgCdn: {
+    concurrency: 0,
     enabled: false,
     imageBed: 'qiniu',
     host: '',

diff --git a/lib/Downloader.js b/lib/Downloader.js
@@ -210,10 +210,21 @@ class Downloader {
     const { _cachedArticles, postBasicPath } = this;
     mkdirp.sync(postBasicPath);
     out.info(`create posts directory (if it not exists): ${postBasicPath}`);
-    const promiseList = _cachedArticles.map(async item => {
-      return await this.generatePost(item);
+    const promiseList = _cachedArticles.map(item => {
+      return async () => {
+        return await this.generatePost(item);
+      };
+    });
+    // 并发数
+    const concurrency = this.config.imgCdn.concurrency || promiseList.length;
+    const queue = new Queue({ concurrency });
+    queue.push(...promiseList);
+    await new Promise((resolve, reject) => {
+      queue.start(function(err) {
+        if (err) return reject(err);
+        resolve();
+      });
     });
-    await Promise.all(promiseList);
   }
 
   // 文章下载 => 增量更新文章到缓存 json 文件 => 全量生成 markdown 文章

diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "yuque-hexo",
-  "version": "1.9.4",
+  "version": "1.9.5",
   "description": "A downloader for articles from yuque",
   "main": "index.js",
   "scripts": {

diff --git a/util/imageBeds/cos.js b/util/imageBeds/cos.js
@@ -3,7 +3,6 @@
 // 腾讯云图床
 const COS = require('cos-nodejs-sdk-v5');
 const out = require('../../lib/out');
-const { transformRes } = require('../index');
 
 const secretId = process.env.SECRET_ID;
 const secretKey = process.env.SECRET_KEY;
@@ -40,7 +39,6 @@ class CosClient {
       });
       return `https://${this.config.bucket}.cos.${this.config.region}.myqcloud.com/${this.config.prefixKey}/${fileName}`;
     } catch (e) {
-      out.warn(`检查图片信息时出错: ${transformRes(e)}`);
       return '';
     }
   }
@@ -63,8 +61,8 @@ class CosClient {
       });
       return `https://${res.Location}`;
     } catch (e) {
-      out.error(`上传图片失败，请检查: ${transformRes(e)}`);
-      process.exit(-1);
+      out.warn(`上传图片失败，请检查: ${e}`);
+      return '';
     }
   }
 }

diff --git a/util/imageBeds/github.js b/util/imageBeds/github.js
@@ -51,6 +51,10 @@ class GithubClient {
           Authorization: `token ${secretKey}`,
         },
       });
+      if (result.status === 409) {
+        out.warn('由于github并发问题，图片上传失败');
+        return '';
+      }
       if (result.status === 200 || result.status === 201) {
         if (this.config.host) {
           return `${this.config.host}/gh/${secretId}/${this.config.bucket}/${this.config.prefixKey}/${fileName}`;
@@ -96,11 +100,8 @@ class GithubClient {
       const base64File = imgBuffer.toString('base64');
       const imgUrl = await this._fetch('PUT', fileName, base64File);
       if (imgUrl) return imgUrl;
-      out.error('上传图片失败，请检查');
-      process.exit(-1);
     } catch (e) {
-      out.error(`上传图片失败，请检查: ${transformRes(e)}`);
-      process.exit(-1);
+      out.warn(`上传图片失败，请检查: ${e}`);
     }
   }
 }

diff --git a/util/imageBeds/oss.js b/util/imageBeds/oss.js
@@ -57,8 +57,8 @@ class OssClient {
       const res = await this.imageBedInstance.put(`${this.config.prefixKey}/${fileName}`, imgBuffer);
       return res.url;
     } catch (e) {
-      out.error(`上传图片失败，请检查: ${transformRes(e)}`);
-      process.exit(-1);
+      out.warn(`上传图片失败，请检查: ${e}`);
+      return '';
     }
   }
 }

diff --git a/util/imageBeds/qiniu.js b/util/imageBeds/qiniu.js
@@ -72,14 +72,14 @@ class QiniuClient {
       this.formUploader.put(this.uploadToken, `${this.config.prefixKey}/${fileName}`, imgBuffer, this.putExtra, (respErr,
         respBody, respInfo) => {
         if (respErr) {
-          out.error(`上传图片失败，请检查: ${transformRes(respErr)}`);
-          process.exit(-1);
+          out.warn(`上传图片失败，请检查: ${transformRes(respErr)}`);
+          resolve('');
         }
         if (respInfo.statusCode === 200) {
           resolve(`${this.config.host}/${this.config.prefixKey}/${fileName}`);
         } else {
-          out.error(`上传图片失败，请检查: ${transformRes(respInfo)}`);
-          process.exit(-1);
+          out.warn(`上传图片失败，请检查: ${transformRes(respInfo)}`);
+          resolve('');
         }
       });
     });

diff --git a/util/imageBeds/upyun.js b/util/imageBeds/upyun.js
@@ -3,7 +3,6 @@
 // 又拍云图床
 const upyun = require('upyun');
 const out = require('../../lib/out');
-const { transformRes } = require('../index');
 
 const secretId = process.env.SECRET_ID;
 const secretKey = process.env.SECRET_KEY;
@@ -47,7 +46,7 @@ class UPClient {
       }
       return '';
     } catch (e) {
-      out.error(`上传图片失败，请检查: ${transformRes(e)}`);
+      out.warn(`上传图片失败，请检查: ${e}`);
       return '';
     }
   }
@@ -65,11 +64,11 @@ class UPClient {
       if (res) {
         return `${this.config.host}/${this.config.prefixKey}/${fileName}`;
       }
-      out.error('上传图片失败，请检查又拍云配置');
-      process.exit(-1);
+      out.warn('上传图片失败，请检查又拍云配置');
+      return '';
     } catch (e) {
-      out.error(`上传图片失败，请检查: ${transformRes(e)}`);
-      process.exit(-1);
+      out.warn(`上传图片失败，请检查: ${e}`);
+      return '';
     }
   }
 }

diff --git a/util/img2cdn.js b/util/img2cdn.js
@@ -5,6 +5,8 @@ const getEtag = require('../lib/qetag');
 const config = require('../config');
 const out = require('../lib/out');
 const ImageBed = require('./imageBeds');
+const Queue = require('queue');
+const lodash = require('lodash');
 
 const imageBed = config.imgCdn.enabled ? ImageBed.getInstance(config.imgCdn) : null;
 
@@ -83,49 +85,67 @@ async function img2Cdn(article) {
   // 1。从文章中获取语雀的图片URL列表
   const matchYuqueImgUrlList = article.body.match(imageUrlRegExp);
   if (!matchYuqueImgUrlList) return article;
-  const promiseList = matchYuqueImgUrlList.map(async matchYuqueImgUrl => {
-    // 获取真正的图片url
-    const yuqueImgUrl = getImgUrl(matchYuqueImgUrl);
-    // 2。将图片转成buffer
-    const imgBuffer = await img2Buffer(yuqueImgUrl);
-    if (!imgBuffer) {
-      return {
-        originalUrl: matchYuqueImgUrl,
-        yuqueRealImgUrl: yuqueImgUrl,
-        url: yuqueImgUrl,
-      };
-    }
-    // 3。根据buffer文件生成唯一的hash文件名
-    const fileName = await getFileName(imgBuffer, yuqueImgUrl);
-    try {
-      // 4。检查图床是否存在该文件
-      let url = await imageBed.hasImage(fileName);
-      let exists = true;
-      // 5。如果图床已经存在，直接替换；如果图床不存在，则先上传到图床，再将原本的语雀url进行替换
-      if (!url) {
-        url = await imageBed.uploadImg(imgBuffer, fileName);
-        exists = false;
+  const promiseList = matchYuqueImgUrlList.map(matchYuqueImgUrl => {
+    return async () => {
+      // 获取真正的图片url
+      const yuqueImgUrl = getImgUrl(matchYuqueImgUrl);
+      // 2。将图片转成buffer
+      const imgBuffer = await img2Buffer(yuqueImgUrl);
+      if (!imgBuffer) {
+        return {
+          originalUrl: matchYuqueImgUrl,
+          yuqueRealImgUrl: yuqueImgUrl,
+          url: yuqueImgUrl,
+        };
       }
-      return {
-        originalUrl: matchYuqueImgUrl,
-        yuqueRealImgUrl: yuqueImgUrl,
-        url,
-        exists,
-      };
-    } catch (e) {
-      out.error(`访问图床出错，请检查配置: ${e}`);
-      process.exit(-1);
-    }
+      // 3。根据buffer文件生成唯一的hash文件名
+      const fileName = await getFileName(imgBuffer, yuqueImgUrl);
+      try {
+        // 4。检查图床是否存在该文件
+        let url = await imageBed.hasImage(fileName);
+        let exists = true;
+        // 5。如果图床已经存在，直接替换；如果图床不存在，则先上传到图床，再将原本的语雀url进行替换
+        if (!url) {
+          url = await imageBed.uploadImg(imgBuffer, fileName);
+          exists = false;
+        }
+        return {
+          originalUrl: matchYuqueImgUrl,
+          yuqueRealImgUrl: yuqueImgUrl,
+          url,
+          exists,
+        };
+      } catch (e) {
+        out.error(`访问图床出错，请检查配置: ${e}`);
+        return {
+          yuqueRealImgUrl: yuqueImgUrl,
+          url: '',
+        };
+      }
+    };
+  });
+  // 并发数
+  const concurrency = config.imgCdn.concurrency || promiseList.length;
+  const queue = new Queue({ concurrency, results: [] });
+  queue.push(...promiseList);
+  await new Promise(resolve => {
+    queue.start(() => {
+      resolve();
+    });
   });
-  const urlList = await Promise.all(promiseList);
+  const _urlList = queue.results;
+  const urlList = lodash.flatten(_urlList);
+
   urlList.forEach(function(url) {
-    if (url) {
+    if (url.url) {
       article.body = article.body.replace(url.originalUrl, `![](${url.url})`);
       if (url.exists) {
         out.info(`图片已存在 skip: ${url.url}`);
       } else {
         out.info(`replace ${url.yuqueRealImgUrl} to ${url.url}`);
       }
+    } else {
+      out.warn(`图片替换失败，将使用原url： ${url.yuqueRealImgUrl}`);
     }
   });
   return article;