Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Sign up
Appearance settings

Commit 804027f

Browse files
author
欧如栋
committed
支持上传文件夹
1 parent d34583c commit 804027f

File tree

4 files changed

+158
-34
lines changed

4 files changed

+158
-34
lines changed

‎bin/index.js

Lines changed: 82 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ const path = require('path');
88
require('winston-daily-rotate-file');
99
const ProgressBar = require('progress');
1010
const BlueBirdPromise = require("bluebird");
11+
const glob = require('glob');
1112

1213
const logger = require('../lib/log');
1314
const { DEFAULT_CHUNK_SIZE, MAX_CHUNK } = require('../lib/constants');
@@ -33,7 +34,7 @@ process.on('uncaughtException', error => {
3334
logger.error(error.stack);
3435
})
3536

36-
const upload = async (filePath, parts = []) => {
37+
const upload = async (filePath, parts = [],requestUrl) => {
3738
const bar = new ProgressBar(':bar [:current/:total] :percent ', { total: totalChunk });
3839
const uploadChunk = async (currentChunk, currentChunkIndex, parts, isRetry) => {
3940
if (parts.some(({ partNumber, size }) => partNumber === currentChunkIndex && size === currentChunk.length)) {
@@ -47,7 +48,7 @@ const upload = async (filePath, parts = []) => {
4748
version,
4849
partNumber: currentChunkIndex,
4950
size: currentChunk.length,
50-
currentChunk
51+
currentChunk
5152
}, {
5253
headers: {
5354
'Content-Type': 'application/octet-stream'
@@ -75,14 +76,14 @@ const upload = async (filePath, parts = []) => {
7576
}
7677
}
7778

78-
console.log(`\n开始上传\n`)
79-
logger.info('开始上传')
79+
console.log(`\n开始上传 (${filePath})\n`);
80+
logger.info(`开始上传 (${filePath})`);
8081

8182
try {
8283

83-
const chunkIndexs = new Array(totalChunk).fill("").map((_,index) => index+1)
84+
const chunkIndexs = new Array(totalChunk).fill("").map((_,index) => index+1)
8485

85-
await BlueBirdPromise.map(chunkIndexs,(currentChunkIndex)=>{
86+
await BlueBirdPromise.map(chunkIndexs,(currentChunkIndex)=>{
8687
const start = (currentChunkIndex - 1) * chunkSize;
8788
const end = ((start + chunkSize) >= fileSize) ? fileSize : start + chunkSize - 1;
8889
const stream = fs.createReadStream(filePath, { start, end })
@@ -113,9 +114,9 @@ const upload = async (filePath, parts = []) => {
113114

114115

115116

116-
117117

118-
const merge = async () => {
118+
119+
const merge = async () => {
119120
console.log(chalk.cyan('正在合并分片,请稍等...'))
120121
return await _mergeAllChunks(requestUrl, {
121122
version,
@@ -126,7 +127,7 @@ const upload = async (filePath, parts = []) => {
126127
Authorization
127128
});
128129
}
129-
130+
130131

131132
try {
132133
const res = await withRetry(merge, 3, 500);
@@ -140,11 +141,11 @@ const upload = async (filePath, parts = []) => {
140141
return;
141142
}
142143

143-
console.log(chalk.green(`\n上传完毕\n`))
144+
console.log(chalk.green(`\n上传完毕 (${filePath})\n`))
144145
logger.info('************************ 上传完毕 ************************')
145146
}
146147

147-
const getFileMD5Success = async (filePath) => {
148+
const getFileMD5Success = async (filePath,requestUrl) => {
148149
try {
149150
const res = await _getExistChunks(requestUrl, {
150151
fileSize,
@@ -160,10 +161,10 @@ const getFileMD5Success = async (filePath) => {
160161

161162
// 上传过一部分
162163
if (Array.isArray(res.data.parts)) {
163-
await upload(filePath, res.data.parts);
164+
await upload(filePath, res.data.parts,requestUrl);
164165
} else {
165166
// 未上传过
166-
await upload(filePath);
167+
await upload(filePath,[],requestUrl);
167168
}
168169
} catch (error) {
169170
logger.error(error.message);
@@ -173,20 +174,20 @@ const getFileMD5Success = async (filePath) => {
173174
}
174175
}
175176

176-
const getFileMD5 = async (filePath) => {
177+
const getFileMD5 = async (filePath,requestUrl) => {
177178
totalChunk = Math.ceil(fileSize / DEFAULT_CHUNK_SIZE);
178179
if (totalChunk > MAX_CHUNK) {
179180
chunkSize = Math.ceil(fileSize / MAX_CHUNK);
180181
totalChunk = Math.ceil(fileSize / chunkSize);
181182
}
182183
const spark = new SparkMD5.ArrayBuffer();
183184
try {
184-
console.log(`\n开始计算 MD5\n`)
185-
logger.info('开始计算 MD5')
185+
console.log(`\n开始计算 MD5 (${filePath})\n`);
186+
logger.info(`开始计算 MD5 (${filePath})`);
186187

187188
const bar = new ProgressBar(':bar [:current/:total] :percent ', { total: totalChunk });
188189
await new Promise(resolve => {
189-
stream = fs.createReadStream(filePath, { highWaterMark: chunkSize })
190+
stream = fs.createReadStream(filePath, { highWaterMark: chunkSize });
190191
stream.on('data', chunk => {
191192
bar.tick();
192193
spark.append(chunk)
@@ -198,7 +199,7 @@ const getFileMD5 = async (filePath) => {
198199
md5 = spark.end();
199200
spark.destroy();
200201
console.log(`\n文件 MD5:${md5}\n`)
201-
await getFileMD5Success(filePath);
202+
await getFileMD5Success(filePath,requestUrl);
202203
resolve();
203204
})
204205
}).catch(error => {
@@ -212,14 +213,70 @@ const getFileMD5 = async (filePath) => {
212213
}
213214
}
214215

216+
const uploadFile = async (filePath, size, requestUrl) => {
217+
fileSize = size;
218+
await getFileMD5(filePath, requestUrl);
219+
md5 = '';
220+
uploadId = '';
221+
fileSize = 0;
222+
chunkSize = DEFAULT_CHUNK_SIZE;
223+
totalChunk = 0;
224+
}
225+
226+
const uploadDir = async (dir) => {
227+
let files = [];
228+
try {
229+
files = await new Promise((resolve, reject) => {
230+
glob("**/**", {
231+
cwd: dir,
232+
root: dir
233+
}, function (error, files = []) {
234+
if (error) {
235+
reject(error);
236+
} else {
237+
resolve(files)
238+
}
239+
})
240+
});
241+
} catch (error) {
242+
if (error) {
243+
console.log(chalk.red((error.response && error.response.data) || error.message));
244+
logger.error(error.message);
245+
logger.error(error.stack);
246+
process.exit(1);
247+
} else {
248+
resolve(files)
249+
}
250+
}
251+
252+
253+
for (const file of files) {
254+
const filePath = path.join(dir, file);
255+
const stat = fs.lstatSync(filePath);
256+
const isDirectory = stat.isDirectory();
257+
if (!isDirectory) {
258+
const url = new URL(`chunks/${dir.split(path.sep).pop()}/${file}`, requestUrl.endsWith('/') ? requestUrl : `${requestUrl}/`).toString();
259+
await uploadFile(filePath, stat.size, url);
260+
console.log('************************ **** ************************');
261+
logger.info('************************ **** ************************');
262+
}
263+
}
264+
}
265+
215266
const beforeUpload = async (filePath) => {
267+
const isUploadDir = argv.dir;
268+
let fSize = 0;
216269
try {
217270
const stat = fs.lstatSync(filePath);
218-
if (stat.isDirectory()) {
271+
const isDirectory = stat.isDirectory();
272+
if (isDirectory && !isUploadDir) {
219273
console.log(chalk.red(`\n${filePath}不合法,需指定一个文件\n`))
220274
process.exit(1);
275+
} else if (!isDirectory && isUploadDir) {
276+
console.log(chalk.red(`\n${filePath}不合法,需指定一个文件夹\n`))
277+
process.exit(1);
221278
}
222-
fileSize = stat.size;
279+
fSize = stat.size;
223280
} catch (error) {
224281
if (error.code === 'ENOENT') {
225282
console.log(chalk.red(`未找到 ${filePath}`));
@@ -230,7 +287,11 @@ const beforeUpload = async (filePath) => {
230287
}
231288
process.exit(1);
232289
}
233-
await getFileMD5(filePath);
290+
if (isUploadDir) {
291+
await uploadDir(filePath);
292+
} else {
293+
await uploadFile(filePath, fSize, requestUrl);
294+
}
234295
}
235296

236297
const onUpload = (_username, _password) => {

‎lib/argv.js

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
const argv = require('yargs')
2-
.usage('用法: coding-generic --username=<USERNAME>[:PASSWORD] --path=<FILE.EXT> --registry=<REGISTRY>')
2+
.usage('上传文件: coding-generic --username=<USERNAME>[:PASSWORD] --path=<FILE.EXT> --registry=<REGISTRY>')
3+
.usage('上传文件夹: coding-generic --username=<USERNAME>[:PASSWORD] --dir --path=<FOLDER> --registry=<REGISTRY>')
34
.options({
45
username: {
56
alias: 'u',
@@ -21,12 +22,18 @@ const argv = require('yargs')
2122
describe: '上传分块并行数',
2223
demandOption: true,
2324
default: 5,
25+
},
26+
dir: {
27+
alias: 'd',
28+
describe: '上传文件夹',
29+
boolean: true,
2430
}
2531
})
2632
.alias('version', 'v')
2733
.help('h')
2834
.alias('h', 'help')
29-
.example('coding-generic --username=coding@coding.com:123456 --path=./test.txt --registry="https://codingcorp-generic.pkg.coding.net/project-name/generic-repo/chunks/test.txt?version=latest"')
35+
.example('上传文件: coding-generic --username=coding@coding.com:123456 --path=./test.txt --registry="https://codingcorp-generic.pkg.coding.net/project-name/generic-repo/chunks/test.txt?version=latest"')
36+
.example('上传文件夹: coding-generic --username=coding@coding.com:123456 --dir --path=./dirname --registry="https://codingcorp-generic.pkg.coding.net/project-name/generic-repo?version=latest"')
3037
.argv;
3138

3239
module.exports = argv;

‎package.json

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "coding-generic",
3-
"version": "1.2.6",
3+
"version": "1.2.7",
44
"description": "",
55
"main": "index.js",
66
"bin": {
@@ -15,6 +15,7 @@
1515
"chalk": "^4.1.0",
1616
"cos-nodejs-sdk-v5": "^2.8.2",
1717
"form-data": "^3.0.0",
18+
"glob": "^8.0.3",
1819
"progress": "^2.0.3",
1920
"prompts": "^2.3.2",
2021
"spark-md5": "^3.0.1",

‎yarn.lock

Lines changed: 65 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -70,12 +70,17 @@ aws4@^1.8.0:
7070
resolved "https://registry.npm.taobao.org/aws4/download/aws4-1.10.1.tgz?cache=0&sync_timestamp=1597236947743&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Faws4%2Fdownload%2Faws4-1.10.1.tgz#e1e82e4f3e999e2cfd61b161280d16a111f86428"
7171
integrity sha1-4eguTz6Zniz9YbFhKA0WoRH4ZCg=
7272

73-
axios@^0.20.0:
74-
version "0.20.0"
75-
resolved "https://registry.npm.taobao.org/axios/download/axios-0.20.0.tgz#057ba30f04884694993a8cd07fa394cff11c50bd"
76-
integrity sha1-BXujDwSIRpSZOozQf6OUz/EcUL0=
73+
axios@^0.21.1:
74+
version "0.21.4"
75+
resolved "https://mirrors.tencent.com/npm/axios/-/axios-0.21.4.tgz#c67b90dc0568e5c1cf2b0b858c43ba28e2eda575"
76+
integrity sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg==
7777
dependencies:
78-
follow-redirects "^1.10.0"
78+
follow-redirects "^1.14.0"
79+
80+
balanced-match@^1.0.0:
81+
version "1.0.2"
82+
resolved "https://mirrors.tencent.com/npm/balanced-match/-/balanced-match-1.0.2.tgz#e83e3a7e3f300b34cb9d87f615fa0cbf357690ee"
83+
integrity sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==
7984

8085
bcrypt-pbkdf@^1.0.0:
8186
version "1.0.2"
@@ -89,6 +94,13 @@ bluebird@^3.7.2:
8994
resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.7.2.tgz#9f229c15be272454ffa973ace0dbee79a1b0c36f"
9095
integrity sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==
9196

97+
brace-expansion@^2.0.1:
98+
version "2.0.1"
99+
resolved "https://mirrors.tencent.com/npm/brace-expansion/-/brace-expansion-2.0.1.tgz#1edc459e0f0c548486ecf9fc99f2221364b9a0ae"
100+
integrity sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==
101+
dependencies:
102+
balanced-match "^1.0.0"
103+
92104
caseless@~0.12.0:
93105
version "0.12.0"
94106
resolved "https://registry.npm.taobao.org/caseless/download/caseless-0.12.0.tgz#1b681c21ff84033c826543090689420d187151dc"
@@ -310,10 +322,10 @@ fn.name@1.x.x:
310322
resolved "https://registry.npm.taobao.org/fn.name/download/fn.name-1.1.0.tgz#26cad8017967aea8731bc42961d04a3d5988accc"
311323
integrity sha1-JsrYAXlnrqhzG8QpYdBKPVmIrMw=
312324

313-
follow-redirects@^1.10.0:
314-
version "1.13.0"
315-
resolved "https://registry.npm.taobao.org/follow-redirects/download/follow-redirects-1.13.0.tgz?cache=0&sync_timestamp=1597057976909&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Ffollow-redirects%2Fdownload%2Ffollow-redirects-1.13.0.tgz#b42e8d93a2a7eea5ed88633676d6597bc8e384db"
316-
integrity sha1-tC6Nk6Kn7qXtiGM2dtZZe8jjhNs=
325+
follow-redirects@^1.14.0:
326+
version "1.15.2"
327+
resolved "https://mirrors.tencent.com/npm/follow-redirects/-/follow-redirects-1.15.2.tgz#b460864144ba63f2681096f274c4e57026da2c13"
328+
integrity sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==
317329

318330
forever-agent@~0.6.1:
319331
version "0.6.1"
@@ -338,6 +350,11 @@ form-data@~2.3.2:
338350
combined-stream "^1.0.6"
339351
mime-types "^2.1.12"
340352

353+
fs.realpath@^1.0.0:
354+
version "1.0.0"
355+
resolved "https://mirrors.tencent.com/npm/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f"
356+
integrity sha1-FQStJSMVjKpA20onh8sBQRmU6k8=
357+
341358
get-caller-file@^2.0.5:
342359
version "2.0.5"
343360
resolved "https://registry.npm.taobao.org/get-caller-file/download/get-caller-file-2.0.5.tgz#4f94412a82db32f36e3b0b9741f8a97feb031f7e"
@@ -350,6 +367,17 @@ getpass@^0.1.1:
350367
dependencies:
351368
assert-plus "^1.0.0"
352369

370+
glob@^8.0.3:
371+
version "8.0.3"
372+
resolved "https://mirrors.tencent.com/npm/glob/-/glob-8.0.3.tgz#415c6eb2deed9e502c68fa44a272e6da6eeca42e"
373+
integrity sha512-ull455NHSHI/Y1FqGaaYFaLGkNMMJbavMrEGFXG/PGrg6y7sutWHUHrz6gy6WEBH6akM1M414dWKCNs+IhKdiQ==
374+
dependencies:
375+
fs.realpath "^1.0.0"
376+
inflight "^1.0.4"
377+
inherits "2"
378+
minimatch "^5.0.1"
379+
once "^1.3.0"
380+
353381
har-schema@^2.0.0:
354382
version "2.0.0"
355383
resolved "https://registry.npm.taobao.org/har-schema/download/har-schema-2.0.0.tgz#a94c2224ebcac04782a0d9035521f24735b7ec92"
@@ -377,7 +405,15 @@ http-signature@~1.2.0:
377405
jsprim "^1.2.2"
378406
sshpk "^1.7.0"
379407

380-
inherits@^2.0.3, inherits@~2.0.3:
408+
inflight@^1.0.4:
409+
version "1.0.6"
410+
resolved "https://mirrors.tencent.com/npm/inflight/-/inflight-1.0.6.tgz#49bd6331d7d02d0c09bc910a1075ba8165b56df9"
411+
integrity sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=
412+
dependencies:
413+
once "^1.3.0"
414+
wrappy "1"
415+
416+
inherits@2, inherits@^2.0.3, inherits@~2.0.3:
381417
version "2.0.4"
382418
resolved "https://registry.npm.taobao.org/inherits/download/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c"
383419
integrity sha1-D6LGT5MpF8NDOg3tVTY6rjdBa3w=
@@ -510,6 +546,13 @@ mimic-fn@^3.0.0:
510546
resolved "https://registry.npm.taobao.org/mimic-fn/download/mimic-fn-3.1.0.tgz?cache=0&sync_timestamp=1596095644798&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Fmimic-fn%2Fdownload%2Fmimic-fn-3.1.0.tgz#65755145bbf3e36954b949c16450427451d5ca74"
511547
integrity sha1-ZXVRRbvz42lUuUnBZFBCdFHVynQ=
512548

549+
minimatch@^5.0.1:
550+
version "5.1.0"
551+
resolved "https://mirrors.tencent.com/npm/minimatch/-/minimatch-5.1.0.tgz#1717b464f4971b144f6aabe8f2d0b8e4511e09c7"
552+
integrity sha512-9TPBGGak4nHfGZsPBohm9AWg6NoT7QTCehS3BIJABslyZbzxfV78QM2Y6+i741OPZIafFAaiiEMh5OyIrJPgtg==
553+
dependencies:
554+
brace-expansion "^2.0.1"
555+
513556
moment@^2.11.2:
514557
version "2.29.1"
515558
resolved "https://registry.npm.taobao.org/moment/download/moment-2.29.1.tgz#b2be769fa31940be9eeea6469c075e35006fa3d3"
@@ -530,6 +573,13 @@ object-hash@^2.0.1:
530573
resolved "https://registry.npm.taobao.org/object-hash/download/object-hash-2.0.3.tgz#d12db044e03cd2ca3d77c0570d87225b02e1e6ea"
531574
integrity sha1-0S2wROA80so9d8BXDYciWwLh5uo=
532575

576+
once@^1.3.0:
577+
version "1.4.0"
578+
resolved "https://mirrors.tencent.com/npm/once/-/once-1.4.0.tgz#583b1aa775961d4b113ac17d9c50baef9dd76bd1"
579+
integrity sha1-WDsap3WWHUsROsF9nFC6753Xa9E=
580+
dependencies:
581+
wrappy "1"
582+
533583
one-time@^1.0.0:
534584
version "1.0.0"
535585
resolved "https://registry.npm.taobao.org/one-time/download/one-time-1.0.0.tgz#e06bc174aed214ed58edede573b433bbf827cb45"
@@ -868,6 +918,11 @@ wrap-ansi@^7.0.0:
868918
string-width "^4.1.0"
869919
strip-ansi "^6.0.0"
870920

921+
wrappy@1:
922+
version "1.0.2"
923+
resolved "https://mirrors.tencent.com/npm/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f"
924+
integrity sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=
925+
871926
xml2js@^0.4.19:
872927
version "0.4.23"
873928
resolved "https://registry.npm.taobao.org/xml2js/download/xml2js-0.4.23.tgz#a0c69516752421eb2ac758ee4d4ccf58843eac66"

0 commit comments

Comments
(0)

AltStyle によって変換されたページ (->オリジナル) /