大文件上传
场景分析
在业务场景中文件上传很普遍，而大文件的上传经常会导致上传时长过久，大量占用带宽资源，而分片上传就解决了目前的问题。
解决方案
前端
Promise限流import React from 'react';
import { uploadFetch } from './utils/upload';
import ConcurrentUtil from './utils/concurrent'
import "./file.css";
export default class FileContainer extends React.Component {
constructor() {
super();
this.concurrent = new ConcurrentUtil(this.uploadRequest,this.mergeRequest);
this.state = {
file: null,
fileSplit: [],
};
}
onChange = (event) => {
event.persist();
const [file] = event.target.files;
this.setState({
file
});
this.concurrent.setState({
file
});
event.preventDefault();
}
handleSubmit = async (event) => {
event.preventDefault();
const startTime = new Date().getTime(),
{ name: file_name } = this.state.file;
console.log("start at: -----> ", startTime);
//1.请求服务端，查找对应文件块
const { data: { list: pathList } } = await uploadFetch({
action: '/v1/resources/blobs',
data: {
file_name
}
})
this.concurrent.handleUpload(pathList).then((res) => {
const endTime = new Date().getTime();
console.log("<------ end at: -----> ", endTime, " duration: ", endTime - startTime);
alert('= = ',res);
});
}
/**
* @description 文件上传
*/
uploadRequest=({file,file_name,file_mark:index})=>{
return uploadFetch({
action: '/v1/resources/upload',
file,
data: {
file_name,
index,
}
})
}
/**
* @description 合并请求
*/
mergeRequest=({file_name})=>{
return uploadFetch({
action: '/v1/resources/merge',
data: {
file_name
},
})
}
render() {
return (
<div className="file-container">
<form onSubmit={this.handleSubmit}>
<label>请上传文件: </label>
<input type="file" name="file" id="file" onChange={this.onChange} />
<input type="submit" value="提交" />
</form>
<hr />
<div>
{this.state.file ? this.state.file.name : ''}
</div>
<ul>
{
this.state.fileSplit.map((item, index) =>
<li key={index}>{index}</li>
)
}
</ul>
</div>
)
}
}
import PromiseLimit from './promiseLimit'
/**
* @description 文件分片上传Util
* @export
* @class Concurrent
*/
export default class Concurrent {
/**
*Creates an instance of Concurrent.
* @param {Function returns object Promise} upload
* @param {Function returns object Promise} merge
* @param {number} [size=1024 * 1024 * 1]
* @param {number} [limit=4]
* @memberof Concurrent
*/
constructor(
upload,
merge,
size = 1024 * 1024 * 1, //默认 1M
limit = 4, //并发限制 默认 4
) {
this.state = {
upload,
merge,
size,
limit,
file: null, //当前文件对象
fileSplit: [], // 文件分片
};
}
/**
* @param {object} obj
* @memberof Concurrent
*/
setState(obj) {
this.state = { ...this.state, ...obj };
}
/**
* @description 文件分片
* @returns {Array}
* @memberof Concurrent
*/
splitZip() {
const { file, size } = this.state;
const fileSplit = [];
if (file.size > size) {
let start_index = 0, end_index = 0;
while (true) {
end_index += size;
const blob = file.slice(start_index, end_index);
start_index += size;
if (!blob.size) break;
fileSplit.push(blob);
}
} else {
fileSplit.push(file);
}
this.setState({
fileSplit
});
return fileSplit;
}
/**
* @description 上传行为；如果 pathList 为 falsy或[] 则上传当前所有分片
* @memberof Concurrent
*/
handleUpload = async (pathList) => {
this.splitZip();
let { fileSplit, file, limit } = this.state,
{ name: file_name } = file,
fileMark = "";
//文件过滤
fileSplit = fileSplit.map((blob, index) => {
fileMark += `${index}` //文件分片标识
if ((Array.isArray(pathList) && !pathList.find(pp => pp.split("_index_")[1] === `${index}`)) || !pathList) {
return { file: blob, file_mark: `${fileMark}_index_${index}`, file_name }
} else {
return null
}
})
fileSplit = fileSplit.filter(item => item);
// 没有缺失的片段 ，发送合并请求
if (Array.isArray(fileSplit) && fileSplit.length === 0) {
return this.state.merge.call(this, { file_name })
}
//2.上传缺失的文件块,PROMISE 限流
const promiseLimit = new PromiseLimit(this.limit, fileSplit, this.state.upload)
await promiseLimit.excute();
return this.state.merge.call(this, { file_name })
}
/**
* @description 分片上传
* @param {Array} fileSequenceItem
* @param {String} file_name
* @memberof Concurrent
*/
sequenceUpload = (fileSequenceItem, file_name) => {
return Promise.all(
fileSequenceItem.map(item =>
this.state.upload.call(this, {
file: item.file,
file_mark: item.index,
file_name,
})
)
)
}
}
/**
* @description Promise限流
*
* @export
* @class PromiseLimit
*/
export default class PromiseLimit {
constructor(limit = 4, params,iteratorFunc) {
this.limit = limit;
this.params = params;
this.i = 0;
this.iteratorFunc= iteratorFunc;
this.sequenceRet = [];
this.sequenceExcuting = [];
}
sequence = async () => {
//遍历结束
if (this.i === this.params.length) {
return Promise.resolve();
}
const paramItem = this.params[this.i++];
const p = Promise.resolve().then(() => this.iteratorFunc(paramItem));
this.sequenceRet.push(p);
// 如果执行完毕，从执行队列中删除
const e = p.then(() => this.sequenceExcuting.splice(this.sequenceExcuting.indexOf(e), 1));
this.sequenceExcuting.push(e);
let r = Promise.resolve();
// 执行队列>= 限制并发数时进行触发，结束后递归假如新的Promise实例
if (this.sequenceExcuting.length >= this.limit) {
r = Promise.race(this.sequenceExcuting);
}
await r;
return this.sequence();
}
excute = async () => {
await this.sequence();
return await Promise.all(this.sequenceRet);
}
}
后端接口(Golang)
- 保存分片
- 已保存的分片info获取
- 合并分片
实现效果：
