大文件上传

使用React生成一个大文件上传功能，此功能我们调用了ant design 的ui。

<FileUpload multiple chunked threadNum={3}>
    <p className="ant-upload-drag-icon">
        <Icon type="inbox" />
    </p>
    <p className="ant-upload-text">点击选择文件或将文件拖拽到这里</p>
    <p className="ant-upload-hint">系统支持在线播放.mp4，.flv，.ogv，.webm格式视频，其他格式视频需要下载后才可观看。</p>
</FileUpload>

这个组件的参数如下：

{
    multiple: PropTypes.bool,       //是否支持多选文件
    chunkSize: PropTypes.number,    //文件分块大小
    threadNum: PropTypes.number,    //并发分块数量
    accept: PropTypes.string,       //接受上传的文件类型
    onChange: PropTypes.func,     //上传完成后的回调
    chunked: PropTypes.bool         //是否分片上传
}

const FileUpload = ({ chunked , ...rest }) => (
    chunked ?  <LargeFileUpload {...rest} /> : <SmallFileUpload {...rest} />
)

我们有两种上传方式，大文件上传，小文件上传

import React from 'react';
import PropTypes from 'prop-types';
import {
    Upload,
    Button,
    Icon,
    Modal
} from 'antd';

import {
    uploadFile
} from '../../../unit/fetch';

const propTypes = {
    multiple: PropTypes.bool,
    accept: PropTypes.string,
    onChange: PropTypes.func
}

const Dragger = Upload.Dragger;

const FILE_STATE = {
    WAITING: Symbol(),
    UPLOADING: Symbol(),
    FINISHED: Symbol(),
    ERRORED: Symbol()
};

class SmallFileUpload extends React.Component {
    constructor({
        multiple = false,
        accept = ''
    }) {
        super();
        this.state = {
            multiple,
            accept,
            disabled: false,
            fileList: []
        };

        this.toUpload = this.toUpload.bind(this);
        this.submit = this.submit.bind(this);
        this.onCompleted = this.onCompleted.bind(this);
    }

    onCompleted() {
        if (typeof this.props.onChange === 'function') {
            this.props.onChange({
                success: this.state.fileList.filter(info => info.state === FILE_STATE['FINISHED']).map(info => {return {token: info.token, name: info.config.file.name}}),
                total: this.state.fileList.length
            });
        }
    }

    toUpload(fileInfo) {
        return new Promise(async (resolve) => { 
            try { 
                const { token } = await uploadFile({
                    file: fileInfo.config.file
                });
                fileInfo.token = token;
                fileInfo.state = FILE_STATE['FINISHED'];
                fileInfo.config.onSuccess();
            } catch(e) {
                fileInfo.state = FILE_STATE['ERRORED'];
                fileInfo.config.onError();
            }
            resolve();
        });
    }

    async submit() {
        if (!this.state.fileList.length) {
            Modal.info({
                title: '提示',
                content: (
                    <div>
                        <p>请选择文件后，再提交！</p>
                    </div>
                )
            });
            return;
        }
        for (let fileInfo of this.state.fileList) {
            if (fileInfo.state === FILE_STATE['WAITING']) {
                fileInfo.state = FILE_STATE['UPLOADING'];
                await this.toUpload(fileInfo);
                this.onCompleted();
            }
        }
    }

    render() {
        const draggerProps = {
            multiple: this.state.multiple,
            accept: this.state.accept,
            disabled: this.state.disabled,
            customRequest: (config) => {
                let fileInfo = {
                    id: config.file.uid,
                    state: FILE_STATE['WAITING'],
                    token: null,
                    config
                };
                this.setState({
                    fileList: [...this.state.fileList, fileInfo],
                    disabled: !this.state.multiple
                });
            },
            onRemove: file => {
                const fileList = [...this.state.fileList.filter(info => info.id !== file.uid)];
                let isCompleted = fileList.every(info => [FILE_STATE['FINISHED'], FILE_STATE['ERRORED']].includes(info.state));
                this.setState({ 
                    fileList,
                    disabled: fileList.length && !this.state.multiple
                }, () => {
                    isCompleted && this.onCompleted();
                });
            }
        };

        return (<div>
            <Dragger {...draggerProps}>
                {this.props.children ? this.props.children :
                <div>
                    <p className="ant-upload-drag-icon"><Icon type="inbox" /></p>
                    <p className="ant-upload-text">点击选择文件或将文件拖拽到这里</p>
                </div>}
            </Dragger>
            <div style={{marginTop: '10px'}}>
                <Button ghost type="primary" size="small" onClick={this.submit}>上传</Button>
            </div>
        </div>);
    }
}

SmallFileUpload.propTypes = propTypes;

export default SmallFileUpload;

使用了ant design 的upload 组件，使用customRequest自己写上传逻辑。
获得的config对象：

{
    action: ""
    data: {}
    file: File {uid: "rc-upload-1551879565599-4", name: "面试.txt", lastModified: 1547794695512, lastModifiedDate: Fri Jan 18 2019 14:58:15 GMT+0800 (中国标准时间), webkitRelativePath: "", …}
    filename: "file"
    headers: {}
    onError: ƒ (r,o)
    onProgress: ƒ (t)
    onSuccess: ƒ (r,o)
    withCredentials: false
}

我们的文件对象：

我们通过fetch上传一个一个小文件， 如果文件太大呢？

我们需要将文件进行分片上传。为了保证文件上传的正确性，不被挟持。我们要传递给后台md5值。

MD5在论坛上、软件发布时经常用，是为了保证文件的正确性，防止一些人盗用程序，加些木马或者篡改版权，设计的一套验证系统。每个文件都可以用MD5验证程序算出一个固定的MD5码来。

我们使用了Spark-MD5 JS， 它的api我们使用如下：

var hexHash = SparkMD5.hash('Hi there'); //二进制hash

// 增量相加
var spark = new SparkMD5();
spark.append('Hi');
spark.append(' there');
var hexHash = spark.end();                      // hex hash
var rawHash = spark.end(true); 

// 传递二进制数据的话
var chunks = Math.ceil(file.size / chunkSize),
    spark = new SparkMD5.ArrayBuffer(),
    currentChunk = 0;
fileReader = new FileReader();
fileReader.onload = function (e) {
    spark.append(e.target.result);                   // Append array buffer
    currentChunk++;

    if (currentChunk < chunks) {
        loadNext();
    } else {
        console.log('finished loading');
        console.info('computed hash', spark.end());  // Compute hash
    }
};

var blobSlice = File.prototype.slice || File.prototype.mozSlice || File.prototype.webkitSlice;
function loadNext() {
    var start = currentChunk * chunkSize,
        end = ((start + chunkSize) >= file.size) ? file.size : start + chunkSize;

    fileReader.readAsArrayBuffer(blobSlice.call(file, start, end));
}

上传的接口我们采用了三个接口，一个是用户根据MD5值，文件大小和名字创建上传Token。
二再根据上传Token，order向文件继续追加内容。
三是大文件合并，参数是Token和全部patch的数量

同时，因为大文件上传缓慢，我们要获取上传进度值。使用了progress event。

Progress Events定义了与客户端服务器通信有关的事件。这些事件最早其实只针对XHR操作，但目前也被其它API借鉴。

loadstart：在接收到相应数据的第一个字节时触发。
progress：在接收相应期间持续不断触发。
error：在请求发生错误时触发。
abort：在因为调用abort()方法而终止链接时触发。
load：在接收到完整的相应数据时触发。
loadend：在通信完成或者触发error、abort或load事件后触

通过xhr.upload.onprogress获取上传进度，lengthComputable是一个表示进度信息是否可用的布尔值

xhr.upload.onprogress = function(evt) {
    // event.total是需要传输的总字节，event.loaded是已经传输的字节。如果event.lengthComputable不为真，则event.total等于0
}

//这是单文件上传
export function uploadFile(options, progressBarFunc) {
    return new Promise((resolve, reject) => {
        let xhr = new XMLHttpRequest();
        let url = address.upload.substring(0, address.upload.length - 1);
        let formdata = new FormData();
        formdata.append('file', options.file);
        xhr.open('POST', url);
        xhr.setRequestHeader('Auth-Token', getAccessToken());
        xhr.upload.onprogress = function(evt) {
            if(evt.lengthComputable) {
                progressBarFunc && progressBarFunc(evt.loaded, evt.total);
            }
        };
        xhr.onreadystatechange = function() {
            if(xhr.readyState === 4) {
                if (xhr.status === 200) {
                    resolve(JSON.parse(xhr.responseText));
                } else {
                    reject(JSON.parse(xhr.responseText))
                }
            }
        };
        xhr.send(formdata);
    });
}

// 分块上传，用户根据MD5值，文件大小和名字创建上传Token。再根据上传Token，order向文件继续追加内容。
export function getUploadTokenAndProgress(option) {
    return executeFetch(address.upload, {
        method: 'POST',
        headers: {
            'Content-Type': 'application/x-www-form-urlencoded',
            'Auth-Token': getAccessToken()
        },
        body: objToQueryString(option).substring(1)
    });
}

export function uploadFileThunk(options, progressBarFunc) {
    return new Promise((resolve, reject) => {
        let xhr = new XMLHttpRequest();
        let url = `${address.upload}/append`;
        let formdata = new FormData();
        formdata.append('uploadToken', options.uploadToken);
        formdata.append('order', options.order);
        formdata.append('file', options.file);
        xhr.open('POST', url);
        xhr.setRequestHeader('Auth-Token', getAccessToken());
        xhr.upload.onprogress = function(evt) {
            if(evt.lengthComputable) {
                progressBarFunc && progressBarFunc(evt.loaded, evt.total);
            }
        };
        xhr.onreadystatechange = function() {
            if(xhr.readyState === 4) {
                if (xhr.status === 200) {
                    resolve();
                } else {
                    reject(JSON.parse(xhr.responseText))
                }
            }
        };
        xhr.send(formdata);
    });
}

// 大文件合并， 参数是Token和全部patch的数量
export function finishFileUpload(option) {
    return executeFetch(`${address.upload}/complete`, {
        method: 'PUT',
        headers: {
            'Content-Type': 'application/x-www-form-urlencoded',
            'Auth-Token': getAccessToken()
        },
        body: objToQueryString(option).substring(1)
    });
}

完整的生成调用代码如下：

import React from 'react';
import PropTypes from 'prop-types';
import { Upload, Button, Icon, Modal } from 'antd';

import { getUploadTokenAndProgress, uploadFileThunk, finishFileUpload } from '../../../unit/fetch';

const propTypes = {
    multiple: PropTypes.bool,
    chunkSize: PropTypes.number,
    threadNum: PropTypes.number,
    accept: PropTypes.string,
    onChange: PropTypes.func
}

const Dragger = Upload.Dragger;

const FILE_STATE = {
    PREPARING: Symbol(),
    WAITING: Symbol(),
    UPLOADING: Symbol(),
    FINISHED: Symbol(),
    ERRORED: Symbol()
};

class LargeFileUpload extends React.Component {
    constructor({
            multiple = false,
            chunkSize = 5 * 1024 * 1024,
            threadNum = 1,
            accept = ''
        }) {
        super();
        this.state = {
            multiple,
            chunkSize,
            threadNum,
            accept,
            disabled: false,
            fileList: [],
            isPrepared: false
        };

        this.submit = this.submit.bind(this);
        this.sliceFile = this.sliceFile.bind(this);
        this.md5File = this.md5File.bind(this);
        this.toUpload = this.toUpload.bind(this);
        this.onCompleted = this.onCompleted.bind(this);

        import('../../../unit/tool/MD5File').then(MD5WorkerUrl => {
            this.MD5Worker = new Worker(MD5WorkerUrl);
            this.MD5Worker.onmessage = ({data}) => {
                const fileInfo = this.state.fileList.find( info => info.id === data.id );
                if(!fileInfo) {
                    return ;
                }
                fileInfo.MD5 = data.hash;
                fileInfo.state = FILE_STATE['WAITING'];
                let isPrepared = false;
                for (let info of this.state.fileList) {
                    if (info.state === FILE_STATE['PREPARING']) {
                        isPrepared = true;
                        break;
                    }
                }
                this.setState({ isPrepared });
            };
        })

    }

    onCompleted() {
        if (typeof this.props.onChange === 'function') {
            this.props.onChange({
                success: this.state.fileList.filter(info => info.state === FILE_STATE['FINISHED']).map(info => {return {token: info.token, name: info.config.file.name}}),
                total: this.state.fileList.length,
                isCompleted: !this.state.fileList.some(info => ![FILE_STATE['FINISHED'], FILE_STATE['ERRORED']].includes(info.state))
            });
        }
    }

    //文件切片
    sliceFile(file, chunkCount) {
        let sliceRst = [];
        let blobSlice = File.prototype.slice || File.prototype.mozSlice || File.prototype.webkitSlice;
        for (let i = 0, start, end; i < chunkCount; i++) {
            start = i * this.state.chunkSize;
            end = Math.min(file.size, start + this.state.chunkSize);
            sliceRst.push(blobSlice.call(file, start, end));
        }
        return sliceRst;
    }

    //文件MD5
    md5File(fileInfo) {
        this.MD5Worker.postMessage({
            'fileChunk': this.sliceFile(fileInfo.config.file, fileInfo.chunkCount),
            'id': fileInfo.id
        });
    }

    toUpload(fileInfo) {
        return new Promise(async (resolve, reject) => {
            const successThunkArr = new Set();
            const fileChunkArray = this.sliceFile(fileInfo.config.file, fileInfo.chunkCount);
            const completeUploadingFile = async () => {
                fileInfo.config.onProgress({percent: 100});
                try{
                    const { token } = await finishFileUpload({
                        uploadToken: fileInfo.uploadToken,
                        patchSum: fileInfo.chunkCount
                    });
                    fileInfo.token = token;
                    fileInfo.state = FILE_STATE['FINISHED'];
                    fileInfo.config.onSuccess();
                    resolve();
                } catch(e) {
                    failureUploadThunk();
                }
            };
            const failureUploadThunk = () => {
                fileInfo.state = FILE_STATE['ERRORED'];
                fileInfo.config.onError();
                resolve();
            };
            const successUploadThunk = curChunkIndex => {
                successThunkArr.add(+curChunkIndex);
                if (successThunkArr.size === fileInfo.chunkCount) {
                    completeUploadingFile();
                } else if (successThunkArr.size < fileInfo.chunkCount) {
                    fileInfo.config.onProgress({percent: (successThunkArr.size / fileInfo.chunkCount).toFixed(2) * 100 });
                    curChunkIndex && uploadThunk(successThunkArr.size);
                }
            };
            //chunkIndex从0开始
            const uploadThunk = (currentSuccessThunkNum, chunkIndex) => {
                return new Promise(async resolve => {
                    const curChunkIndex = chunkIndex + 1 ? chunkIndex : currentSuccessThunkNum + Math.min(this.state.threadNum, fileInfo.chunkCount) - 1;
                    let failureTimes = 0;
                    function toUploadThunk() {
                        return new Promise(async resolve => {
                            try {
                                await uploadFileThunk({
                                    uploadToken: fileInfo.uploadToken,
                                    order: curChunkIndex,
                                    file: fileChunkArray[curChunkIndex]
                                });
                                successUploadThunk(curChunkIndex);
                                resolve();
                            } catch(e) {
                                failureTimes++;
                                if (failureTimes < 3) {
                                    toUploadThunk();
                                } else {
                                    //文件上传失败，结束上传
                                    failureUploadThunk();
                                }
                            }
                        })
                    }

                    if(curChunkIndex >= fileInfo.chunkCount) {
                        return
                    }

                    try {
                        const fileSize = fileInfo.config.file.size;
                        const { uploadToken, thunkArr } = await getUploadTokenAndProgress({
                            md5: fileInfo.MD5,
                            fileSize,
                            fileName: encodeURIComponent(fileInfo.config.file.name)
                        });
                        fileInfo.uploadToken = uploadToken;
                        thunkArr.forEach( index => successThunkArr.add(+index) );

                        if (successThunkArr.has(+curChunkIndex)) { //当前要上传的分片已存在服务器中
                            successUploadThunk(curChunkIndex);
                        } else {
                            await toUploadThunk();
                        }
                        resolve()
                    } catch(e) {
                        failureUploadThunk();
                    }
                })
            };

            await uploadThunk(0, 0);

            for(let i = 1, len = Math.min(this.state.threadNum + 1, fileInfo.chunkCount); i < len; i++) {
                uploadThunk(successThunkArr.size, i);
            }
        });
    }

    async submit() {
        if(!this.state.fileList.length) {
            Modal.info({
                title: '提示',
                content: (
                  <div>
                    <p>请选择文件后，再提交！</p>
                  </div>
                )
            });
            return;
        }
        for(let fileInfo of this.state.fileList) {
            if (fileInfo.state === FILE_STATE['WAITING']) {
                fileInfo.state = FILE_STATE['UPLOADING'];
                await this.toUpload(fileInfo);
            }
        }
        this.onCompleted();
    }

    render() {
        const draggerProps = {
            multiple: this.state.multiple,
            accept: this.state.accept,
            disabled: this.state.disabled,
            customRequest: config => {
                let fileInfo = {
                    id : config.file.uid,
                    state: FILE_STATE['PREPARING'],
                    chunkCount: Math.ceil(config.file.size / this.state.chunkSize),
                    MD5: null,
                    uploadToken: null,
                    token: null,
                    config
                };
                console.log(config);
                this.setState({
                    fileList: [...this.state.fileList, fileInfo],
                    disabled: !this.state.multiple,
                    isPrepared: true
                }, this.onCompleted);
                this.md5File(fileInfo);
            },
            onRemove: file => {
                const fileList = [...this.state.fileList.filter(info => info.id !== file.uid)];
                let isCompleted = !fileList.some(info => ![FILE_STATE['FINISHED'], FILE_STATE['ERRORED']].includes(info.state));
                this.setState({ 
                    fileList,
                    disabled: fileList.length && !this.state.multiple
                }, () => {
                    isCompleted && this.onCompleted();
                });
            }
        };

        return (<div>
            <Dragger {...draggerProps}>
                {this.props.children ? this.props.children :
                <div>
                    <p className="ant-upload-drag-icon"><Icon type="inbox" /></p>
                    <p className="ant-upload-text">点击选择文件或将文件拖拽到这里</p>
                </div>}
            </Dragger>
            <div style={{marginTop: '10px'}}>
                <Button ghost type="primary" size="small" onClick={this.submit} loading={this.state.isPrepared}>上传</Button>
                { this.state.isPrepared && <span style={{marginLeft: '7px'}}>正在读取文件，请稍后！</span>}
            </div>
        </div>);
    }
}

LargeFileUpload.propTypes = propTypes;

export default LargeFileUpload;

逻辑是：比如并发的线程是3，我们的文件分成9块传送，
第一次：向后台请求我们文件发送的token，以及已经上传成功的块的order假设是Set[0,1,3]。第0块已有跳过。
第二步：根据线程数对[1,2,3]order的块重新发送上传请求，第一个请求order:1已经存在服务器了，就把【当前已经存储的数目+并发线程-1】=2+3-1 = 4，order为4的进行传递; order为2的不存在，继续上传。
第三步：直到上传完成。