Electron文件分片上传

发表于 | 分类于
Electron文件分片上传

浏览器选择文件上传

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
const chunkSize = 2 * 1024 * 1024; // 每个chunk的大小，设置为2兆
// 使用Blob.slice方法来对文件进行分割。
// 同时该方法在不同的浏览器使用方式不同。
const blobSlice = File.prototype.slice || File.prototype.mozSlice || File.prototype.webkitSlice;

let file = new File();
if (!file) {
    alert('没有获取文件');
    return;
}
const blockCount = Math.ceil(file.size / chunkSize); // 分片总数
const axiosPromiseArray = []; // axiosPromise数组
const hash = await hashFile(file); //文件 hash
// 获取文件hash之后，如果需要做断点续传，可以根据hash值去后台进行校验。
// 看看是否已经上传过该文件，并且是否已经传送完成以及已经上传的切片。
console.log(hash);

for (let i = 0; i < blockCount; i++) {
    const start = i * chunkSize;
    const end = Math.min(file.size, start + chunkSize);
    // 构建表单
    const form = new FormData();
    form.append('file', blobSlice.call(file, start, end));
    form.append('identifier', file.name);
    form.append('filename', file.name);
    form.append('chunkNumber', i+1);
    form.append('size', file.size);
    form.append('hash', hash);
    // ajax提交 分片，此时 content-type 为 multipart/form-data
    const axiosOptions = {
        onUploadProgress: e => {
            // 处理上传的进度
            console.log(blockCount, i, e, file);
        },
    };
    // 加入到 Promise 数组中
    axiosPromiseArray.push(axios.post('/file/upload', form, axiosOptions));
}
// 所有分片上传后，请求合并分片文件
await axios.all(axiosPromiseArray).then(() => {
    // 合并chunks
    const data = {
        size: file.size,
        name: file.name,
        total: blockCount,
        hash
    };
    axios.post('/file/merge_chunks', data).then(res => {
        console.log('上传成功');
        console.log(res.data, file);
        alert('上传成功');
    }).catch(err => {
        console.log(err);
    });
});

本地文件上传

获取文件分片

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
let stats = fs.statSync(filepath);//读取文件信息
let chunkSize = 3*1024*1024;//每片分块的大小3M
let size = stats.size;//文件大小
let pieces = Math.ceil(size / chunkSize);//总共的分片数  
function uploadPiece (i){ 
    //计算每块的结束位置
    let startdata = i * chunkSize;
    let enddata = Math.min(size, (i + 1) * chunkSize);
    let arr = [];
    //创建一个readStream对象，根据文件起始位置和结束位置读取固定的分片
    let readStream = fs.createReadStream(
        filepath,
        {start: startdata, end: enddata - 1}
    );
    //on data读取数据
    readStream.on('data', (data)=>{
        arr.push(data)
    }) 
    //on end在该分片读取完成时触发
    readStream.on('end', ()=>{
        //这里服务端只接受blob对象，需要把原始的数据流转成blob对象，这块为了配合后端才转
        let blob = new Blob(arr)
        //新建formdata数据对象
        var formdata = new FormData();
        let md5Val = md5(Buffer.concat(arr));
        formdata.append("file", blob);
        console.log('blob.size',blob.size)
        formdata.append("md5", md5Val);
        formdata.append("size", size + ''); // 数字30被转换成字符串"30"
        formdata.append("chunk", i + '');//第几个分片，从0开始
        formdata.append("chunks", pieces + '');//分片数
        formdata.append("name", name);//文件名
        post(formdata)//这里是伪代码，实现上传，开发者自己实现
    })
}

获取文件Hash值

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
const hashFile = (file) => {
    return new Promise((resolve, reject) => {
        const chunks = Math.ceil(file.size / chunkSize);
        let currentChunk = 0;
        const spark = new SparkMD5.ArrayBuffer();
        const fileReader = new FileReader();

        function loadNext() {
            const start = currentChunk * chunkSize;
            const end = start + chunkSize >= file.size ? file.size : start + chunkSize;
            fileReader.readAsArrayBuffer(blobSlice.call(file, start, end));
        }

        fileReader.onload = e => {
            spark.append(e.target.result); // Append array buffer
            currentChunk += 1;
            if (currentChunk < chunks) {
                loadNext();
            } else {
                console.log('finished loading');
                const result = spark.end();
                // 如果单纯的使用result 作为hash值的时候, 如果文件内容相同，而名称不同的时候
                // 想保留两个文件无法保留。所以把文件名称加上。
                const sparkMd5 = new SparkMD5();
                sparkMd5.append(result);
                sparkMd5.append(file.name);
                const hexHash = sparkMd5.end();
                resolve(hexHash);
            }
        };
        
        fileReader.onerror = () => {
            console.warn('文件读取失败！');
        };
        loadNext();
    }).catch(err => {
        console.log(err);
    });
}

Node上传本地文件

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
let that = this;
let filepath = this.filepath;
let filename = filepath.split("/").reverse()[0];
let stats = fs.statSync(filepath);//读取文件信息
let chunkSize = 3 * 1024 * 1024;//每片分块的大小3M
let size = stats.size;//文件大小
let piecesAll = Math.ceil(size / chunkSize);//总共的分片数
let identifier = parseInt(new Date().getTime() / 1000 + "");
let savefolder = this.userInfo.schoolid + "/live";
const axiosPromiseArray = []; // axiosPromise数组
console.info("文件大小:" + size);
console.info("文件分片:" + piecesAll);
console.info("identifier", identifier)
console.info("filename", filename)
console.info("savefolder", this.userInfo.schoolid + "/live")
let readPieces = 0;//已读取的文件片数
for (let i = 0; i < piecesAll; i++) {
    let startdata = i * chunkSize;
    let enddata = Math.min(size, (i + 1) * chunkSize);
    let arr = [];
    //创建一个readStream对象，根据文件起始位置和结束位置读取固定的分片
    let readStream = fs.createReadStream(filepath,
                                         {start: startdata, end: enddata - 1}
                                        );
    //on data读取数据
    readStream.on('data', (data) => {
        arr.push(data)
    })

    //on end在该分片读取完成时触发
    readStream.on('end', () => {
        //这里服务端只接受blob对象，需要把原始的数据流转成blob对象，这块为了配合后端才转
        let blob = new Blob(arr)
        //新建formdata数据对象
        const form = new FormData();
        form.append('file', blob);
        form.append('identifier', identifier);
        form.append('filename', filename);
        form.append('chunkNumber', i + 1);
        form.append('savefolder', savefolder);
        //加入到 Promise 数组中
        axiosPromiseArray.push(axios.post(fileuploadUrl + 'chunk/up', form, {
            headers:
            {'Content-Type': 'application/x-www-form-urlencoded'}
        }));

        readPieces += 1;
        that.tipmsg.progress = parseInt((readPieces * 100 / piecesAll) + "")
        if (readPieces === piecesAll) {
            axios.all(axiosPromiseArray).then(() => {
                // 合并chunks
                const form = new FormData();
                form.append('savefolder', savefolder);
                form.append('identifier', identifier + "");
                form.append('filename', filename);
                form.append('thumed', 1 + "");
                form.append('rename', 1 + "");
                axios.post(fileuploadUrl + 'chunk/mergevideo', form, {
                    headers: {
                        'Content-Type': 'application/x-www-form-urlencoded'
                    }
                }
                          ).then(res => {
                    console.log('上传成功');
                    console.log("res.data", res.data);
                    axios.post(apiUrl + "section/update_playback", {
                        sectionid: that.sectionid,
                        mp4code: res.data.obj.mp4code || "h264",
                        playback: res.data.obj.videopath
                    }).then(res3 => {
                        win.close()
                    }).catch(err3 => {
                        win.close()
                    })
                    // win.close()
                }).catch(err => {
                    console.log(err);
                    win.close()
                });
            });
        }
    })
}

关键点

记录已经读取的分片数量 当所有分片都已经读取后再调用合并接口

录制屏幕后上传

获取流

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
async record_screen_audio() {
    let stream = await navigator.mediaDevices.getUserMedia({
        audio: false,
        video: {
            mandatory: {
                chromeMediaSource: "desktop",
                maxWidth: window.screen.width,
                maxHeight: window.screen.height,
            },
        },
    });
    let audioStream = await navigator.mediaDevices.getUserMedia({
        audio: true,
        video: false,
    });
    let audioTracks = audioStream.getAudioTracks()[0];
    stream.addTrack(audioTracks);
    this.createRecorder(stream);
},

创建录制器

1
2
3
4
5
6
7
8
9
10
11
12
13
14
createRecorder(stream) {
    let recorder = new MediaRecorder(stream);
    this.recorder = recorder;
    recorder.start();
    recorder.ondataavailable = (event) => {
        let blob = new Blob([event.data], {
            type: "video/mp4",
        });
        this.saveMedia(blob);
    };
    recorder.onerror = (err) => {
        console.error(err);
    };
},

保存

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
saveMedia(blob) {
    let reader = new FileReader();
    const fs = window.require("fs");
    reader.onload = () => {
        let buffer = new Buffer(reader.result);
        let filename = parseInt(new Date().getTime() / 1000 + "") + ".mp4";
        const filepath = path.join(app.getPath("downloads"), filename);
        this.filepath = filepath;
        console.info("filepath", filepath);
        fs.writeFile(filepath, buffer, {}, (err, res) => {
            this.uploadfile();
        });
    };
    reader.onerror = (err) => console.error(err);
    reader.readAsArrayBuffer(blob);
},

获取文件分片

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
async uploadfile() {
  let isupload = false;
  let that = this;
  let filepath = this.filepath;
  let filename = filepath.split("/").reverse()[0];
  let stats = fs.statSync(filepath); //读取文件信息
  let chunkSize = 3 * 1024 * 1024; //每片分块的大小3M
  let size = stats.size; //文件大小
  let piecesAll = Math.ceil(size / chunkSize); //总共的分片数
  let identifier = parseInt(new Date().getTime() / 1000 + "");
  let savefolder = this.userInfo.schoolid + "/live";
  this.uploadinfo.filepath = filepath;
  this.uploadinfo.filename = filename;
  this.uploadinfo.stats = stats;
  this.uploadinfo.chunkSize = chunkSize;
  this.uploadinfo.size = size;
  this.uploadinfo.piecesAll = piecesAll;
  this.uploadinfo.identifier = identifier;
  this.uploadinfo.savefolder = savefolder;
  this.uploadinfo.readPieces = 0;
  console.info("文件大小:" + size);
  console.info("文件分片:" + piecesAll);
  console.info("identifier", identifier);
  console.info("filename", filename);
  console.info("savefolder", this.userInfo.schoolid + "/live");
  this.uploadfile_pian();
},

上传分片

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
uploadfile_pian() {
    let that = this;
    let i = this.uploadinfo.readPieces;
    let chunkSize = this.uploadinfo.chunkSize;
    let size = this.uploadinfo.size;
    let filepath = this.uploadinfo.filepath;
    let savefolder = this.uploadinfo.savefolder;
    let identifier = this.uploadinfo.identifier;
    let filename = this.uploadinfo.filename;
    let startdata = i * chunkSize;
    let enddata = Math.min(size, (i + 1) * chunkSize);
    let arr = [];
    //创建一个readStream对象，根据文件起始位置和结束位置读取固定的分片
    let readStream = fs.createReadStream(filepath, {
        start: startdata,
        end: enddata - 1,
    });
    //on data读取数据
    readStream.on("data", (data) => {
        arr.push(data);
    });

    //on end在该分片读取完成时触发
    readStream.on("end", () => {
        //这里服务端只接受blob对象，需要把原始的数据流转成blob对象，这块为了配合后端才转
        let blob = new Blob(arr);
        //新建formdata数据对象
        const form = new FormData();
        form.append("file", blob);
        form.append("identifier", identifier);
        form.append("filename", filename);
        form.append("chunkNumber", i + 1);
        form.append("savefolder", savefolder);
        // 加入到 Promise 数组中
        axios
            .post(fileuploadUrl + "chunk/up", form, {
            headers: {"Content-Type": "application/x-www-form-urlencoded"},
        })
            .then((data) => {
            that.uploadinfo.readPieces += 1;
            that.tipmsg.progress = parseInt(
                (that.uploadinfo.readPieces * 100 / that.uploadinfo.piecesAll) +
                ""
            );
            if (that.tipmsg.progress > 100) {
                that.tipmsg.progress = 100;
            }
            if (that.uploadinfo.readPieces < that.uploadinfo.piecesAll) {
                that.uploadfile_pian();
            } else {
                const form = new FormData();
                form.append("savefolder", savefolder);
                form.append("identifier", identifier + "");
                form.append("filename", filename);
                form.append("thumed", 1 + "");
                form.append("rename", 1 + "");
                axios
                    .post(fileuploadUrl + "chunk/mergevideo", form, {
                    headers: {
                        "Content-Type": "application/x-www-form-urlencoded",
                    },
                })
                    .then((res) => {
                    console.log("上传成功");
                    console.log("res.data", res.data);
                    axios
                        .post(apiUrl + "section/update_playback", {
                        sectionid: that.sectionid,
                        mp4code: res.data.obj.mp4code || "h264",
                        playback: res.data.obj.videopath,
                    })
                        .then((res3) => {
                        win.close();
                    })
                        .catch((err3) => {
                        win.close();
                    });
                    // win.close()
                })
                    .catch((err) => {
                    console.log(err);
                    win.close();
                });
            }
        });
    });
},

录制屏幕同时上传

推荐这种方式

属性

1
2
3
4
5
6
uploadinfo: {
    savefolder: "",
    identifier: "",
    filename: "",
    piecenow: 0
},

录制屏幕

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
async record_screen_audio() {
    this.uploadinfo.savefolder = this.userInfo.schoolid + "/live";
    this.uploadinfo.filename = parseInt(new Date().getTime() / 1000 + "") + ".mp4";
    this.uploadinfo.identifier = parseInt(new Date().getTime() / 1000 + "");
    this.uploadinfo.piecenow = 0;
    let stream = await navigator.mediaDevices.getUserMedia({
        audio: false,
        video: {
            mandatory: {
                chromeMediaSource: "desktop",
                maxWidth: window.screen.width,
                maxHeight: window.screen.height,
            },
        },
    });
    let audioStream = await navigator.mediaDevices.getUserMedia({
        audio: true,
        video: false,
    });
    let audioTracks = audioStream.getAudioTracks()[0];
    stream.addTrack(audioTracks);
    this.createRecorder(stream);
},

录制器

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
createRecorder(stream) {
    let that = this;
    let recorder = new MediaRecorder(stream);
    this.recorder = recorder;
    recorder.start(3000);
    recorder.ondataavailable = (event) => {
        let blob = new Blob([event.data], {
            type: "video/mp4",
        });
        that.uploadinfo.piecenow += 1;
        this.uploadByBlob(blob, that.uploadinfo.piecenow);
    };
    recorder.onerror = (err) => {
        console.error(err);
    };
},

上传分片

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
uploadByBlob(blob, piece) {
    let savefolder = this.uploadinfo.savefolder;
    let identifier = this.uploadinfo.identifier;
    let filename = this.uploadinfo.filename;
    const form = new FormData();
    form.append("file", blob);
    form.append("identifier", identifier);
    form.append("filename", filename);
    form.append("chunkNumber", piece);
    form.append("savefolder", savefolder);
    // 加入到 Promise 数组中
    axios
        .post(fileuploadUrl + "chunk/up", form, {
        headers: {"Content-Type": "application/x-www-form-urlencoded"},
    })
        .then((data) => {
        console.info(data);
    })
},

停止并合并

1
2
3
4
5
6
stopRecord() {
    this.mergevideo()
    if (this.recorder) {
        this.recorder.pause();
    }
},

注意

这里没有调用recorder.stop()而是调用recorder.pause()原因是调用stop后所有的分片都会再触发一次上传

合并

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
mergevideo() {
    let that = this;
    if (that.uploadinfo.piecenow === 0) {
        win.close();
        return
    }
    let savefolder = this.uploadinfo.savefolder;
    let identifier = this.uploadinfo.identifier;
    let filename = this.uploadinfo.filename;
    const form = new FormData();
    form.append("savefolder", savefolder);
    form.append("identifier", identifier + "");
    form.append("filename", filename);
    form.append("thumed", 1 + "");
    form.append("rename", 1 + "");
    axios
        .post(fileuploadUrl + "chunk/mergevideo", form, {
        headers: {
            "Content-Type": "application/x-www-form-urlencoded",
        },
    })
        .then((res) => {
        console.log("上传成功");
        console.log("res.data", res.data);
        axios
            .post(apiUrl + "section/update_playback", {
            sectionid: that.sectionid,
            mp4code: res.data.obj.mp4code || "h264",
            playback: res.data.obj.videopath,
        })
            .then((res3) => {
            win.close();
        })
            .catch((err3) => {
            win.close();
        });
        // win.close()
    })
        .catch((err) => {
        console.log(err);
        win.close();
    });
}

相关

Blob说明https://developer.mozilla.org/zh-CN/docs/Web/API/Blob

https://developer.mozilla.org/zh-CN/docs/Web/API/MediaRecorder/ondataavailable

MediaRecorder.ondataavailable事件处理程序(part of the MediaStream记录API)处理event("dataavailable")事件，让您在响应运行代码数据被提供使用。

dataavailable当MediaRecorder将媒体数据传递到您的应用程序以供使用时，将触发该事件。数据在包含数据的Blob对象中提供。这在四种情况下发生：

  1. 媒体流结束时，所有尚未传递到ondataavailable处理程序的媒体数据都将在单个Blob中传递。

  2. 当调用MediaRecorder.stop()时，自记录开始或dataavailable事件最后一次发生以来已捕获的所有媒体数据都将传递到Blob中；此后，捕获结束。

  3. 调用MediaRecorder.requestData() dataavailable时，将传递自记录开始或事件最后一次发生以来捕获的所有媒体数据；然后Blob创建一个新文件，并将媒体捕获继续到该blob中。

  4. 如果将timeslice属性传递到开始媒体捕获的MediaRecorder.start()方法中，dataavailable则每timeslice毫秒触发一次事件。这意味着每个Blob都有特定的持续时间（最后一个Blob除外，后者可能更短，因为它将是自上次事件以来剩下的所有东西）。因此，如果该方法调用看起来像这样- recorder.start(1000);的dataavailable事件将媒体捕捉的每一秒发生火灾后，我们的事件处理程序将被称为与媒体数据的BLOB每秒即坚持一个第二长。

    您可以timeslice与MediaRecorder.stop()和MediaRecorder.requestData()一起使用，以产生多个相同长度的Blob，以及其他较短的Blob。