如何处理大体积 XLSX/CSV/TXT 文件？

function sliceInPiece(file, piece = 1024 * 1024 * 5) {  let totalSize = file.size; // 文件总大小  let start = 0; // 每次上传的开始字节  let end = start + piece; // 每次上传的结尾字节  let chunks = []  while (start < totalSize) {    // 根据长度截取每次需要上传的数据    // File对象继承自Blob对象，因此包含slice方法    let blob = file.slice(start, end);     chunks.push(blob)
    start = end;    end = start + piece;  }  return chunks}

let file =  document.querySelector("[name=file]").files[0];
const LENGTH = 1024 * 1024 * 0.1;let chunks = sliceInPiece(file, LENGTH); // 首先拆分切片
chunks.forEach(chunk=>{  let fd = new FormData();  fd.append("file", chunk);  post('/upload', fd)})

export function createElement (tagName: string, vnode: VNode): Element {  const elm = document.createElement(tagName)  if (tagName !== 'select') {    return elm  }  // false or null will remove the attribute but undefined will not  if (vnode.data && vnode.data.attrs && vnode.data.attrs.multiple !== undefined) {    elm.setAttribute('multiple', 'multiple')  }  return elm}

export function createElement (tagName: string): WeexElement {  return document.createElement(tagName)}

import * as nodeOps from 'web/runtime/node-ops'import { createPatchFunction } from 'core/vdom/patch'import baseModules from 'core/vdom/modules/index'import platformModules from 'web/runtime/modules/index'
// the directive module should be applied last, after all// built-in modules have been applied.const modules = platformModules.concat(baseModules)
// nodeops 封装了一系列DOM操作方法。modules定义了一些模块的钩子函数的实现export const patch: Function = createPatchFunction({ nodeOps, modules })

import * as nodeOps from 'weex/runtime/node-ops'import { createPatchFunction } from 'core/vdom/patch'import baseModules from 'core/vdom/modules/index'import platformModules from 'weex/runtime/modules/index'
// the directive module should be applied last, after all// built-in modules have been applied.const modules = platformModules.concat(baseModules)
export const patch: Function = createPatchFunction({  nodeOps,  modules,  LONG_LIST_THRESHOLD: 10})

import { readFile } from 'fs';
readFile('/etc/passwd','utf-8', (err, data) => {  if (err) throw err;  console.log(data);});

// pipe 消费readable.pipe(writable)
// 或者// 事件消费readable.on('data', (chunk) => {  writable.write(chunk);});readable.on('end', () => {  writable.end();});

async readOneByOne () {   try {    for (let i = 0; i < needParsePathArr.length; i++) {      const filePath = needParsePathArr[i]      console.log(`解析到第${i}个文件，文件名：${filePath}`)      await streamInsertDB(filePath)    }  } catch (err) {
  }}

async function streamInsertDB (filePath) {  return new Promise((resolve, reject) => {    const ext = path.extname(filePath)    // 判断了下文件类型    if (ext === '.csv') {      // 解析csv      parseAndInsertFromCSV(filePath, resolve, reject)    } else if (ext === '.xlsx') {      // 自执行函数      (async function getName () {        try {          // 先转换成csv。也可以不转换，直接解析xlsx，后文会详细解释。          const csvFileName = await convertXlsx2Csv(filePath)          // 复用解析csv的逻辑          parseAndInsertFromCSV(csvFileName, resolve, reject)        } catch (error) {          reject(`error: ${error.message || error}`)        }      })()    }  })}

const chardet = require('chardet');const csv = require('fast-csv'); // 比较快解析csv的速度的工具const iconv = require('iconv-lite');
const arrayFromParseCSV = []  // 存放解析出来的一行行csv数据的let count = 0 // 计数// resolve, reject 是外部函数传进来的，用以判断函数执行的状态，以便正确的进行后续逻辑处理function parseAndInsertFromCSV (filePath, resolve, reject) {  const rs = fs.createReadStream(filePath)  // 创建可读流  // 这里的防抖和柯里化  const delayInsert = debounce((isEnd, cb = () => {}) => insertInBlock(isEnd, cb, rs, resolve, reject), 300)  /// sampleSize: 5120 表示值读取文件前5120个字节的数据，就可以判断出文件的编码类型了，不需要全部读取  chardet.detectFile(filePath, { sampleSize: 5120 }).then(encoding => {    // 如果不是UTF-8编码，转换为utf8编码    if (encoding !== 'UTF-8') {      rs.pipe(iconv.decodeStream(encoding))        .pipe(iconv.encodeStream('UTF-8'))        .pipe(csv.parse({ header: false, ignoreEmpty: true, trim: true })) // 解析csv        .on('error', error => {          reject(`解析csv error: ${error}`)        })        .on('data', rows => {          count++ // 计数，因为我们要分块读取和操作          arrayFromParseCSV.push(rows) // 读到就推送到数组中          if (count > 30000) { // 已经读了30000行，我们就要先把这3w行处理掉，避免占用过多内存。            rs.pause() // 暂停可读流            delayInsert(false) // false 还没有结束。注意：即使rs.pause, 流的读取也不是立即暂停的，所以需要防抖。          }                  }).on('end', rowCount => {          console.log(`解析完${filePath}文件一共${rowCount}行`)          delayInsert(true, () => {            rs.destroy() // 销毁流            resolve('ok') // 一个文件读取完毕了          })        })    }  })}

function insertInBlock (isEnd, cb, filePath, resolve, reject) {  const arr = doSomethingWithData() // 可能会有一些清洗数据的操作  // 假如我们后续的需求是将数据写入数据库  const batchInsert = () => {    batchInsertDatabasePromise().then(() => {      if (cb && typeof cb === 'function') cb()      !isEnd && rs.resume() // 这一个片段的数据写入完毕，可以恢复流继续读了    })  }    const truely = schemaHasTable() // 比如判断数据库中有没有某个表，有就写入。没有先建表再写入。  if (truely) { //     batchInsert()   } else {     // 建表或者其他操作，然后再写入     doSomething().then(() => batchInsert())  }}

const { XLSX } = require('xlsx-extract')new XLSX().convert('path/to/file.xlsx', 'path/to/destfile.tsv')    .on('error', function (err) {        console.error(err);    })    .on('end', function () {        console.log('written');    })

import { parse } from '@fast-csv/parse';const stream = parse({ delimiter: '\t' })    .on('error', error => console.error(error))    .on('data', row => console.log(row))    .on('end', (rowCount: number) => console.log(`Parsed ${rowCount} rows`));

const { xslx } = require('xlsx-extract') // 流式解析xlsx文件工具// parser: expat, 需要额外安装node-expat,可以提高解析速度。new XLSX().extract(filePath, { sheet_nr: 1, parser: 'expat' })    .on('row', function (row) {        // 每一行数据获取到时都可以触发      }).on('error', function (err) {        // error     });

const readStream = fs.createReadableStream('path/to/xlsx.xlsx')

const unzip = require('unzipper')const zip = unzip.Parse();rs.pipe(zip)  .on('entry', function (entry) {    console.log('entry ---', entry);    const fileName = entry.path;    const { type } = entry; // 'Directory' or 'File'    const size = entry.vars.uncompressedSize; // There is also compressedSize;    if (fileName === "this IS the file I'm looking for") {      entry.pipe(fs.createWriteStream('output/path'));    } else {      entry.autodrain();    }  })

const saxStream = require('sax').createStream(false);saxStream.on('error', function (e) {  console.error('error!', e);});saxStream.on('opentag', function (node) {  console.log('node ---', node);});saxStream.on('text', (text) => console.log('text ---', typeof text, text));

const unzip = require('unzipper');const saxStream = require('sax').createStream(false);const zip = unzip.Parse();
saxStream.on('error', function (e) {  console.error('error!', e);});saxStream.on('opentag', function (node) {  console.log('node ---', node);});saxStream.on('text', (text) => {    console.log('text ---', typeof text, text)});
rs.pipe(zip)  .on('entry', function (entry) {    console.log('entry ---', entry);    entry.pipe(saxStream)  })