neil/build-push-action
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update generated content

Browse Source 
Signed-off-by: CrazyMax <crazy-max@users.noreply.github.com>
This commit is contained in:
CrazyMax 2020-11-06 14:56:57 +01:00
parent b6150991af
commit 35f1834293
No known key found for this signature in database
GPG Key ID: 3248E46B6BB8C7F7
1 changed files with 201 additions and 118 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

285
dist/index.js generated vendored
View File

@ -10817,16 +10817,44 @@ const nl = 10
const np = 12 const np = 12
const cr = 13 const cr = 13
const space = 32 const space = 32
const bom_utf8 = Buffer.from([239, 187, 191]) const boms = {
// Note, the following are equals:
// Buffer.from("\ufeff")
// Buffer.from([239, 187, 191])
// Buffer.from('EFBBBF', 'hex')
'utf8': Buffer.from([239, 187, 191]),
// Note, the following are equals:
// Buffer.from "\ufeff", 'utf16le
// Buffer.from([255, 254])
'utf16le': Buffer.from([255, 254])
}
class Parser extends Transform { class Parser extends Transform {
constructor(opts = {}){ constructor(opts = {}){
super({...{readableObjectMode: true}, ...opts}) super({...{readableObjectMode: true}, ...opts, encoding: null})
this.__originalOptions = opts
this.__normalizeOptions(opts)
}
__normalizeOptions(opts){
const options = {} const options = {}
// Merge with user options // Merge with user options
for(let opt in opts){ for(let opt in opts){
options[underscore(opt)] = opts[opt] options[underscore(opt)] = opts[opt]
} }
// Normalize option `encoding`
// Note: defined first because other options depends on it
// to convert chars/strings into buffers.
if(options.encoding === undefined || options.encoding === true){
options.encoding = 'utf8'
}else if(options.encoding === null || options.encoding === false){
options.encoding = null
}else if(typeof options.encoding !== 'string' && options.encoding !== null){
throw new CsvError('CSV_INVALID_OPTION_ENCODING', [
'Invalid option encoding:',
'encoding must be a string or null to return a buffer,',
`got ${JSON.stringify(options.encoding)}`
], options)
}
// Normalize option `bom` // Normalize option `bom`
if(options.bom === undefined || options.bom === null || options.bom === false){ if(options.bom === undefined || options.bom === null || options.bom === false){
options.bom = false options.bom = false
@ -10834,7 +10862,7 @@ class Parser extends Transform {
throw new CsvError('CSV_INVALID_OPTION_BOM', [ throw new CsvError('CSV_INVALID_OPTION_BOM', [
'Invalid option bom:', 'bom must be true,', 'Invalid option bom:', 'bom must be true,',
`got ${JSON.stringify(options.bom)}` `got ${JSON.stringify(options.bom)}`
]) ], options)
} }
// Normalize option `cast` // Normalize option `cast`
let fnCastField = null let fnCastField = null
@ -10847,7 +10875,7 @@ class Parser extends Transform {
throw new CsvError('CSV_INVALID_OPTION_CAST', [ throw new CsvError('CSV_INVALID_OPTION_CAST', [
'Invalid option cast:', 'cast must be true or a function,', 'Invalid option cast:', 'cast must be true or a function,',
`got ${JSON.stringify(options.cast)}` `got ${JSON.stringify(options.cast)}`
]) ], options)
} }
// Normalize option `cast_date` // Normalize option `cast_date`
if(options.cast_date === undefined || options.cast_date === null || options.cast_date === false || options.cast_date === ''){ if(options.cast_date === undefined || options.cast_date === null || options.cast_date === false || options.cast_date === ''){
@ -10861,7 +10889,7 @@ class Parser extends Transform {
throw new CsvError('CSV_INVALID_OPTION_CAST_DATE', [ throw new CsvError('CSV_INVALID_OPTION_CAST_DATE', [
'Invalid option cast_date:', 'cast_date must be true or a function,', 'Invalid option cast_date:', 'cast_date must be true or a function,',
`got ${JSON.stringify(options.cast_date)}` `got ${JSON.stringify(options.cast_date)}`
]) ], options)
} }
// Normalize option `columns` // Normalize option `columns`
let fnFirstLineToHeaders = null let fnFirstLineToHeaders = null
@ -10880,7 +10908,7 @@ class Parser extends Transform {
'Invalid option columns:', 'Invalid option columns:',
'expect an object, a function or true,', 'expect an object, a function or true,',
`got ${JSON.stringify(options.columns)}` `got ${JSON.stringify(options.columns)}`
]) ], options)
} }
// Normalize option `columns_duplicates_to_array` // Normalize option `columns_duplicates_to_array`
if(options.columns_duplicates_to_array === undefined || options.columns_duplicates_to_array === null || options.columns_duplicates_to_array === false){ if(options.columns_duplicates_to_array === undefined || options.columns_duplicates_to_array === null || options.columns_duplicates_to_array === false){
@ -10890,21 +10918,21 @@ class Parser extends Transform {
'Invalid option columns_duplicates_to_array:', 'Invalid option columns_duplicates_to_array:',
'expect an boolean,', 'expect an boolean,',
`got ${JSON.stringify(options.columns_duplicates_to_array)}` `got ${JSON.stringify(options.columns_duplicates_to_array)}`
]) ], options)
} }
// Normalize option `comment` // Normalize option `comment`
if(options.comment === undefined || options.comment === null || options.comment === false || options.comment === ''){ if(options.comment === undefined || options.comment === null || options.comment === false || options.comment === ''){
options.comment = null options.comment = null
}else{ }else{
if(typeof options.comment === 'string'){ if(typeof options.comment === 'string'){
options.comment = Buffer.from(options.comment) options.comment = Buffer.from(options.comment, options.encoding)
} }
if(!Buffer.isBuffer(options.comment)){ if(!Buffer.isBuffer(options.comment)){
throw new CsvError('CSV_INVALID_OPTION_COMMENT', [ throw new CsvError('CSV_INVALID_OPTION_COMMENT', [
'Invalid option comment:', 'Invalid option comment:',
'comment must be a buffer or a string,', 'comment must be a buffer or a string,',
`got ${JSON.stringify(options.comment)}` `got ${JSON.stringify(options.comment)}`
]) ], options)
} }
} }
// Normalize option `delimiter` // Normalize option `delimiter`
@ -10915,39 +10943,35 @@ class Parser extends Transform {
'Invalid option delimiter:', 'Invalid option delimiter:',
'delimiter must be a non empty string or buffer or array of string|buffer,', 'delimiter must be a non empty string or buffer or array of string|buffer,',
`got ${delimiter_json}` `got ${delimiter_json}`
]) ], options)
} }
options.delimiter = options.delimiter.map(function(delimiter){ options.delimiter = options.delimiter.map(function(delimiter){
if(delimiter === undefined || delimiter === null || delimiter === false){ if(delimiter === undefined || delimiter === null || delimiter === false){
return Buffer.from(',') return Buffer.from(',', options.encoding)
} }
if(typeof delimiter === 'string'){ if(typeof delimiter === 'string'){
delimiter = Buffer.from(delimiter) delimiter = Buffer.from(delimiter, options.encoding)
} }
if( !Buffer.isBuffer(delimiter) || delimiter.length === 0){ if( !Buffer.isBuffer(delimiter) || delimiter.length === 0){
throw new CsvError('CSV_INVALID_OPTION_DELIMITER', [ throw new CsvError('CSV_INVALID_OPTION_DELIMITER', [
'Invalid option delimiter:', 'Invalid option delimiter:',
'delimiter must be a non empty string or buffer or array of string|buffer,', 'delimiter must be a non empty string or buffer or array of string|buffer,',
`got ${delimiter_json}` `got ${delimiter_json}`
]) ], options)
} }
return delimiter return delimiter
}) })
// Normalize option `escape` // Normalize option `escape`
if(options.escape === undefined || options.escape === true){ if(options.escape === undefined || options.escape === true){
options.escape = Buffer.from('"') options.escape = Buffer.from('"', options.encoding)
}else if(typeof options.escape === 'string'){ }else if(typeof options.escape === 'string'){
options.escape = Buffer.from(options.escape) options.escape = Buffer.from(options.escape, options.encoding)
}else if (options.escape === null || options.escape === false){ }else if (options.escape === null || options.escape === false){
options.escape = null options.escape = null
} }
if(options.escape !== null){ if(options.escape !== null){
if(!Buffer.isBuffer(options.escape)){ if(!Buffer.isBuffer(options.escape)){
throw new Error(`Invalid Option: escape must be a buffer, a string or a boolean, got ${JSON.stringify(options.escape)}`) throw new Error(`Invalid Option: escape must be a buffer, a string or a boolean, got ${JSON.stringify(options.escape)}`)
}else if(options.escape.length !== 1){
throw new Error(`Invalid Option Length: escape must be one character, got ${options.escape.length}`)
}else{
options.escape = options.escape[0]
} }
} }
// Normalize option `from` // Normalize option `from`
@ -11003,7 +11027,11 @@ class Parser extends Transform {
if(options.objname.length === 0){ if(options.objname.length === 0){
throw new Error(`Invalid Option: objname must be a non empty buffer`) throw new Error(`Invalid Option: objname must be a non empty buffer`)
} }
options.objname = options.objname.toString() if(options.encoding === null){
// Don't call `toString`, leave objname as a buffer
}else{
options.objname = options.objname.toString(options.encoding)
}
}else if(typeof options.objname === 'string'){ }else if(typeof options.objname === 'string'){
if(options.objname.length === 0){ if(options.objname.length === 0){
throw new Error(`Invalid Option: objname must be a non empty string`) throw new Error(`Invalid Option: objname must be a non empty string`)
@ -11020,23 +11048,19 @@ class Parser extends Transform {
'Invalid option `on_record`:', 'Invalid option `on_record`:',
'expect a function,', 'expect a function,',
`got ${JSON.stringify(options.on_record)}` `got ${JSON.stringify(options.on_record)}`
]) ], options)
} }
// Normalize option `quote` // Normalize option `quote`
if(options.quote === null || options.quote === false || options.quote === ''){ if(options.quote === null || options.quote === false || options.quote === ''){
options.quote = null options.quote = null
}else{ }else{
if(options.quote === undefined || options.quote === true){ if(options.quote === undefined || options.quote === true){
options.quote = Buffer.from('"') options.quote = Buffer.from('"', options.encoding)
}else if(typeof options.quote === 'string'){ }else if(typeof options.quote === 'string'){
options.quote = Buffer.from(options.quote) options.quote = Buffer.from(options.quote, options.encoding)
} }
if(!Buffer.isBuffer(options.quote)){ if(!Buffer.isBuffer(options.quote)){
throw new Error(`Invalid Option: quote must be a buffer or a string, got ${JSON.stringify(options.quote)}`) throw new Error(`Invalid Option: quote must be a buffer or a string, got ${JSON.stringify(options.quote)}`)
}else if(options.quote.length !== 1){
throw new Error(`Invalid Option Length: quote must be one character, got ${options.quote.length}`)
}else{
options.quote = options.quote[0]
} }
} }
// Normalize option `raw` // Normalize option `raw`
@ -11053,7 +11077,7 @@ class Parser extends Transform {
} }
options.record_delimiter = options.record_delimiter.map( function(rd){ options.record_delimiter = options.record_delimiter.map( function(rd){
if(typeof rd === 'string'){ if(typeof rd === 'string'){
rd = Buffer.from(rd) rd = Buffer.from(rd, options.encoding)
} }
return rd return rd
}) })
@ -11182,13 +11206,24 @@ class Parser extends Transform {
bomSkipped: false, bomSkipped: false,
castField: fnCastField, castField: fnCastField,
commenting: false, commenting: false,
// Current error encountered by a record
error: undefined,
enabled: options.from_line === 1, enabled: options.from_line === 1,
escaping: false, escaping: false,
escapeIsQuote: options.escape === options.quote, // escapeIsQuote: options.escape === options.quote,
escapeIsQuote: Buffer.isBuffer(options.escape) && Buffer.isBuffer(options.quote) && Buffer.compare(options.escape, options.quote) === 0,
expectedRecordLength: options.columns === null ? 0 : options.columns.length, expectedRecordLength: options.columns === null ? 0 : options.columns.length,
field: new ResizeableBuffer(20), field: new ResizeableBuffer(20),
firstLineToHeaders: fnFirstLineToHeaders, firstLineToHeaders: fnFirstLineToHeaders,
info: Object.assign({}, this.info), info: Object.assign({}, this.info),
needMoreDataSize: Math.max(
// Skip if the remaining buffer smaller than comment
options.comment !== null ? options.comment.length : 0,
// Skip if the remaining buffer can be delimiter
...options.delimiter.map( (delimiter) => delimiter.length),
// Skip if the remaining buffer can be escape sequence
options.quote !== null ? options.quote.length : 0,
),
previousBuf: undefined, previousBuf: undefined,
quoting: false, quoting: false,
stop: false, stop: false,
@ -11197,7 +11232,7 @@ class Parser extends Transform {
recordHasError: false, recordHasError: false,
record_length: 0, record_length: 0,
recordDelimiterMaxLength: options.record_delimiter.length === 0 ? 2 : Math.max(...options.record_delimiter.map( (v) => v.length)), recordDelimiterMaxLength: options.record_delimiter.length === 0 ? 2 : Math.max(...options.record_delimiter.map( (v) => v.length)),
trimChars: [Buffer.from(' ')[0], Buffer.from('\t')[0]], trimChars: [Buffer.from(' ', options.encoding)[0], Buffer.from('\t', options.encoding)[0]],
wasQuoting: false, wasQuoting: false,
wasRowDelimiter: false wasRowDelimiter: false
} }
@ -11251,11 +11286,15 @@ class Parser extends Transform {
this.state.previousBuf = buf this.state.previousBuf = buf
return return
} }
// skip BOM detect because data length < 3
}else{ }else{
if(bom_utf8.compare(buf, 0, 3) === 0){ for(let encoding in boms){
if(boms[encoding].compare(buf, 0, boms[encoding].length) === 0){
// Skip BOM // Skip BOM
buf = buf.slice(3) buf = buf.slice(boms[encoding].length)
// Renormalize original options with the new encoding
this.__normalizeOptions({...this.__originalOptions, encoding: encoding})
break
}
} }
this.state.bomSkipped = true this.state.bomSkipped = true
} }
@ -11301,35 +11340,37 @@ class Parser extends Transform {
}else{ }else{
// Escape is only active inside quoted fields // Escape is only active inside quoted fields
// We are quoting, the char is an escape chr and there is a chr to escape // We are quoting, the char is an escape chr and there is a chr to escape
if(escape !== null && this.state.quoting === true && chr === escape && pos + 1 < bufLen){ // if(escape !== null && this.state.quoting === true && chr === escape && pos + 1 < bufLen){
if(escape !== null && this.state.quoting === true && this.__isEscape(buf, pos, chr) && pos + escape.length < bufLen){
if(escapeIsQuote){ if(escapeIsQuote){
if(buf[pos+1] === quote){ if(this.__isQuote(buf, pos+escape.length)){
this.state.escaping = true this.state.escaping = true
pos += escape.length - 1
continue continue
} }
}else{ }else{
this.state.escaping = true this.state.escaping = true
pos += escape.length - 1
continue continue
} }
} }
// Not currently escaping and chr is a quote // Not currently escaping and chr is a quote
// TODO: need to compare bytes instead of single char // TODO: need to compare bytes instead of single char
if(this.state.commenting === false && chr === quote){ if(this.state.commenting === false && this.__isQuote(buf, pos)){
if(this.state.quoting === true){ if(this.state.quoting === true){
const nextChr = buf[pos+1] const nextChr = buf[pos+quote.length]
const isNextChrTrimable = rtrim && this.__isCharTrimable(nextChr) const isNextChrTrimable = rtrim && this.__isCharTrimable(nextChr)
// const isNextChrComment = nextChr === comment const isNextChrComment = comment !== null && this.__compareBytes(comment, buf, pos+quote.length, nextChr)
const isNextChrComment = comment !== null && this.__compareBytes(comment, buf, pos+1, nextChr) const isNextChrDelimiter = this.__isDelimiter(buf, pos+quote.length, nextChr)
const isNextChrDelimiter = this.__isDelimiter(nextChr, buf, pos+1) const isNextChrRowDelimiter = record_delimiter.length === 0 ? this.__autoDiscoverRowDelimiter(buf, pos+quote.length) : this.__isRecordDelimiter(nextChr, buf, pos+quote.length)
const isNextChrRowDelimiter = record_delimiter.length === 0 ? this.__autoDiscoverRowDelimiter(buf, pos+1) : this.__isRecordDelimiter(nextChr, buf, pos+1)
// Escape a quote // Escape a quote
// Treat next char as a regular character // Treat next char as a regular character
// TODO: need to compare bytes instead of single char if(escape !== null && this.__isEscape(buf, pos, chr) && this.__isQuote(buf, pos + escape.length)){
if(escape !== null && chr === escape && nextChr === quote){ pos += escape.length - 1
pos++
}else if(!nextChr || isNextChrDelimiter || isNextChrRowDelimiter || isNextChrComment || isNextChrTrimable){ }else if(!nextChr || isNextChrDelimiter || isNextChrRowDelimiter || isNextChrComment || isNextChrTrimable){
this.state.quoting = false this.state.quoting = false
this.state.wasQuoting = true this.state.wasQuoting = true
pos += quote.length - 1
continue continue
}else if(relax === false){ }else if(relax === false){
const err = this.__error( const err = this.__error(
@ -11339,14 +11380,14 @@ class Parser extends Transform {
`at line ${this.info.lines}`, `at line ${this.info.lines}`,
'instead of delimiter, row delimiter, trimable character', 'instead of delimiter, row delimiter, trimable character',
'(if activated) or comment', '(if activated) or comment',
], this.__context()) ], this.options, this.__context())
) )
if(err !== undefined) return err if(err !== undefined) return err
}else{ }else{
this.state.quoting = false this.state.quoting = false
this.state.wasQuoting = true this.state.wasQuoting = true
// continue
this.state.field.prepend(quote) this.state.field.prepend(quote)
pos += quote.length - 1
} }
}else{ }else{
if(this.state.field.length !== 0){ if(this.state.field.length !== 0){
@ -11356,7 +11397,7 @@ class Parser extends Transform {
new CsvError('INVALID_OPENING_QUOTE', [ new CsvError('INVALID_OPENING_QUOTE', [
'Invalid Opening Quote:', 'Invalid Opening Quote:',
`a quote is found inside a field at line ${this.info.lines}`, `a quote is found inside a field at line ${this.info.lines}`,
], this.__context(), { ], this.options, this.__context(), {
field: this.state.field, field: this.state.field,
}) })
) )
@ -11364,6 +11405,7 @@ class Parser extends Transform {
} }
}else{ }else{
this.state.quoting = true this.state.quoting = true
pos += quote.length - 1
continue continue
} }
} }
@ -11414,7 +11456,7 @@ class Parser extends Transform {
this.state.commenting = true this.state.commenting = true
continue continue
} }
let delimiterLength = this.__isDelimiter(chr, buf, pos) let delimiterLength = this.__isDelimiter(buf, pos, chr)
if(delimiterLength !== 0){ if(delimiterLength !== 0){
const errField = this.__onField() const errField = this.__onField()
if(errField !== undefined) return errField if(errField !== undefined) return errField
@ -11431,7 +11473,7 @@ class Parser extends Transform {
'record exceed the maximum number of tolerated bytes', 'record exceed the maximum number of tolerated bytes',
`of ${max_record_size}`, `of ${max_record_size}`,
`at line ${this.info.lines}`, `at line ${this.info.lines}`,
], this.__context()) ], this.options, this.__context())
) )
if(err !== undefined) return err if(err !== undefined) return err
} }
@ -11448,7 +11490,7 @@ class Parser extends Transform {
'Invalid Closing Quote:', 'Invalid Closing Quote:',
'found non trimable byte after quote', 'found non trimable byte after quote',
`at line ${this.info.lines}`, `at line ${this.info.lines}`,
], this.__context()) ], this.options, this.__context())
) )
if(err !== undefined) return err if(err !== undefined) return err
} }
@ -11460,7 +11502,7 @@ class Parser extends Transform {
new CsvError('CSV_QUOTE_NOT_CLOSED', [ new CsvError('CSV_QUOTE_NOT_CLOSED', [
'Quote Not Closed:', 'Quote Not Closed:',
`the parsing is finished with an opening quote at line ${this.info.lines}`, `the parsing is finished with an opening quote at line ${this.info.lines}`,
], this.__context()) ], this.options, this.__context())
) )
if(err !== undefined) return err if(err !== undefined) return err
}else{ }else{
@ -11489,7 +11531,7 @@ class Parser extends Transform {
return chr === space || chr === tab || chr === cr || chr === nl || chr === np return chr === space || chr === tab || chr === cr || chr === nl || chr === np
} }
__onRow(){ __onRow(){
const {columns, columns_duplicates_to_array, info, from, relax_column_count, relax_column_count_less, relax_column_count_more, raw, skip_lines_with_empty_values} = this.options const {columns, columns_duplicates_to_array, encoding, info, from, relax_column_count, relax_column_count_less, relax_column_count_more, raw, skip_lines_with_empty_values} = this.options
const {enabled, record} = this.state const {enabled, record} = this.state
if(enabled === false){ if(enabled === false){
return this.__resetRow() return this.__resetRow()
@ -11507,35 +11549,38 @@ class Parser extends Transform {
this.state.expectedRecordLength = recordLength this.state.expectedRecordLength = recordLength
} }
if(recordLength !== this.state.expectedRecordLength){ if(recordLength !== this.state.expectedRecordLength){
if(relax_column_count === true || const err = columns === false ?
(relax_column_count_less === true && recordLength < this.state.expectedRecordLength) || this.__error(
(relax_column_count_more === true && recordLength > this.state.expectedRecordLength) ){ // Todo: rename CSV_INCONSISTENT_RECORD_LENGTH to
this.info.invalid_field_length++ // CSV_RECORD_INCONSISTENT_FIELDS_LENGTH
}else{
if(columns === false){
const err = this.__error(
new CsvError('CSV_INCONSISTENT_RECORD_LENGTH', [ new CsvError('CSV_INCONSISTENT_RECORD_LENGTH', [
'Invalid Record Length:', 'Invalid Record Length:',
`expect ${this.state.expectedRecordLength},`, `expect ${this.state.expectedRecordLength},`,
`got ${recordLength} on line ${this.info.lines}`, `got ${recordLength} on line ${this.info.lines}`,
], this.__context(), { ], this.options, this.__context(), {
record: record, record: record,
}) })
) )
if(err !== undefined) return err :
}else{ this.__error(
const err = this.__error( // Todo: rename CSV_RECORD_DONT_MATCH_COLUMNS_LENGTH to
// CSV_INVALID_RECORD_LENGTH_DONT_MATCH_COLUMNS // CSV_RECORD_INCONSISTENT_COLUMNS
new CsvError('CSV_RECORD_DONT_MATCH_COLUMNS_LENGTH', [ new CsvError('CSV_RECORD_DONT_MATCH_COLUMNS_LENGTH', [
'Invalid Record Length:', 'Invalid Record Length:',
`columns length is ${columns.length},`, // rename columns `columns length is ${columns.length},`, // rename columns
`got ${recordLength} on line ${this.info.lines}`, `got ${recordLength} on line ${this.info.lines}`,
], this.__context(), { ], this.options, this.__context(), {
record: record, record: record,
}) })
) )
if(err !== undefined) return err if(relax_column_count === true ||
} (relax_column_count_less === true && recordLength < this.state.expectedRecordLength) ||
(relax_column_count_more === true && recordLength > this.state.expectedRecordLength) ){
this.info.invalid_field_length++
this.state.error = err
// Error is undefined with skip_lines_with_error
}else if(err !== undefined){
return err
} }
} }
if(skip_lines_with_empty_values === true){ if(skip_lines_with_empty_values === true){
@ -11556,7 +11601,6 @@ class Parser extends Transform {
// Transform record array to an object // Transform record array to an object
for(let i = 0, l = record.length; i < l; i++){ for(let i = 0, l = record.length; i < l; i++){
if(columns[i] === undefined || columns[i].disabled) continue if(columns[i] === undefined || columns[i].disabled) continue
// obj[columns[i].name] = record[i]
// Turn duplicate columns into an array // Turn duplicate columns into an array
if (columns_duplicates_to_array === true && obj[columns[i].name]) { if (columns_duplicates_to_array === true && obj[columns[i].name]) {
if (Array.isArray(obj[columns[i].name])) { if (Array.isArray(obj[columns[i].name])) {
@ -11573,7 +11617,7 @@ class Parser extends Transform {
if(raw === true || info === true){ if(raw === true || info === true){
const err = this.__push(Object.assign( const err = this.__push(Object.assign(
{record: obj}, {record: obj},
(raw === true ? {raw: this.state.rawBuffer.toString()}: {}), (raw === true ? {raw: this.state.rawBuffer.toString(encoding)}: {}),
(info === true ? {info: this.state.info}: {}) (info === true ? {info: this.state.info}: {})
)) ))
if(err){ if(err){
@ -11589,7 +11633,7 @@ class Parser extends Transform {
if(raw === true || info === true){ if(raw === true || info === true){
const err = this.__push(Object.assign( const err = this.__push(Object.assign(
{record: [obj[objname], obj]}, {record: [obj[objname], obj]},
raw === true ? {raw: this.state.rawBuffer.toString()}: {}, raw === true ? {raw: this.state.rawBuffer.toString(encoding)}: {},
info === true ? {info: this.state.info}: {} info === true ? {info: this.state.info}: {}
)) ))
if(err){ if(err){
@ -11606,7 +11650,7 @@ class Parser extends Transform {
if(raw === true || info === true){ if(raw === true || info === true){
const err = this.__push(Object.assign( const err = this.__push(Object.assign(
{record: record}, {record: record},
raw === true ? {raw: this.state.rawBuffer.toString()}: {}, raw === true ? {raw: this.state.rawBuffer.toString(encoding)}: {},
info === true ? {info: this.state.info}: {} info === true ? {info: this.state.info}: {}
)) ))
if(err){ if(err){
@ -11632,7 +11676,7 @@ class Parser extends Transform {
'Invalid Column Mapping:', 'Invalid Column Mapping:',
'expect an array from column function,', 'expect an array from column function,',
`got ${JSON.stringify(headers)}` `got ${JSON.stringify(headers)}`
], this.__context(), { ], this.options, this.__context(), {
headers: headers, headers: headers,
}) })
) )
@ -11650,17 +11694,18 @@ class Parser extends Transform {
if(this.options.raw === true){ if(this.options.raw === true){
this.state.rawBuffer.reset() this.state.rawBuffer.reset()
} }
this.state.error = undefined
this.state.record = [] this.state.record = []
this.state.record_length = 0 this.state.record_length = 0
} }
__onField(){ __onField(){
const {cast, rtrim, max_record_size} = this.options const {cast, encoding, rtrim, max_record_size} = this.options
const {enabled, wasQuoting} = this.state const {enabled, wasQuoting} = this.state
// Short circuit for the from_line options // Short circuit for the from_line options
if(enabled === false){ /* this.options.columns !== true && */ if(enabled === false){ /* this.options.columns !== true && */
return this.__resetField() return this.__resetField()
} }
let field = this.state.field.toString() let field = this.state.field.toString(encoding)
if(rtrim === true && wasQuoting === false){ if(rtrim === true && wasQuoting === false){
field = field.trimRight() field = field.trimRight()
} }
@ -11727,38 +11772,30 @@ class Parser extends Transform {
__isFloat(value){ __isFloat(value){
return (value - parseFloat( value ) + 1) >= 0 // Borrowed from jquery return (value - parseFloat( value ) + 1) >= 0 // Borrowed from jquery
} }
__compareBytes(sourceBuf, targetBuf, pos, firtByte){ __compareBytes(sourceBuf, targetBuf, targetPos, firstByte){
if(sourceBuf[0] !== firtByte) return 0 if(sourceBuf[0] !== firstByte) return 0
const sourceLength = sourceBuf.length const sourceLength = sourceBuf.length
for(let i = 1; i < sourceLength; i++){ for(let i = 1; i < sourceLength; i++){
if(sourceBuf[i] !== targetBuf[pos+i]) return 0 if(sourceBuf[i] !== targetBuf[targetPos+i]) return 0
} }
return sourceLength return sourceLength
} }
__needMoreData(i, bufLen, end){ __needMoreData(i, bufLen, end){
if(end){ if(end) return false
return false const {quote} = this.options
} const {quoting, needMoreDataSize, recordDelimiterMaxLength} = this.state
const {comment, delimiter} = this.options
const {quoting, recordDelimiterMaxLength} = this.state
const numOfCharLeft = bufLen - i - 1 const numOfCharLeft = bufLen - i - 1
const requiredLength = Math.max( const requiredLength = Math.max(
// Skip if the remaining buffer smaller than comment needMoreDataSize,
comment ? comment.length : 0, // Skip if the remaining buffer smaller than record delimiter
// Skip if the remaining buffer smaller than row delimiter
recordDelimiterMaxLength, recordDelimiterMaxLength,
// Skip if the remaining buffer can be row delimiter following the closing quote // Skip if the remaining buffer can be row delimiter following the closing quote
// 1 is for quote.length // 1 is for quote.length
quoting ? (1 + recordDelimiterMaxLength) : 0, quoting ? (quote.length + recordDelimiterMaxLength) : 0,
// Skip if the remaining buffer can be delimiter
delimiter.length,
// Skip if the remaining buffer can be escape sequence
// 1 is for escape.length
1
) )
return numOfCharLeft < requiredLength return numOfCharLeft < requiredLength
} }
__isDelimiter(chr, buf, pos){ __isDelimiter(buf, pos, chr){
const {delimiter} = this.options const {delimiter} = this.options
loop1: for(let i = 0; i < delimiter.length; i++){ loop1: for(let i = 0; i < delimiter.length; i++){
const del = delimiter[i] const del = delimiter[i]
@ -11789,20 +11826,46 @@ class Parser extends Transform {
} }
return 0 return 0
} }
__isEscape(buf, pos, chr){
const {escape} = this.options
if(escape === null) return false
const l = escape.length
if(escape[0] === chr){
for(let i = 0; i < l; i++){
if(escape[i] !== buf[pos+i]){
return false
}
}
return true
}
return false
}
__isQuote(buf, pos){
const {quote} = this.options
if(quote === null) return false
const l = quote.length
for(let i = 0; i < l; i++){
if(quote[i] !== buf[pos+i]){
return false
}
}
return true
}
__autoDiscoverRowDelimiter(buf, pos){ __autoDiscoverRowDelimiter(buf, pos){
const {encoding} = this.options
const chr = buf[pos] const chr = buf[pos]
if(chr === cr){ if(chr === cr){
if(buf[pos+1] === nl){ if(buf[pos+1] === nl){
this.options.record_delimiter.push(Buffer.from('\r\n')) this.options.record_delimiter.push(Buffer.from('\r\n', encoding))
this.state.recordDelimiterMaxLength = 2 this.state.recordDelimiterMaxLength = 2
return 2 return 2
}else{ }else{
this.options.record_delimiter.push(Buffer.from('\r')) this.options.record_delimiter.push(Buffer.from('\r', encoding))
this.state.recordDelimiterMaxLength = 1 this.state.recordDelimiterMaxLength = 1
return 1 return 1
} }
}else if(chr === nl){ }else if(chr === nl){
this.options.record_delimiter.push(Buffer.from('\n')) this.options.record_delimiter.push(Buffer.from('\n', encoding))
this.state.recordDelimiterMaxLength = 1 this.state.recordDelimiterMaxLength = 1
return 1 return 1
} }
@ -11830,6 +11893,7 @@ class Parser extends Transform {
) : ) :
this.state.record.length, this.state.record.length,
empty_lines: this.info.empty_lines, empty_lines: this.info.empty_lines,
error: this.state.error,
header: columns === true, header: columns === true,
index: this.state.record.length, index: this.state.record.length,
invalid_field_length: this.info.invalid_field_length, invalid_field_length: this.info.invalid_field_length,
@ -11855,7 +11919,7 @@ const parse = function(){
throw new CsvError('CSV_INVALID_ARGUMENT', [ throw new CsvError('CSV_INVALID_ARGUMENT', [
'Invalid argument:', 'Invalid argument:',
`got ${JSON.stringify(argument)} at index ${i}` `got ${JSON.stringify(argument)} at index ${i}`
]) ], this.options)
} }
} }
const parser = new Parser(options) const parser = new Parser(options)
@ -11894,7 +11958,7 @@ const parse = function(){
} }
class CsvError extends Error { class CsvError extends Error {
constructor(code, message, ...contexts) { constructor(code, message, options, ...contexts) {
if(Array.isArray(message)) message = message.join(' ') if(Array.isArray(message)) message = message.join(' ')
super(message) super(message)
if(Error.captureStackTrace !== undefined){ if(Error.captureStackTrace !== undefined){
@ -11904,7 +11968,7 @@ class CsvError extends Error {
for(const context of contexts){ for(const context of contexts){
for(const key in context){ for(const key in context){
const value = context[key] const value = context[key]
this[key] = Buffer.isBuffer(value) ? value.toString() : value == null ? value : JSON.parse(JSON.stringify(value)) this[key] = Buffer.isBuffer(value) ? value.toString(options.encoding) : value == null ? value : JSON.parse(JSON.stringify(value))
} }
} }
} }
@ -13192,6 +13256,20 @@ class ResizeableBuffer{
this.buf = Buffer.alloc(size) this.buf = Buffer.alloc(size)
} }
prepend(val){ prepend(val){
if(Buffer.isBuffer(val)){
const length = this.length + val.length
if(length >= this.size){
this.resize()
if(length >= this.size){
throw Error('INVALID_BUFFER_STATE')
}
}
const buf = this.buf
this.buf = Buffer.alloc(this.size)
val.copy(this.buf, 0)
buf.copy(this.buf, val.length)
this.length += val.length
}else{
const length = this.length++ const length = this.length++
if(length === this.size){ if(length === this.size){
this.resize() this.resize()
@ -13200,6 +13278,7 @@ class ResizeableBuffer{
this.buf[0] = val this.buf[0] = val
buf.copy(this.buf,1, 0, length) buf.copy(this.buf,1, 0, length)
} }
}
append(val){ append(val){
const length = this.length++ const length = this.length++
if(length === this.size){ if(length === this.size){
@ -13217,11 +13296,15 @@ class ResizeableBuffer{
this.buf.copy(buf,0, 0, length) this.buf.copy(buf,0, 0, length)
this.buf = buf this.buf = buf
} }
toString(){ toString(encoding){
return this.buf.slice(0, this.length).toString() if(encoding){
return this.buf.slice(0, this.length).toString(encoding)
}else{
return Uint8Array.prototype.slice.call(this.buf.slice(0, this.length))
}
} }
toJSON(){ toJSON(){
return this.toString() return this.toString('utf8')
} }
reset(){ reset(){
this.length = 0 this.length = 0