Initial commit
This commit is contained in:
chinahu-woker
+191
@@ -0,0 +1,191 @@
|
||||
const Stream = require('./'); const { DEFAULT_ENCODING, getEncoding } = Stream;
|
||||
const { end_of_stream, finished, codePointsToString } = require('../utils');
|
||||
const { decoders } = require('../table');
|
||||
|
||||
// 8.1 Interface TextDecoder
|
||||
|
||||
class TextDecoder {
|
||||
/**
|
||||
* @param {string=} label The label of the encoding; defaults to 'utf-8'.
|
||||
* @param {Object=} options
|
||||
*/
|
||||
constructor(label = DEFAULT_ENCODING, options = {}) {
|
||||
// A TextDecoder object has an associated encoding, decoder,
|
||||
// stream, ignore BOM flag (initially unset), BOM seen flag
|
||||
// (initially unset), error mode (initially replacement), and do
|
||||
// not flush flag (initially unset).
|
||||
|
||||
/** @private */
|
||||
this._encoding = null
|
||||
/** @private @type {?Decoder} */
|
||||
this._decoder = null
|
||||
/** @private @type {boolean} */
|
||||
this._ignoreBOM = false
|
||||
/** @private @type {boolean} */
|
||||
this._BOMseen = false
|
||||
/** @private @type {string} */
|
||||
this._error_mode = 'replacement'
|
||||
/** @private @type {boolean} */
|
||||
this._do_not_flush = false
|
||||
|
||||
|
||||
// 1. Let encoding be the result of getting an encoding from
|
||||
// label.
|
||||
const encoding = getEncoding(label)
|
||||
|
||||
// 2. If encoding is failure or replacement, throw a RangeError.
|
||||
if (encoding === null || encoding.name == 'replacement')
|
||||
throw RangeError('Unknown encoding: ' + label)
|
||||
if (!decoders[encoding.name]) {
|
||||
throw Error('Decoder not present.' +
|
||||
' Did you forget to include encoding-indexes.js first?')
|
||||
}
|
||||
|
||||
// 4. Set dec's encoding to encoding.
|
||||
this._encoding = encoding
|
||||
|
||||
// 5. If options's fatal member is true, set dec's error mode to
|
||||
// fatal.
|
||||
if (options['fatal'])
|
||||
this._error_mode = 'fatal'
|
||||
|
||||
// 6. If options's ignoreBOM member is true, set dec's ignore BOM
|
||||
// flag.
|
||||
if (options['ignoreBOM'])
|
||||
this._ignoreBOM = true
|
||||
}
|
||||
|
||||
get encoding() {
|
||||
return this._encoding.name.toLowerCase()
|
||||
}
|
||||
get fatal() {
|
||||
return this._error_mode === 'fatal'
|
||||
}
|
||||
get ignoreBOM() {
|
||||
return this._ignoreBOM
|
||||
}
|
||||
/**
|
||||
* @param {BufferSource=} input The buffer of bytes to decode.
|
||||
* @param {Object=} options
|
||||
* @return The decoded string.
|
||||
*/
|
||||
decode(input, options = {}) {
|
||||
let bytes
|
||||
if (typeof input === 'object' && input instanceof ArrayBuffer) {
|
||||
bytes = new Uint8Array(input)
|
||||
} else if (typeof input === 'object' && 'buffer' in input &&
|
||||
input.buffer instanceof ArrayBuffer) {
|
||||
bytes = new Uint8Array(input.buffer,
|
||||
input.byteOffset,
|
||||
input.byteLength)
|
||||
} else {
|
||||
bytes = new Uint8Array(0)
|
||||
}
|
||||
|
||||
// 1. If the do not flush flag is unset, set decoder to a new
|
||||
// encoding's decoder, set stream to a new stream, and unset the
|
||||
// BOM seen flag.
|
||||
if (!this._do_not_flush) {
|
||||
this._decoder = decoders[this._encoding.name]({
|
||||
fatal: this._error_mode === 'fatal' })
|
||||
this._BOMseen = false
|
||||
}
|
||||
|
||||
// 2. If options's stream is true, set the do not flush flag, and
|
||||
// unset the do not flush flag otherwise.
|
||||
this._do_not_flush = Boolean(options['stream'])
|
||||
|
||||
// 3. If input is given, push a copy of input to stream.
|
||||
// TODO: Align with spec algorithm - maintain stream on instance.
|
||||
const input_stream = new Stream(bytes)
|
||||
|
||||
// 4. Let output be a new stream.
|
||||
const output = []
|
||||
|
||||
/** @type {?(number|!Array.<number>)} */
|
||||
let result
|
||||
|
||||
// 5. While true:
|
||||
while (true) {
|
||||
// 1. Let token be the result of reading from stream.
|
||||
const token = input_stream.read()
|
||||
|
||||
// 2. If token is end-of-stream and the do not flush flag is
|
||||
// set, return output, serialized.
|
||||
// TODO: Align with spec algorithm.
|
||||
if (token === end_of_stream)
|
||||
break
|
||||
|
||||
// 3. Otherwise, run these subsubsteps:
|
||||
|
||||
// 1. Let result be the result of processing token for decoder,
|
||||
// stream, output, and error mode.
|
||||
result = this._decoder.handler(input_stream, token)
|
||||
|
||||
// 2. If result is finished, return output, serialized.
|
||||
if (result === finished)
|
||||
break
|
||||
|
||||
if (result !== null) {
|
||||
if (Array.isArray(result))
|
||||
output.push.apply(output, /**@type {!Array.<number>}*/(result))
|
||||
else
|
||||
output.push(result)
|
||||
}
|
||||
|
||||
// 3. Otherwise, if result is error, throw a TypeError.
|
||||
// (Thrown in handler)
|
||||
|
||||
// 4. Otherwise, do nothing.
|
||||
}
|
||||
// TODO: Align with spec algorithm.
|
||||
if (!this._do_not_flush) {
|
||||
do {
|
||||
result = this._decoder.handler(input_stream, input_stream.read())
|
||||
if (result === finished)
|
||||
break
|
||||
if (result === null)
|
||||
continue
|
||||
if (Array.isArray(result))
|
||||
output.push.apply(output, /**@type {!Array.<number>}*/(result))
|
||||
else
|
||||
output.push(result)
|
||||
} while (!input_stream.endOfStream())
|
||||
this._decoder = null
|
||||
}
|
||||
|
||||
return this.serializeStream(output)
|
||||
}
|
||||
// A TextDecoder object also has an associated serialize stream
|
||||
// algorithm...
|
||||
/**
|
||||
* @param {!Array.<number>} stream
|
||||
*/
|
||||
serializeStream(stream) {
|
||||
// 1. Let token be the result of reading from stream.
|
||||
// (Done in-place on array, rather than as a stream)
|
||||
|
||||
// 2. If encoding is UTF-8, UTF-16BE, or UTF-16LE, and ignore
|
||||
// BOM flag and BOM seen flag are unset, run these subsubsteps:
|
||||
if (['UTF-8', 'UTF-16LE', 'UTF-16BE'].includes(this._encoding.name) &&
|
||||
!this._ignoreBOM && !this._BOMseen) {
|
||||
if (stream.length > 0 && stream[0] === 0xFEFF) {
|
||||
// 1. If token is U+FEFF, set BOM seen flag.
|
||||
this._BOMseen = true
|
||||
stream.shift()
|
||||
} else if (stream.length > 0) {
|
||||
// 2. Otherwise, if token is not end-of-stream, set BOM seen
|
||||
// flag and append token to stream.
|
||||
this._BOMseen = true
|
||||
} else {
|
||||
// 3. Otherwise, if token is not end-of-stream, append token
|
||||
// to output.
|
||||
// (no-op)
|
||||
}
|
||||
}
|
||||
// 4. Otherwise, return output.
|
||||
return codePointsToString(stream)
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = TextDecoder
|
||||
+108
@@ -0,0 +1,108 @@
|
||||
const Stream = require('./'); const { DEFAULT_ENCODING, getEncoding } = Stream;
|
||||
const { end_of_stream, finished, stringToCodePoints } = require('../utils');
|
||||
const { encoders } = require('../table');
|
||||
|
||||
// 8.2 Interface TextEncoder
|
||||
|
||||
class TextEncoder {
|
||||
/**
|
||||
* @param {string=} label The label of the encoding. NONSTANDARD.
|
||||
* @param {Object=} [options] NONSTANDARD.
|
||||
*/
|
||||
constructor(label, options = {}) {
|
||||
// A TextEncoder object has an associated encoding and encoder.
|
||||
|
||||
/** @private */
|
||||
this._encoding = null
|
||||
/** @private @type {?Encoder} */
|
||||
this._encoder = null
|
||||
|
||||
// Non-standard
|
||||
/** @private @type {boolean} */
|
||||
this._do_not_flush = false
|
||||
/** @private @type {string} */
|
||||
this._fatal = options['fatal'] ? 'fatal' : 'replacement'
|
||||
|
||||
// 2. Set enc's encoding to UTF-8's encoder.
|
||||
if (options['NONSTANDARD_allowLegacyEncoding']) {
|
||||
// NONSTANDARD behavior.
|
||||
label = label !== undefined ? String(label) : DEFAULT_ENCODING
|
||||
var encoding = getEncoding(label)
|
||||
if (encoding === null || encoding.name === 'replacement')
|
||||
throw RangeError('Unknown encoding: ' + label)
|
||||
if (!encoders[encoding.name]) {
|
||||
throw Error('Encoder not present.' +
|
||||
' Did you forget to include encoding-indexes.js first?')
|
||||
}
|
||||
this._encoding = encoding
|
||||
} else {
|
||||
// Standard behavior.
|
||||
this._encoding = getEncoding('utf-8')
|
||||
|
||||
if (label !== undefined && 'console' in global) {
|
||||
console.warn('TextEncoder constructor called with encoding label, '
|
||||
+ 'which is ignored.')
|
||||
}
|
||||
}
|
||||
}
|
||||
get encoding() {
|
||||
return this._encoding.name.toLowerCase()
|
||||
}
|
||||
/**
|
||||
* @param {string=} opt_string The string to encode.
|
||||
* @param {Object=} options
|
||||
*/
|
||||
encode(opt_string = '', options = {}) {
|
||||
// NOTE: This option is nonstandard. None of the encodings
|
||||
// permitted for encoding (i.e. UTF-8, UTF-16) are stateful when
|
||||
// the input is a USVString so streaming is not necessary.
|
||||
if (!this._do_not_flush)
|
||||
this._encoder = encoders[this._encoding.name]({
|
||||
fatal: this._fatal === 'fatal' })
|
||||
this._do_not_flush = Boolean(options['stream'])
|
||||
|
||||
// 1. Convert input to a stream.
|
||||
const input = new Stream(stringToCodePoints(opt_string))
|
||||
|
||||
// 2. Let output be a new stream
|
||||
const output = []
|
||||
|
||||
/** @type {?(number|!Array.<number>)} */
|
||||
var result
|
||||
// 3. While true, run these substeps:
|
||||
while (true) {
|
||||
// 1. Let token be the result of reading from input.
|
||||
var token = input.read()
|
||||
if (token === end_of_stream)
|
||||
break
|
||||
// 2. Let result be the result of processing token for encoder,
|
||||
// input, output.
|
||||
result = this._encoder.handler(input, token)
|
||||
if (result === finished)
|
||||
break
|
||||
if (Array.isArray(result))
|
||||
output.push.apply(output, /**@type {!Array.<number>}*/(result))
|
||||
else
|
||||
output.push(result)
|
||||
}
|
||||
// TODO: Align with spec algorithm.
|
||||
if (!this._do_not_flush) {
|
||||
while (true) {
|
||||
result = this._encoder.handler(input, input.read())
|
||||
if (result === finished)
|
||||
break
|
||||
if (Array.isArray(result))
|
||||
output.push.apply(output, /**@type {!Array.<number>}*/(result))
|
||||
else
|
||||
output.push(result)
|
||||
}
|
||||
this._encoder = null
|
||||
}
|
||||
// 3. If result is finished, convert output into a byte sequence,
|
||||
// and then return a Uint8Array object wrapping an ArrayBuffer
|
||||
// containing output.
|
||||
return new Uint8Array(output)
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = TextEncoder
|
||||
+122
@@ -0,0 +1,122 @@
|
||||
const { end_of_stream } = require('../utils');
|
||||
const { label_to_encoding } = require('../table');
|
||||
|
||||
class Stream {
|
||||
/**
|
||||
* A stream represents an ordered sequence of tokens.
|
||||
* @param {!(Array.<number>|Uint8Array)} tokens Array of tokens that provide
|
||||
* the stream.
|
||||
*/
|
||||
constructor(tokens) {
|
||||
this.tokens = [...tokens]
|
||||
// Reversed as push/pop is more efficient than shift/unshift.
|
||||
this.tokens.reverse()
|
||||
}
|
||||
/**
|
||||
* @returns True if end-of-stream has been hit.
|
||||
*/
|
||||
endOfStream() {
|
||||
return !this.tokens.length
|
||||
}
|
||||
/**
|
||||
* When a token is read from a stream, the first token in the
|
||||
* stream must be returned and subsequently removed, and
|
||||
* end-of-stream must be returned otherwise.
|
||||
*
|
||||
* @return Get the next token from the stream, or end_of_stream.
|
||||
*/
|
||||
read() {
|
||||
if (!this.tokens.length)
|
||||
return end_of_stream
|
||||
return this.tokens.pop()
|
||||
}
|
||||
/**
|
||||
* When one or more tokens are prepended to a stream, those tokens
|
||||
* must be inserted, in given order, before the first token in the
|
||||
* stream.
|
||||
*
|
||||
* @param {(number|!Array.<number>)} token The token(s) to prepend to the
|
||||
* stream.
|
||||
*/
|
||||
prepend(token) {
|
||||
if (Array.isArray(token)) {
|
||||
var tokens = /**@type {!Array.<number>}*/(token)
|
||||
while (tokens.length)
|
||||
this.tokens.push(tokens.pop())
|
||||
} else {
|
||||
this.tokens.push(token)
|
||||
}
|
||||
}
|
||||
/**
|
||||
* When one or more tokens are pushed to a stream, those tokens
|
||||
* must be inserted, in given order, after the last token in the
|
||||
* stream.
|
||||
*
|
||||
* @param {(number|!Array.<number>)} token The tokens(s) to push to the
|
||||
* stream.
|
||||
*/
|
||||
push(token) {
|
||||
if (Array.isArray(token)) {
|
||||
const tokens = /**@type {!Array.<number>}*/(token)
|
||||
while (tokens.length)
|
||||
this.tokens.unshift(tokens.shift())
|
||||
} else {
|
||||
this.tokens.unshift(token)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const DEFAULT_ENCODING = 'utf-8'
|
||||
|
||||
|
||||
/**
|
||||
* Returns the encoding for the label.
|
||||
* @param {string} label The encoding label.
|
||||
*/
|
||||
function getEncoding(label) {
|
||||
// 1. Remove any leading and trailing ASCII whitespace from label.
|
||||
label = String(label).trim().toLowerCase()
|
||||
|
||||
// 2. If label is an ASCII case-insensitive match for any of the
|
||||
// labels listed in the table below, return the corresponding
|
||||
// encoding, and failure otherwise.
|
||||
if (Object.prototype.hasOwnProperty.call(label_to_encoding, label)) {
|
||||
return label_to_encoding[label]
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
//
|
||||
// 5. Encodings
|
||||
//
|
||||
|
||||
// 5.1 Encoders and decoders
|
||||
|
||||
// /** @interface */
|
||||
// function Decoder() {}
|
||||
// Decoder.prototype = {
|
||||
// /**
|
||||
// * @param {Stream} stream The stream of bytes being decoded.
|
||||
// * @param {number} bite The next byte read from the stream.
|
||||
// * @return {?(number|!Array.<number>)} The next code point(s)
|
||||
// * decoded, or null if not enough data exists in the input
|
||||
// * stream to decode a complete code point, or |finished|.
|
||||
// */
|
||||
// handler: function(stream, bite) {},
|
||||
// }
|
||||
|
||||
// /** @interface */
|
||||
// function Encoder() {}
|
||||
// Encoder.prototype = {
|
||||
// /**
|
||||
// * @param {Stream} stream The stream of code points being encoded.
|
||||
// * @param {number} code_point Next code point read from the stream.
|
||||
// * @return {(number|!Array.<number>)} Byte(s) to emit, or |finished|.
|
||||
// */
|
||||
// handler: function(stream, code_point) {},
|
||||
// }
|
||||
|
||||
module.exports = Stream
|
||||
module.exports.DEFAULT_ENCODING = DEFAULT_ENCODING
|
||||
module.exports.getEncoding = getEncoding
|
||||
Reference in New Issue
Block a user