diff --git a/.nycrc.json b/.nycrc.json index 69f58e8d..89492926 100644 --- a/.nycrc.json +++ b/.nycrc.json @@ -1,6 +1,6 @@ { - "include": ["src/**/*.ts"], - "extension": [".ts"], + "include": ["src/**/*.ts", "src/**/*.mts"], + "extension": [".ts", ".mtx"], "reporter": [], "sourceMap": true, "instrument": true diff --git a/.vscode/settings.json b/.vscode/settings.json index 0bfd00ab..5a870670 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -6,6 +6,7 @@ "source.fixAll.eslint": true }, "cSpell.words": [ - "tsdoc" + "tsdoc", + "whatwg" ] } diff --git a/CHANGELOG.md b/CHANGELOG.md index 4cd42fa0..b29eaa2b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # This is the revision history of @msgpack/msgpack +## NEXT + +* Drop IE11 support ([#221](https://github.com/msgpack/msgpack-javascript/pull/221)) + * It also fixes [feature request: option to disable TEXT_ENCODING env check #219](https://github.com/msgpack/msgpack-javascript/issues/219) +* + ## 2.8.0 2022-09-02 * Let `Encoder#encode()` return a copy of the internal buffer, instead of the reference of the buffer (fix #212). diff --git a/benchmark/decode-string.ts b/benchmark/decode-string.ts index 7e57d043..a6ea1467 100644 --- a/benchmark/decode-string.ts +++ b/benchmark/decode-string.ts @@ -5,7 +5,7 @@ import { utf8EncodeJs, utf8Count, utf8DecodeJs, utf8DecodeTD } from "../src/util import Benchmark from "benchmark"; for (const baseStr of ["A", "あ", "🌏"]) { - const dataSet = [10, 100, 200, 1_000, 10_000, 100_000].map((n) => { + const dataSet = [10, 100, 500, 1_000].map((n) => { return baseStr.repeat(n); }); @@ -14,7 +14,7 @@ for (const baseStr of ["A", "あ", "🌏"]) { const bytes = new Uint8Array(new ArrayBuffer(byteLength)); utf8EncodeJs(str, bytes, 0); - console.log(`\n## string "${baseStr}" x ${str.length} (byteLength=${byteLength})\n`); + console.log(`\n## string "${baseStr}" (strLength=${str.length}, byteLength=${byteLength})\n`); const suite = new Benchmark.Suite(); diff --git a/benchmark/encode-string.ts b/benchmark/encode-string.ts index 47662dd1..3f6aac6c 100644 --- a/benchmark/encode-string.ts +++ b/benchmark/encode-string.ts @@ -5,7 +5,7 @@ import { utf8EncodeJs, utf8Count, utf8EncodeTE } from "../src/utils/utf8"; import Benchmark from "benchmark"; for (const baseStr of ["A", "あ", "🌏"]) { - const dataSet = [10, 100, 200, 1_000, 10_000, 100_000].map((n) => { + const dataSet = [10, 30, 50, 100].map((n) => { return baseStr.repeat(n); }); @@ -13,7 +13,7 @@ for (const baseStr of ["A", "あ", "🌏"]) { const byteLength = utf8Count(str); const buffer = new Uint8Array(byteLength); - console.log(`\n## string "${baseStr}" x ${str.length} (byteLength=${byteLength})\n`); + console.log(`\n## string "${baseStr}" (strLength=${str.length}, byteLength=${byteLength})\n`); const suite = new Benchmark.Suite(); diff --git a/package.json b/package.json index f75863b7..587615af 100644 --- a/package.json +++ b/package.json @@ -16,12 +16,7 @@ "prepublishOnly": "run-p 'test:dist:*' && npm run test:browser", "clean": "rimraf build dist dist.*", "test": "mocha 'test/**/*.test.ts'", - "test:purejs": "TEXT_ENCODING=never mocha 'test/**/*.test.ts'", - "test:te": "TEXT_ENCODING=force mocha 'test/**/*.test.ts'", - "test:dist:purejs": "TS_NODE_PROJECT=tsconfig.test-dist-es5-purejs.json npm run test:purejs -- --reporter=dot", - "test:cover": "npm run cover:clean && npm-run-all 'test:cover:*' && npm run cover:report", - "test:cover:purejs": "npx nyc --no-clean npm run test:purejs", - "test:cover:te": "npx nyc --no-clean npm run test:te", + "test:cover": "npm run cover:clean && npx nyc --no-clean npm run 'test' && npm run cover:report", "test:deno": "deno test test/deno_test.ts", "test:fuzz": "npm exec --yes -- jsfuzz@git+https://gitlab.com/gitlab-org/security-products/analyzers/fuzzers/jsfuzz.git --fuzzTime 60 --no-versifier test/decode.jsfuzz.js corpus", "cover:clean": "rimraf .nyc_output coverage/", diff --git a/src/Decoder.ts b/src/Decoder.ts index 7045d437..173142c7 100644 --- a/src/Decoder.ts +++ b/src/Decoder.ts @@ -1,7 +1,7 @@ import { prettyByte } from "./utils/prettyByte"; import { ExtensionCodec, ExtensionCodecType } from "./ExtensionCodec"; import { getInt64, getUint64, UINT32_MAX } from "./utils/int"; -import { utf8DecodeJs, TEXT_DECODER_THRESHOLD, utf8DecodeTD } from "./utils/utf8"; +import { utf8Decode } from "./utils/utf8"; import { createDataView, ensureUint8Array } from "./utils/typedArrays"; import { CachedKeyDecoder, KeyDecoder } from "./CachedKeyDecoder"; import { DecodeError } from "./DecodeError"; @@ -38,18 +38,16 @@ const HEAD_BYTE_REQUIRED = -1; const EMPTY_VIEW = new DataView(new ArrayBuffer(0)); const EMPTY_BYTES = new Uint8Array(EMPTY_VIEW.buffer); -// IE11: Hack to support IE11. -// IE11: Drop this hack and just use RangeError when IE11 is obsolete. -export const DataViewIndexOutOfBoundsError: typeof Error = (() => { - try { - // IE11: The spec says it should throw RangeError, - // IE11: but in IE11 it throws TypeError. - EMPTY_VIEW.getInt8(0); - } catch (e: any) { - return e.constructor; +try { + // IE11: The spec says it should throw RangeError, + // IE11: but in IE11 it throws TypeError. + EMPTY_VIEW.getInt8(0); +} catch (e) { + if (!(e instanceof RangeError)) { + throw new Error("This module is not supported in the current JavaScript engine because DataView does not throw RangeError on out-of-bounds access"); } - throw new Error("never reached"); -})(); +} +export const DataViewIndexOutOfBoundsError = RangeError; const MORE_DATA = new DataViewIndexOutOfBoundsError("Insufficient data"); @@ -507,10 +505,8 @@ export class Decoder { let object: string; if (this.stateIsMapKey() && this.keyDecoder?.canBeCached(byteLength)) { object = this.keyDecoder.decode(this.bytes, offset, byteLength); - } else if (byteLength > TEXT_DECODER_THRESHOLD) { - object = utf8DecodeTD(this.bytes, offset, byteLength); } else { - object = utf8DecodeJs(this.bytes, offset, byteLength); + object = utf8Decode(this.bytes, offset, byteLength); } this.pos += headerOffset + byteLength; return object; diff --git a/src/Encoder.ts b/src/Encoder.ts index 14a16322..c2fdb4ea 100644 --- a/src/Encoder.ts +++ b/src/Encoder.ts @@ -1,4 +1,4 @@ -import { utf8EncodeJs, utf8Count, TEXT_ENCODER_THRESHOLD, utf8EncodeTE } from "./utils/utf8"; +import { utf8Count, utf8Encode } from "./utils/utf8"; import { ExtensionCodec, ExtensionCodecType } from "./ExtensionCodec"; import { setInt64, setUint64 } from "./utils/int"; import { ensureUint8Array } from "./utils/typedArrays"; @@ -177,21 +177,12 @@ export class Encoder { private encodeString(object: string) { const maxHeaderSize = 1 + 4; - const strLength = object.length; - - if (strLength > TEXT_ENCODER_THRESHOLD) { - const byteLength = utf8Count(object); - this.ensureBufferSizeToWrite(maxHeaderSize + byteLength); - this.writeStringHeader(byteLength); - utf8EncodeTE(object, this.bytes, this.pos); - this.pos += byteLength; - } else { - const byteLength = utf8Count(object); - this.ensureBufferSizeToWrite(maxHeaderSize + byteLength); - this.writeStringHeader(byteLength); - utf8EncodeJs(object, this.bytes, this.pos); - this.pos += byteLength; - } + + const byteLength = utf8Count(object); + this.ensureBufferSizeToWrite(maxHeaderSize + byteLength); + this.writeStringHeader(byteLength); + utf8Encode(object, this.bytes, this.pos); + this.pos += byteLength; } private encodeObject(object: unknown, depth: number) { diff --git a/src/utils/utf8.ts b/src/utils/utf8.ts index 4d08690c..f4068c34 100644 --- a/src/utils/utf8.ts +++ b/src/utils/utf8.ts @@ -1,10 +1,3 @@ -/* eslint-disable @typescript-eslint/no-unnecessary-condition */ -import { UINT32_MAX } from "./int"; - -const TEXT_ENCODING_AVAILABLE = - (typeof process === "undefined" || process?.env?.["TEXT_ENCODING"] !== "never") && - typeof TextEncoder !== "undefined" && - typeof TextDecoder !== "undefined"; export function utf8Count(str: string): number { const strLength = str.length; @@ -89,22 +82,30 @@ export function utf8EncodeJs(str: string, output: Uint8Array, outputOffset: numb } } -const sharedTextEncoder = TEXT_ENCODING_AVAILABLE ? new TextEncoder() : undefined; -export const TEXT_ENCODER_THRESHOLD = !TEXT_ENCODING_AVAILABLE - ? UINT32_MAX - : typeof process !== "undefined" && process?.env?.["TEXT_ENCODING"] !== "force" - ? 200 - : 0; +// TextEncoder and TextDecoder are standardized in whatwg encoding: +// https://encoding.spec.whatwg.org/ +// and available in all the modern browsers: +// https://caniuse.com/textencoder +// They are available in Node.js since v12 LTS as well: +// https://nodejs.org/api/globals.html#textencoder -function utf8EncodeTEencode(str: string, output: Uint8Array, outputOffset: number): void { - output.set(sharedTextEncoder!.encode(str), outputOffset); -} +const sharedTextEncoder = new TextEncoder(); + +// This threshold should be determined by benchmarking, which might vary in engines and input data. +// Run `npx ts-node benchmark/encode-string.ts` for details. +const TEXT_ENCODER_THRESHOLD = 50; -function utf8EncodeTEencodeInto(str: string, output: Uint8Array, outputOffset: number): void { - sharedTextEncoder!.encodeInto(str, output.subarray(outputOffset)); +export function utf8EncodeTE(str: string, output: Uint8Array, outputOffset: number): void { + sharedTextEncoder.encodeInto(str, output.subarray(outputOffset)); } -export const utf8EncodeTE = sharedTextEncoder?.encodeInto ? utf8EncodeTEencodeInto : utf8EncodeTEencode; +export function utf8Encode(str: string, output: Uint8Array, outputOffset: number): void { + if (str.length > TEXT_ENCODER_THRESHOLD) { + utf8EncodeTE(str, output, outputOffset); + } else { + utf8EncodeJs(str, output, outputOffset); + } +} const CHUNK_SIZE = 0x1_000; @@ -157,14 +158,21 @@ export function utf8DecodeJs(bytes: Uint8Array, inputOffset: number, byteLength: return result; } -const sharedTextDecoder = TEXT_ENCODING_AVAILABLE ? new TextDecoder() : null; -export const TEXT_DECODER_THRESHOLD = !TEXT_ENCODING_AVAILABLE - ? UINT32_MAX - : typeof process !== "undefined" && process?.env?.["TEXT_DECODER"] !== "force" - ? 200 - : 0; +const sharedTextDecoder = new TextDecoder(); + +// This threshold should be determined by benchmarking, which might vary in engines and input data. +// Run `npx ts-node benchmark/decode-string.ts` for details. +const TEXT_DECODER_THRESHOLD = 200; export function utf8DecodeTD(bytes: Uint8Array, inputOffset: number, byteLength: number): string { const stringBytes = bytes.subarray(inputOffset, inputOffset + byteLength); - return sharedTextDecoder!.decode(stringBytes); + return sharedTextDecoder.decode(stringBytes); +} + +export function utf8Decode(bytes: Uint8Array, inputOffset: number, byteLength: number): string { + if (byteLength > TEXT_DECODER_THRESHOLD) { + return utf8DecodeTD(bytes, inputOffset, byteLength); + } else { + return utf8DecodeJs(bytes, inputOffset, byteLength); + } } diff --git a/tsconfig.test-dist-es5-purejs.json b/tsconfig.test-dist-es5-purejs.json deleted file mode 100644 index f431a1e8..00000000 --- a/tsconfig.test-dist-es5-purejs.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "extends": "./tsconfig.es5.json", - "compilerOptions": { - "outDir": "./build/test-dist-es5-purejs", - "noImplicitAny": false, - "paths": { - "@msgpack/msgpack": ["./dist.es5+umd/msgpack"] - } - } -}