diff --git a/NOTICE b/NOTICE index a260a70483..bf9e9abd64 100644 --- a/NOTICE +++ b/NOTICE @@ -24,6 +24,7 @@ under the licensing terms detailed in LICENSE: * Julien Letellier * Guido Zuidhof * ncave <777696+ncave@users.noreply.github.com> +* Andrew Davis Portions of this software are derived from third-party works licensed under the following terms: diff --git a/src/tokenizer.ts b/src/tokenizer.ts index bcd9055931..02f8dae7da 100644 --- a/src/tokenizer.ts +++ b/src/tokenizer.ts @@ -1494,37 +1494,78 @@ export class Tokenizer extends DiagnosticEmitter { } readDecimalFloat(): f64 { - // TODO: numeric separators (parseFloat can't handle these) var text = this.source.text; - var pos = this.pos; var end = this.end; - var start = pos; - while (pos < end && isDecimalDigit(text.charCodeAt(pos))) { - ++pos; - } - if (pos < end && text.charCodeAt(pos) == CharCode.DOT) { - ++pos; - while (pos < end && isDecimalDigit(text.charCodeAt(pos))) { - ++pos; - } + var start = this.pos; + var sepCount = 0; + + sepCount += this.readDecimalFloatPartial(false); + if (this.pos < end && text.charCodeAt(this.pos) == CharCode.DOT) { + ++this.pos; + sepCount += this.readDecimalFloatPartial(); } - if (pos < end) { - let c = text.charCodeAt(pos); + if (this.pos < end) { + let c = text.charCodeAt(this.pos); if ((c | 32) == CharCode.e) { if ( - ++pos < end && - (c = text.charCodeAt(pos)) == CharCode.MINUS || c == CharCode.PLUS && - isDecimalDigit(text.charCodeAt(pos + 1)) + ++this.pos < end && + (c = text.charCodeAt(this.pos)) == CharCode.MINUS || c == CharCode.PLUS && + isDecimalDigit(text.charCodeAt(this.pos + 1)) ) { - ++pos; + ++this.pos; } - while (pos < end && isDecimalDigit(text.charCodeAt(pos))) { - ++pos; + sepCount += this.readDecimalFloatPartial(); + } + } + let result = text.substring(start, this.pos); + if (sepCount > 0) result = result.replaceAll("_", ""); + return parseFloat(result); + } + + /** Reads past one section of a decimal float literal. Returns the number of separators encountered. */ + private readDecimalFloatPartial(allowLeadingZeroSep: bool = true): u32 { + var text = this.source.text; + var pos = this.pos; + var start = pos; + var end = this.end; + var sepEnd = start; + var sepCount = 0; + + while (pos < end) { + let c = text.charCodeAt(pos); + + if (c == CharCode._) { + if (sepEnd == pos) { + this.error( + sepEnd == start + ? DiagnosticCode.Numeric_separators_are_not_allowed_here + : DiagnosticCode.Multiple_consecutive_numeric_separators_are_not_permitted, + this.range(pos) + ); + } else if (!allowLeadingZeroSep && pos - 1 == start && text.charCodeAt(pos - 1) == CharCode._0) { + this.error( + DiagnosticCode.Numeric_separators_are_not_allowed_here, + this.range(pos) + ); } + sepEnd = pos + 1; + ++sepCount; + } else if (!isDecimalDigit(c)) { + break; } + + ++pos; + } + + if (pos != start && sepEnd == pos) { + this.error( + DiagnosticCode.Numeric_separators_are_not_allowed_here, + this.range(sepEnd - 1) + ); } + this.pos = pos; - return parseFloat(text.substring(start, pos)); + return sepCount; } readHexFloat(): f64 { diff --git a/src/tsconfig.json b/src/tsconfig.json index 23d36c7d41..7100628f62 100644 --- a/src/tsconfig.json +++ b/src/tsconfig.json @@ -4,7 +4,7 @@ "outDir": "../out", "allowJs": false, "sourceMap": true, - "target": "ES2016", + "target": "esnext", "strict": true }, "include": [ diff --git a/std/portable.json b/std/portable.json index e29a93d27e..1a1e427d21 100644 --- a/std/portable.json +++ b/std/portable.json @@ -7,6 +7,7 @@ "downlevelIteration": true, "preserveConstEnums": true, "typeRoots": [ "types" ], - "types": [ "portable" ] + "types": [ "portable" ], + "lib": ["esnext", "esnext.string"] } } diff --git a/tests/parser/numeric-separators.ts b/tests/parser/numeric-separators.ts index c296672c09..3341aef2fa 100644 --- a/tests/parser/numeric-separators.ts +++ b/tests/parser/numeric-separators.ts @@ -2,6 +2,12 @@ 0b01_01_01; 0o12_12_12; 0x23_23_23; +1_000_000.1234_1234; +1_0e1_0; +1_000_000e-1_0; +0.0_0; +1_0e0_0; +1_0e0_1; // error cases that should still continue parsing: @@ -16,3 +22,19 @@ 0x23_23_23_; // 6188 0x23__23_23; // 6189 + +1000_.1234; // 6188 +1000._1234; // 6188 +1000.1234_; // 6188 + +10__00.1234; // 6189 +1000.12__34; // 6189 + +1_e2; // 6188 +1e_2; // 6188 +1e2_; // 6188 +1e-1__0; // 6189 + +0_0.0; // 6188 +0_0.0_0; // 6188 +0_0e0_0; // 6188 \ No newline at end of file diff --git a/tests/parser/numeric-separators.ts.fixture.ts b/tests/parser/numeric-separators.ts.fixture.ts index 57af00bb90..a0cb362767 100644 --- a/tests/parser/numeric-separators.ts.fixture.ts +++ b/tests/parser/numeric-separators.ts.fixture.ts @@ -2,6 +2,12 @@ 21; 41610; 2302755; +1000000.12341234; +100000000000; +0.0001; +0; +10; +100; 111111; 111111; 21; @@ -10,11 +16,35 @@ 41610; 2302755; 2302755; -// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(8,9+0) -// ERROR 6189: "Multiple consecutive numeric separators are not permitted." in numeric-separators.ts(9,4+0) -// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(11,11+0) -// ERROR 6189: "Multiple consecutive numeric separators are not permitted." in numeric-separators.ts(12,6+0) -// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(14,11+0) -// ERROR 6189: "Multiple consecutive numeric separators are not permitted." in numeric-separators.ts(15,6+0) +1000.1234; +1000.1234; +1000.1234; +1000.1234; +1000.1234; +100; +100; +100; +1e-10; +0; +0; +0; +// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(14,9+0) +// ERROR 6189: "Multiple consecutive numeric separators are not permitted." in numeric-separators.ts(15,4+0) // ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(17,11+0) // ERROR 6189: "Multiple consecutive numeric separators are not permitted." in numeric-separators.ts(18,6+0) +// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(20,11+0) +// ERROR 6189: "Multiple consecutive numeric separators are not permitted." in numeric-separators.ts(21,6+0) +// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(23,11+0) +// ERROR 6189: "Multiple consecutive numeric separators are not permitted." in numeric-separators.ts(24,6+0) +// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(26,5+0) +// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(27,6+0) +// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(28,10+0) +// ERROR 6189: "Multiple consecutive numeric separators are not permitted." in numeric-separators.ts(30,4+0) +// ERROR 6189: "Multiple consecutive numeric separators are not permitted." in numeric-separators.ts(31,9+0) +// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(33,2+0) +// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(34,3+0) +// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(35,4+0) +// ERROR 6189: "Multiple consecutive numeric separators are not permitted." in numeric-separators.ts(36,6+0) +// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(38,2+0) +// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(39,2+0) +// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(40,2+0)