From 82d64d8597e80bb3391a399282ade54260253bbb Mon Sep 17 00:00:00 2001 From: BuildTools Date: Fri, 10 Jul 2020 14:17:20 -0400 Subject: [PATCH 1/9] Support separators in floating point literals --- NOTICE | 1 + src/tokenizer.ts | 48 +++++++++++++++---- tests/parser/numeric-separators.ts | 15 ++++++ tests/parser/numeric-separators.ts.fixture.ts | 29 +++++++++-- 4 files changed, 79 insertions(+), 14 deletions(-) diff --git a/NOTICE b/NOTICE index a260a70483..bf9e9abd64 100644 --- a/NOTICE +++ b/NOTICE @@ -24,6 +24,7 @@ under the licensing terms detailed in LICENSE: * Julien Letellier * Guido Zuidhof * ncave <777696+ncave@users.noreply.github.com> +* Andrew Davis Portions of this software are derived from third-party works licensed under the following terms: diff --git a/src/tokenizer.ts b/src/tokenizer.ts index 5dbb891e00..c0e90d2939 100644 --- a/src/tokenizer.ts +++ b/src/tokenizer.ts @@ -1462,18 +1462,13 @@ export class Tokenizer extends DiagnosticEmitter { } readDecimalFloat(): f64 { - // TODO: numeric separators (parseFloat can't handle these) var start = this.pos; var end = this.end; var text = this.source.text; - while (this.pos < end && isDecimalDigit(text.charCodeAt(this.pos))) { - ++this.pos; - } + this.readDecimalFloatPartial(); if (this.pos < end && text.charCodeAt(this.pos) == CharCode.DOT) { ++this.pos; - while (this.pos < end && isDecimalDigit(text.charCodeAt(this.pos))) { - ++this.pos; - } + this.readDecimalFloatPartial(); } if (this.pos < end) { let c = text.charCodeAt(this.pos); @@ -1485,12 +1480,45 @@ export class Tokenizer extends DiagnosticEmitter { ) { ++this.pos; } - while (this.pos < end && isDecimalDigit(text.charCodeAt(this.pos))) { - ++this.pos; + this.readDecimalFloatPartial(); + } + } + let floatString = text.substring(start, this.pos).replace(/_/g, ''); + return parseFloat(floatString); + } + + private readDecimalFloatPartial() { + let start = this.pos; + let end = this.end; + let text = this.source.text; + var sepEnd = start; + + while (this.pos < end) { + let c = text.charCodeAt(this.pos); + + if (c == CharCode._) { + if (sepEnd == this.pos) { + this.error( + sepEnd == start + ? DiagnosticCode.Numeric_separators_are_not_allowed_here + : DiagnosticCode.Multiple_consecutive_numeric_separators_are_not_permitted, + this.range(this.pos) + ); } + sepEnd = this.pos + 1; + } else if (!isDecimalDigit(c)) { + break; } + + ++this.pos; + } + + if (this.pos != start && sepEnd == this.pos) { + this.error( + DiagnosticCode.Numeric_separators_are_not_allowed_here, + this.range(sepEnd - 1) + ); } - return parseFloat(text.substring(start, this.pos)); } readHexFloat(): f64 { diff --git a/tests/parser/numeric-separators.ts b/tests/parser/numeric-separators.ts index c296672c09..5191f1db13 100644 --- a/tests/parser/numeric-separators.ts +++ b/tests/parser/numeric-separators.ts @@ -2,6 +2,9 @@ 0b01_01_01; 0o12_12_12; 0x23_23_23; +1_000_000.1234_1234; +1_0e1_0; +1_000_000e-1_0; // error cases that should still continue parsing: @@ -16,3 +19,15 @@ 0x23_23_23_; // 6188 0x23__23_23; // 6189 + +1000_.1234; // 6188 +1000._1234; // 6188 +1000.1234_; // 6188 + +10__00.1234; // 6189 +1000.12__34; // 6189 + +1_e2; // 6188 +1e_2; // 6188 +1e2_; // 6188 +1e-1__0; // 6189 \ No newline at end of file diff --git a/tests/parser/numeric-separators.ts.fixture.ts b/tests/parser/numeric-separators.ts.fixture.ts index 57af00bb90..dec2e83785 100644 --- a/tests/parser/numeric-separators.ts.fixture.ts +++ b/tests/parser/numeric-separators.ts.fixture.ts @@ -2,6 +2,9 @@ 21; 41610; 2302755; +1000000.12341234; +100000000000; +0.0001; 111111; 111111; 21; @@ -10,11 +13,29 @@ 41610; 2302755; 2302755; -// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(8,9+0) -// ERROR 6189: "Multiple consecutive numeric separators are not permitted." in numeric-separators.ts(9,4+0) -// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(11,11+0) -// ERROR 6189: "Multiple consecutive numeric separators are not permitted." in numeric-separators.ts(12,6+0) +1000.1234; +1000.1234; +1000.1234; +1000.1234; +1000.1234; +100; +100; +100; +1e-10; +// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(11,9+0) +// ERROR 6189: "Multiple consecutive numeric separators are not permitted." in numeric-separators.ts(12,4+0) // ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(14,11+0) // ERROR 6189: "Multiple consecutive numeric separators are not permitted." in numeric-separators.ts(15,6+0) // ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(17,11+0) // ERROR 6189: "Multiple consecutive numeric separators are not permitted." in numeric-separators.ts(18,6+0) +// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(20,11+0) +// ERROR 6189: "Multiple consecutive numeric separators are not permitted." in numeric-separators.ts(21,6+0) +// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(23,5+0) +// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(24,6+0) +// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(25,10+0) +// ERROR 6189: "Multiple consecutive numeric separators are not permitted." in numeric-separators.ts(27,4+0) +// ERROR 6189: "Multiple consecutive numeric separators are not permitted." in numeric-separators.ts(28,9+0) +// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(30,2+0) +// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(31,3+0) +// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(32,4+0) +// ERROR 6189: "Multiple consecutive numeric separators are not permitted." in numeric-separators.ts(33,6+0) From bc05fbfbc91ed2dc589626ca28e2b02ac33808dc Mon Sep 17 00:00:00 2001 From: BuildTools Date: Fri, 10 Jul 2020 14:34:56 -0400 Subject: [PATCH 2/9] Add missing return type and use replaceAll --- src/tokenizer.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tokenizer.ts b/src/tokenizer.ts index c0e90d2939..0ce0da2731 100644 --- a/src/tokenizer.ts +++ b/src/tokenizer.ts @@ -1483,11 +1483,11 @@ export class Tokenizer extends DiagnosticEmitter { this.readDecimalFloatPartial(); } } - let floatString = text.substring(start, this.pos).replace(/_/g, ''); + let floatString = text.substring(start, this.pos).replaceAll("_", ""); return parseFloat(floatString); } - private readDecimalFloatPartial() { + private readDecimalFloatPartial(): void { let start = this.pos; let end = this.end; let text = this.source.text; From d889fd12150af878346677189aad143ab7dab9bb Mon Sep 17 00:00:00 2001 From: BuildTools Date: Fri, 10 Jul 2020 15:03:07 -0400 Subject: [PATCH 3/9] @ts-ignore a replaceAll call --- src/tokenizer.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/tokenizer.ts b/src/tokenizer.ts index 0ce0da2731..edadc98df6 100644 --- a/src/tokenizer.ts +++ b/src/tokenizer.ts @@ -1483,6 +1483,7 @@ export class Tokenizer extends DiagnosticEmitter { this.readDecimalFloatPartial(); } } + // @ts-ignore let floatString = text.substring(start, this.pos).replaceAll("_", ""); return parseFloat(floatString); } From 0b91fe9bd2818ab46e711cda6f9c090a832f39c5 Mon Sep 17 00:00:00 2001 From: BuildTools Date: Fri, 10 Jul 2020 15:42:12 -0400 Subject: [PATCH 4/9] Update ts target to esnext --- src/tokenizer.ts | 1 - src/tsconfig.json | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/tokenizer.ts b/src/tokenizer.ts index edadc98df6..0ce0da2731 100644 --- a/src/tokenizer.ts +++ b/src/tokenizer.ts @@ -1483,7 +1483,6 @@ export class Tokenizer extends DiagnosticEmitter { this.readDecimalFloatPartial(); } } - // @ts-ignore let floatString = text.substring(start, this.pos).replaceAll("_", ""); return parseFloat(floatString); } diff --git a/src/tsconfig.json b/src/tsconfig.json index 23d36c7d41..7100628f62 100644 --- a/src/tsconfig.json +++ b/src/tsconfig.json @@ -4,7 +4,7 @@ "outDir": "../out", "allowJs": false, "sourceMap": true, - "target": "ES2016", + "target": "esnext", "strict": true }, "include": [ From 6c2440e070aba1182566eb20461703014cecc72f Mon Sep 17 00:00:00 2001 From: BuildTools Date: Fri, 10 Jul 2020 16:28:21 -0400 Subject: [PATCH 5/9] Disallow leading 0_ --- src/tokenizer.ts | 9 +++-- tests/parser/numeric-separators.ts | 9 ++++- tests/parser/numeric-separators.ts.fixture.ts | 35 ++++++++++++------- 3 files changed, 37 insertions(+), 16 deletions(-) diff --git a/src/tokenizer.ts b/src/tokenizer.ts index 0ce0da2731..61b023001e 100644 --- a/src/tokenizer.ts +++ b/src/tokenizer.ts @@ -1465,7 +1465,7 @@ export class Tokenizer extends DiagnosticEmitter { var start = this.pos; var end = this.end; var text = this.source.text; - this.readDecimalFloatPartial(); + this.readDecimalFloatPartial(false); if (this.pos < end && text.charCodeAt(this.pos) == CharCode.DOT) { ++this.pos; this.readDecimalFloatPartial(); @@ -1487,7 +1487,7 @@ export class Tokenizer extends DiagnosticEmitter { return parseFloat(floatString); } - private readDecimalFloatPartial(): void { + private readDecimalFloatPartial(allowLeadingZeroSep: boolean = true): void { let start = this.pos; let end = this.end; let text = this.source.text; @@ -1504,6 +1504,11 @@ export class Tokenizer extends DiagnosticEmitter { : DiagnosticCode.Multiple_consecutive_numeric_separators_are_not_permitted, this.range(this.pos) ); + } else if (!allowLeadingZeroSep && this.pos - 1 == start && text.charCodeAt(this.pos - 1) == CharCode._0) { + this.error( + DiagnosticCode.Numeric_separators_are_not_allowed_here, + this.range(this.pos) + ); } sepEnd = this.pos + 1; } else if (!isDecimalDigit(c)) { diff --git a/tests/parser/numeric-separators.ts b/tests/parser/numeric-separators.ts index 5191f1db13..3341aef2fa 100644 --- a/tests/parser/numeric-separators.ts +++ b/tests/parser/numeric-separators.ts @@ -5,6 +5,9 @@ 1_000_000.1234_1234; 1_0e1_0; 1_000_000e-1_0; +0.0_0; +1_0e0_0; +1_0e0_1; // error cases that should still continue parsing: @@ -30,4 +33,8 @@ 1_e2; // 6188 1e_2; // 6188 1e2_; // 6188 -1e-1__0; // 6189 \ No newline at end of file +1e-1__0; // 6189 + +0_0.0; // 6188 +0_0.0_0; // 6188 +0_0e0_0; // 6188 \ No newline at end of file diff --git a/tests/parser/numeric-separators.ts.fixture.ts b/tests/parser/numeric-separators.ts.fixture.ts index dec2e83785..a0cb362767 100644 --- a/tests/parser/numeric-separators.ts.fixture.ts +++ b/tests/parser/numeric-separators.ts.fixture.ts @@ -5,6 +5,9 @@ 1000000.12341234; 100000000000; 0.0001; +0; +10; +100; 111111; 111111; 21; @@ -22,20 +25,26 @@ 100; 100; 1e-10; -// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(11,9+0) -// ERROR 6189: "Multiple consecutive numeric separators are not permitted." in numeric-separators.ts(12,4+0) -// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(14,11+0) -// ERROR 6189: "Multiple consecutive numeric separators are not permitted." in numeric-separators.ts(15,6+0) +0; +0; +0; +// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(14,9+0) +// ERROR 6189: "Multiple consecutive numeric separators are not permitted." in numeric-separators.ts(15,4+0) // ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(17,11+0) // ERROR 6189: "Multiple consecutive numeric separators are not permitted." in numeric-separators.ts(18,6+0) // ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(20,11+0) // ERROR 6189: "Multiple consecutive numeric separators are not permitted." in numeric-separators.ts(21,6+0) -// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(23,5+0) -// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(24,6+0) -// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(25,10+0) -// ERROR 6189: "Multiple consecutive numeric separators are not permitted." in numeric-separators.ts(27,4+0) -// ERROR 6189: "Multiple consecutive numeric separators are not permitted." in numeric-separators.ts(28,9+0) -// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(30,2+0) -// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(31,3+0) -// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(32,4+0) -// ERROR 6189: "Multiple consecutive numeric separators are not permitted." in numeric-separators.ts(33,6+0) +// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(23,11+0) +// ERROR 6189: "Multiple consecutive numeric separators are not permitted." in numeric-separators.ts(24,6+0) +// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(26,5+0) +// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(27,6+0) +// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(28,10+0) +// ERROR 6189: "Multiple consecutive numeric separators are not permitted." in numeric-separators.ts(30,4+0) +// ERROR 6189: "Multiple consecutive numeric separators are not permitted." in numeric-separators.ts(31,9+0) +// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(33,2+0) +// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(34,3+0) +// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(35,4+0) +// ERROR 6189: "Multiple consecutive numeric separators are not permitted." in numeric-separators.ts(36,6+0) +// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(38,2+0) +// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(39,2+0) +// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(40,2+0) From 8117c06c070498a354b074111bfd444bb4ad754c Mon Sep 17 00:00:00 2001 From: BuildTools Date: Fri, 10 Jul 2020 16:49:10 -0400 Subject: [PATCH 6/9] Add ESNext libs to tsconfig --- src/tsconfig.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/tsconfig.json b/src/tsconfig.json index 7100628f62..a28bc8e325 100644 --- a/src/tsconfig.json +++ b/src/tsconfig.json @@ -5,7 +5,8 @@ "allowJs": false, "sourceMap": true, "target": "esnext", - "strict": true + "strict": true, + "lib": ["esnext", "esnext.string"] }, "include": [ "./**/*.ts" From bd6b5eb6d65751d0c91f21db6703499be2f3c5f8 Mon Sep 17 00:00:00 2001 From: BuildTools Date: Sat, 11 Jul 2020 16:15:12 -0400 Subject: [PATCH 7/9] Optimize tokenizing floats with separators --- src/tokenizer.ts | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/src/tokenizer.ts b/src/tokenizer.ts index 61b023001e..14dcfa6cf3 100644 --- a/src/tokenizer.ts +++ b/src/tokenizer.ts @@ -1465,10 +1465,11 @@ export class Tokenizer extends DiagnosticEmitter { var start = this.pos; var end = this.end; var text = this.source.text; - this.readDecimalFloatPartial(false); + + let hasSep = this.readDecimalFloatPartial(false); if (this.pos < end && text.charCodeAt(this.pos) == CharCode.DOT) { ++this.pos; - this.readDecimalFloatPartial(); + hasSep = this.readDecimalFloatPartial() || hasSep; } if (this.pos < end) { let c = text.charCodeAt(this.pos); @@ -1480,18 +1481,21 @@ export class Tokenizer extends DiagnosticEmitter { ) { ++this.pos; } - this.readDecimalFloatPartial(); + hasSep = this.readDecimalFloatPartial() || hasSep; } } - let floatString = text.substring(start, this.pos).replaceAll("_", ""); - return parseFloat(floatString); + let result = text.substring(start, this.pos); + if (hasSep) result = result.replaceAll("_", ""); + return parseFloat(result); } - private readDecimalFloatPartial(allowLeadingZeroSep: boolean = true): void { + /** Reads one decimal section of a float literal, returning true iff the literal had any separators. */ + private readDecimalFloatPartial(allowLeadingZeroSep: bool = true): bool { let start = this.pos; let end = this.end; let text = this.source.text; - var sepEnd = start; + let sepEnd = start; + let hasSep = false; while (this.pos < end) { let c = text.charCodeAt(this.pos); @@ -1511,6 +1515,7 @@ export class Tokenizer extends DiagnosticEmitter { ); } sepEnd = this.pos + 1; + hasSep = true; } else if (!isDecimalDigit(c)) { break; } @@ -1524,6 +1529,8 @@ export class Tokenizer extends DiagnosticEmitter { this.range(sepEnd - 1) ); } + + return hasSep; } readHexFloat(): f64 { From 3e709c3783167f11acf1141aa89b31078f05a211 Mon Sep 17 00:00:00 2001 From: BuildTools Date: Sat, 11 Jul 2020 16:18:03 -0400 Subject: [PATCH 8/9] Move esnext libs to portable compiler options --- src/tsconfig.json | 3 +-- std/portable.json | 3 ++- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/tsconfig.json b/src/tsconfig.json index a28bc8e325..7100628f62 100644 --- a/src/tsconfig.json +++ b/src/tsconfig.json @@ -5,8 +5,7 @@ "allowJs": false, "sourceMap": true, "target": "esnext", - "strict": true, - "lib": ["esnext", "esnext.string"] + "strict": true }, "include": [ "./**/*.ts" diff --git a/std/portable.json b/std/portable.json index e29a93d27e..1a1e427d21 100644 --- a/std/portable.json +++ b/std/portable.json @@ -7,6 +7,7 @@ "downlevelIteration": true, "preserveConstEnums": true, "typeRoots": [ "types" ], - "types": [ "portable" ] + "types": [ "portable" ], + "lib": ["esnext", "esnext.string"] } } From 21b1bb1a15ecf875f4ced3094f7760ff662c4939 Mon Sep 17 00:00:00 2001 From: BuildTools Date: Mon, 13 Jul 2020 10:17:36 -0400 Subject: [PATCH 9/9] Add integer to count separators --- src/tokenizer.ts | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/tokenizer.ts b/src/tokenizer.ts index 2af26b1ec4..02f8dae7da 100644 --- a/src/tokenizer.ts +++ b/src/tokenizer.ts @@ -1497,12 +1497,12 @@ export class Tokenizer extends DiagnosticEmitter { var text = this.source.text; var end = this.end; var start = this.pos; - var hasSep = false; + var sepCount = 0; - hasSep = this.readDecimalFloatPartial(false) || hasSep; + sepCount += this.readDecimalFloatPartial(false); if (this.pos < end && text.charCodeAt(this.pos) == CharCode.DOT) { ++this.pos; - hasSep = this.readDecimalFloatPartial() || hasSep; + sepCount += this.readDecimalFloatPartial(); } if (this.pos < end) { let c = text.charCodeAt(this.pos); @@ -1514,22 +1514,22 @@ export class Tokenizer extends DiagnosticEmitter { ) { ++this.pos; } - hasSep = this.readDecimalFloatPartial() || hasSep; + sepCount += this.readDecimalFloatPartial(); } } let result = text.substring(start, this.pos); - if (hasSep) result = result.replaceAll("_", ""); + if (sepCount > 0) result = result.replaceAll("_", ""); return parseFloat(result); } - /** Reads past one section of a decimal float literal. Returns true iff there were any separators. */ - private readDecimalFloatPartial(allowLeadingZeroSep: bool = true): bool { + /** Reads past one section of a decimal float literal. Returns the number of separators encountered. */ + private readDecimalFloatPartial(allowLeadingZeroSep: bool = true): u32 { var text = this.source.text; var pos = this.pos; var start = pos; var end = this.end; var sepEnd = start; - var hasSep = false; + var sepCount = 0; while (pos < end) { let c = text.charCodeAt(pos); @@ -1549,7 +1549,7 @@ export class Tokenizer extends DiagnosticEmitter { ); } sepEnd = pos + 1; - hasSep = true; + ++sepCount; } else if (!isDecimalDigit(c)) { break; } @@ -1565,7 +1565,7 @@ export class Tokenizer extends DiagnosticEmitter { } this.pos = pos; - return hasSep; + return sepCount; } readHexFloat(): f64 {