Decode unicode group names to canonical forms

Author: rbuckton

README.md

@@ -1336,6 +1336,34 @@ We have the following node (the `name` property with value `foo` is added):
   type: 'Group',
   capturing: true,
   name: 'foo',
+  nameRaw: 'foo',
+  number: 1,
+  expression: {
+    type: 'Char',
+    value: 'x',
+    symbol: 'x',
+    kind: 'simple',
+    codePoint: 120
+  }
+}
+```
+
+Note: The `nameRaw` property represents the name *as parsed from the original source*, including escape sequences. The `name` property represents the canonical decoded form of the name.
+
+For example, given the `/u` flag and the following group:
+
+```regexp
+(?<\u{03C0}>x)
+```
+
+We would have the following node:
+
+```js
+{
+  type: 'Group',
+  capturing: true,
+  name: 'π',
+  nameRaw: '\\u{03C0}',
   number: 1,
   expression: {
     type: 'Char',
@@ -1465,6 +1493,7 @@ A node:
       type: 'Group',
       capturing: true,
       name: 'foo',
+      nameRaw: 'foo',
       number: 1,
       expression: {
         type: 'Char',
@@ -1478,7 +1507,8 @@ A node:
       type: 'Backreference',
       kind: 'name',
       number: 1,
-      reference: 'foo'
+      reference: 'foo',
+      referenceRaw: 'foo'
     },
     {
       type: 'Backreference',
@@ -1490,6 +1520,52 @@ A node:
 }
 ```
 
+Note: The `referenceRaw` property represents the reference *as parsed from the original source*, including escape sequences. The `reference` property represents the canonical decoded form of the reference.
+
+For example, given the `/u` flag and the following pattern (matches `www`):
+
+```regexp
+(?<π>w)\k<\u{03C0}>\1
+```
+
+We would have the following node:
+
+```js
+{
+  type: 'Alternative',
+  expressions: [
+    {
+      type: 'Group',
+      capturing: true,
+      name: 'π',
+      nameRaw: 'π',
+      number: 1,
+      expression: {
+        type: 'Char',
+        value: 'w',
+        symbol: 'w',
+        kind: 'simple',
+        codePoint: 119
+      }
+    },
+    {
+      type: 'Backreference',
+      kind: 'name',
+      number: 1,
+      reference: 'π',
+      referenceRaw: '\\u{03C0}'
+    },
+    {
+      type: 'Backreference',
+      kind: 'number',
+      number: 1,
+      reference: 1
+    }
+  ]
+}
+```
+
+
 #### Quantifiers
 
 Quantifiers specify _repetition_ of a regular expression (or of its part). Below are the quantifiers which _wrap_ a parsed expression into a `Repetition` node. The quantifier itself can be of different _kinds_, and has `Quantifier` node type.

index.d.ts

@@ -56,6 +56,7 @@ declare module 'regexp-tree/ast' {
     capturing: true;
     number: number;
     name?: string;
+    nameRaw?: string;
     expression: Expression | null;
   }
 
@@ -76,6 +77,7 @@ declare module 'regexp-tree/ast' {
     kind: 'name';
     number: number;
     reference: string;
+    referenceRaw: string;
   }
 
   export type Backreference = NumericBackreference | NamedBackreference;

scripts/generate-unicode-id-parts.js

@@ -137,7 +137,7 @@ const isContinue = c => /\p{ID_Continue}/u.test(c);
 let idStartWriter = new Writer();
 let idContinueWriter = new Writer();
 
-for (let cp = 0; cp < MAX_UNICODE_CODEPOINT; cp++) {
+for (let cp = 0; cp <= MAX_UNICODE_CODEPOINT; cp++) {
   const ch = String.fromCodePoint(cp);
   if (isStart(ch)) {
     idStartWriter.push(cp);
@@ -151,7 +151,7 @@ console.log(`/**
 * Generated by scripts/generate-unicode-id-parts.js on node ${
   process.version
 } with unicode ${process.versions.unicode}
-* based on http://www.unicode.org/reports/tr31/ and https://www.ecma-international.org/ecma-262/6.0/#sec-names-and-keywords
+* based on http://www.unicode.org/reports/tr31/ and https://tc39.es/ecma262/#sec-names-and-keywords
 * U_ID_START corresponds to the ID_Start property, and U_ID_CONTINUE corresponds to ID_Continue property.
 */`);
 console.log('U_ID_START              ' + idStartWriter.toString());

src/compat-transpiler/transforms/compat-named-capturing-groups-transform.js

@@ -9,7 +9,6 @@
  * A regexp-tree plugin to translate `/(?<name>a)\k<name>/` to `/(a)\1/`.
  */
 module.exports = {
-
   // To track the names of the groups, and return them
   // in the transform result state.
   //
@@ -41,6 +40,7 @@ module.exports = {
     this._groupNames[node.name] = node.number;
 
     delete node.name;
+    delete node.nameRaw;
   },
 
   Backreference(path) {
@@ -52,5 +52,6 @@ module.exports = {
 
     node.kind = 'number';
     node.reference = node.number;
+    delete node.referenceRaw;
   },
-};
+};

src/generator/__tests__/generator-basic-test.js

@@ -14,7 +14,6 @@ function test(re) {
 }
 
 describe('generator-basic', () => {
-
   it('simple char', () => {
     test(/a/);
   });
@@ -81,11 +80,11 @@ describe('generator-basic', () => {
   });
 
   it('named group', () => {
-    test('/(?<foo\\u003B\\u{003B}>bar)/');
+    test('/(?<foo\\u003B\\u{003B}>bar)/u');
   });
 
   it('empty named group', () => {
-    test('/(?<foo\\u003B\\u{003B}>)/');
+    test('/(?<foo\\u003B\\u{003B}>)/u');
   });
 
   it('empty non-capturing group', () => {
@@ -97,7 +96,7 @@ describe('generator-basic', () => {
   });
 
   it('named backreference', () => {
-    test('/(?<foo\\u003B\\u{003B}>)\\k<foo\\u003B\\u{003B}>/');
+    test('/(?<foo\\u003B\\u{003B}>)\\k<foo\\u003B\\u{003B}>/u');
   });
 
   it('basic-assertion', () => {
@@ -179,5 +178,4 @@ describe('generator-basic', () => {
     test(/a{1,}?/);
     test(/a{1,3}?/);
   });
-
-});
+});

src/generator/index.js

@@ -34,7 +34,7 @@ const generator = {
     if (node.capturing) {
       // A named group.
       if (node.name) {
-        return `(?<${node.name}>${expression})`;
+        return `(?<${node.nameRaw || node.name}>${expression})`;
       }
 
       return `(${expression})`;
@@ -48,7 +48,7 @@ const generator = {
       case 'number':
         return `\\${node.reference}`;
       case 'name':
-        return `\\k<${node.reference}>`;
+        return `\\k<${node.referenceRaw || node.reference}>`;
       default:
         throw new TypeError(`Unknown Backreference kind: ${node.kind}`);
     }

src/parser/__tests__/parser-extended-test.js

@@ -25,6 +25,7 @@ describe('extended', () => {
           {
             type: 'Group',
             name: 'year',
+            nameRaw: 'year',
             number: 1,
             capturing: true,
             expression: {
@@ -55,6 +56,7 @@ describe('extended', () => {
           {
             type: 'Group',
             name: 'month',
+            nameRaw: 'month',
             number: 2,
             capturing: true,
             expression: {
@@ -85,6 +87,7 @@ describe('extended', () => {
           {
             type: 'Group',
             name: 'day',
+            nameRaw: 'day',
             number: 3,
             capturing: true,
             expression: {
@@ -263,6 +266,7 @@ describe('extended', () => {
             type: 'Group',
             capturing: true,
             name: 'c',
+            nameRaw: 'c',
             number: 1,
             expression: {
               type: 'Alternative',
@@ -271,6 +275,7 @@ describe('extended', () => {
                   type: 'Group',
                   capturing: true,
                   name: 'b',
+                  nameRaw: 'b',
                   number: 2,
                   expression: {
                     type: 'Alternative',
@@ -279,6 +284,7 @@ describe('extended', () => {
                         type: 'Group',
                         capturing: true,
                         name: 'a',
+                        nameRaw: 'a',
                         number: 3,
                         expression: {
                           type: 'Char',
@@ -312,6 +318,7 @@ describe('extended', () => {
             type: 'Group',
             capturing: true,
             name: 'd',
+            nameRaw: 'd',
             number: 4,
             expression: {
               type: 'Char',
@@ -325,6 +332,7 @@ describe('extended', () => {
             type: 'Group',
             capturing: true,
             name: 'e',
+            nameRaw: 'e',
             number: 5,
             expression: {
               type: 'Char',

src/parser/__tests__/parser-test262-test.js

@@ -138,6 +138,7 @@ describe('test262', () => {
   it('unicode group names', () => {
     valid('/(?<π>a)/u');
     valid('/(?<\\u{03C0}>a)/u');
+    invalid('/(?<π>a)(?<\\u{03C0}>a)/u', 'Duplicate of the named group');
     valid('/(?<$𐒤>a)/u');
     valid('/(?<_\\u200C>a)/u');
     valid('/(?<_\\u200D>a)/u');