Merge pull request #184 from rbuckton/fixUnicodeGroupNames

Improve parser suppport for regexp group names

Author: DmitrySoshnikov

README.md

@@ -1336,6 +1336,34 @@ We have the following node (the `name` property with value `foo` is added):
   type: 'Group',
   capturing: true,
   name: 'foo',
+  nameRaw: 'foo',
+  number: 1,
+  expression: {
+    type: 'Char',
+    value: 'x',
+    symbol: 'x',
+    kind: 'simple',
+    codePoint: 120
+  }
+}
+```
+
+Note: The `nameRaw` property represents the name *as parsed from the original source*, including escape sequences. The `name` property represents the canonical decoded form of the name.
+
+For example, given the `/u` flag and the following group:
+
+```regexp
+(?<\u{03C0}>x)
+```
+
+We would have the following node:
+
+```js
+{
+  type: 'Group',
+  capturing: true,
+  name: 'π',
+  nameRaw: '\\u{03C0}',
   number: 1,
   expression: {
     type: 'Char',
@@ -1465,6 +1493,7 @@ A node:
       type: 'Group',
       capturing: true,
       name: 'foo',
+      nameRaw: 'foo',
       number: 1,
       expression: {
         type: 'Char',
@@ -1478,7 +1507,8 @@ A node:
       type: 'Backreference',
       kind: 'name',
       number: 1,
-      reference: 'foo'
+      reference: 'foo',
+      referenceRaw: 'foo'
     },
     {
       type: 'Backreference',
@@ -1490,6 +1520,52 @@ A node:
 }
 ```
 
+Note: The `referenceRaw` property represents the reference *as parsed from the original source*, including escape sequences. The `reference` property represents the canonical decoded form of the reference.
+
+For example, given the `/u` flag and the following pattern (matches `www`):
+
+```regexp
+(?<π>w)\k<\u{03C0}>\1
+```
+
+We would have the following node:
+
+```js
+{
+  type: 'Alternative',
+  expressions: [
+    {
+      type: 'Group',
+      capturing: true,
+      name: 'π',
+      nameRaw: 'π',
+      number: 1,
+      expression: {
+        type: 'Char',
+        value: 'w',
+        symbol: 'w',
+        kind: 'simple',
+        codePoint: 119
+      }
+    },
+    {
+      type: 'Backreference',
+      kind: 'name',
+      number: 1,
+      reference: 'π',
+      referenceRaw: '\\u{03C0}'
+    },
+    {
+      type: 'Backreference',
+      kind: 'number',
+      number: 1,
+      reference: 1
+    }
+  ]
+}
+```
+
+
 #### Quantifiers
 
 Quantifiers specify _repetition_ of a regular expression (or of its part). Below are the quantifiers which _wrap_ a parsed expression into a `Repetition` node. The quantifier itself can be of different _kinds_, and has `Quantifier` node type.

index.d.ts

@@ -62,6 +62,7 @@ declare module 'regexp-tree/ast' {
     capturing: true;
     number: number;
     name?: string;
+    nameRaw?: string;
     expression: Expression | null;
   }
 
@@ -84,6 +85,7 @@ declare module 'regexp-tree/ast' {
     kind: 'name';
     number: number;
     reference: string;
+    referenceRaw: string;
   }
 
   export type Backreference =

scripts/generate-unicode-id-parts.js

@@ -0,0 +1,158 @@
+// based on https://github.com/microsoft/TypeScript/tree/master/scripts/regenerate-unicode-identifier-parts.js
+
+/** @param {number} i */
+function toHex4Digits(i) {
+  let s = i.toString(16);
+  while (s.length < 4) {
+    s = '0' + s;
+  }
+  if (s.length > 4) throw new Error('Invalid Hex4Digits value');
+  return s;
+}
+
+class NonSurrogateRange {
+  /** @param {number} codePoint */
+  constructor(codePoint) {
+    this.firstCodePoint = codePoint;
+    this.lastCodePoint = codePoint;
+  }
+  toString() {
+    let text = '\\u' + toHex4Digits(this.firstCodePoint);
+    if (this.lastCodePoint !== this.firstCodePoint) {
+      text += '-\\u' + toHex4Digits(this.lastCodePoint);
+    }
+    return text;
+  }
+}
+
+class LeadSurrogateRange {
+  /** @param {number} leadSurrogate */
+  constructor(leadSurrogate) {
+    this.leadSurrogate = leadSurrogate;
+    /** @type {TrailSurrogateRange[]} */
+    this.ranges = [];
+  }
+
+  toString() {
+    return (
+      '\\u' +
+      toHex4Digits(this.leadSurrogate) +
+      '[' +
+      this.ranges.join('') +
+      ']'
+    );
+  }
+}
+
+class TrailSurrogateRange {
+  /** @param {number} trailSurrogate */
+  constructor(trailSurrogate) {
+    this.firstTrailSurrogate = trailSurrogate;
+    this.lastTrailSurrogate = trailSurrogate;
+  }
+  toString() {
+    let text = '\\u' + toHex4Digits(this.firstTrailSurrogate);
+    if (this.lastTrailSurrogate !== this.firstTrailSurrogate) {
+      text += '-\\u' + toHex4Digits(this.lastTrailSurrogate);
+    }
+    return text;
+  }
+}
+
+class Writer {
+  constructor() {
+    /** @type {number} */
+    this.lastCodePoint = -1;
+    /** @type {NonSurrogateRange[]} */
+    this.nonSurrogateRanges = [];
+    /** @type {LeadSurrogateRange[]} */
+    this.surrogateRanges = [];
+    /** @type {NonSurrogateRange} */
+    this.nonSurrogateRange;
+    /** @type {LeadSurrogateRange} */
+    this.leadSurrogateRange;
+    /** @type {TrailSurrogateRange} */
+    this.trailSurrogateRange;
+  }
+
+  /** @param {number} codePoint */
+  push(codePoint) {
+    if (codePoint <= this.lastCodePoint)
+      throw new Error('Code points must be added in order.');
+    this.lastCodePoint = codePoint;
+
+    if (codePoint < MAX_UNICODE_NON_SURROGATE) {
+      if (
+        this.nonSurrogateRange &&
+        this.nonSurrogateRange.lastCodePoint === codePoint - 1
+      ) {
+        this.nonSurrogateRange.lastCodePoint = codePoint;
+        return;
+      }
+      this.nonSurrogateRange = new NonSurrogateRange(codePoint);
+      this.nonSurrogateRanges.push(this.nonSurrogateRange);
+    } else {
+      const leadSurrogate = Math.floor((codePoint - 0x10000) / 0x400) + 0xd800;
+      const trailSurrogate = ((codePoint - 0x10000) % 0x400) + 0xdc00;
+      if (
+        !this.leadSurrogateRange ||
+        this.leadSurrogateRange.leadSurrogate !== leadSurrogate
+      ) {
+        this.trailSurrogateRange = undefined;
+        this.leadSurrogateRange = new LeadSurrogateRange(leadSurrogate);
+        this.surrogateRanges.push(this.leadSurrogateRange);
+      }
+
+      if (
+        this.trailSurrogateRange &&
+        this.trailSurrogateRange.lastTrailSurrogate === trailSurrogate - 1
+      ) {
+        this.trailSurrogateRange.lastTrailSurrogate = trailSurrogate;
+        return;
+      }
+
+      this.trailSurrogateRange = new TrailSurrogateRange(trailSurrogate);
+      this.leadSurrogateRange.ranges.push(this.trailSurrogateRange);
+    }
+  }
+
+  toString() {
+    let first = this.nonSurrogateRanges.join('');
+    let second = this.surrogateRanges.join('|');
+    return first && second
+      ? `([${first}]|${second})`
+      : first
+      ? `[${first}]`
+      : second
+      ? `(${second})`
+      : '';
+  }
+}
+
+const MAX_UNICODE_NON_SURROGATE = 0xffff;
+const MAX_UNICODE_CODEPOINT = 0x10ffff;
+const isStart = c => /\p{ID_Start}/u.test(c);
+const isContinue = c => /\p{ID_Continue}/u.test(c);
+
+let idStartWriter = new Writer();
+let idContinueWriter = new Writer();
+
+for (let cp = 0; cp <= MAX_UNICODE_CODEPOINT; cp++) {
+  const ch = String.fromCodePoint(cp);
+  if (isStart(ch)) {
+    idStartWriter.push(cp);
+  }
+  if (isContinue(ch)) {
+    idContinueWriter.push(cp);
+  }
+}
+
+console.log(`/**
+* Generated by scripts/generate-unicode-id-parts.js on node ${
+  process.version
+} with unicode ${process.versions.unicode}
+* based on http://www.unicode.org/reports/tr31/ and https://tc39.es/ecma262/#sec-names-and-keywords
+* U_ID_START corresponds to the ID_Start property, and U_ID_CONTINUE corresponds to ID_Continue property.
+*/`);
+console.log('U_ID_START              ' + idStartWriter.toString());
+console.log('U_ID_CONTINUE           ' + idContinueWriter.toString());

src/compat-transpiler/transforms/compat-named-capturing-groups-transform.js

@@ -9,7 +9,6 @@
  * A regexp-tree plugin to translate `/(?<name>a)\k<name>/` to `/(a)\1/`.
  */
 module.exports = {
-
   // To track the names of the groups, and return them
   // in the transform result state.
   //
@@ -41,6 +40,7 @@ module.exports = {
     this._groupNames[node.name] = node.number;
 
     delete node.name;
+    delete node.nameRaw;
   },
 
   Backreference(path) {
@@ -52,5 +52,6 @@ module.exports = {
 
     node.kind = 'number';
     node.reference = node.number;
+    delete node.referenceRaw;
   },
-};
+};

src/generator/__tests__/generator-basic-test.js

@@ -14,7 +14,6 @@ function test(re) {
 }
 
 describe('generator-basic', () => {
-
   it('simple char', () => {
     test(/a/);
   });
@@ -81,11 +80,11 @@ describe('generator-basic', () => {
   });
 
   it('named group', () => {
-    test('/(?<foo\\u003B\\u{003B}>bar)/');
+    test('/(?<foo\\u003B\\u{003B}>bar)/u');
   });
 
   it('empty named group', () => {
-    test('/(?<foo\\u003B\\u{003B}>)/');
+    test('/(?<foo\\u003B\\u{003B}>)/u');
   });
 
   it('empty non-capturing group', () => {
@@ -97,7 +96,7 @@ describe('generator-basic', () => {
   });
 
   it('named backreference', () => {
-    test('/(?<foo\\u003B\\u{003B}>)\\k<foo\\u003B\\u{003B}>/');
+    test('/(?<foo\\u003B\\u{003B}>)\\k<foo\\u003B\\u{003B}>/u');
   });
 
   it('basic-assertion', () => {
@@ -179,5 +178,4 @@ describe('generator-basic', () => {
     test(/a{1,}?/);
     test(/a{1,3}?/);
   });
-
-});
+});

src/generator/index.js

@@ -34,7 +34,7 @@ const generator = {
     if (node.capturing) {
       // A named group.
       if (node.name) {
-        return `(?<${node.name}>${expression})`;
+        return `(?<${node.nameRaw || node.name}>${expression})`;
       }
 
       return `(${expression})`;
@@ -48,7 +48,7 @@ const generator = {
       case 'number':
         return `\\${node.reference}`;
       case 'name':
-        return `\\k<${node.reference}>`;
+        return `\\k<${node.referenceRaw || node.reference}>`;
       default:
         throw new TypeError(`Unknown Backreference kind: ${node.kind}`);
     }