Move character set parsing logic into the parser

Author: ColinEberhardt

assembly/char.ts

@@ -9,8 +9,9 @@ export const enum Char {
   LeftParenthesis = 0x28,
   RightParenthesis = 0x29,
   Asterisk = 0x2a, // "*"
-  Comma = 0x2c, // "*"
   Plus = 0x2b, // "+"
+  Comma = 0x2c, // "*"
+  Minus = 0x2d, // "-"
   Dot = 0x2e, // "."
   Zero = 0x30,
   Question = 0x3f, // "?"

assembly/nfa/matcher.ts

@@ -4,19 +4,42 @@ import {
   CharacterNode,
   CharacterSetNode,
   CharacterClassNode,
+  CharacterRangeNode,
 } from "../parser/node";
+import { Match } from "../regexp";
 
-export abstract class Matcher {
-  abstract matches(code: u32): bool;
+export class Matcher {
+  matches(code: u32): bool {
+    return false;
+  }
 
   static fromCharacterClassNode(
     node: CharacterClassNode
   ): CharacterClassMatcher {
     return new CharacterClassMatcher(node.charClass);
   }
 
+  static fromCharacterRangeNode(
+    node: CharacterRangeNode
+  ): CharacterRangeMatcher {
+    return new CharacterRangeMatcher(node.from, node.to);
+  }
+
   static fromCharacterSetNode(node: CharacterSetNode): CharacterSetMatcher {
-    return new CharacterSetMatcher(node.chars, node.negated);
+    const matchers = new Array<Matcher>();
+    for (let i = 0; i < node.expressions.length; i++) {
+      const exp = node.expressions[i];
+      if (CharacterRangeNode.is(exp)) {
+        matchers.push(
+          Matcher.fromCharacterRangeNode(exp as CharacterRangeNode)
+        );
+      } else if (CharacterNode.is(exp)) {
+        matchers.push(Matcher.fromCharacterNode(exp as CharacterNode));
+      } else {
+        throw new Error("unsupported node type within character set");
+      }
+    }
+    return new CharacterSetMatcher(matchers, node.negated);
   }
 
   static fromCharacterNode(node: CharacterNode): CharacterMatcher {
@@ -34,6 +57,16 @@ export class CharacterMatcher extends Matcher {
   }
 }
 
+export class CharacterRangeMatcher extends Matcher {
+  constructor(public from: u32, public to: u32) {
+    super();
+  }
+
+  matches(code: u32): bool {
+    return code >= this.from && code <= this.to;
+  }
+}
+
 export class CharacterClassMatcher extends Matcher {
   constructor(public charClass: Char) {
     super();
@@ -79,28 +112,20 @@ export class CharacterClassMatcher extends Matcher {
   }
 }
 
+// no closure support
+let _code: u32;
+
 export class CharacterSetMatcher extends Matcher {
-  constructor(public set: string, public negated: bool) {
+  constructor(public matchers: Matcher[], public negated: bool) {
     super();
   }
 
-  matchesSet(set: string, code: u32): bool {
-    for (let i = 0, len = set.length; i < len; i++) {
-      // TODO - perform the set parsing logic in the constructor?
-      // TODO - move into the parser?
-      if (i < len - 2 && set.charCodeAt(i + 1) == 45 /*-*/) {
-        const from = set.charCodeAt(i) as u32;
-        const to = set.charCodeAt(i + 2) as u32;
-        if (code >= from && code <= to) return true;
-      } else {
-        if (set.charCodeAt(i) == code) return true;
-      }
-    }
-    return false;
-  }
-
   matches(code: u32): bool {
-    const matches = this.matchesSet(this.set, code);
-    return this.negated ? !matches : matches;
+    _code = code;
+    if (!this.negated) {
+      return this.matchers.some((m) => m.matches(_code));
+    } else {
+      return !this.matchers.some((m) => m.matches(_code));
+    }
   }
 }

assembly/parser/node.ts

@@ -9,6 +9,7 @@ export const enum NodeType {
   Character,
   CharacterSet,
   CharacterClass,
+  CharacterRange,
   Repetition,
   RangeRepetition,
   Group,
@@ -79,7 +80,7 @@ export class ConcatenationNode extends Node {
 }
 
 export class CharacterSetNode extends Node {
-  constructor(public chars: string, public negated: bool) {
+  constructor(public expressions: Node[], public negated: bool) {
     super(NodeType.CharacterSet);
   }
 
@@ -88,7 +89,24 @@ export class CharacterSetNode extends Node {
   }
 
   clone(): Node {
-    return new CharacterSetNode(this.chars, this.negated);
+    return new CharacterSetNode(
+      this.expressions.slice(0).map<Node>((s) => s.clone()),
+      this.negated
+    );
+  }
+}
+
+export class CharacterRangeNode extends Node {
+  constructor(public from: u32, public to: u32) {
+    super(NodeType.CharacterRange);
+  }
+
+  static is(node: Node): bool {
+    return node.type == NodeType.CharacterRange;
+  }
+
+  clone(): Node {
+    return new CharacterRangeNode(this.from, this.to);
   }
 }
 

assembly/parser/parser.ts

@@ -11,6 +11,7 @@ import {
   ConcatenationNode,
   RepetitionNode,
   CharacterSetNode,
+  CharacterRangeNode,
 } from "./node";
 
 function isQuantifier(code: Char): bool {
@@ -117,7 +118,7 @@ export class Parser {
     let firstDigit = true;
     let digitStr = "";
     while (this.more()) {
-      let token = this.currentToken.charCodeAt(0);
+      const token = this.currentToken.charCodeAt(0);
       if (token == Char.RightParenthesis) break;
       if (firstDigit) {
         if (isDigit(token)) {
@@ -170,7 +171,7 @@ export class Parser {
   private parseSequence(): Node {
     let nodes = new Array<Node>();
     while (this.more()) {
-      let token = this.currentToken.charCodeAt(0);
+      const token = this.currentToken.charCodeAt(0);
       if (token == Char.RightParenthesis) break;
       // @ts-ignore
       if (token == Char.VerticalBar) {
@@ -207,23 +208,38 @@ export class Parser {
     return nodes.length > 1 ? new ConcatenationNode(nodes) : nodes[0];
   }
 
+  private parseCharacterRange(): Node {
+    const from = this.eatToken();
+    this.eatToken(Char.Minus);
+    const to = this.eatToken();
+    return new CharacterRangeNode(from, to);
+  }
+
   private parseCharacterSet(): CharacterSetNode {
-    let chars = "";
     this.eatToken(Char.LeftSquareBracket);
-    const negated = this.currentToken == "^";
+    const token = this.currentToken.charCodeAt(0);
+
+    const negated = token == Char.Caret;
     if (negated) {
       this.eatToken(Char.Caret);
     }
-    while (
-      this.currentToken != "]" ||
-      (chars.length == 0 && this.currentToken == "]")
-    ) {
-      // TODO characters set can contain character classes
-      chars += this.currentToken;
-      this.eatToken();
+
+    const nodes = new Array<Node>();
+    while (this.currentToken != "]" || nodes.length == 0) {
+      // lookahead for character range
+      if (
+        this.cursor + 1 < u32(this.input.length) &&
+        this.input.charCodeAt(this.cursor + 1) == Char.Minus &&
+        this.input.charCodeAt(this.cursor + 2) != Char.RightSquareBracket
+      ) {
+        nodes.push(this.parseCharacterRange());
+      } else {
+        nodes.push(this.parseCharacter());
+      }
+
       // TODO error if we run out of chars?
     }
     this.eatToken(Char.RightSquareBracket);
-    return new CharacterSetNode(chars, negated);
+    return new CharacterSetNode(nodes, negated);
   }
 }

assembly/regexp.ts

@@ -1,10 +1,17 @@
-import { State, Automata, toNFAFromAST, GroupEndMarkerState } from "./nfa/nfa";
+import {
+  State,
+  Automata,
+  toNFAFromAST,
+  GroupEndMarkerState,
+  MatcherState,
+} from "./nfa/nfa";
 import { walker as nfaWalker } from "./nfa/walker";
 import { ConcatenationNode, AssertionNode } from "./parser/node";
 import { Char } from "./char";
 import { Parser } from "./parser/parser";
 import { first, last } from "./util";
 import { walker as astWalker, expandRepetitions } from "./parser/walker";
+import { CharacterMatcher, CharacterSetMatcher, Matcher } from "./nfa/matcher";
 
 function recursiveBacktrackingSearch(
   state: State,
@@ -145,5 +152,32 @@ export class RegExp {
 // TODO: do we need this factory function, or can we invoke
 // the ctr via the loader?
 export function createRegExp(regex: string, flags: string): RegExp {
+
+  /* ---------------- */
+  /*
+    This block of code is needed to avoid the following runtime error ...
+
+    RuntimeError: unreachable
+        at assembly/nfa/matcher/Matcher#matches@virtual (wasm-function[240]:1)
+        at assembly/nfa/matcher/CharacterSetMatcher#matches~anonymous|0 (wasm-function[241]:19)
+        at ~lib/array/Array<assembly/nfa/matcher/Matcher>#some (wasm-function[242]:85)
+        at assembly/nfa/matcher/CharacterSetMatcher#matches (wasm-function[244]:21)
+        at assembly/nfa/nfa/MatcherState<assembly/nfa/matcher/CharacterSetMatcher>#matches (wasm-function[245]:8)
+        at assembly/nfa/nfa/State#matches@virtual (wasm-function[250]:58)
+        at assembly/regexp/recursiveBacktrackingSearch (wasm-function[184]:121)
+        at assembly/regexp/recursiveBacktrackingSearch@varargs (wasm-function[185]:56)
+        at assembly/regexp/RegExp#exec (wasm-function[192]:307)
+  */
+  const matchers = new Array<Matcher>();
+  matchers.push(new CharacterMatcher(Char.A));
+  const charMatcher = new CharacterSetMatcher(matchers, false);
+  const state = new MatcherState<CharacterSetMatcher>(
+    charMatcher,
+    new State(true)
+  );
+  const char = "a".charCodeAt(0);
+  const doesMatch = state.matches(char) != null;
+  /* ---------------- */
+
   return new RegExp(regex, flags);
 }

ts/index.ts

@@ -5,7 +5,7 @@ globalAny.log = console.log;
 
 import { RegExp } from "../assembly/regexp";
 
-const regexObj = new RegExp("(a*)+");
-const match = regexObj.exec("-");
+const regexObj = new RegExp("[]a]");
+const match = regexObj.exec("]");
 
 console.log(match);