Address some literal pattern printing issues

This fixes some issues, all regex match tests now succeed with the
round-tripped regex, when that regex is supported.

Author: natecook1000

Sources/_StringProcessing/LiteralPrinter.swift

@@ -118,7 +118,7 @@ extension LiteralPrinter {
     case let .atom(atom):
       outputAtom(atom)
     case let .quotedLiteral(literal):
-      outputQuotedLiteral(literal)
+      output(prepareQuotedLiteral(literal))
 
     case .trivia(_):
       // TODO: Include trivia?
@@ -157,13 +157,40 @@ extension LiteralPrinter {
     output(")")
   }
 
+  func requiresGrouping(_ node: DSLTree.Node) -> Bool {
+    switch node {
+    case .concatenation(let children):
+      switch children.count {
+      case 0:
+        return false
+      case 1:
+        return requiresGrouping(children.first!)
+      default:
+        return true
+      }
+      
+    case .quotedLiteral(let literal):
+      return prepareQuotedLiteral(literal).count > 1
+      
+    default:
+      return false
+    }
+  }
+
   mutating func outputQuantification(
     _ amount: DSLTree._AST.QuantificationAmount,
     _ kind: DSLTree.QuantificationKind,
     _ child: DSLTree.Node
   ) {
-    outputNode(child)
-    
+    // RegexBuilder regexes can have children that need 
+    if requiresGrouping(child) {
+      output("(?:")
+      outputNode(child)
+      output(")")
+    } else {
+      outputNode(child)
+    }
+
     switch amount.ast {
     case .zeroOrMore:
       output("*")
@@ -254,7 +281,7 @@ extension LiteralPrinter {
   mutating func outputAtom(_ atom: DSLTree.Atom) {
     switch atom {
     case .char(let char):
-      output(char.escapingConfusable)
+      output(char.escapingForLiteral)
     case .scalar(let scalar):
       output(scalar.escapedString)
     case .any:
@@ -284,11 +311,12 @@ extension LiteralPrinter {
     case .symbolicReference(_):
       // RegexBuilder only
       saveInconvertible(.atom(atom))
-    case .changeMatchingOptions(let options):
-      output(options.ast._patternString)
+    case .changeMatchingOptions(let optionSequence):
+      output(optionSequence.ast._patternString)
       output(")")
+      options.apply(optionSequence.ast)
     case .unconverted(let atom):
-      output(atom.ast._dumpBase)
+      outputUnconvertedAST(atom.ast)
     }
   }
 
@@ -316,12 +344,11 @@ extension LiteralPrinter {
     }
   }
 
-  mutating func outputQuotedLiteral(_ literal: String) {
-    // TODO: Look for confusable characters
-    if literal.containsRegexMetaCharacters {
-      output(#"\Q\#(literal)\E"#)
+  func prepareQuotedLiteral(_ literal: String) -> String {
+    if options.usesExtendedWhitespace || literal.containsRegexMetaCharacters {
+      return #"\Q\#(literal)\E"#
     } else {
-      output(literal.escapingConfusableCharacters())
+      return literal.escapingConfusableCharacters()
     }
   }
 
@@ -350,7 +377,7 @@ extension LiteralPrinter {
       case let .custom(charClass):
         outputCustomCharacterClass(charClass)
       case let .quotedLiteral(literal):
-        if literal.containsRegexMetaCharacters {
+        if options.usesExtendedWhitespace || literal.containsRegexMetaCharacters {
           output(#"\Q\#(literal)\E"#)
         } else {
           output(literal)
@@ -374,14 +401,34 @@ extension LiteralPrinter {
     }
     output("]")
   }
+  
+  mutating func outputUnconvertedAST(_ ast: AST.Atom) {
+    switch ast.kind {
+    case let .property(property):
+      if let base = property._regexBase {
+        output(base)
+      } else {
+        saveInconvertible(.atom(.unconverted(.init(ast: ast))))
+      }
+    case let .namedCharacter(name):
+      output("\\N{\(name)}")
+    default:
+      saveInconvertible(.atom(.unconverted(.init(ast: ast))))
+    }
+  }
 }
 
 // MARK: - Supporting extensions
 
 fileprivate let metachars = Set(#"\[](){}|+*?^$.-"#)
+
 extension String {
   var containsRegexMetaCharacters: Bool {
-    contains { metachars.contains($0) }
+    contains(where: \.isRegexMetaCharacter)
+  }
+  
+  func escapingConfusableCharacters() -> String {
+    lazy.map(\.escapingConfusable).joined()
   }
 }
 
@@ -392,19 +439,25 @@ extension UnicodeScalar {
 }
 
 extension Character {
+  var isRegexMetaCharacter: Bool {
+    metachars.contains(self)
+  }
+  
   var escapingConfusable: String {
     if isConfusable {
-      String(unicodeScalars.first!) +
+      return String(unicodeScalars.first!) +
         unicodeScalars.dropFirst().lazy.map(\.escapedString).joined()
     } else {
-      String(self)
+      return String(self)
     }
   }
-}
-
-extension String {
-  func escapingConfusableCharacters() -> String {
-    lazy.map(\.escapingConfusable).joined()
+  
+  var escapingForLiteral: String {
+    if isRegexMetaCharacter {
+      return "\\\(self)"
+    } else {
+      return escapingConfusable
+    }
   }
 }
 
@@ -493,11 +546,10 @@ extension AST.MatchingOptionSequence {
 
     if resetsCurrentOptions {
       assert(removing.isEmpty)
-      return "(?^\(adding):"
+      return "(?^\(adding)"
     } else {
       return "(?\(adding)"
       + (removing.isEmpty ? "" : "-\(removing)")
-      + ":"
     }
   }
 }
@@ -521,7 +573,7 @@ extension DSLTree._AST.GroupKind {
     case .atomicScriptRun:        return "(*asr:"
 
     case let .changeMatchingOptions(sequence):
-      return sequence._patternString
+      return sequence._patternString + ":"
     }
   }
 }

Sources/_StringProcessing/MatchingOptions.swift

@@ -110,6 +110,11 @@ extension MatchingOptions {
     !stack.last!.contains(.unicodeWordBoundaries)
   }
 
+  var usesExtendedWhitespace: Bool {
+    stack.last!.contains(.extended)
+      || stack.last!.contains(.extraExtended)
+  }
+  
   enum SemanticLevel {
     case graphemeCluster
     case unicodeScalar
@@ -160,6 +165,10 @@ extension MatchingOptions {
 
     // Swift-only default possessive quantifier
     case possessiveByDefault
+    
+    // Whitespace options
+    case extended
+    case extraExtended
 
     init?(_ astKind: AST.MatchingOption.Kind) {
       switch astKind {
@@ -197,10 +206,10 @@ extension MatchingOptions {
         self = .byteSemantics
       case .possessiveByDefault:
         self = .possessiveByDefault
-        
-      // Whitespace options are only relevant during parsing, not compilation.
-      case .extended, .extraExtended:
-        return nil
+      case .extended:
+        self = .extended
+      case .extraExtended:
+        self = .extraExtended
       }
     }
 

Sources/_StringProcessing/PrintAsPattern.swift

@@ -847,7 +847,7 @@ extension AST.Atom.CharacterProperty {
   // TODO: Some way to integrate this with conversion...
   var _patternBase: String {
     if isUnprintableProperty {
-      return _regexBase
+      return _regexBase ?? " // TODO: Property \(self)"
     }
 
     return _dslBase
@@ -886,25 +886,19 @@ extension AST.Atom.CharacterProperty {
     }
   }
 
-  var _regexBase: String {
+  var _regexBase: String? {
+    let prefix = isInverted ? "\\P" : "\\p"
     switch kind {
     case .ascii:
       return "[:\(isInverted ? "^" : "")ascii:]"
 
-    case .binary(let b, value: _):
-      if isInverted {
-        return "[^\\p{\(b.rawValue)}]"
-      } else {
-        return "\\p{\(b.rawValue)}"
-      }
+    case .binary(let b, value: let value):
+      let suffix = value ? "" : "=false"
+      return "\(prefix){\(b.rawValue)\(suffix)}"
 
     case .generalCategory(let gc):
-      if isInverted {
-        return "[^\\p{\(gc.rawValue)}]"
-      } else {
-        return "\\p{\(gc.rawValue)}"
-      }
-      
+      return "\(prefix){\(gc.rawValue)}"
+
     case .posix(let p):
       return "[:\(isInverted ? "^" : "")\(p.rawValue):]"
 
@@ -914,8 +908,16 @@ extension AST.Atom.CharacterProperty {
     case .scriptExtension(let s):
       return "[:\(isInverted ? "^" : "")scx=\(s.rawValue):]"
 
+    case .any:
+      return "\(prefix){Any}"
+    case .assigned:
+      return "\(prefix){Assigned}"
+
+    case .named(let name):
+      return "\\N{\(name)}"
+
     default:
-      return " // TODO: Property \(self)"
+      return nil
     }
   }
 }
@@ -1066,7 +1068,7 @@ extension AST.Atom {
       return "<#value#>"
 
     case let .property(p):
-      return p._regexBase
+      return p._regexBase ?? " // TODO: Property \(p)"
 
     case let .escaped(e):
       return "\\\(e.character)"

Tests/RegexTests/LiteralPrinterTests.swift

@@ -27,18 +27,18 @@ extension RegexTests {
 
   func testPrintableDSLRegex() throws {
     let regex = Regex {
-      OneOrMore("a", .reluctant)
+      OneOrMore("aaa", .reluctant)
       Regex {
         ChoiceOf {
-          ZeroOrMore("b")
+          ZeroOrMore("bbb")
           OneOrMore("d")
           Repeat("e", 3...)
         }
       }.dotMatchesNewlines()
       Optionally("c")
     }.ignoresCase()
     let pattern = try XCTUnwrap(regex._literalPattern)
-    XCTAssertEqual("(?i:a+?(?s:b*|d+|e{3,})c?)", pattern)
+    XCTAssertEqual("(?i:(?:aaa)+?(?s:(?:bbb)*|d+|e{3,})c?)", pattern)
 
     let nonPrintableRegex = Regex {
       OneOrMore("a")

Tests/RegexTests/MatchTests.swift

@@ -25,14 +25,15 @@ struct MatchError: Error {
 // literal patterns round trip correctly.
 func _roundTripLiteral(
   _ regexStr: String,
-  syntax: SyntaxOptions = .traditional
-) throws {
+  syntax: SyntaxOptions
+) throws -> Regex<AnyRegexOutput>? {
   guard let pattern = try Regex(regexStr, syntax: syntax)._literalPattern else {
-    return
+    return nil
   }
 
   let remadeRegex = try Regex(pattern)
   XCTAssertEqual(pattern, remadeRegex._literalPattern)
+  return remadeRegex
 }
 
 func _firstMatch(
@@ -44,7 +45,33 @@ func _firstMatch(
 ) throws -> (String, [String?])? {
   var regex = try Regex(regexStr, syntax: syntax).matchingSemantics(semanticLevel)
   let result = try regex.firstMatch(in: input)
-  try _roundTripLiteral(regexStr, syntax: syntax)
+  do {
+    let roundTripRegex = try? _roundTripLiteral(regexStr, syntax: syntax)
+    let roundTripResult = try? roundTripRegex?
+      .matchingSemantics(semanticLevel)
+      .firstMatch(in: input)?[0]
+      .substring
+    switch (result?[0].substring, roundTripResult) {
+    case let (match?, rtMatch?):
+      XCTAssertEqual(match, rtMatch)
+    case (nil, nil):
+      break // okay
+    case let (match?, _):
+      XCTFail("""
+        Didn't match in round-tripped version of '\(regexStr)'
+        For input '\(input)'
+        Original: '\(regexStr)'
+        _literalPattern: '\(roundTripRegex?._literalPattern ?? "<no pattern>")'
+        """)
+    case let (_, rtMatch?):
+      XCTFail("""
+        Incorrectly matched as '\(rtMatch)'
+        For input '\(input)'
+        Original: '\(regexStr)'
+        _literalPattern: '\(roundTripRegex!._literalPattern!)'
+        """)
+    }
+  }
 
   if validateOptimizations {
     assert(regex._forceAction(.addOptions(.disableOptimizations)))