Merge pull request #40 from kareman/addSkipTests

Add `Skip` tests

Author: kareman

Sources/Patterns/Grammar.swift

@@ -49,22 +49,33 @@ public class Grammar: Pattern {
 		var instructions = finalInstructions
 		let startIndex = instructions.endIndex
 		instructions.append(
-			.openCall(name: try firstPattern ?? Parser<Input>.InitError.message("Grammar is empty")))
+			.openCall(name: try firstPattern ?? Parser<Input>.InitError.message("Grammar is empty.")))
 		instructions.append(.jump(offset: .max)) // replaced later
-		var callTable = [String: Instructions.Index]()
+		var callTable = [String: Range<Instructions.Index>]()
 		for (name, pattern) in patterns {
-			callTable[name] = instructions.endIndex
+			let startIndex = instructions.endIndex
 			try pattern.createInstructions(&instructions)
-			precondition(callTable[name] != instructions.endIndex,
-			             "Pattern '\(name) <- \(pattern)' was empty")
 			instructions.append(.return)
+			guard (startIndex ..< instructions.endIndex).count > 1 else {
+				throw Parser<Input>.InitError.message("Pattern '\(name) <- \(pattern)' was empty.")
+			}
+			callTable[name] = startIndex ..< instructions.endIndex
 		}
 
 		for i in instructions.indices[startIndex...] {
 			if case let .openCall(name) = instructions[i] {
-				let address = try callTable[name]
-					?? Parser<Input>.InitError.message("Pattern '\(name)' was never defined with ´<-´ operator.")
-				instructions[i] = .call(offset: address - i)
+				guard let subpatternRange = callTable[name] else {
+					throw Parser<Input>.InitError.message("Pattern '\(name)' was never defined with ´<-´ operator.")
+				}
+				// If the last non-dummy (i.e. .choiceEnd) instruction in a subpattern is a call to itself we perform
+				// a tail call optimisation by jumping directly instead.
+				// The very last instruction is a .return, so skip that.
+				if subpatternRange.upperBound - 2 == i
+					|| (subpatternRange.upperBound - 3 == i && instructions[i + 1].doesNotDoAnything) {
+					instructions[i] = .jump(offset: subpatternRange.lowerBound - i)
+				} else {
+					instructions[i] = .call(offset: subpatternRange.lowerBound - i)
+				}
 			}
 		}
 		instructions[startIndex + 1] = .jump(offset: instructions.endIndex - startIndex - 1)

Sources/Patterns/Operations on Patterns/Skip.swift

@@ -13,13 +13,12 @@ public struct Skip: Pattern {
 	@inlinable
 	public func createInstructions(_ instructions: inout Instructions) throws {
 		instructions.append(.skip)
-		instructions.append(.jump(offset: 1)) // dummy
 	}
 }
 
 import SE0270_RangeSet
 
-extension MutableCollection where Self: RandomAccessCollection, Index == Int {
+extension MutableCollection where Self: RandomAccessCollection, Self: RangeReplaceableCollection, Index == Int {
 	@usableFromInline
 	mutating func replaceSkips<Input>() where Element == Instruction<Input> {
 		for i in self.indices {
@@ -33,7 +32,7 @@ extension MutableCollection where Self: RandomAccessCollection, Index == Int {
 
 	@usableFromInline
 	mutating func setupSkip<Input>(at skipIndex: Index) where Element == Instruction<Input> {
-		let searchablesStartAt = skipIndex + 2
+		let searchablesStartAt = skipIndex + 1
 		switch self[searchablesStartAt] {
 		case let .checkIndex(function, atIndexOffset: 0):
 			self[skipIndex] = .search { input, index in
@@ -42,7 +41,7 @@ extension MutableCollection where Self: RandomAccessCollection, Index == Int {
 			}
 			self[searchablesStartAt] = .choice(offset: -1, atIndexOffset: 1)
 		case .checkIndex(_, atIndexOffset: _):
-			fatalError("Cannot see a valid reason for a `.checkIndex` with a non-zero offset to be located right after a `.skip` instruction. Correct me if I'm wrong.")
+			fatalError("Cannot see a valid reason for a `.checkIndex` with a non-zero offset to be located right after a `.skip` instruction.") // Correct me if I'm wrong.
 		case let .checkElement(test):
 			self[skipIndex] = .search { input, index in
 				input[index...].firstIndex(where: test)
@@ -75,14 +74,14 @@ extension MutableCollection where Self: RandomAccessCollection, Index == Int {
 			}
 		default:
 			self[skipIndex] = .choice(offset: 0, atIndexOffset: +1)
-			self.placeSkipCommit(dummyIsAt: skipIndex + 1, startSearchFrom: skipIndex + 2)
+			self.placeSkipCommit(startSearchFrom: skipIndex + 1)
 			return
 		}
-		self.placeSkipCommit(dummyIsAt: skipIndex + 1, startSearchFrom: skipIndex + 3)
+		self.placeSkipCommit(startSearchFrom: skipIndex + 2)
 	}
 
 	@usableFromInline
-	mutating func placeSkipCommit<Input>(dummyIsAt dummyIndex: Index, startSearchFrom: Index)
+	mutating func placeSkipCommit<Input>(startSearchFrom: Index)
 		where Element == Instruction<Input> {
 		var i = startSearchFrom
 		loop: while true {
@@ -97,17 +96,48 @@ extension MutableCollection where Self: RandomAccessCollection, Index == Int {
 				} else {
 					i += offset
 				}
-			case let .jump(offset):
+			case let .jump(offset) where offset > 0: // If we jump backwards we are likely to enter an infinite loop.
 				i += offset
-			case .elementEquals, .checkElement, .checkIndex, .moveIndex, .captureStart, .captureEnd, .call:
+			case .elementEquals, .checkElement, .checkIndex, .moveIndex, .captureStart, .captureEnd, .call, .jump:
 				i += 1
 			case .commit, .choiceEnd, .return, .match, .skip, .search:
-				moveSubranges(RangeSet(dummyIndex ..< (dummyIndex + 1)), to: i)
-				self[i - 1] = .commit
+				insertInstructions(.commit, at: i)
 				return
 			case .fail, .openCall:
 				fatalError()
 			}
 		}
 	}
+
+	/// Inserts new instructions at `location`. Adjusts the offsets of other instructions accordingly.
+	@usableFromInline
+	mutating func insertInstructions<Input>(_ newInstructions: Element..., at location: Index)
+		where Element == Instruction<Input> {
+		insert(contentsOf: newInstructions, at: location)
+		let insertedRange = location ..< (location + newInstructions.count + 1)
+		for i in startIndex ..< insertedRange.lowerBound {
+			switch self[i] {
+			case let .call(offset) where offset > (location - i):
+				self[i] = .call(offset: offset + newInstructions.count)
+			case let .jump(offset) where offset > (location - i):
+				self[i] = .jump(offset: offset + newInstructions.count)
+			case let .choice(offset, atIndexOffset) where offset > (location - i):
+				self[i] = .choice(offset: offset + newInstructions.count, atIndexOffset: atIndexOffset)
+			default:
+				break
+			}
+		}
+		for i in insertedRange.upperBound ..< endIndex {
+			switch self[i] {
+			case let .call(offset) where offset < (location - i):
+				self[i] = .call(offset: offset - newInstructions.count)
+			case let .jump(offset) where offset < (location - i):
+				self[i] = .jump(offset: offset - newInstructions.count)
+			case let .choice(offset, atIndexOffset) where offset < (location - i):
+				self[i] = .choice(offset: offset - newInstructions.count, atIndexOffset: atIndexOffset)
+			default:
+				break
+			}
+		}
+	}
 }

Sources/Patterns/Parser.swift

@@ -33,6 +33,7 @@ public struct Parser<Input: BidirectionalCollection> where Input.Element: Hashab
 		try self.init(Skip() • pattern)
 	}
 
+	@inlinable
 	public func ranges(in input: Input, from startindex: Input.Index? = nil)
 		-> AnySequence<Range<Input.Index>> {
 		AnySequence(matches(in: input, from: startindex).lazy.map { $0.range })

Sources/Patterns/Pattern And Instruction.swift

@@ -130,6 +130,17 @@ public enum Instruction<Input: BidirectionalCollection> where Input.Element: Has
 			return nil
 		}
 	}
+
+	/// Returns false only if instruction has no effect.
+	@usableFromInline
+	var doesNotDoAnything: Bool {
+		switch self {
+		case .choiceEnd, .jump(+1):
+			return true
+		default:
+			return false
+		}
+	}
 }
 
 extension Sequence where Element == Instruction<Pattern.Input> {

Tests/LongTests/LongTests.swift

@@ -22,7 +22,7 @@ class LongTests: XCTestCase {
 	}
 
 	func testPlaygroundExample() throws {
-		let text = """
+		let text = #"""
 		0   0.0   0.01
 		-0   +0   -0.0   +0.0
 		-123.456e+00   -123.456E+00   -123.456e-00   -123.456E-00
@@ -33,7 +33,7 @@ class LongTests: XCTestCase {
 		0x123E   0x123e
 		0x0123456789abcdef
 		0b0   0b1   0b0000   0b0001   0b11110000   0b0000_1111   0b1010_00_11
-		"""
+		"""#
 
 		let unsigned = digit+
 		let sign = "-" / "+"
@@ -48,4 +48,93 @@ class LongTests: XCTestCase {
 
 		XCTAssertEqual(Array(parser.matches(in: text)).count, 44)
 	}
+
+	// from http://www.inf.puc-rio.br/~roberto/docs/peg.pdf, page 2 and 5
+	static let pegGrammar = Grammar { g in
+		//g.all     <- g.pattern • !any
+		g.pattern <- g.grammar / g.simplepatt
+		g.grammar <- (g.nonterminal • "<-" • g.sp • g.simplepatt)+
+		g.simplepatt <- g.alternative • ("/" • g.sp • g.alternative)*
+		g.alternative <- (OneOf("!&")¿ • g.sp • g.suffix)+
+		g.suffix <- g.primary • (OneOf("*+?") • g.sp)*
+		g.primary <- "(" • g.sp • g.pattern • ")" • g.sp / "." • g.sp / g.literal / g.charclass / g.nonterminal • !"<-"
+		g.literal <- "’" • (!"’" • any)* • "’" • g.sp
+		g.charclass <- "[" • (!"]" • (any • "-" • any / any))* • "]" • g.sp
+		g.nonterminal <- OneOf("a" ... "z", "A" ... "Z")+ • g.sp
+		g.sp <- OneOf(" \t\n")*
+	}
+
+	static let pegGrammarParser = { try! Parser(pegGrammar) }()
+
+	func testPEGGrammar() throws {
+		// page 5
+		let grammar1Text = """
+		pattern      <- grammar / simplepatt
+		grammar      <- (nonterminal ’<-’ sp simplepatt)+
+		simplepatt   <- alternative (’/’ sp alternative)*
+		alternative  <- ([!&]? sp suffix)+
+		suffix       <- primary ([*+?] sp)*
+		primary      <- ’(’ sp pattern ’)’ sp / ’.’ sp / literal / charclass / nonterminal !’<-’
+		literal      <- [’] (![’] .)* [’] sp
+		charclass    <- ’[’ (!’]’ (. ’-’ . / . ))* ’]’ sp
+		nonterminal  <- [a-zA-Z]+ sp
+		sp           <- [ \t\n]*
+		"""
+		XCTAssertEqual(Self.pegGrammarParser.match(in: grammar1Text)?.endIndex, grammar1Text.endIndex)
+
+		// page 2
+		let grammar2Text = """
+		grammar      <- (nonterminal ’<-’ sp pattern)+
+		pattern      <- alternative (’/’ sp alternative)*
+		alternative  <- ([!&]? sp suffix)+
+		suffix       <- primary ([*+?] sp)*
+		primary      <- ’(’ sp pattern ’)’ sp / ’.’ sp / literal / charclass / nonterminal !’<-’
+		literal      <- [’] (![’] .)* [’] sp
+		charclass    <- ’[’ (!’]’ (. ’-’ . / . ))* ’]’ sp
+		nonterminal  <- [a-zA-Z]+ sp
+		sp           <- [ \t\n]*
+		"""
+		XCTAssertEqual(Self.pegGrammarParser.match(in: grammar2Text)?.endIndex, grammar2Text.endIndex)
+	}
+
+	func testOriginalPEGGrammar() throws {
+		try XCTSkipIf(true, "pegGrammar does not support escaping characters.")
+
+		// https://bford.info/pub/lang/peg.pdf Page 2, Figure 1.
+		let origPEGGrammarText = """
+		# Hierarchical syntax
+		Grammar <- Spacing Definition+ EndOfFile
+		Definition <- Identifier LEFTARROW Expression
+		Expression <- Sequence (SLASH Sequence)*
+		Sequence <- Prefix*
+		Prefix <- (AND / NOT)? Suffix
+		Suffix <- Primary (QUESTION / STAR / PLUS)?
+		Primary <- Identifier !LEFTARROW / OPEN Expression CLOSE / Literal / Class / DOT
+
+		# Lexical syntax
+		Identifier <- IdentStart IdentCont* Spacing
+		IdentStart <- [a-zA-Z_]
+		IdentCont <- IdentStart / [0-9]
+		Literal <- [’] (![’] Char)* [’] Spacing / ["] (!["] Char)* ["] Spacing
+		Class <- ’[’ (!’]’ Range)* ’]’ Spacing
+		Range <- Char ’-’ Char / Char
+		Char <- ’\\’ [nrt’"[]\\] / ’\\’ [0-2][0-7][0-7] / ’\\’ [0-7][0-7]? / !’\\’ .
+		LEFTARROW <- ’<-’ Spacing
+		SLASH <- ’/’ Spacing
+		AND <- ’&’ Spacing
+		NOT <- ’!’ Spacing
+		QUESTION <- ’?’ Spacing
+		STAR <- ’*’ Spacing
+		PLUS <- ’+’ Spacing
+		OPEN <- ’(’ Spacing
+		CLOSE <- ’)’ Spacing
+		DOT <- ’.’ Spacing
+		Spacing <- (Space / Comment)*
+		Comment <- ’#’ (!EndOfLine .)* EndOfLine
+		Space <- ’ ’ / ’\t’ / EndOfLine
+		EndOfLine <- ’\r\n’ / ’\n’ / ’\r’
+		EndOfFile <- !.
+		"""
+		XCTAssertEqual(Self.pegGrammarParser.match(in: origPEGGrammarText)?.endIndex, origPEGGrammarText.endIndex)
+	}
 }

Tests/PatternsTests/ConcatenationTests.swift

@@ -69,44 +69,6 @@ class ConcatenationTests: XCTestCase {
 			result: ["FMA026712 TECNOAUTOMOTRIZ ATLACOMULCO S"])
 	}
 
-	func testSkip1() throws {
-		let text = "This is a test text."
-		assertParseAll(
-			Capture(" " • Skip() • " "),
-			input: text, result: [" is ", " test "])
-
-		assertParseAll(
-			Capture(" " • Skip() • "d"),
-			input: " ad d", result: [" ad", " d"])
-	}
-
-	func testSkipAndCapture() throws {
-		let text = "This is a test text."
-		assertParseAll(
-			" " • Capture(letter • Skip()) • " ",
-			input: text, result: ["is", "a", "test"])
-		assertParseAll(
-			" " • Capture(Skip() • letter+) • " ",
-			input: text, result: ["is", "a", "test"])
-		assertParseAll(
-			" " • Capture(Skip()) • " ",
-			input: text, result: ["is", "a", "test"])
-
-		assertParseAll(
-			Line.start • Capture(Skip()) • Line.end,
-			input: """
-			1
-			2
-
-			3
-			""",
-			result: ["1", "2", "", "3"])
-
-		// undefined (Skip at end)
-		_ = try Parser(search: " " • Capture(Skip()))
-			.matches(in: text)
-	}
-
 	func testMatchFullRange() throws {
 		let text = """
 		line 1

Tests/PatternsTests/GrammarTests.swift

@@ -5,7 +5,7 @@
 //  Created by Kåre Morstøl on 27/05/2020.
 //
 
-import Patterns
+@testable import Patterns
 import XCTest
 
 class GrammarTests: XCTestCase {
@@ -54,4 +54,21 @@ class GrammarTests: XCTestCase {
 		assertParseMarkers(p, input: "1+2-3*(4+3)|")
 		assertParseAll(p, input: "1+2(", count: 0)
 	}
+
+	func testOptimisesTailCall() throws {
+		let g = Grammar { g in
+			g.a <- " " / Skip() • g.a
+		}
+
+		func isCall(_ inst: Instruction<String>) -> Bool {
+			switch inst {
+			case .call:
+				return true
+			default: return false
+			}
+		}
+
+		XCTAssertEqual(try Parser(g).matcher.instructions.filter(isCall(_:)).count, 1)
+		XCTAssertEqual(try Parser(search: g).matcher.instructions.filter(isCall(_:)).count, 1)
+	}
 }