Revise expectation syntax, fix regular expression bug, refactor implementation (#30)

* Remove LinuxMain

* Update expectations to type, value, match, and error

* Refactor scanning logic into Scanner type

* Fix Scanner regular expression pattern

Add test for Scanner

* Internalize inessential APIs

* Extract tests into separate files

* Add changelog entries for #30

* Update README

* Add CI badge to README

* Feed source file into REPL when not running in package context

* Log notice for blocks without any import statements when running through SPM

Author: mattt

Changelog.md

@@ -11,6 +11,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - Changed Swift version requirement to 5.3.
   #29 by @mattt.
+- Changed expression syntax.
+  #30 by @mattt.
+
+### Fixed
+
+- Fixed a bug that caused DocTest annotations to be missed.
+  #30 by @mattt.
 
 ## [0.1.0] - 2020-05-04
 

Package.swift

@@ -34,14 +34,14 @@ let package = Package(
             name: "swift-doctest",
             dependencies: [
                 .target(name: "DocTest"),
-                .product(name: "StringLocationConverter", package: "StringLocationConverter"),
                 .product(name: "ArgumentParser", package: "swift-argument-parser"),
                 .product(name: "Logging", package: "swift-log"),
             ]),
         .target(
             name: "DocTest",
             dependencies: [
                 .product(name: "SwiftSyntax", package: "SwiftSyntax"),
+                .product(name: "StringLocationConverter", package: "StringLocationConverter"),
                 .product(name: "TAP", package: "TAP"),
             ]),
         .testTarget(

README.md

@@ -1,5 +1,7 @@
 # DocTest
 
+![CI][ci badge]
+
 **DocTest** is an experimental tool
 for testing Swift example code in documentation.
 
@@ -73,7 +75,7 @@ OVERVIEW: A utility for syntax testing documentation in Swift code.
 USAGE: swift-doctest <input> [--swift-launch-path <swift-launch-path>] [--package] [--assumed-filename <assumed-filename>]
 
 ARGUMENTS:
-  <input>                 Swift code or a path to a Swift file 
+  <input>                 Swift code or a path to a Swift file
 
 OPTIONS:
   --swift-launch-path <swift-launch-path>
@@ -130,30 +132,28 @@ This tells the documentation test runner to evaluate the code sample.
 ```
 
 By adding an annotation in the format
-`=> (Type) = (Value)`,
+`=> <#Value#>`,
 we can test the expected type and value
 of the expression.
 
 ```diff
 - add(1 1) // 3.0
-+ add(1 1) // => Double = 3.0
++ add(1 1) // => 3.0
 ```
 
 Run the `swift-doctest` command
-from the root directory of the Swift package,
-specifying the `--package` flag
-(to invoke the Swift REPL via the Swift Package Manager)
+from the root directory of the Swift package
 and passing the path to the file containing the `add(_:_:)` function.
 This will scan for all of code blocks annotated with
-<code>```swift doctest</code>
+<code>```swift doctest</code>,
 run them through the Swift REPL,
 and test the output with any annotated expectations.
 
 ```terminal
 $ swift doctest --package path/to/file.swift
 TAP version 13
 1..1
-not ok 1 - `add(1 1)` did not produce `Double = 3.0`
+not ok 1 - `add(1 1)` did not produce `3.0`
   ---
   column: 1
   file: path/to/file.swift.md
@@ -172,7 +172,7 @@ we update the documentation to fix the example.
     Returns the sum of two integers.
 
     ```swift doctest
-    add(1, 1) // => Int = 2
+    add(1, 1) // => 2
     ```
 */
 func add(_ a: Int, _ b: Int) -> Int { ... }
@@ -185,7 +185,7 @@ the tests now pass as expected.
 $ swift doctest --package path/to/file.swift
 TAP version 13
 1..1
-ok 1 - `add(1, 1)` produces `Int = 2`
+ok 1 - `add(1, 1)` produces `2`
   ---
   column: 1
   file: path/to/file.swift.md
@@ -210,3 +210,5 @@ Mattt ([@mattt](https://twitter.com/mattt))
 
 [seccomp]: https://docs.docker.com/engine/security/seccomp/
 [apparmor]: https://docs.docker.com/engine/security/apparmor/
+
+[ci badge]: https://github.com/SwiftDocOrg/DocTest/workflows/CI/badge.svg

Sources/DocTest/Expectation.swift

@@ -1,19 +1,42 @@
 import Foundation
 
 public enum Expectation: Hashable {
-    case value(String)
     case error
+    case type(String)
+    case value(String)
+    case match(String)
 
     public init?(_ string: String?) {
-        guard let string = string?.trimmingCharacters(in: .whitespacesAndNewlines) else { return nil }
-        if string.starts(with: "=>"),
-            let index = string.firstIndex(where: { $0.isWhitespace })
-        {
-            self = .value(string.suffix(from: index).trimmingCharacters(in: .whitespaces))
-        } else if string.starts(with: "!!") {
+        guard let string = string?.trimmed,
+              let index = string.index(string.startIndex, offsetBy: 2, limitedBy: string.endIndex)
+        else { return nil }
+
+        switch string.prefix(upTo: index) {
+        case "!!":
             self = .error
-        } else {
+        case "->":
+            self = .type(string.suffix(from: index).trimmed)
+        case "=>":
+            self = .value(string.suffix(from: index).trimmed)
+        case "~>":
+            self = .match(string.suffix(from: index).trimmed)
+        default:
             return nil
         }
     }
+
+    public func evaluate(_ output: String) -> Bool {
+        let output = output.trimmed
+
+        switch self {
+        case .error:
+            return output.hasPrefix("error:")
+        case .type(let type):
+            return output.hasPrefix("\(type) =")
+        case .value(let value):
+            return output.hasSuffix("= \(value)")
+        case .match(let pattern):
+            return output.range(of: pattern, options: .regularExpression) != nil
+        }
+    }
 }

Sources/DocTest/Extensions/StringProtocol+Extensions.swift

@@ -0,0 +1,7 @@
+import Foundation
+
+extension StringProtocol {
+    var trimmed: String {
+        trimmingCharacters(in: .whitespacesAndNewlines)
+    }
+}

Sources/DocTest/REPL.swift

@@ -17,9 +17,9 @@ public class REPL {
         public var description: String
 
         public init?(_ description: String) {
-            let trimmedDescription = description.trimmingCharacters(in: .whitespacesAndNewlines)
-            guard !trimmedDescription.isEmpty else { return nil }
-            self.description = trimmedDescription
+            let description = description.trimmed
+            guard !description.isEmpty else { return nil }
+            self.description = description
         }
     }
 
@@ -41,7 +41,7 @@ public class REPL {
 
     public var evaluationHandler: ((Statement, Result<String, Error>) -> Void)?
 
-    public init(configuration: Configuration) {
+    init(configuration: Configuration) {
         process = Process()
 
         if #available(OSX 10.13, *) {
@@ -108,7 +108,7 @@ public class REPL {
         }
     }
 
-    public func evaluate(_ statement: Statement) {
+    func evaluate(_ statement: Statement) {
         if !process.isRunning {
             if #available(OSX 10.13, *) {
                 try! process.run()
@@ -133,12 +133,11 @@ public class REPL {
         outputPipe.fileHandleForReading.readabilityHandler = nil
     }
 
-    public func waitUntilExit() {
+    func waitUntilExit() {
         process.waitUntilExit()
     }
 
-    public func close() {
-
+    func close() {
         if #available(OSX 10.15, *) {
             try! self.inputPipe.fileHandleForWriting.close()
         }

Sources/DocTest/Runner.swift

@@ -37,14 +37,14 @@ public final class Runner {
 
         let repl = REPL(configuration: configuration)
 
-        repl.evaluationHandler = { (statement, result) in
-            tests.append(contentsOf: statement.tests(with: result))
-        }
-
         for statement in statements {
             repl.evaluate(statement)
         }
 
+        repl.evaluationHandler = { (statement, result) in
+            tests.append(contentsOf: statement.tests(with: result))
+        }
+
         repl.close()
         repl.waitUntilExit()
 

Sources/DocTest/Scanner.swift

@@ -0,0 +1,48 @@
+import Foundation
+import StringLocationConverter
+
+public class Scanner {
+    public typealias Match = (line: Int, column: Int, content: String)
+
+    private var regularExpression: NSRegularExpression
+
+    public init() throws {
+        let pattern = #"""
+        ^
+        \h* \`{3} \h* swift \h+ doctest \h* \n
+        (.+)\n
+        \h* \`{3} \h*
+        $
+        """#
+        self.regularExpression = try NSRegularExpression(pattern: pattern,
+                                                         options: [
+                                                            .allowCommentsAndWhitespace,
+                                                            .anchorsMatchLines,
+                                                            .caseInsensitive,
+                                                            .dotMatchesLineSeparators
+                                                         ])
+    }
+
+    public func matches(in source: String) -> [Match] {
+        let range = NSRange(source.startIndex..<source.endIndex, in: source)
+        return regularExpression.matches(in: source, options: [], range: range).compactMap { result in
+            guard result.numberOfRanges == 2,
+                let range = Range(result.range(at: 1), in: source)
+                else { return nil }
+            let content = source[range]
+
+            let converter = StringLocationConverter(for: source)
+
+            let line: Int, column: Int
+            if let location = converter.location(for: range.lowerBound, in: source) {
+                line = location.line
+                column = location.column
+            } else {
+                line = 0
+                column = range.lowerBound.utf16Offset(in: source)
+            }
+
+            return (line, column, content.trimmed)
+        }
+    }
+}

Sources/DocTest/Statement.swift

@@ -8,14 +8,14 @@ public class Statement {
     public internal(set) var expectations: [Expectation] = []
 
     public init?(_ node: CodeBlockItemSyntax, _ sourceLocation: SourceLocation) {
-        let code = node.withoutTrivia().description.trimmingCharacters(in: .whitespacesAndNewlines)
+        let code = node.withoutTrivia().description.trimmed
         guard !code.isEmpty else { return nil }
 
         self.code = code
         self.sourceLocation = sourceLocation
     }
 
-    public func tests(with result: Result<String, REPL.Error>) -> [Test] {
+    func tests(with result: Result<String, REPL.Error>) -> [Test] {
         var metadata: [String: Any] = [
             "file": self.sourceLocation.file as Any?,
             "line": self.sourceLocation.line as Any?,
@@ -31,16 +31,18 @@ public class Statement {
             } else {
                 return expectations.map { expectation in
                     switch expectation {
-                    case .value(let expected):
+                    case .error:
+                        return test {
+                            .success("- `\(self.code)` produced an error, as expected", directive: nil, metadata: metadata)
+                        }
+                    case .type(let expected),
+                         .value(let expected),
+                         .match(let expected):
                         metadata["expected"] = expected
 
                         return test {
                             .failure("- `\(self.code)` produced an error", directive: nil, metadata: metadata)
                         }
-                    case .error:
-                        return test {
-                            .success("- `\(self.code)` produced an error, as expected", directive: nil, metadata: metadata)
-                        }
                     }
                 }
             }
@@ -49,10 +51,12 @@ public class Statement {
 
             return expectations.map { expectation in
                 switch expectation {
-                case .value(let expected):
+                case .type(let expected),
+                     .value(let expected),
+                     .match(let expected):
                     metadata["expected"] = expected
 
-                    if actual == expected {
+                    if expectation.evaluate(actual) {
                         return test {
                             .success("- `\(self.code)` produces `\(actual)`", directive: nil, metadata: metadata)
                         }

Sources/swift-doctest/main.swift

@@ -66,8 +66,7 @@ struct SwiftDocTest: ParsableCommand {
         logger.debug("Swift launch path: \(configuration.launchPath)")
         logger.debug("Swift launch arguments: \(configuration.arguments)")
 
-        let pattern = #"^\`{3}\s*swift\s+doctest\s*\n(.+)\n\`{3}$"#
-        let regex = try NSRegularExpression(pattern: pattern, options: [.caseInsensitive, .anchorsMatchLines, .dotMatchesLineSeparators])
+        let scanner = try Scanner()
 
         let source: String
         let assumedFileName: String
@@ -82,29 +81,24 @@ struct SwiftDocTest: ParsableCommand {
             logger.trace("Scanning standard input for DocTest blocks")
         }
 
-        let converter = StringLocationConverter(for: source)
-
         var reports: [Report] = []
 
         let group = DispatchGroup()
-        regex.enumerateMatches(in: source, options: [], range: NSRange(source.startIndex..<source.endIndex, in: source)) { (result, _, _) in
-            guard let result = result, result.numberOfRanges == 2,
-                let range = Range(result.range(at: 1), in: source)
-                else { return }
-            let match = source[range]
-
-            let position: String
-            var lineOffset: Int = 0
-            if let location = converter.location(for: range.lowerBound, in: source) {
-                lineOffset = location.line
-                position = "\(location.line):\(location.column)"
+        for match in scanner.matches(in: source) {
+            logger.info("Found DocTest block at \(assumedFileName)#\(match.line):\(match.column)\n\(match.content)")
+
+            var lineOffset = match.line
+            var code = match.content
+            if options.runThroughPackageManager {
+                if source.range(of: #"^import \w"#, options: [.regularExpression]) == nil {
+                    logger.notice("No import statements found at \(assumedFileName)#\(match.line):\(match.column). This may cause unexpected API resolution failures when running through Swift Package Manager.")
+                }
             } else {
-                position = "\(range.lowerBound.utf16Offset(in: source)) – \(range.upperBound.utf16Offset(in: source))"
+                code = "\(source)\n\(code)"
+                lineOffset -= source.split(whereSeparator: { $0.isNewline }).count + 1
             }
-            logger.info("Found DocTest block at \(assumedFileName)#\(position)\n\(match)")
-
-            let runner = try! Runner(source: String(match), assumedFileName: assumedFileName, lineOffset: lineOffset)
 
+            let runner = try Runner(source: code, assumedFileName: assumedFileName, lineOffset: lineOffset)
             group.enter()
             runner.run(with: configuration) { result in
                 switch result {