From 8d2ac0001580fc4ce7f881d9f1e858c053dc989a Mon Sep 17 00:00:00 2001 From: Rintaro Ishizaki Date: Thu, 23 Jan 2025 17:55:28 -0800 Subject: [PATCH] [XcodeGen] Handle 'rule' declarations and generate command line args * Rename 'BuildRule to 'BuildEdige' because it is the official term * NinjaParser to handle 'include' and 'rule' directives * NinjaParser to handle parse "rule name" in 'build' correctly * Make variable table a simple `[String: String]` and keep any bindings to make the substitutions possible. * Generate command line argumets using 'command' variable in the 'rule' and use it as the source of truth, istead of using random known bindings like 'FLAGS'. --- .../BuildArgs/RunnableTargets.swift | 4 +- .../BuildArgs/SwiftDriverUtils.swift | 126 ------------- .../BuildArgs/SwiftTargets.swift | 74 +++----- .../SwiftXcodeGen/Ninja/NinjaBuildFile.swift | 156 +++++++++++++---- .../SwiftXcodeGen/Ninja/NinjaParser.swift | 165 ++++++++++++------ .../SwiftXcodeGen/Ninja/RepoBuildDir.swift | 2 +- .../Sources/SwiftXcodeGen/Path/AnyPath.swift | 9 + .../SwiftXcodeGen/Path/RelativePath.swift | 5 + .../SwiftXcodeGenTest/NinjaParserTests.swift | 162 +++++++++++++---- 9 files changed, 403 insertions(+), 300 deletions(-) delete mode 100644 utils/swift-xcodegen/Sources/SwiftXcodeGen/BuildArgs/SwiftDriverUtils.swift diff --git a/utils/swift-xcodegen/Sources/SwiftXcodeGen/BuildArgs/RunnableTargets.swift b/utils/swift-xcodegen/Sources/SwiftXcodeGen/BuildArgs/RunnableTargets.swift index b776f96d6304b..6f5f133aab77a 100644 --- a/utils/swift-xcodegen/Sources/SwiftXcodeGen/BuildArgs/RunnableTargets.swift +++ b/utils/swift-xcodegen/Sources/SwiftXcodeGen/BuildArgs/RunnableTargets.swift @@ -22,7 +22,7 @@ struct RunnableTargets { private var targets: [RunnableTarget] = [] init(from buildDir: RepoBuildDir) throws { - for rule in try buildDir.ninjaFile.buildRules { + for rule in try buildDir.ninjaFile.buildEdges { tryAddTarget(rule, buildDir: buildDir) } } @@ -59,7 +59,7 @@ extension RunnableTargets { } private mutating func tryAddTarget( - _ rule: NinjaBuildFile.BuildRule, buildDir: RepoBuildDir + _ rule: NinjaBuildFile.BuildEdge, buildDir: RepoBuildDir ) { guard let (name, path) = getRunnablePath(for: rule.outputs), addedPaths.insert(path).inserted else { return } diff --git a/utils/swift-xcodegen/Sources/SwiftXcodeGen/BuildArgs/SwiftDriverUtils.swift b/utils/swift-xcodegen/Sources/SwiftXcodeGen/BuildArgs/SwiftDriverUtils.swift deleted file mode 100644 index 667e2498883d0..0000000000000 --- a/utils/swift-xcodegen/Sources/SwiftXcodeGen/BuildArgs/SwiftDriverUtils.swift +++ /dev/null @@ -1,126 +0,0 @@ -//===--- SwiftDriverUtils.swift -------------------------------------------===// -// -// This source file is part of the Swift.org open source project -// -// Copyright (c) 2024 Apple Inc. and the Swift project authors -// Licensed under Apache License v2.0 with Runtime Library Exception -// -// See https://swift.org/LICENSE.txt for license information -// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors -// -//===----------------------------------------------------------------------===// - -// https://github.com/swiftlang/swift-driver/blob/661e0bc74bdae4d9f6ea8a7a54015292febb0059/Sources/SwiftDriver/Utilities/StringAdditions.swift -extension String { - /// Whether this string is a Swift identifier. - var isValidSwiftIdentifier: Bool { - guard let start = unicodeScalars.first else { - return false - } - - let continuation = unicodeScalars.dropFirst() - - return start.isValidSwiftIdentifierStart && - continuation.allSatisfy { $0.isValidSwiftIdentifierContinuation } - } -} - -extension Unicode.Scalar { - - var isValidSwiftIdentifierStart: Bool { - guard isValidSwiftIdentifierContinuation else { return false } - - if isASCIIDigit || self == "$" { - return false - } - - // N1518: Recommendations for extended identifier characters for C and C++ - // Proposed Annex X.2: Ranges of characters disallowed initially - if (0x0300...0x036F).contains(value) || - (0x1DC0...0x1DFF).contains(value) || - (0x20D0...0x20FF).contains(value) || - (0xFE20...0xFE2F).contains(value) { - return false - } - - return true - } - - var isValidSwiftIdentifierContinuation: Bool { - if isASCII { - return isCIdentifierBody(allowDollar: true) - } - - // N1518: Recommendations for extended identifier characters for C and C++ - // Proposed Annex X.1: Ranges of characters allowed - return value == 0x00A8 || - value == 0x00AA || - value == 0x00AD || - value == 0x00AF || - - (0x00B2...0x00B5).contains(value) || - (0x00B7...0x00BA).contains(value) || - (0x00BC...0x00BE).contains(value) || - (0x00C0...0x00D6).contains(value) || - (0x00D8...0x00F6).contains(value) || - (0x00F8...0x00FF).contains(value) || - - (0x0100...0x167F).contains(value) || - (0x1681...0x180D).contains(value) || - (0x180F...0x1FFF).contains(value) || - - (0x200B...0x200D).contains(value) || - (0x202A...0x202E).contains(value) || - (0x203F...0x2040).contains(value) || - value == 0x2054 || - (0x2060...0x206F).contains(value) || - - (0x2070...0x218F).contains(value) || - (0x2460...0x24FF).contains(value) || - (0x2776...0x2793).contains(value) || - (0x2C00...0x2DFF).contains(value) || - (0x2E80...0x2FFF).contains(value) || - - (0x3004...0x3007).contains(value) || - (0x3021...0x302F).contains(value) || - (0x3031...0x303F).contains(value) || - - (0x3040...0xD7FF).contains(value) || - - (0xF900...0xFD3D).contains(value) || - (0xFD40...0xFDCF).contains(value) || - (0xFDF0...0xFE44).contains(value) || - (0xFE47...0xFFF8).contains(value) || - - (0x10000...0x1FFFD).contains(value) || - (0x20000...0x2FFFD).contains(value) || - (0x30000...0x3FFFD).contains(value) || - (0x40000...0x4FFFD).contains(value) || - (0x50000...0x5FFFD).contains(value) || - (0x60000...0x6FFFD).contains(value) || - (0x70000...0x7FFFD).contains(value) || - (0x80000...0x8FFFD).contains(value) || - (0x90000...0x9FFFD).contains(value) || - (0xA0000...0xAFFFD).contains(value) || - (0xB0000...0xBFFFD).contains(value) || - (0xC0000...0xCFFFD).contains(value) || - (0xD0000...0xDFFFD).contains(value) || - (0xE0000...0xEFFFD).contains(value) - } - - /// `true` if this character is an ASCII digit: [0-9] - var isASCIIDigit: Bool { (0x30...0x39).contains(value) } - - /// `true` if this is a body character of a C identifier, - /// which is [a-zA-Z0-9_]. - func isCIdentifierBody(allowDollar: Bool = false) -> Bool { - if (0x41...0x5A).contains(value) || - (0x61...0x7A).contains(value) || - isASCIIDigit || - self == "_" { - return true - } else { - return allowDollar && self == "$" - } - } -} diff --git a/utils/swift-xcodegen/Sources/SwiftXcodeGen/BuildArgs/SwiftTargets.swift b/utils/swift-xcodegen/Sources/SwiftXcodeGen/BuildArgs/SwiftTargets.swift index 8ffe13bee694a..01bc7fda399c9 100644 --- a/utils/swift-xcodegen/Sources/SwiftXcodeGen/BuildArgs/SwiftTargets.swift +++ b/utils/swift-xcodegen/Sources/SwiftXcodeGen/BuildArgs/SwiftTargets.swift @@ -24,7 +24,7 @@ struct SwiftTargets { init(for buildDir: RepoBuildDir) throws { log.debug("[*] Reading Swift targets from build.ninja") - for rule in try buildDir.ninjaFile.buildRules { + for rule in try buildDir.ninjaFile.buildEdges { try tryAddTarget(for: rule, buildDir: buildDir) } targets.sort(by: { $0.name < $1.name }) @@ -65,33 +65,17 @@ struct SwiftTargets { } private mutating func computeBuildArgs( - for rule: NinjaBuildFile.BuildRule + for edge: NinjaBuildFile.BuildEdge, + in ninja: NinjaBuildFile ) throws -> BuildArgs? { - var buildArgs = BuildArgs(for: .swiftc) - if let commandAttr = rule.attributes[.command] { - // We have a custom command, parse it looking for a swiftc invocation. - let command = try CommandParser.parseKnownCommandOnly(commandAttr.value) - guard let command, command.executable.knownCommand == .swiftc else { - return nil - } - buildArgs += command.args - } else if rule.attributes[.flags] != nil { - // Ninja separates out other arguments we need, splice them back in. - for key: NinjaBuildFile.Attribute.Key in [.flags, .includes, .defines] { - guard let attr = rule.attributes[key] else { continue } - buildArgs.append(attr.value) - } - // Add a module name argument if one is specified, validating to - // ensure it's correct since we currently have some targets with - // invalid module names, e.g swift-plugin-server. - if let moduleName = rule.attributes[.swiftModuleName]?.value, - moduleName.isValidSwiftIdentifier { - buildArgs.append("-module-name \(moduleName)") - } - } else { + let commandLine = try ninja.commandLine(for: edge) + let command = try CommandParser.parseKnownCommandOnly(commandLine) + guard let command, command.executable.knownCommand == .swiftc else { return nil } + var buildArgs = BuildArgs(for: .swiftc, args: command.args) + // Only include known flags for now. buildArgs = buildArgs.filter { arg in if arg.flag != nil { @@ -125,17 +109,9 @@ struct SwiftTargets { } func getSources( - from rule: NinjaBuildFile.BuildRule, buildDir: RepoBuildDir + from edge: NinjaBuildFile.BuildEdge, buildDir: RepoBuildDir ) throws -> SwiftTarget.Sources { - // If we have SWIFT_SOURCES defined, use it, otherwise check the rule - // inputs. - let files: [AnyPath] - if let sourcesStr = rule.attributes[.swiftSources]?.value { - files = try CommandParser.parseArguments(sourcesStr, for: .swiftc) - .compactMap(\.value).map(AnyPath.init) - } else { - files = rule.inputs.map(AnyPath.init) - } + let files: [AnyPath] = edge.inputs.map(AnyPath.init) // Split the files into repo sources and external sources. Repo sources // are those under the repo path, external sources are outside that path, @@ -166,29 +142,29 @@ struct SwiftTargets { } private mutating func tryAddTarget( - for rule: NinjaBuildFile.BuildRule, + for edge: NinjaBuildFile.BuildEdge, buildDir: RepoBuildDir ) throws { // Phonies are only used to track aliases. - if rule.isPhony { - for output in rule.outputs { - outputAliases[output, default: []] += rule.inputs + if edge.isPhony { + for output in edge.outputs { + outputAliases[output, default: []] += edge.inputs } return } // Ignore build rules that don't have object file or swiftmodule outputs. - let forBuild = rule.outputs.contains( + let forBuild = edge.outputs.contains( where: { $0.hasExtension(.o) } ) - let forModule = rule.outputs.contains( + let forModule = edge.outputs.contains( where: { $0.hasExtension(.swiftmodule) } ) guard forBuild || forModule else { return } - let primaryOutput = rule.outputs.first! - let sources = try getSources(from: rule, buildDir: buildDir) + let primaryOutput = edge.outputs.first! + let sources = try getSources(from: edge, buildDir: buildDir) let repoSources = sources.repoSources let externalSources = sources.externalSources @@ -198,32 +174,30 @@ struct SwiftTargets { return } - guard let buildArgs = try computeBuildArgs(for: rule) else { return } + guard let buildArgs = try computeBuildArgs(for: edge, in: buildDir.ninjaFile) else { return } // Pick up the module name from the arguments, or use an explicitly // specified module name if we have one. The latter might be invalid so // may not be part of the build args (e.g 'swift-plugin-server'), but is // fine for generation. - let moduleName = buildArgs.lastValue(for: .moduleName) ?? - rule.attributes[.swiftModuleName]?.value + let moduleName = buildArgs.lastValue(for: .moduleName) ?? edge.bindings[.swiftModuleName] guard let moduleName else { log.debug("! Skipping Swift target with output \(primaryOutput); no module name") return } - let moduleLinkName = rule.attributes[.swiftLibraryName]?.value ?? - buildArgs.lastValue(for: .moduleLinkName) + let moduleLinkName = buildArgs.lastValue(for: .moduleLinkName) ?? edge.bindings[.swiftLibraryName] let name = moduleLinkName ?? moduleName // Add the dependencies. We track dependencies for any input files, along // with any recorded swiftmodule dependencies. dependenciesByTargetName.withValue(for: name, default: []) { deps in deps.formUnion( - rule.inputs.filter { + edge.inputs.filter { $0.hasExtension(.swiftmodule) || $0.hasExtension(.o) } ) deps.formUnion( - rule.dependencies.filter { $0.hasExtension(.swiftmodule) } + edge.dependencies.filter { $0.hasExtension(.swiftmodule) } ) } @@ -258,7 +232,7 @@ struct SwiftTargets { targets.append(target) return target }() - for output in rule.outputs { + for output in edge.outputs { targetsByOutput[output] = target } if buildRule == nil || target.buildRule == nil { diff --git a/utils/swift-xcodegen/Sources/SwiftXcodeGen/Ninja/NinjaBuildFile.swift b/utils/swift-xcodegen/Sources/SwiftXcodeGen/Ninja/NinjaBuildFile.swift index 8e2c3cd66560d..64b088df10499 100644 --- a/utils/swift-xcodegen/Sources/SwiftXcodeGen/Ninja/NinjaBuildFile.swift +++ b/utils/swift-xcodegen/Sources/SwiftXcodeGen/Ninja/NinjaBuildFile.swift @@ -11,88 +11,179 @@ //===----------------------------------------------------------------------===// struct NinjaBuildFile { - var attributes: [Attribute.Key: Attribute] - var buildRules: [BuildRule] = [] + var bindings: Bindings + var rules: [String: Rule] + var buildEdges: [BuildEdge] = [] init( - attributes: [Attribute.Key: Attribute], - buildRules: [BuildRule] + bindings: [String: String], + rules: [String: Rule], + buildEdges: [BuildEdge] ) { - self.attributes = attributes - self.buildRules = buildRules + self.bindings = Bindings(storage: bindings) + self.buildEdges = buildEdges + self.rules = rules } } extension NinjaBuildFile { var buildConfiguration: BuildConfiguration? { - attributes[.configuration] - .flatMap { BuildConfiguration(rawValue: $0.value) } + bindings[.configuration] + .flatMap { BuildConfiguration(rawValue: $0) } } } extension NinjaBuildFile { - struct BuildRule: Hashable { + + struct Bindings: Hashable { + let values: [String: String] + + init(storage: [String : String]) { + self.values = storage + } + + subscript(key: String) -> String? { + values[key] + } + } + + struct Rule: Equatable { + let name: String + var bindings: Bindings + + init(name: String, bindings: [String: String]) { + self.name = name + self.bindings = Bindings(storage: bindings) + } + } + + struct BuildEdge: Hashable { + let ruleName: String let inputs: [String] let outputs: [String] let dependencies: [String] + var bindings: Bindings - let attributes: [Attribute.Key: Attribute] - private(set) var isPhony = false + var isPhony: Bool { + ruleName == "phony" + } init( + ruleName: String, inputs: [String], outputs: [String], dependencies: [String], - attributes: [Attribute.Key : Attribute] + bindings: [String: String] ) { + self.ruleName = ruleName self.inputs = inputs self.outputs = outputs self.dependencies = dependencies - self.attributes = attributes + self.bindings = Bindings(storage: bindings) } static func phony(for outputs: [String], inputs: [String]) -> Self { - var rule = Self( - inputs: inputs, outputs: outputs, dependencies: [], attributes: [:] + return Self( + ruleName: "phony", inputs: inputs, outputs: outputs, dependencies: [], bindings: [:] ) - rule.isPhony = true - return rule } } } + +fileprivate enum NinjaCommandLineError: Error { + case unknownRule(String) + case missingCommandBinding +} + extension NinjaBuildFile { - struct Attribute: Hashable { - var key: Key - var value: String + + func commandLine(for edge: BuildEdge) throws -> String { + guard let rule = self.rules[edge.ruleName] else { + throw NinjaCommandLineError.unknownRule(edge.ruleName) + } + + // Helper to get a substitution value for ${key}. + // Note that we don't do built-in substitutions (e.g. $in, $out) for now. + func value(for key: String) -> String? { + edge.bindings[key] ?? rule.bindings[key] ?? self.bindings[key] + } + + func eval(string: String) -> String { + var result = "" + string.scanningUTF8 { scanner in + while scanner.hasInput { + if let prefix = scanner.eat(while: { $0 != "$" }) { + result += String(utf8: prefix) + } + guard scanner.tryEat("$") else { + // Reached the end. + break + } + + let substituted: String? = scanner.tryEating { scanner in + // Parse the variable name. + let key: String + if scanner.tryEat("{"), let keyName = scanner.eat(while: { $0 != "}" }), scanner.tryEat("}") { + key = String(utf8: keyName) + } else if let keyName = scanner.eat(while: { $0.isNinjaVarName }) { + key = String(utf8: keyName) + } else { + return nil + } + + return value(for: key) + } + + if let substituted { + // Recursive substitutions. + result += eval(string: substituted) + } else { + // Was not a variable, restore '$' and move on. + result += "$" + } + } + } + return result + } + + guard let commandLine = rule.bindings["command"] else { + throw NinjaCommandLineError.missingCommandBinding + } + return eval(string: commandLine) + } +} + +extension Byte { + fileprivate var isNinjaVarName: Bool { + switch self.scalar { + case "0"..."9", "a"..."z", "A"..."Z", "_", "-": + return true + default: + return false + } } } extension NinjaBuildFile: CustomDebugStringConvertible { var debugDescription: String { - buildRules.map(\.debugDescription).joined(separator: "\n") + buildEdges.map(\.debugDescription).joined(separator: "\n") } } -extension NinjaBuildFile.BuildRule: CustomDebugStringConvertible { +extension NinjaBuildFile.BuildEdge: CustomDebugStringConvertible { var debugDescription: String { """ { inputs: \(inputs) outputs: \(outputs) dependencies: \(dependencies) - attributes: \(attributes) + bindings: \(bindings) isPhony: \(isPhony) } """ } } -extension NinjaBuildFile.Attribute: CustomStringConvertible { - var description: String { - "\(key.rawValue) = \(value)" - } -} - -extension NinjaBuildFile.Attribute { +extension NinjaBuildFile.Bindings { enum Key: String { case configuration = "CONFIGURATION" case defines = "DEFINES" @@ -102,6 +193,9 @@ extension NinjaBuildFile.Attribute { case swiftModuleName = "SWIFT_MODULE_NAME" case swiftLibraryName = "SWIFT_LIBRARY_NAME" case swiftSources = "SWIFT_SOURCES" - case command = "COMMAND" + } + + subscript(key: Key) -> String? { + return self[key.rawValue] } } diff --git a/utils/swift-xcodegen/Sources/SwiftXcodeGen/Ninja/NinjaParser.swift b/utils/swift-xcodegen/Sources/SwiftXcodeGen/Ninja/NinjaParser.swift index 41ab6d1a2fb65..0c78da2f760a5 100644 --- a/utils/swift-xcodegen/Sources/SwiftXcodeGen/Ninja/NinjaParser.swift +++ b/utils/swift-xcodegen/Sources/SwiftXcodeGen/Ninja/NinjaParser.swift @@ -13,22 +13,26 @@ import Foundation struct NinjaParser { + private let filePath: AbsolutePath + private let fileReader: (AbsolutePath) throws -> Data private var lexer: Lexer - private init(_ input: UnsafeRawBufferPointer) throws { + private init(input: UnsafeRawBufferPointer, filePath: AbsolutePath, fileReader: @escaping (AbsolutePath) throws -> Data) throws { + self.filePath = filePath + self.fileReader = fileReader self.lexer = Lexer(ByteScanner(input)) } - static func parse(_ input: Data) throws -> NinjaBuildFile { - try input.withUnsafeBytes { bytes in - var parser = try Self(bytes) + static func parse(filePath: AbsolutePath, fileReader: @escaping (AbsolutePath) throws -> Data = { try $0.read() }) throws -> NinjaBuildFile { + + try fileReader(filePath).withUnsafeBytes { bytes in + var parser = try Self(input: bytes, filePath: filePath, fileReader: fileReader) return try parser.parse() } } } fileprivate enum NinjaParseError: Error { - case badAttribute case expected(NinjaParser.Lexeme) } @@ -66,18 +70,20 @@ fileprivate extension ByteScanner { } fileprivate extension NinjaParser { - typealias BuildRule = NinjaBuildFile.BuildRule - typealias Attribute = NinjaBuildFile.Attribute + typealias Rule = NinjaBuildFile.Rule + typealias BuildEdge = NinjaBuildFile.BuildEdge - struct ParsedAttribute: Hashable { + struct ParsedBinding: Hashable { var key: String var value: String } enum Lexeme: Hashable { - case attribute(ParsedAttribute) + case binding(ParsedBinding) case element(String) + case rule case build + case include case newline case colon case equal @@ -132,14 +138,6 @@ fileprivate extension Byte { } } -fileprivate extension NinjaBuildFile.Attribute { - init?(_ parsed: NinjaParser.ParsedAttribute) { - // Ignore unknown attributes for now. - guard let key = Key(rawValue: parsed.key) else { return nil } - self.init(key: key, value: parsed.value) - } -} - extension NinjaParser.Lexer { typealias Lexeme = NinjaParser.Lexeme @@ -170,7 +168,7 @@ extension NinjaParser.Lexer { }) } - private mutating func tryConsumeAttribute(key: String) -> Lexeme? { + private mutating func tryConsumeBinding(key: String) -> Lexeme? { input.tryEating { input in input.skip(while: \.isSpaceOrTab) guard input.tryEat("=") else { return nil } @@ -178,7 +176,7 @@ extension NinjaParser.Lexer { guard let value = input.consumeUnescaped(while: { !$0.isNewline }) else { return nil } - return .attribute(.init(key: key, value: value)) + return .binding(.init(key: key, value: value)) } } @@ -204,15 +202,24 @@ extension NinjaParser.Lexer { if c.isNinjaOperator { return consumeOperator() } - if isAtStartOfLine && input.tryEat(utf8: "build") { - return .build + if isAtStartOfLine { + // decl keywords. + if input.tryEat(utf8: "build") { + return .build + } + if input.tryEat(utf8: "rule") { + return .rule + } + if input.tryEat(utf8: "include") { + return .include + } } guard let element = consumeElement() else { return nil } - // If we're on a newline, check to see if we can lex an attribute. + // If we're on a newline, check to see if we can lex a binding. if isAtStartOfLine { - if let attr = tryConsumeAttribute(key: element) { - return attr + if let binding = tryConsumeBinding(key: element) { + return binding } } return .element(element) @@ -233,14 +240,42 @@ fileprivate extension NinjaParser { while let lexeme = eat(), lexeme != .newline {} } - mutating func parseAttribute() throws -> ParsedAttribute? { - guard case let .attribute(attr) = peek else { return nil } + mutating func parseBinding() throws -> ParsedBinding? { + guard case let .binding(binding) = peek else { return nil } eat() tryEat(.newline) - return attr + return binding } - mutating func parseBuildRule() throws -> BuildRule? { + /// ``` + /// rule rulename + /// command = ... + /// var = ... + /// ``` + mutating func parseRule() throws -> Rule? { + let indent = lexer.leadingTriviaCount + guard tryEat(.rule) else { return nil } + + guard let ruleName = tryEatElement() else { + throw NinjaParseError.expected(.element("")) + } + guard tryEat(.newline) else { + throw NinjaParseError.expected(.newline) + } + + var bindings: [String: String] = [:] + while indent < lexer.leadingTriviaCount, let binding = try parseBinding() { + bindings[binding.key] = binding.value + } + + return Rule(name: ruleName, bindings: bindings) + } + + /// ``` + /// build out1... | implicit-out... : rulename input... | dep... || order-only-dep... + /// var = ... + /// ``` + mutating func parseBuildEdge() throws -> BuildEdge? { let indent = lexer.leadingTriviaCount guard tryEat(.build) else { return nil } @@ -258,17 +293,16 @@ fileprivate extension NinjaParser { throw NinjaParseError.expected(.colon) } - var isPhony = false + guard let ruleName = tryEatElement() else { + throw NinjaParseError.expected(.element("")) + } + var inputs: [String] = [] while let str = tryEatElement() { - if str == "phony" { - isPhony = true - } else { - inputs.append(str) - } + inputs.append(str) } - if isPhony { + if ruleName == "phony" { skipLine() return .phony(for: outputs, inputs: inputs) } @@ -280,7 +314,7 @@ fileprivate extension NinjaParser { continue } if tryEat(.pipe) || tryEat(.doublePipe) { - // Currently we don't distinguish between implicit and explicit deps. + // Currently we don't distinguish between implicit deps and order-only deps. continue } break @@ -289,38 +323,61 @@ fileprivate extension NinjaParser { // We're done with the line, skip to the next. skipLine() - var attributes: [Attribute.Key: Attribute] = [:] - while indent < lexer.leadingTriviaCount, let attr = try parseAttribute() { - if let attr = Attribute(attr) { - attributes[attr.key] = attr - } + var bindings: [String: String] = [:] + while indent < lexer.leadingTriviaCount, let binding = try parseBinding() { + bindings[binding.key] = binding.value } - return BuildRule( - inputs: inputs, + return BuildEdge( + ruleName: ruleName, + inputs: inputs, outputs: outputs, dependencies: dependencies, - attributes: attributes + bindings: bindings ) } + /// ``` + /// include path/to/sub.ninja + /// ``` + mutating func parseInclude() throws -> NinjaBuildFile? { + guard tryEat(.include) else { return nil } + + guard let fileName = tryEatElement() else { + throw NinjaParseError.expected(.element("")) + } + + let baseDirectory = self.filePath.parentDir! + let path = AnyPath(fileName).absolute(in: baseDirectory) + return try NinjaParser.parse(filePath: path, fileReader: fileReader) + } + mutating func parse() throws -> NinjaBuildFile { - var buildRules: [BuildRule] = [] - var attributes: [Attribute.Key: Attribute] = [:] + var bindings: [String: String] = [:] + var rules: [String: Rule] = [:] + var buildEdges: [BuildEdge] = [] while peek != nil { - if let rule = try parseBuildRule() { - buildRules.append(rule) + if let rule = try parseRule() { + rules[rule.name] = rule continue } - if let attr = try parseAttribute() { - if let attr = Attribute(attr) { - attributes[attr.key] = attr - } + if let edge = try parseBuildEdge() { + buildEdges.append(edge) + continue + } + if let binding = try parseBinding() { + bindings[binding.key] = binding.value + continue + } + if let included = try parseInclude() { + bindings.merge(included.bindings.values, uniquingKeysWith: { _, other in other }) + rules.merge(included.rules, uniquingKeysWith: { _, other in other }) + buildEdges.append(contentsOf: included.buildEdges) continue } - // Ignore unknown bits of syntax like 'include' for now. + // Ignore unknown bits of syntax like 'subninja' for now. eat() } - return NinjaBuildFile(attributes: attributes, buildRules: buildRules) + return NinjaBuildFile(bindings: bindings, rules: rules, buildEdges: buildEdges) } } diff --git a/utils/swift-xcodegen/Sources/SwiftXcodeGen/Ninja/RepoBuildDir.swift b/utils/swift-xcodegen/Sources/SwiftXcodeGen/Ninja/RepoBuildDir.swift index 49b9a77ff1665..16027047cab5a 100644 --- a/utils/swift-xcodegen/Sources/SwiftXcodeGen/Ninja/RepoBuildDir.swift +++ b/utils/swift-xcodegen/Sources/SwiftXcodeGen/Ninja/RepoBuildDir.swift @@ -82,7 +82,7 @@ extension RepoBuildDir { } log.debug("[*] Reading '\(fileName)'") - let ninjaFile = try NinjaParser.parse(fileName.read()) + let ninjaFile = try NinjaParser.parse(filePath: fileName) _ninjaFile = ninjaFile return ninjaFile } diff --git a/utils/swift-xcodegen/Sources/SwiftXcodeGen/Path/AnyPath.swift b/utils/swift-xcodegen/Sources/SwiftXcodeGen/Path/AnyPath.swift index 1d2bf3dcf0e5c..5ba0651f01739 100644 --- a/utils/swift-xcodegen/Sources/SwiftXcodeGen/Path/AnyPath.swift +++ b/utils/swift-xcodegen/Sources/SwiftXcodeGen/Path/AnyPath.swift @@ -52,6 +52,15 @@ extension AnyPath { a } } + + public func absolute(in base: AbsolutePath) -> AbsolutePath { + switch self { + case .relative(let r): + r.absolute(in: base) + case .absolute(let a): + a + } + } } extension AnyPath: Decodable { diff --git a/utils/swift-xcodegen/Sources/SwiftXcodeGen/Path/RelativePath.swift b/utils/swift-xcodegen/Sources/SwiftXcodeGen/Path/RelativePath.swift index b1a5f3982552a..693fe3c615403 100644 --- a/utils/swift-xcodegen/Sources/SwiftXcodeGen/Path/RelativePath.swift +++ b/utils/swift-xcodegen/Sources/SwiftXcodeGen/Path/RelativePath.swift @@ -37,6 +37,11 @@ public extension RelativePath { .init(FileManager.default.currentDirectoryPath).appending(self) } + func absolute(in base: AbsolutePath) -> AbsolutePath { + precondition(base.isDirectory, "Expected '\(base)' to be a directory") + return base.appending(self) + } + init(_ component: Component) { self.init(FilePath(root: nil, components: component)) } diff --git a/utils/swift-xcodegen/Tests/SwiftXcodeGenTest/NinjaParserTests.swift b/utils/swift-xcodegen/Tests/SwiftXcodeGenTest/NinjaParserTests.swift index 5cf3644d2569b..3f0c2c39ea9a3 100644 --- a/utils/swift-xcodegen/Tests/SwiftXcodeGenTest/NinjaParserTests.swift +++ b/utils/swift-xcodegen/Tests/SwiftXcodeGenTest/NinjaParserTests.swift @@ -53,30 +53,50 @@ fileprivate func expectEqual( fileprivate func assertParse( _ str: String, - attributes: [NinjaBuildFile.Attribute] = [], - rules: [NinjaBuildFile.BuildRule], + bindings: [String: String] = [:], + rules: [String: NinjaBuildFile.Rule] = [:], + edges: [NinjaBuildFile.BuildEdge], + file: StaticString = #file, line: UInt = #line +) { + let filePath: AbsolutePath = "/tmp/build.ninja" + let files: [AbsolutePath: String] = [ + filePath: str + ] + assertParse(filePath, in: files, bindings: bindings, rules: rules, edges: edges, file: file, line: line) +} + +fileprivate func assertParse( + _ filePath: AbsolutePath, + in fileSystem: [AbsolutePath: String], + bindings: [String: String] = [:], + rules: [String: NinjaBuildFile.Rule] = [:], + edges: [NinjaBuildFile.BuildEdge], file: StaticString = #file, line: UInt = #line ) { do { - let buildFile = try NinjaParser.parse(Data(str.utf8)) - guard rules.count == buildFile.buildRules.count else { + let buildFile = try NinjaParser.parse(filePath: filePath, fileReader: { Data(fileSystem[$0]!.utf8) }) + guard edges.count == buildFile.buildEdges.count else { XCTFail( - "Expected \(rules.count) rules, got \(buildFile.buildRules.count)", + "Expected \(edges.count) edges, got \(buildFile.buildEdges.count)", file: file, line: line ) return } XCTAssertEqual( - Dictionary(uniqueKeysWithValues: attributes.map { ($0.key, $0) }), - buildFile.attributes, + bindings, + buildFile.bindings.values, + file: file, line: line + ) + XCTAssertEqual( + rules, buildFile.rules, file: file, line: line ) - for (expected, actual) in zip(rules, buildFile.buildRules) { + for (expected, actual) in zip(edges, buildFile.buildEdges) { + expectEqual(expected, actual, \.ruleName, file: file, line: line) expectEqual(expected, actual, \.inputs, file: file, line: line) expectEqual(expected, actual, \.outputs, file: file, line: line) expectEqual(expected, actual, \.dependencies, file: file, line: line) - expectEqual(expected, actual, \.attributes, file: file, line: line) - expectEqual(expected, actual, \.isPhony, file: file, line: line) + expectEqual(expected, actual, \.bindings, file: file, line: line) XCTAssertEqual(expected, actual, file: file, line: line) } @@ -86,27 +106,86 @@ fileprivate func assertParse( } class NinjaParserTests: XCTestCase { - func testBuildRule() throws { - assertParse(""" + func testBuildEdge() throws { + assertParse( + """ # ignore comment, build foo.o: a.swift | dep || orderdep #another build comment - build foo.o foo.swiftmodule: a.swift | dep || orderdep + build foo.o foo.swiftmodule: SWIFTC a.swift | dep || orderdep notpartofthebuildrule - """, rules: [ + """, + edges: [ .init( + ruleName: "SWIFTC", inputs: ["a.swift"], outputs: ["foo.o", "foo.swiftmodule"], dependencies: ["dep", "orderdep"], - attributes: [:] + bindings: [:] + ) + ] + ) + } + + func testRule() throws { + assertParse( + """ + rule SWIFTC + command = /bin/switfc -wmo -target unknown + other = whatever + notpartoftherule + """, + rules: [ + "SWIFTC": .init( + name: "SWIFTC", + bindings: [ + "command": "/bin/switfc -wmo -target unknown", + "other": "whatever", + ]) + ], + edges: [] + ) + } + + func testInclude() throws { + let files: [AbsolutePath: String] = [ + "/tmp/build.ninja": """ + include path/to/sub.ninja + + build foo.swiftmodule : SWIFTC foo.swift + """, + "/tmp/path/to/sub.ninja": """ + rule SWIFTC + command = /bin/swiftc $in -o $out + """ + ] + assertParse( + "/tmp/build.ninja", + in: files, + rules: [ + "SWIFTC": .init( + name: "SWIFTC", + bindings: [ + "command": "/bin/swiftc $in -o $out", + ]) + ], + edges: [ + .init( + ruleName: "SWIFTC", + inputs: ["foo.swift"], + outputs: ["foo.swiftmodule"], + dependencies: [], + bindings: [:] ) ] ) } func testPhonyRule() throws { - assertParse(""" + assertParse( + """ build foo.swiftmodule : phony bar.swiftmodule - """, rules: [ + """, + edges: [ .phony( for: ["foo.swiftmodule"], inputs: ["bar.swiftmodule"] @@ -115,13 +194,14 @@ class NinjaParserTests: XCTestCase { ) } - func testAttributes() throws { - assertParse(""" + func testBindings() throws { + assertParse( + """ x = y CONFIGURATION = Debug - build foo.o: xyz foo.swift | baz.o + build foo.o: SWIFTC xyz foo.swift | baz.o UNKNOWN = foobar SWIFT_MODULE_NAME = foobar @@ -137,28 +217,35 @@ class NinjaParserTests: XCTestCase { COMMAND = /bin/swiftc -I /a/b -wmo FLAGS = -I /c/d -wmo - """, attributes: [ - .init(key: .configuration, value: "Debug"), + """, + bindings: [ + "x": "y", + + "CONFIGURATION": "Debug", // This is considered top-level since it's not indented. - .init(key: .flags, value: "-I /c/d -wmo") + "FLAGS": "-I /c/d -wmo" ], - rules: [ + edges: [ .init( + ruleName: "SWIFTC", inputs: ["xyz", "foo.swift"], outputs: ["foo.o"], dependencies: ["baz.o"], - attributes: [ - .swiftModuleName: .init(key: .swiftModuleName, value: "foobar"), - .flags: .init(key: .flags, value: "-I /a/b -wmo"), + bindings: [ + "UNKNOWN": "foobar", + "SWIFT_MODULE_NAME": "foobar", + "FLAGS": "-I /a/b -wmo", + "ANOTHER_UNKNOWN": "a b c", ] ), .init( - inputs: ["CUSTOM_COMMAND", "baz.swift"], + ruleName: "CUSTOM_COMMAND", + inputs: ["baz.swift"], outputs: ["baz.o"], dependencies: [], - attributes: [ - .command: .init(key: .command, value: "/bin/swiftc -I /a/b -wmo"), + bindings: [ + "COMMAND": "/bin/swiftc -I /a/b -wmo", ] ) ] @@ -167,19 +254,22 @@ class NinjaParserTests: XCTestCase { func testEscape() throws { for newline in ["\n", "\r", "\r\n"] { - assertParse(""" - build foo.o$:: xyz$ foo$$.swift | baz$ bar.o + assertParse( + """ + build foo.o$:: SWIFTC xyz$ foo$$.swift | baz$ bar.o FLAGS = -I /a$\(newline)\ /b -wmo COMMAND = swiftc$$ - """, rules: [ + """, + edges: [ .init( + ruleName: "SWIFTC", inputs: ["xyz foo$.swift"], outputs: ["foo.o:"], dependencies: ["baz bar.o"], - attributes: [ - .flags: .init(key: .flags, value: "-I /a/b -wmo"), - .command: .init(key: .command, value: "swiftc$") + bindings: [ + "FLAGS": "-I /a/b -wmo", + "COMMAND": "swiftc$", ] ) ]