diff --git a/Sources/SwiftDriver/IncrementalCompilation/DependencyKey.swift b/Sources/SwiftDriver/IncrementalCompilation/DependencyKey.swift index e8df445da..f7de0ae64 100644 --- a/Sources/SwiftDriver/IncrementalCompilation/DependencyKey.swift +++ b/Sources/SwiftDriver/IncrementalCompilation/DependencyKey.swift @@ -24,17 +24,6 @@ import Foundation } } -/// Since the integration surfaces all externalDependencies to be processed later, -/// a combination of the dependency and fingerprint are needed. -public struct FingerprintedExternalDependency: Hashable, Equatable { - let externalDependency: ExternalDependency - let fingerprint: String? - init(_ externalDependency: ExternalDependency, _ fingerprint: String?) { - self.externalDependency = externalDependency - self.fingerprint = fingerprint - } - var isIncremental: Bool { fingerprint != nil } -} /// A `DependencyKey` carries all of the information necessary to uniquely /// identify a dependency node in the graph, and serves as a point of identity @@ -124,13 +113,6 @@ public struct DependencyKey: Hashable, CustomStringConvertible { /// /// The `name` of the file is a path to the `swiftdeps` file named in /// the output file map for a given Swift file. - /// - /// If the filename is a swiftmodule, and if it was built by a compilation with - /// `-enable-experimental-cross-module-incremental-build`, the swiftmodule file - /// contains a special section with swiftdeps information for the module - /// in it. In that case the enclosing node should have a fingerprint. - /// - case sourceFileProvide(name: String) /// A "nominal" type that is used, or defined by this file. /// diff --git a/Sources/SwiftDriver/IncrementalCompilation/IncrementalCompilationState.swift b/Sources/SwiftDriver/IncrementalCompilation/IncrementalCompilationState.swift index 54cd50e55..4fe328b95 100644 --- a/Sources/SwiftDriver/IncrementalCompilation/IncrementalCompilationState.swift +++ b/Sources/SwiftDriver/IncrementalCompilation/IncrementalCompilationState.swift @@ -442,6 +442,14 @@ extension IncrementalCompilationState { reporter: reporter) let externallyChangedInputs = computeExternallyChangedInputs( + forIncrementalExternalDependencies: false, + buildTime: outOfDateBuildRecord.buildTime, + fileSystem: fileSystem, + moduleDependencyGraph: moduleDependencyGraph, + reporter: moduleDependencyGraph.reporter) + + let incrementallyExternallyChangedInputs = computeExternallyChangedInputs( + forIncrementalExternalDependencies: true, buildTime: outOfDateBuildRecord.buildTime, fileSystem: fileSystem, moduleDependencyGraph: moduleDependencyGraph, @@ -456,9 +464,9 @@ extension IncrementalCompilationState { // Combine to obtain the inputs that definitely must be recompiled. let definitelyRequiredInputs = Set(changedInputs.map({ $0.filePath }) + - externallyChangedInputs + - inputsHavingMalformedDependencySources + - inputsMissingOutputs) + externallyChangedInputs + incrementallyExternallyChangedInputs + + inputsHavingMalformedDependencySources + + inputsMissingOutputs) if let reporter = reporter { for scheduledInput in definitelyRequiredInputs.sorted(by: {$0.file.name < $1.file.name}) { reporter.report("Queuing (initial):", scheduledInput) @@ -555,23 +563,26 @@ extension IncrementalCompilationState { /// Any files dependent on modified files from other modules must be compiled, too. private static func computeExternallyChangedInputs( + forIncrementalExternalDependencies: Bool, buildTime: Date, fileSystem: FileSystem, moduleDependencyGraph: ModuleDependencyGraph, reporter: IncrementalCompilationState.Reporter? - ) -> [TypedVirtualPath] { - var externalDependencySources = Set() - for extDepAndPrint in moduleDependencyGraph.fingerprintedExternalDependencies { - let extDep = extDepAndPrint.externalDependency + ) -> [TypedVirtualPath] { + var externalDependencySources = Set() + let extDeps = forIncrementalExternalDependencies + ? moduleDependencyGraph.incrementalExternalDependencies + : moduleDependencyGraph.externalDependencies + for extDep in extDeps { let extModTime = extDep.file.flatMap {try? fileSystem.getFileInfo($0).modTime} ?? Date.distantFuture if extModTime >= buildTime { - for dependent in moduleDependencyGraph.untracedDependents(of: extDepAndPrint) { + for dependent in moduleDependencyGraph.untracedDependents(of: extDep, isIncremental: forIncrementalExternalDependencies) { guard let dependencySource = dependent.dependencySource else { fatalError("Dependent \(dependent) does not have dependencies file!") } reporter?.report( - "Queuing because of external dependency on newer \(extDep.file?.basename ?? "extDep?")", + "Queuing because of \(forIncrementalExternalDependencies ? "incremental " : "")external dependency on newer \(extDep.file?.basename ?? "extDep?")", dependencySource.typedFile) externalDependencySources.insert(dependencySource) } diff --git a/Sources/SwiftDriver/IncrementalCompilation/KeyAndFingerprintEnforcer.swift b/Sources/SwiftDriver/IncrementalCompilation/KeyAndFingerprintEnforcer.swift deleted file mode 100644 index e427c8f06..000000000 --- a/Sources/SwiftDriver/IncrementalCompilation/KeyAndFingerprintEnforcer.swift +++ /dev/null @@ -1,55 +0,0 @@ -//===------- ExternalDependencyHolder.swift -------------------------------===// -// -// This source file is part of the Swift.org open source project -// -// Copyright (c) 2020 Apple Inc. and the Swift project authors -// Licensed under Apache License v2.0 with Runtime Library Exception -// -// See https://swift.org/LICENSE.txt for license information -// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors -// -//===----------------------------------------------------------------------===// - -import Foundation - -/// Encapsulates the invariant required for anything with a DependencyKey and an fingerprint -protocol KeyAndFingerprintEnforcer { - var key: DependencyKey {get} - var fingerprint: String? {get} -} -extension KeyAndFingerprintEnforcer { - func verifyKeyAndFingerprint() throws { - guard case .externalDepend(let externalDependency) = key.designator - else { - return - } - guard key.aspect == .interface else { - throw KeyAndFingerprintEnforcerError.externalDepsMustBeInterface(externalDependency) - } - guard let file = externalDependency.file else { - throw KeyAndFingerprintEnforcerError.noFile(externalDependency) - } - guard let fingerprint = self.fingerprint, - file.extension == FileType.swiftModule.rawValue - else { - return - } - throw KeyAndFingerprintEnforcerError.onlySwiftModulesHaveFingerprints(externalDependency, fingerprint) - } -} -enum KeyAndFingerprintEnforcerError: LocalizedError { - case externalDepsMustBeInterface(ExternalDependency) - case noFile(ExternalDependency) - case onlySwiftModulesHaveFingerprints(ExternalDependency, String) - - var errorDescription: String? { - switch self { - case let .externalDepsMustBeInterface(externalDependency): - return "Aspect of external dependency must be interface: \(externalDependency)" - case let .noFile(externalDependency): - return "External dependency must point to a file: \(externalDependency)" - case let .onlySwiftModulesHaveFingerprints(externalDependency, fingerprint): - return "An external dependency with a fingerprint (\(fingerprint)) must point to a swiftmodule file: \(externalDependency)" - } - } -} diff --git a/Sources/SwiftDriver/IncrementalCompilation/ModuleDependencyGraph.swift b/Sources/SwiftDriver/IncrementalCompilation/ModuleDependencyGraph.swift index b8afec248..add84b7e4 100644 --- a/Sources/SwiftDriver/IncrementalCompilation/ModuleDependencyGraph.swift +++ b/Sources/SwiftDriver/IncrementalCompilation/ModuleDependencyGraph.swift @@ -18,7 +18,7 @@ import SwiftOptions // MARK: - ModuleDependencyGraph /*@_spi(Testing)*/ public final class ModuleDependencyGraph { - + var nodeFinder = NodeFinder() /// When integrating a change, want to find untraced nodes so we can kick off jobs that have not been @@ -29,24 +29,24 @@ import SwiftOptions private(set) var inputDependencySourceMap = BidirectionalMap() // The set of paths to external dependencies known to be in the graph - public internal(set) var fingerprintedExternalDependencies = Set() + public internal(set) var externalDependencies = Set() + + // The set of paths to incremental external dependencies known to be in the graph + public internal(set) var incrementalExternalDependencies = Set() let options: IncrementalCompilationState.Options let reporter: IncrementalCompilationState.Reporter? - let fileSystem: FileSystem private let diagnosticEngine: DiagnosticsEngine public init( diagnosticEngine: DiagnosticsEngine, reporter: IncrementalCompilationState.Reporter?, - fileSystem: FileSystem, options: IncrementalCompilationState.Options ) { self.reporter = reporter self.diagnosticEngine = diagnosticEngine self.options = options - self.fileSystem = fileSystem } var isCrossModuleIncrementalBuildEnabled: Bool { @@ -108,7 +108,6 @@ extension ModuleDependencyGraph { let graph = Self( diagnosticEngine: diagnosticEngine, reporter: reporter, - fileSystem: fileSystem, options: options) let inputsAndSwiftdeps = inputs.map { input in @@ -137,96 +136,84 @@ extension ModuleDependencyGraph { // do not try to read swiftdeps of a new input return nil } - guard let _ = graph.integrate(swiftDeps: typedSwiftDepsFile) + guard let results = Integrator.integrate( + dependencySource: dependencySource, + into: graph, + input: input, + reporter: reporter, + diagnosticEngine: diagnosticEngine, + fileSystem: fileSystem, + options: options) else { - return (input, swiftDepsFile) // remember the errorred files + return (input, swiftDepsFile) } + // for initial build, don't care about changedNodes + _ = integrate( + incrementalExternalDependencies: results.discoveredIncrementalExternalDependencies, + into: graph, + reporter: reporter, + diagnosticEngine: diagnosticEngine, + fileSystem: fileSystem) return nil } return (graph, inputsAndMalformedSwiftDeps: inputsAndMalformedSwiftDeps) } -} - -// MARK: - Integrating -extension ModuleDependencyGraph { - /// Integrate one swiftDeps, corresponding to an input, and return the changed nodes - /// Return nil for errror - public func integrate(swiftDeps: TypedVirtualPath) -> Set? { - precondition(swiftDeps.type == .swiftDeps) - let dependencySource = DependencySource(swiftDeps) - return integrate(from: dependencySource) - } - - public func integrate(from dependencySource: DependencySource) -> Set? { - precondition(dependencySource.typedFile.type == .swiftDeps) - guard let sourceGraph = dependencySource.read( - in: fileSystem, - reporter: reporter) - else { - return nil - } - return integrate(sourceGraph: sourceGraph) - } - - public func integrate(sourceGraph: SourceFileDependencyGraph) -> Set? { - let results = Integrator.integrateAndCollectExternalDepNodes(from: sourceGraph, - into: self) - guard let externallyCausedChanges = - integrate(discoveredExternalDependencies: results.discoveredExternalDependencies) - else { - return nil - } - return results.changedNodes.union(externallyCausedChanges) - } /// Returns changed nodes - private func integrate( - discoveredExternalDependencies: Set - ) -> Set? + private static func integrate( + incrementalExternalDependencies: Set, + into graph: ModuleDependencyGraph, + reporter: IncrementalCompilationState.Reporter?, + diagnosticEngine: DiagnosticsEngine, + fileSystem: FileSystem + ) -> Set? { - var workList = discoveredExternalDependencies + var remainingIEDs = incrementalExternalDependencies var changedNodes = Set() - while let edp = workList.first { - guard fingerprintedExternalDependencies.insert(edp).inserted else { + while let ied = remainingIEDs.first { + guard !graph.incrementalExternalDependencies.contains(ied) else { continue } let resultIfOK = integrate( - externalDependency: edp, + incrementalExternalDependency: ied, + into: graph, + reporter: reporter, + diagnosticEngine: diagnosticEngine, fileSystem: fileSystem) if let result = resultIfOK { changedNodes.formUnion(result.changedNodes) - workList.formUnion(result.discoveredExternalDependencies) + remainingIEDs.formUnion(result.discoveredIncrementalExternalDependencies) + graph.incrementalExternalDependencies.insert(ied) } else { return nil // give up } - workList.remove(edp) + remainingIEDs.remove(ied) } return changedNodes } - - private func integrate( - externalDependency: FingerprintedExternalDependency, + private static func integrate( + incrementalExternalDependency: ExternalDependency, + into graph: ModuleDependencyGraph, + reporter: IncrementalCompilationState.Reporter?, + diagnosticEngine: DiagnosticsEngine, fileSystem: FileSystem ) -> Integrator.Results? { - // Save time; don't even try - guard externalDependency.isIncremental else { - return Integrator.Results() - } - let file = externalDependency.externalDependency.file! - let dependencySource = DependencySource(file) - reporter?.report("integrating incremental external dependency", - dependencySource.typedFile) - guard let sourceGraph = dependencySource.read( - in: fileSystem, - reporter: reporter) - else { + guard let file = incrementalExternalDependency.file + else { + diagnosticEngine.emit(warning: "Cannot get file for externalDependency \(incrementalExternalDependency.fileName)") return nil } - let results = Integrator.integrateAndCollectExternalDepNodes( - from: sourceGraph, - into: self - ) + let dependencySource = DependencySource(TypedVirtualPath(file: file, type: .swiftModule)) + reporter?.report("integrating incrementalExperimentalDependency", dependencySource.typedFile) + let results = Integrator.integrate( + dependencySource: dependencySource, + into: graph, + input: nil, + reporter: reporter, + diagnosticEngine: diagnosticEngine, + fileSystem: fileSystem, + options: graph.options) return results } @@ -289,12 +276,29 @@ extension ModuleDependencyGraph { dependencySource: DependencySource, on fileSystem: FileSystem ) -> [TypedVirtualPath]? { - let swiftDeps = inputDependencySourceMap[input].typedFile - let changedNodesIfOK = integrate(swiftDeps: swiftDeps) + let resultsIfOK = Integrator.integrate( + dependencySource: dependencySource, + into: self, + input: input, + reporter: self.reporter, + diagnosticEngine: diagnosticEngine, + fileSystem: fileSystem, + options: self.options) + guard let results = resultsIfOK else { + return nil + } + let changedNodesIfOK = + Self.integrate( + incrementalExternalDependencies: results.discoveredIncrementalExternalDependencies, + into: self, + reporter: reporter, + diagnosticEngine: diagnosticEngine, + fileSystem: fileSystem) guard let changedNodes = changedNodesIfOK else { return nil } - return findSwiftDepsToRecompileWhenNodesChange(changedNodes) + let allChangedNodes = results.changedNodes.union(changedNodes) + return findSwiftDepsToRecompileWhenNodesChange(allChangedNodes) .map { inputDependencySourceMap[$0] } } } @@ -316,16 +320,12 @@ extension ModuleDependencyGraph { } /*@_spi(Testing)*/ public func untracedDependents( - of extDepAndPrint: FingerprintedExternalDependency + of externalSwiftDeps: ExternalDependency, + isIncremental: Bool ) -> [ModuleDependencyGraph.Node] { // These nodes will depend on the *interface* of the external Decl. - let key = DependencyKey(interfaceFor: extDepAndPrint.externalDependency) - // DependencySource is OK as a nil placeholder because it's only used to find - // the corresponding implementation node and there won't be any for an - // external dependency node. - let node = Node(key: key, - fingerprint: extDepAndPrint.fingerprint, - dependencySource: nil) + let key = DependencyKey(interfaceFor: externalSwiftDeps) + let node = Node(key: key, fingerprint: nil, dependencySource: nil) return nodeFinder .orderedUses(of: node) .filter({ use in isUntraced(use) }) @@ -382,7 +382,8 @@ extension ModuleDependencyGraph { } // MARK: - debugging extension ModuleDependencyGraph { - func emitDotFile(_ g: SourceFileDependencyGraph) { + func emitDotFile(_ g: SourceFileDependencyGraph, + _ dependencySource: DependencySource) { // TODO: Incremental emitDotFIle fatalError("unimplmemented, writing dot file of dependency graph") } @@ -480,13 +481,11 @@ extension ModuleDependencyGraph { init( diagnosticEngine: DiagnosticsEngine, reporter: IncrementalCompilationState.Reporter?, - fileSystem: FileSystem, options: IncrementalCompilationState.Options ) { self.graph = ModuleDependencyGraph( diagnosticEngine: diagnosticEngine, reporter: reporter, - fileSystem: fileSystem, options: options) } @@ -561,7 +560,7 @@ extension ModuleDependencyGraph { let fingerprint = hasFingerprint ? fingerprintStr : nil let dependencySource = try swiftDepsStr .map({ try VirtualPath(path: $0) }) - .map(DependencySource.init) + .map(ModuleDependencyGraph.DependencySource.init) self.finalize(node: Node(key: key, fingerprint: fingerprint, dependencySource: dependencySource)) @@ -595,13 +594,11 @@ extension ModuleDependencyGraph { let path = identifiers[Int(record.fields[0])] let hasFingerprint = Int(record.fields[1]) != 0 let fingerprint = hasFingerprint ? fingerprintStr : nil - self.graph.fingerprintedExternalDependencies.insert( - FingerprintedExternalDependency(ExternalDependency(path), fingerprint)) + self.graph.externalDependencies.insert(ExternalDependency(path, fingerprint: fingerprint)) case .identifierNode: guard record.fields.count == 0, case .blob(let identifierBlob) = record.payload, - let identifier = String(data: identifierBlob, encoding: .utf8) - else { + let identifier = String(data: identifierBlob, encoding: .utf8) else { throw ReadError.malformedIdentifierRecord } identifiers.append(identifier) @@ -617,7 +614,6 @@ extension ModuleDependencyGraph { let data = Data(bytesNoCopy: baseAddr, count: buf.count, deallocator: .none) var visitor = Visitor(diagnosticEngine: diagnosticEngine, reporter: reporter, - fileSystem: fileSystem, options: options) try Bitcode.read(stream: data, using: &visitor) guard let major = visitor.majorVersion, @@ -757,10 +753,10 @@ extension ModuleDependencyGraph { if let dependencySourceFileName = node.dependencySource?.file.name { self.addIdentifier(dependencySourceFileName) } - if let context = node.key.designator.context { + if let context = node.dependencyKey.designator.context { self.addIdentifier(context) } - if let name = node.key.designator.name { + if let name = node.dependencyKey.designator.name { self.addIdentifier(name) } } @@ -774,8 +770,8 @@ extension ModuleDependencyGraph { } } - for edF in graph.fingerprintedExternalDependencies { - self.addIdentifier(edF.externalDependency.fileName) + for path in graph.externalDependencies { + self.addIdentifier(path.fileName) } for str in self.identifiersToWrite { @@ -873,12 +869,12 @@ extension ModuleDependencyGraph { graph.nodeFinder.forEachNode { node in serializer.stream.writeRecord(serializer.abbreviations[.moduleDepGraphNode]!, { $0.append(RecordID.moduleDepGraphNode) - $0.append(node.key.designator.code) - $0.append(node.key.aspect.code) + $0.append(node.dependencyKey.designator.code) + $0.append(node.dependencyKey.aspect.code) $0.append(serializer.lookupIdentifierCode( - for: node.key.designator.context ?? "")) + for: node.dependencyKey.designator.context ?? "")) $0.append(serializer.lookupIdentifierCode( - for: node.key.designator.name ?? "")) + for: node.dependencyKey.designator.name ?? "")) $0.append((node.dependencySource != nil) ? UInt32(1) : UInt32(0)) $0.append(serializer.lookupIdentifierCode( for: node.dependencySource?.file.name ?? "")) @@ -908,14 +904,12 @@ extension ModuleDependencyGraph { } } - for fingerprintedExternalDependency in graph.fingerprintedExternalDependencies { + for dep in graph.externalDependencies { serializer.stream.writeRecord(serializer.abbreviations[.externalDepNode]!, { $0.append(RecordID.externalDepNode) - $0.append(serializer.lookupIdentifierCode( - for: fingerprintedExternalDependency.externalDependency.fileName)) - $0.append((fingerprintedExternalDependency.fingerprint != nil) ? UInt32(1) : UInt32(0)) - }, - blob: (fingerprintedExternalDependency.fingerprint ?? "")) + $0.append(serializer.lookupIdentifierCode(for: dep.fileName)) + $0.append((dep.fingerprint != nil) ? UInt32(1) : UInt32(0)) + }, blob: dep.fingerprint ?? "") } } return ByteString(serializer.stream.data) @@ -1051,7 +1045,7 @@ extension ModuleDependencyGraph { guard nodeFinder.matches(other.nodeFinder), tracedNodes.matches(other.tracedNodes), inputDependencySourceMap.matches(other.inputDependencySourceMap), - fingerprintedExternalDependencies.matches(other.fingerprintedExternalDependencies) + externalDependencies.matches(other.externalDependencies) else { return false } @@ -1065,13 +1059,13 @@ extension Set where Element == ModuleDependencyGraph.Node { } } -extension BidirectionalMap where T1 == TypedVirtualPath, T2 == DependencySource { +extension BidirectionalMap where T1 == TypedVirtualPath, T2 == ModuleDependencyGraph.DependencySource { fileprivate func matches(_ other: Self) -> Bool { self == other } } -extension Set where Element == FingerprintedExternalDependency { +extension Set where Element == ExternalDependency { fileprivate func matches(_ other: Self) -> Bool { self == other } diff --git a/Sources/SwiftDriver/IncrementalCompilation/ModuleDependencyGraphParts/DependencySource.swift b/Sources/SwiftDriver/IncrementalCompilation/ModuleDependencyGraphParts/DependencySource.swift index 98c24c6e2..d29e349f1 100644 --- a/Sources/SwiftDriver/IncrementalCompilation/ModuleDependencyGraphParts/DependencySource.swift +++ b/Sources/SwiftDriver/IncrementalCompilation/ModuleDependencyGraphParts/DependencySource.swift @@ -13,73 +13,46 @@ import Foundation import TSCBasic // MARK: - DependencySource -/// Points to the source of dependencies, i.e. the file read to obtain the information. -/*@_spi(Testing)*/ -public struct DependencySource: Hashable, CustomStringConvertible { +extension ModuleDependencyGraph { + /// Points to the source of dependencies, i.e. the file read to obtain the information. + /*@_spi(Testing)*/ + public struct DependencySource: Hashable, CustomStringConvertible { - let typedFile: TypedVirtualPath + let typedFile: TypedVirtualPath - init(_ typedFile: TypedVirtualPath) { - assert( typedFile.type == .swiftDeps || - typedFile.type == .swiftModule) + init(_ typedFile: TypedVirtualPath) { + assert( typedFile.type == .swiftDeps || + typedFile.type == .swiftModule + ) self.typedFile = typedFile - } - - /*@_spi(Testing)*/ - public init(_ file: VirtualPath) { - let ext = file.extension - let typeIfExpected = - ext == FileType.swiftDeps .rawValue ? FileType.swiftDeps : - ext == FileType.swiftModule.rawValue ? FileType.swiftModule - : nil - guard let type = typeIfExpected else { - fatalError("unexpected dependencySource extension: \(String(describing: ext))") } - self.init(TypedVirtualPath(file: file, type: type)) - } - - var file: VirtualPath { typedFile.file } - - public var description: String { - file.description - } -} - -// MARK: - mocking -extension DependencySource { - /*@_spi(Testing)*/ public init(mock i: Int) { - self.init(try! VirtualPath(path: String(i) + "." + FileType.swiftDeps.rawValue)) - } - - /*@_spi(Testing)*/ public var mockID: Int { - Int(file.basenameWithoutExt)! - } -} - -// MARK: - reading -extension DependencySource { - /// Throws if a read error - /// Returns nil if no dependency info there. - public func read( - in fileSystem: FileSystem, - reporter: IncrementalCompilationState.Reporter? - ) -> SourceFileDependencyGraph? { - let graphIfPresent: SourceFileDependencyGraph? - do { - graphIfPresent = try SourceFileDependencyGraph.read( - from: self, - on: fileSystem) + init(_ file: VirtualPath) { + let ext = file.extension + let typeIfExpected = + ext == FileType.swiftDeps .rawValue ? FileType.swiftDeps : + ext == FileType.swiftModule.rawValue ? FileType.swiftModule + : nil + guard let type = typeIfExpected else { + fatalError("unexpected dependencySource extension: \(String(describing: ext))") + } + self.init(TypedVirtualPath(file: file, type: type)) } - catch { - let msg = "Could not read \(file) \(error.localizedDescription)" - reporter?.report(msg, typedFile) - return nil + /*@_spi(Testing)*/ public init(mock i: Int) { + self.init(try! VirtualPath(path: String(i) + "." + FileType.swiftDeps.rawValue)) + } + /*@_spi(Testing)*/ public var mockID: Int { + Int(file.basenameWithoutExt)! + } + var file: VirtualPath { typedFile.file } + + public var description: String { + file.description } - return graphIfPresent } } + // MARK: - testing -extension DependencySource { +extension ModuleDependencyGraph.DependencySource { /*@_spi(Testing)*/ public var sourceFileProvideNameForMockDependencySource: String { file.name @@ -90,7 +63,7 @@ extension DependencySource { } } // MARK: - comparing -extension DependencySource: Comparable { +extension ModuleDependencyGraph.DependencySource: Comparable { public static func < (lhs: Self, rhs: Self) -> Bool { lhs.file.name < rhs.file.name } diff --git a/Sources/SwiftDriver/IncrementalCompilation/ModuleDependencyGraphParts/Integrator.swift b/Sources/SwiftDriver/IncrementalCompilation/ModuleDependencyGraphParts/Integrator.swift index ce3b5a1a0..261caa626 100644 --- a/Sources/SwiftDriver/IncrementalCompilation/ModuleDependencyGraphParts/Integrator.swift +++ b/Sources/SwiftDriver/IncrementalCompilation/ModuleDependencyGraphParts/Integrator.swift @@ -25,12 +25,15 @@ extension ModuleDependencyGraph { /*@_spi(Testing)*/ public struct Results { var changedNodes = Set() - var discoveredExternalDependencies = Set() + var discoveredIncrementalExternalDependencies = Set() } public private(set) var results = Results() /// the graph to be integrated - let sourceGraph: SourceFileDependencyGraph + let source: SourceFileDependencyGraph + + /// the source (file) of the graph to be integrated + let dependencySource: DependencySource /// the graph to be integrated into let destination: ModuleDependencyGraph @@ -39,13 +42,14 @@ extension ModuleDependencyGraph { /// After integration is complete, contains the nodes that have disappeared. var disappearedNodes = [DependencyKey: Graph.Node]() - init(sourceGraph: SourceFileDependencyGraph, + init(source: SourceFileDependencyGraph, + dependencySource: DependencySource, destination: ModuleDependencyGraph) { - self.sourceGraph = sourceGraph - self .destination = destination - self.disappearedNodes = destination.nodeFinder - .findNodes(for: sourceGraph.dependencySource) + self.source = source + self.dependencySource = dependencySource + self.destination = destination + self.disappearedNodes = destination.nodeFinder.findNodes(for: dependencySource) ?? [:] } @@ -54,17 +58,43 @@ extension ModuleDependencyGraph { } } } + +// MARK: - integrate a file containing dependency information +extension ModuleDependencyGraph.Integrator { + /// returns nil for error + static func integrate( + dependencySource: Graph.DependencySource, + into destination: Graph, + input: TypedVirtualPath?, // just for reporting + reporter: IncrementalCompilationState.Reporter?, + diagnosticEngine: DiagnosticsEngine, + fileSystem: FileSystem, + options: IncrementalCompilationState.Options + ) -> Results? { + guard let sfdg = try? SourceFileDependencyGraph.read( + from: dependencySource, on: fileSystem) + else { + reporter?.report("Could not read \(dependencySource)", input) + return nil + } + return integrate(from: sfdg, + dependencySource: dependencySource, + into: destination) + } +} // MARK: - integrate a graph extension ModuleDependencyGraph.Integrator { /// Integrate a SourceFileDepGraph into the receiver. /// Integration happens when the driver needs to read SourceFileDepGraph. /// Returns changed nodes - /*@_spi(Testing)*/ static func integrateAndCollectExternalDepNodes( + /*@_spi(Testing)*/ public static func integrate( from g: SourceFileDependencyGraph, + dependencySource: Graph.DependencySource, into destination: Graph ) -> Results { - var integrator = Self(sourceGraph: g, + var integrator = Self(source: g, + dependencySource: dependencySource, destination: destination) integrator.integrate() @@ -72,7 +102,7 @@ extension ModuleDependencyGraph.Integrator { integrator.verifyAfterImporting() } if destination.options.contains(.emitDependencyDotFileAfterEveryImport) { - destination.emitDotFile(g) + destination.emitDotFile(g, dependencySource) } return integrator.results } @@ -83,7 +113,7 @@ extension ModuleDependencyGraph.Integrator { destination.ensureGraphWillRetraceDependents(of: results.changedNodes) } private mutating func integrateEachSourceNode() { - sourceGraph.forEachNode { integrate(oneNode: $0) } + source.forEachNode { integrate(oneNode: $0) } } private mutating func handleDisappearedNodes() { for (_, node) in disappearedNodes { @@ -116,25 +146,25 @@ extension ModuleDependencyGraph.Integrator { /// and return the merged node. Remember that the merged node has changed if it has. private mutating func integrateWithNodeHere( _ integrand: SourceFileDependencyGraph.Node, - _ nodesMatchingKey: [DependencySource?: Graph.Node] + _ nodesMatchingKey: [Graph.DependencySource?: Graph.Node] ) -> Graph.Node? { - guard let matchHere = nodesMatchingKey[sourceGraph.dependencySource] else { + guard let matchHere = nodesMatchingKey[dependencySource] else { return nil } - assert(matchHere.dependencySource == sourceGraph.dependencySource) + assert(matchHere.dependencySource == dependencySource) // Node was and still is. Do not remove it. - disappearedNodes.removeValue(forKey: matchHere.key) + disappearedNodes.removeValue(forKey: matchHere.dependencyKey) if matchHere.fingerprint != integrand.fingerprint { results.changedNodes.insert(matchHere) } return matchHere } - /// If there is an expat node with this key, replace it with a node for this dependencySource + /// If there is an expat node with this key, replace it with a ndoe for this dependencySource /// and return the replacement. Remember that the replace has changed. private mutating func integrateWithExpat( _ integrand: SourceFileDependencyGraph.Node, - _ nodesMatchingKey: [DependencySource?: Graph.Node] + _ nodesMatchingKey: [Graph.DependencySource?: Graph.Node] ) -> Graph.Node? { guard let expat = nodesMatchingKey[nil] else { return nil @@ -143,7 +173,7 @@ extension ModuleDependencyGraph.Integrator { "If an expat exists, then must not be any matches in other files") let integratedNode = destination.nodeFinder .replace(expat, - newDependencySource: sourceGraph.dependencySource, + newDependencySource: dependencySource, newFingerprint: integrand.fingerprint) results.changedNodes.insert(integratedNode) return integratedNode @@ -157,7 +187,7 @@ extension ModuleDependencyGraph.Integrator { let newNode = Graph.Node( key: integrand.key, fingerprint: integrand.fingerprint, - dependencySource: sourceGraph.dependencySource) + dependencySource: dependencySource) let oldNode = destination.nodeFinder.insert(newNode) assert(oldNode == nil, "Should be new!") results.changedNodes.insert(newNode) @@ -171,22 +201,45 @@ extension ModuleDependencyGraph.Integrator { _ sourceFileUseNode: SourceFileDependencyGraph.Node, _ moduleUseNode: Graph.Node ) { - sourceGraph.forEachDefDependedUpon(by: sourceFileUseNode) { def in + source.forEachDefDependedUpon(by: sourceFileUseNode) { def in let isNewUse = destination.nodeFinder.record(def: def.key, use: moduleUseNode) guard isNewUse else { return } - guard let externalDependency = - def.key.designator.externalDependency - else { + guard let externalDependency = def.key.designator.externalDependency else { return } - // Check both in case we reread a prior ModDepGraph from a different mode - let extDepAndPrint = FingerprintedExternalDependency(externalDependency, - def.fingerprint) - let isKnown = destination.fingerprintedExternalDependencies.contains(extDepAndPrint) + let isIncremental = def.fingerprint != nil + let isKnown = (isIncremental + ? destination.incrementalExternalDependencies + : destination.externalDependencies) + .contains(externalDependency) guard !isKnown else {return} - - results.discoveredExternalDependencies.insert(extDepAndPrint) + if !isIncremental { + // Nice for debugging, but too much otherwise: + // destination.reporter?.report("found externalDependency", externalDependency.file) + + // no integration to do for these, so just remember them here + destination.externalDependencies.insert(externalDependency) + } + else if !isCrossModuleIncrementalBuildEnabled { + destination.reporter?.report( + "found incrementalExternalDependency but treating as non-incremental", + externalDependency.file) + // treat like nonincremental + let key = DependencyKey( + aspect: .interface, + designator: .externalDepend(externalDependency)) + let isNewUse = destination.nodeFinder.record(def: key, use: moduleUseNode) + if isNewUse { + destination.externalDependencies.insert(externalDependency) + results.changedNodes.insert(moduleUseNode) + } + } + else { + destination.reporter?.report( "found incrementalExternalDependency", + externalDependency.file) + results.discoveredIncrementalExternalDependencies.insert(externalDependency) + } results.changedNodes.insert(moduleUseNode) } } @@ -196,10 +249,10 @@ extension ModuleDependencyGraph.Integrator { extension ModuleDependencyGraph.Integrator { @discardableResult func verifyAfterImporting() -> Bool { - guard let nodesInFile = destination.nodeFinder.findNodes(for: sourceGraph.dependencySource), + guard let nodesInFile = destination.nodeFinder.findNodes(for: dependencySource), !nodesInFile.isEmpty else { - fatalError("Just imported \(sourceGraph.dependencySource), should have nodes") + fatalError("Just imported \(dependencySource), should have nodes") } return destination.verifyGraph() } diff --git a/Sources/SwiftDriver/IncrementalCompilation/ModuleDependencyGraphParts/Node.swift b/Sources/SwiftDriver/IncrementalCompilation/ModuleDependencyGraphParts/Node.swift index 2b26a4845..6184b89d8 100644 --- a/Sources/SwiftDriver/IncrementalCompilation/ModuleDependencyGraphParts/Node.swift +++ b/Sources/SwiftDriver/IncrementalCompilation/ModuleDependencyGraphParts/Node.swift @@ -24,13 +24,12 @@ extension ModuleDependencyGraph { /// /// Use a class, not a struct because otherwise it would be duplicated for each thing it uses - /*@_spi(Testing)*/ - public final class Node: KeyAndFingerprintEnforcer { + /*@_spi(Testing)*/ public final class Node { /*@_spi(Testing)*/ public typealias Graph = ModuleDependencyGraph /// Def->use arcs go by DependencyKey. There may be >1 node for a given key. - let key: DependencyKey + let dependencyKey: DependencyKey /// The frontend records in the fingerprint, all of the information about an /// entity, such that any uses need be rebuilt only if the fingerprint @@ -62,23 +61,21 @@ extension ModuleDependencyGraph { /// SourceFileDependencyGraph or the DependencyKeys init(key: DependencyKey, fingerprint: String?, dependencySource: DependencySource?) { - self.key = key + self.dependencyKey = key self.fingerprint = fingerprint self.dependencySource = dependencySource - - try! verifyKeyAndFingerprint() } } } // MARK: - comparing, hashing extension ModuleDependencyGraph.Node: Equatable, Hashable { public static func == (lhs: Graph.Node, rhs: Graph.Node) -> Bool { - lhs.key == rhs.key && lhs.fingerprint == rhs.fingerprint + lhs.dependencyKey == rhs.dependencyKey && lhs.fingerprint == rhs.fingerprint && lhs.dependencySource == rhs.dependencySource } public func hash(into hasher: inout Hasher) { - hasher.combine(key) + hasher.combine(dependencyKey) hasher.combine(fingerprint) hasher.combine(dependencySource) } @@ -94,7 +91,7 @@ extension ModuleDependencyGraph.Node: Comparable { case (_?, nil): return false } } - return lhs.key != rhs.key ? lhs.key < rhs.key : + return lhs.dependencyKey != rhs.dependencyKey ? lhs.dependencyKey < rhs.dependencyKey : lhs.dependencySource != rhs.dependencySource ? lt(lhs.dependencySource, rhs.dependencySource) : lt(lhs.fingerprint, rhs.fingerprint) @@ -104,14 +101,14 @@ extension ModuleDependencyGraph.Node: Comparable { extension ModuleDependencyGraph.Node: CustomStringConvertible { public var description: String { - "\(key) \( dependencySource.map { "in \($0.description)" } ?? "" )" + "\(dependencyKey) \( dependencySource.map { "in \($0.description)" } ?? "" )" } } extension ModuleDependencyGraph.Node { public func verify() { verifyExpatsHaveNoFingerprints() - key.verify() + dependencyKey.verify() } public func verifyExpatsHaveNoFingerprints() { diff --git a/Sources/SwiftDriver/IncrementalCompilation/ModuleDependencyGraphParts/NodeFinder.swift b/Sources/SwiftDriver/IncrementalCompilation/ModuleDependencyGraphParts/NodeFinder.swift index 992015970..2bdf962c6 100644 --- a/Sources/SwiftDriver/IncrementalCompilation/ModuleDependencyGraphParts/NodeFinder.swift +++ b/Sources/SwiftDriver/IncrementalCompilation/ModuleDependencyGraphParts/NodeFinder.swift @@ -41,24 +41,24 @@ extension ModuleDependencyGraph { extension ModuleDependencyGraph.NodeFinder { func findFileInterfaceNode( - forMock dependencySource: DependencySource + forMock dependencySource: ModuleDependencyGraph.DependencySource ) -> Graph.Node? { let fileKey = DependencyKey(fileKeyForMockDependencySource: dependencySource) return findNode((dependencySource, fileKey)) } - func findNode(_ mapKey: (DependencySource?, DependencyKey)) -> Graph.Node? { + func findNode(_ mapKey: (Graph.DependencySource?, DependencyKey)) -> Graph.Node? { nodeMap[mapKey] } func findCorrespondingImplementation(of n: Graph.Node) -> Graph.Node? { - n.key.correspondingImplementation + n.dependencyKey.correspondingImplementation .flatMap {findNode((n.dependencySource, $0))} } - func findNodes(for dependencySource: DependencySource?) + func findNodes(for dependencySource: Graph.DependencySource?) -> [DependencyKey: Graph.Node]? { nodeMap[dependencySource] } - func findNodes(for key: DependencyKey) -> [DependencySource?: Graph.Node]? { + func findNodes(for key: DependencyKey) -> [Graph.DependencySource?: Graph.Node]? { nodeMap[key] } @@ -81,7 +81,7 @@ extension ModuleDependencyGraph.NodeFinder { /// - Returns: A set of nodes corresponding to the uses of the given /// definition node. func uses(of def: Graph.Node) -> Set { - var uses = usesByDef[def.key, default: Set()].values + var uses = usesByDef[def.dependencyKey, default: Set()].values if let impl = findCorrespondingImplementation(of: def) { uses.insert(impl) } @@ -105,9 +105,9 @@ extension ModuleDependencyGraph.NodeFinder { return self.uses(of: def).sorted() } - func mappings(of n: Graph.Node) -> [(DependencySource?, DependencyKey)] { + func mappings(of n: Graph.Node) -> [(Graph.DependencySource?, DependencyKey)] { nodeMap.compactMap { k, _ in - guard k.0 == n.dependencySource && k.1 == n.key else { + guard k.0 == n.dependencySource && k.1 == n.dependencyKey else { return nil } return k @@ -120,8 +120,8 @@ extension ModuleDependencyGraph.NodeFinder { } fileprivate extension ModuleDependencyGraph.Node { - var mapKey: (DependencySource?, DependencyKey) { - return (dependencySource, key) + var mapKey: (Graph.DependencySource?, DependencyKey) { + return (dependencySource, dependencyKey) } } @@ -173,13 +173,13 @@ extension ModuleDependencyGraph.NodeFinder { /// /// Now that nodes are immutable, this function needs to replace the node mutating func replace(_ original: Graph.Node, - newDependencySource: DependencySource, + newDependencySource: Graph.DependencySource, newFingerprint: String? ) -> Graph.Node { - let replacement = Graph.Node(key: original.key, + let replacement = Graph.Node(key: original.dependencyKey, fingerprint: newFingerprint, dependencySource: newDependencySource) - usesByDef.replace(original, with: replacement, forKey: original.key) + usesByDef.replace(original, with: replacement, forKey: original.dependencyKey) nodeMap.removeValue(forKey: original.mapKey) nodeMap.updateValue(replacement, forKey: replacement.mapKey) return replacement @@ -235,14 +235,14 @@ extension ModuleDependencyGraph.NodeFinder { // MARK: - key helpers fileprivate extension DependencyKey { - init(fileKeyForMockDependencySource dependencySource: DependencySource) { + init(fileKeyForMockDependencySource dependencySource: ModuleDependencyGraph.DependencySource) { self.init(aspect: .interface, designator: .sourceFileProvide(name: dependencySource.sourceFileProvidesNameForMocking) ) } } -fileprivate extension DependencySource { +fileprivate extension ModuleDependencyGraph.DependencySource { var sourceFileProvidesNameForMocking: String { // Only when mocking are these two guaranteed to be the same file.name diff --git a/Sources/SwiftDriver/IncrementalCompilation/ModuleDependencyGraphParts/Tracer.swift b/Sources/SwiftDriver/IncrementalCompilation/ModuleDependencyGraphParts/Tracer.swift index 1dff17471..5fc6326e9 100644 --- a/Sources/SwiftDriver/IncrementalCompilation/ModuleDependencyGraphParts/Tracer.swift +++ b/Sources/SwiftDriver/IncrementalCompilation/ModuleDependencyGraphParts/Tracer.swift @@ -127,7 +127,7 @@ extension ModuleDependencyGraph.Tracer { .compactMap { node in node.dependencySource .flatMap {graph.inputDependencySourceMap[$0] } - .map { "\(node.key) in \($0.file.basename)"} + .map { "\(node.dependencyKey) in \($0.file.basename)"} } .joined(separator: " -> ") ].joined(separator: " ") diff --git a/Sources/SwiftDriver/IncrementalCompilation/SourceFileDependencyGraph.swift b/Sources/SwiftDriver/IncrementalCompilation/SourceFileDependencyGraph.swift index 99dc93f8e..6e1ceff80 100644 --- a/Sources/SwiftDriver/IncrementalCompilation/SourceFileDependencyGraph.swift +++ b/Sources/SwiftDriver/IncrementalCompilation/SourceFileDependencyGraph.swift @@ -21,9 +21,6 @@ import TSCUtility public var minorVersion: UInt64 public var compilerVersionString: String private var allNodes: [Node] - - /// Holds the filename fromwhich the graph was read. - public let dependencySource: DependencySource public var sourceFileNodePair: (interface: Node, implementation: Node) { (interface: allNodes[SourceFileDependencyGraph.sourceFileProvidesInterfaceSequenceNumber], @@ -57,8 +54,9 @@ import TSCUtility } } + extension SourceFileDependencyGraph { - public struct Node: KeyAndFingerprintEnforcer { + public struct Node { public var key: DependencyKey public var fingerprint: String? public var sequenceNumber: Int @@ -71,19 +69,16 @@ extension SourceFileDependencyGraph { sequenceNumber: Int, defsIDependUpon: [Int], isProvides: Bool - ) throws { + ) { self.key = key self.fingerprint = fingerprint self.sequenceNumber = sequenceNumber self.defsIDependUpon = defsIDependUpon self.isProvides = isProvides - - try verifyKeyAndFingerprint() } public func verify() { key.verify() - try! verifyKeyAndFingerprint() if case .sourceFileProvide = key.designator { switch key.aspect { @@ -94,14 +89,6 @@ extension SourceFileDependencyGraph { } } } - - public var isIncrementalExternalDependency: Bool { - if case .externalDepend = key.designator, - fingerprint != nil { - return true - } - return false - } } } @@ -116,7 +103,6 @@ extension SourceFileDependencyGraph { fileprivate enum ReadError: Error { case badMagic - case swiftModuleHasNoDependencies case noRecordBlock case malformedMetadataRecord case unexpectedMetadataRecord @@ -130,36 +116,30 @@ extension SourceFileDependencyGraph { case unknownKind } - /// Returns nil if there was no dependency info static func read( - from dependencySource: DependencySource, + from dependencySource: ModuleDependencyGraph.DependencySource, on fileSystem: FileSystem - ) throws -> Self? { - try self.init(contentsOf: dependencySource, on: fileSystem) + ) throws -> Self { + try self.init(contentsOf: dependencySource.typedFile, on: fileSystem) } - /*@_spi(Testing)*/ public init(from dependencySource: DependencySource, - nodesForTesting: [Node]) { + /*@_spi(Testing)*/ public init(nodesForTesting: [Node]) { majorVersion = 0 minorVersion = 0 compilerVersionString = "" allNodes = nodesForTesting - self.dependencySource = dependencySource } - /*@_spi(Testing)*/ public init?( - contentsOf dependencySource: DependencySource, + /*@_spi(Testing)*/ public init( + contentsOf path: TypedVirtualPath, on filesystem: FileSystem ) throws { - let data = try filesystem.readFileContents(dependencySource.file) - try self.init(data: data, from: dependencySource, - fromSwiftModule: dependencySource.typedFile.type == .swiftModule) + let data = try filesystem.readFileContents(path.file) + try self.init(data: data, fromSwiftModule: path.type == .swiftModule) } - /// Returns nil for a swiftmodule with no depenencies - /*@_spi(Testing)*/ public init?( + /*@_spi(Testing)*/ public init( data: ByteString, - from dependencySource: DependencySource, fromSwiftModule extractFromSwiftModule: Bool = false ) throws { struct Visitor: BitstreamVisitor { @@ -180,7 +160,7 @@ extension SourceFileDependencyGraph { func validate(signature: Bitcode.Signature) throws { if extractFromSwiftModule { - guard signature == .init(value: 0x0EA89CE2) else { throw ReadError.swiftModuleHasNoDependencies } + guard signature == .init(value: 0x0EA89CE2) else { throw ReadError.badMagic } } else { guard signature == .init(string: "DEPS") else { throw ReadError.badMagic } } @@ -240,11 +220,11 @@ extension SourceFileDependencyGraph { let designator = try DependencyKey.Designator( kindCode: kindCode, context: context, name: identifier) let key = DependencyKey(aspect: declAspect, designator: designator) - node = try Node(key: key, - fingerprint: nil, - sequenceNumber: sequenceNumber, - defsIDependUpon: [], - isProvides: isProvides) + node = Node(key: key, + fingerprint: nil, + sequenceNumber: sequenceNumber, + defsIDependUpon: [], + isProvides: isProvides) sequenceNumber += 1 case .fingerprintNode: guard node != nil, @@ -270,18 +250,13 @@ extension SourceFileDependencyGraph { } var visitor = Visitor(extractFromSwiftModule: extractFromSwiftModule) - do { - try data.contents.withUnsafeBytes { buf in - // SAFETY: The bitcode reader does not mutate the data stream we give it. - // FIXME: Let's avoid this altogether and traffic in ByteString/[UInt8] - // if possible. There's no real reason to use `Data` in this API. - let baseAddr = UnsafeMutableRawPointer(mutating: buf.baseAddress!) - let data = Data(bytesNoCopy: baseAddr, count: buf.count, deallocator: .none) - try Bitcode.read(stream: data, using: &visitor) - } - } - catch ReadError.swiftModuleHasNoDependencies { - return nil + try data.contents.withUnsafeBytes { buf in + // SAFETY: The bitcode reader does not mutate the data stream we give it. + // FIXME: Let's avoid this altogether and traffic in ByteString/[UInt8] + // if possible. There's no real reason to use `Data` in this API. + let baseAddr = UnsafeMutableRawPointer(mutating: buf.baseAddress!) + let data = Data(bytesNoCopy: baseAddr, count: buf.count, deallocator: .none) + try Bitcode.read(stream: data, using: &visitor) } guard let major = visitor.majorVersion, let minor = visitor.minorVersion, @@ -292,7 +267,6 @@ extension SourceFileDependencyGraph { self.minorVersion = minor self.compilerVersionString = versionString self.allNodes = visitor.nodes - self.dependencySource = dependencySource } } diff --git a/Tests/SwiftDriverTests/DependencyGraphSerializationTests.swift b/Tests/SwiftDriverTests/DependencyGraphSerializationTests.swift index 9f8785701..2b5be596e 100644 --- a/Tests/SwiftDriverTests/DependencyGraphSerializationTests.swift +++ b/Tests/SwiftDriverTests/DependencyGraphSerializationTests.swift @@ -41,8 +41,7 @@ class DependencyGraphSerializationTests: XCTestCase { XCTAssertEqual(graph.nodeFinder.usesByDef, deserializedGraph.nodeFinder.usesByDef) XCTAssertEqual(graph.inputDependencySourceMap, deserializedGraph.inputDependencySourceMap) - XCTAssertEqual(graph.fingerprintedExternalDependencies, - deserializedGraph.fingerprintedExternalDependencies) + XCTAssertEqual(graph.externalDependencies, deserializedGraph.externalDependencies) } func testRoundTripFixtures() throws { diff --git a/Tests/SwiftDriverTests/IncrementalCompilationTests.swift b/Tests/SwiftDriverTests/IncrementalCompilationTests.swift index 42819ed3e..d2ae47527 100644 --- a/Tests/SwiftDriverTests/IncrementalCompilationTests.swift +++ b/Tests/SwiftDriverTests/IncrementalCompilationTests.swift @@ -68,11 +68,10 @@ final class NonincrementalCompilationTests: XCTestCase { func testReadBinarySourceFileDependencyGraph() throws { let absolutePath = try XCTUnwrap(Fixture.fixturePath(at: RelativePath("Incremental"), for: "main.swiftdeps")) - let dependencySource = DependencySource(VirtualPath.absolute(absolutePath)) - let graph = try XCTUnwrap( - try SourceFileDependencyGraph( - contentsOf: dependencySource, - on: localFileSystem)) + let graph = try SourceFileDependencyGraph( + contentsOf: TypedVirtualPath(file: VirtualPath.absolute(absolutePath), + type: .swiftDeps), + on: localFileSystem) XCTAssertEqual(graph.majorVersion, 1) XCTAssertEqual(graph.minorVersion, 0) XCTAssertEqual(graph.compilerVersionString, "Swift version 5.3-dev (LLVM f516ac602c, Swift c39f31febd)") @@ -115,10 +114,10 @@ final class NonincrementalCompilationTests: XCTestCase { func testReadComplexSourceFileDependencyGraph() throws { let absolutePath = try XCTUnwrap(Fixture.fixturePath(at: RelativePath("Incremental"), for: "hello.swiftdeps")) - let graph = try XCTUnwrap( - try SourceFileDependencyGraph( - contentsOf: DependencySource(VirtualPath.absolute(absolutePath)), - on: localFileSystem)) + let graph = try SourceFileDependencyGraph( + contentsOf: TypedVirtualPath(file: VirtualPath.absolute(absolutePath), + type: .swiftDeps), + on: localFileSystem) XCTAssertEqual(graph.majorVersion, 1) XCTAssertEqual(graph.minorVersion, 0) XCTAssertEqual(graph.compilerVersionString, "Swift version 5.3-dev (LLVM 4510748e505acd4, Swift 9f07d884c97eaf4)") @@ -169,10 +168,7 @@ final class NonincrementalCompilationTests: XCTestCase { let absolutePath = try XCTUnwrap(Fixture.fixturePath(at: RelativePath("Incremental"), for: "hello.swiftmodule")) let data = try localFileSystem.readFileContents(absolutePath) - let graph = try XCTUnwrap( - try SourceFileDependencyGraph(data: data, - from: DependencySource(.absolute(absolutePath)), - fromSwiftModule: true)) + let graph = try SourceFileDependencyGraph(data: data, fromSwiftModule: true) XCTAssertEqual(graph.majorVersion, 1) XCTAssertEqual(graph.minorVersion, 0) XCTAssertEqual(graph.compilerVersionString, "Apple Swift version 5.3-dev (LLVM 240312aa7333e90, Swift 15bf0478ad7c47c)") diff --git a/Tests/SwiftDriverTests/ModuleDependencyGraphTests.swift b/Tests/SwiftDriverTests/ModuleDependencyGraphTests.swift index a2a68b1aa..68181c503 100644 --- a/Tests/SwiftDriverTests/ModuleDependencyGraphTests.swift +++ b/Tests/SwiftDriverTests/ModuleDependencyGraphTests.swift @@ -595,8 +595,8 @@ class ModuleDependencyGraphTests: XCTestCase { graph.simulateLoad(0, [.externalDepend: ["/foo->", "/bar->"]]) - XCTAssertTrue(graph.containsExternalDependency( "/foo")) - XCTAssertTrue(graph.containsExternalDependency( "/bar")) + XCTAssertTrue(graph.externalDependencies.contains( "/foo")) + XCTAssertTrue(graph.externalDependencies.contains( "/bar")) do { let swiftDeps = graph.findUntracedSwiftDepsDependent(onExternal: "/foo") @@ -633,8 +633,8 @@ class ModuleDependencyGraphTests: XCTestCase { 1, [.externalDepend: ["/bar->"], .topLevel: ["a->"]]) - XCTAssertTrue(graph.containsExternalDependency( "/foo")) - XCTAssertTrue(graph.containsExternalDependency( "/bar")) + XCTAssertTrue(graph.externalDependencies.contains( "/foo")) + XCTAssertTrue(graph.externalDependencies.contains( "/bar")) do { let swiftDeps = graph.findUntracedSwiftDepsDependent(onExternal: "/foo") @@ -882,7 +882,6 @@ extension ModuleDependencyGraph { ) { self.init(diagnosticEngine: diagnosticEngine, reporter: nil, - fileSystem: localFileSystem, options: options) } @@ -907,14 +906,14 @@ extension ModuleDependencyGraph { hadCompilationError: Bool = false) -> [Int] { - let changedNodes = getChangesForSimulatedLoad( + let results = getChangesForSimulatedLoad( swiftDepsIndex, dependencyDescriptions, interfaceHash, includePrivateDeps: includePrivateDeps, hadCompilationError: hadCompilationError) - return findSwiftDepsToRecompileWhenNodesChange(changedNodes) + return findSwiftDepsToRecompileWhenNodesChange(results.changedNodes) .map { $0.mockID } } @@ -925,7 +924,7 @@ extension ModuleDependencyGraph { _ interfaceHashIfPresent: String? = nil, includePrivateDeps: Bool = true, hadCompilationError: Bool = false - ) -> Set { + ) -> Integrator.Results { let dependencySource = DependencySource(mock: swiftDepsIndex) let interfaceHash = interfaceHashIfPresent ?? dependencySource.interfaceHashForMockDependencySource @@ -936,21 +935,24 @@ extension ModuleDependencyGraph { dependencySource: dependencySource, interfaceHash: interfaceHash, dependencyDescriptions) - return try! XCTUnwrap(integrate(sourceGraph: sfdg)) + + return Integrator.integrate(from: sfdg, + dependencySource: dependencySource, + into: self) } func findUntracedSwiftDepsDependent(onExternal s: String) -> [Int] { - findUntracedSwiftDepsDependent( - on: FingerprintedExternalDependency(s.asExternal, nil)) + findUntracedSwiftDepsDependent(on: s.asExternal, isIncremental: false) .map { $0.mockID } } /// Can return duplicates func findUntracedSwiftDepsDependent( - on fingerprintedExternalDependency: FingerprintedExternalDependency + on externalDependency: ExternalDependency, + isIncremental: Bool ) -> [DependencySource] { var foundSources = [DependencySource]() - for dependent in self.untracedDependents(of: fingerprintedExternalDependency) { + for dependent in self.untracedDependents(of: externalDependency, isIncremental: isIncremental) { let dependencySource = dependent.dependencySource! foundSources.append(dependencySource) // findSwiftDepsToRecompileWhenWholeSwiftDepChanges is reflexive @@ -968,13 +970,6 @@ extension ModuleDependencyGraph { findSwiftDepsToRecompileWhenDependencySourceChanges(DependencySource(mock: i)) .map { $0.mockID } } - - func containsExternalDependency(_ path: String, fingerprint: String? = nil) - -> Bool { - fingerprintedExternalDependencies.contains( - FingerprintedExternalDependency(ExternalDependency(path), - fingerprint)) - } } /// *Dependency info format:* @@ -1008,7 +1003,7 @@ fileprivate struct SourceFileDependencyGraphMocker { private let includePrivateDeps: Bool private let hadCompilationError: Bool - private let dependencySource: DependencySource + private let dependencySource: ModuleDependencyGraph.DependencySource private let interfaceHash: String private let dependencyDescriptions: [(MockDependencyKind, String)] @@ -1019,7 +1014,7 @@ fileprivate struct SourceFileDependencyGraphMocker { static func mock( includePrivateDeps: Bool, hadCompilationError: Bool, - dependencySource: DependencySource, + dependencySource: ModuleDependencyGraph.DependencySource, interfaceHash: String, _ dependencyDescriptions: [MockDependencyKind: [String]] ) -> SourceFileDependencyGraph @@ -1037,8 +1032,7 @@ fileprivate struct SourceFileDependencyGraphMocker { private mutating func mock() -> SourceFileDependencyGraph { buildNodes() - return SourceFileDependencyGraph(from: dependencySource, - nodesForTesting: allNodes) + return SourceFileDependencyGraph(nodesForTesting: allNodes) } private mutating func buildNodes() { @@ -1103,10 +1097,10 @@ fileprivate struct SourceFileDependencyGraphMocker { private mutating func findExistingNodeOrCreateIfNew(_ key: DependencyKey, _ fingerprint: String?, isProvides: Bool) -> Node { func createNew() -> Node { - let n = try! Node(key: key, fingerprint: fingerprint, - sequenceNumber: allNodes.count, - defsIDependUpon: [], - isProvides: isProvides) + let n = Node(key: key, fingerprint: fingerprint, + sequenceNumber: allNodes.count, + defsIDependUpon: [], + isProvides: isProvides) allNodes.append(n) memoizedNodes[key] = n return n @@ -1222,7 +1216,7 @@ fileprivate extension DependencyKey { _ s: String, _ kind: MockDependencyKind, includePrivateDeps: Bool, - dependencySource: DependencySource + dependencySource: ModuleDependencyGraph.DependencySource ) -> (def: Self, use: Self)? { let noncascadingPrefix = "#" let privateHolderPrefix = "~" @@ -1250,7 +1244,7 @@ fileprivate extension DependencyKey { static func computeUseKey( _ s: String, isCascadingUse: Bool, includePrivateDeps: Bool, - dependencySource: DependencySource + dependencySource: ModuleDependencyGraph.DependencySource ) -> Self { // For now, in unit tests, mock uses are always nominal let aspectOfUse: DeclAspect = isCascadingUse ? .interface : .implementation @@ -1312,7 +1306,7 @@ fileprivate extension SourceFileDependencyGraph.Node { fileprivate extension DependencyKey { static func createKeyForWholeSourceFile( _ aspect: DeclAspect, - _ dependencySource: DependencySource + _ dependencySource: ModuleDependencyGraph.DependencySource ) -> Self { return Self(aspect: aspect, designator: Designator(kind: .sourceFileProvide,