Cache lld binaries to avoid redundant rebuilds

Docker/Dockerfile

@@ -1,4 +1,4 @@
-ARG swift_version=5.7
+ARG swift_version=5.9
 ARG ubuntu_version=jammy
 ARG base_image=swift:$swift_version-$ubuntu_version
 FROM $base_image
@@ -7,7 +7,7 @@ ARG swift_version
 ARG ubuntu_version
 
 # set as UTF-8
-RUN apt-get update && apt-get install -y locales locales-all
+RUN apt-get update && apt-get install -y locales locales-all libsqlite3-dev
 ENV LC_ALL en_US.UTF-8
 ENV LANG en_US.UTF-8
 ENV LANGUAGE en_US.UTF-8

Package.swift

@@ -1,6 +1,7 @@
 // swift-tools-version: 5.9
 // The swift-tools-version declares the minimum version of Swift required to build this package.
 
+import CompilerPluginSupport
 import PackageDescription
 
 let package = Package(
@@ -20,10 +21,12 @@ let package = Package(
     .package(url: "https://github.com/apple/swift-argument-parser", from: "1.2.2"),
     .package(url: "https://github.com/apple/swift-async-algorithms.git", exact: "1.0.0-alpha"),
     .package(url: "https://github.com/apple/swift-atomics.git", from: "1.1.0"),
+    .package(url: "https://github.com/apple/swift-collections.git", from: "1.0.4"),
+    .package(url: "https://github.com/apple/swift-crypto.git", from: "3.1.0"),
     .package(url: "https://github.com/apple/swift-nio.git", from: "2.58.0"),
     .package(url: "https://github.com/apple/swift-nio-extras.git", from: "1.19.0"),
     .package(url: "https://github.com/apple/swift-log.git", from: "1.5.3"),
-    .package(url: "https://github.com/apple/swift-collections.git", from: "1.0.4"),
+    .package(url: "https://github.com/apple/swift-syntax.git", from: "509.0.1"),
   ],
   targets: [
     // Targets are the basic building blocks of a package. A target can define a module or a test suite.
@@ -33,7 +36,7 @@ let package = Package(
       dependencies: [
         "SwiftSDKGenerator",
         .product(name: "ArgumentParser", package: "swift-argument-parser"),
-      ],      
+      ],
       swiftSettings: [
         .enableExperimentalFeature("StrictConcurrency=complete"),
       ]
@@ -44,7 +47,9 @@ let package = Package(
         .target(name: "AsyncProcess"),
         .product(name: "AsyncAlgorithms", package: "swift-async-algorithms"),
         .product(name: "AsyncHTTPClient", package: "async-http-client"),
+        .product(name: "Logging", package: "swift-log"),
         .product(name: "SystemPackage", package: "swift-system"),
+        "GeneratorEngine",
       ],
       exclude: ["Dockerfiles"],
       swiftSettings: [
@@ -54,12 +59,47 @@ let package = Package(
     .testTarget(
       name: "SwiftSDKGeneratorTests",
       dependencies: [
-        .target(name: "SwiftSDKGenerator"),
+        "SwiftSDKGenerator",
       ],
       swiftSettings: [
         .enableExperimentalFeature("StrictConcurrency=complete"),
       ]
     ),
+    .target(
+      name: "GeneratorEngine",
+      dependencies: [
+        .product(name: "AsyncHTTPClient", package: "async-http-client"),
+        .product(name: "Crypto", package: "swift-crypto"),
+        .product(name: "Logging", package: "swift-log"),
+        .product(name: "SystemPackage", package: "swift-system"),
+        "Macros",
+        "SystemSQLite",
+      ]
+    ),
+    .testTarget(
+      name: "GeneratorEngineTests",
+      dependencies: [
+        "GeneratorEngine",
+      ]
+    ),
+    .macro(
+      name: "Macros",
+      dependencies: [
+        .product(name: "SwiftCompilerPlugin", package: "swift-syntax"),
+        .product(name: "SwiftSyntax", package: "swift-syntax"),
+        .product(name: "SwiftSyntaxBuilder", package: "swift-syntax"),
+        .product(name: "SwiftSyntaxMacros", package: "swift-syntax"),
+        .product(name: "SwiftDiagnostics", package: "swift-syntax"),
+      ]
+    ),
+    .testTarget(
+      name: "MacrosTests",
+      dependencies: [
+        "Macros",
+        .product(name: "SwiftSyntaxMacros", package: "swift-syntax"),
+        .product(name: "SwiftSyntaxMacrosTestSupport", package: "swift-syntax"),
+      ]
+    ),
     .systemLibrary(name: "SystemSQLite", pkgConfig: "sqlite3"),
     .target(
       name: "AsyncProcess",

Sources/GeneratorCLI/GeneratorCLI.swift

@@ -88,7 +88,7 @@ struct GeneratorCLI: AsyncParsableCommand {
     let linuxDistribution = try LinuxDistribution(name: linuxDistributionName, version: linuxDistributionVersion)
 
     let elapsed = try await ContinuousClock().measure {
-      try await SwiftSDKGenerator(
+      let generator = try await SwiftSDKGenerator(
         hostCPUArchitecture: self.hostArch,
         targetCPUArchitecture: self.targetArch,
         swiftVersion: self.swiftVersion,
@@ -98,7 +98,13 @@ struct GeneratorCLI: AsyncParsableCommand {
         shouldUseDocker: self.withDocker,
         isVerbose: self.verbose
       )
-      .generateBundle(shouldGenerateFromScratch: !self.incremental)
+      do {
+        try await generator.generateBundle(shouldGenerateFromScratch: !self.incremental)
+        try await generator.shutDown()
+      } catch {
+        try await generator.shutDown()
+        throw error
+      }
     }
 
     print("\nTime taken for this generator run: \(elapsed.intervalString).")

Sources/GeneratorEngine/Cache/CacheKeyProtocol.swift

@@ -0,0 +1,97 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift open source project
+//
+// Copyright (c) 2023 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+import Macros
+
+@_exported import protocol Crypto.HashFunction
+import struct Foundation.URL
+import struct SystemPackage.FilePath
+
+/// Indicates that values of a conforming type can be hashed with an arbitrary hashing function. Unlike `Hashable`,
+/// this protocol doesn't utilize random seed values and produces consistent hash values across process launches.
+public protocol CacheKeyProtocol {
+  func hash(with hashFunction: inout some HashFunction)
+}
+
+extension Bool: CacheKeyProtocol {
+  public func hash(with hashFunction: inout some HashFunction) {
+    String(reflecting: Self.self).hash(with: &hashFunction)
+    hashFunction.update(data: self ? [1] : [0])
+  }
+}
+
+extension Int: CacheKeyProtocol {
+  public func hash(with hashFunction: inout some HashFunction) {
+    String(reflecting: Self.self).hash(with: &hashFunction)
+    withUnsafeBytes(of: self) {
+      hashFunction.update(bufferPointer: $0)
+    }
+  }
+}
+
+extension String: CacheKeyProtocol {
+  public func hash(with hashFunction: inout some HashFunction) {
+    var t = String(reflecting: Self.self)
+    t.withUTF8 {
+      hashFunction.update(bufferPointer: .init($0))
+    }
+    var x = self
+    x.withUTF8 {
+      hashFunction.update(bufferPointer: .init($0))
+    }
+  }
+}
+
+extension FilePath: CacheKeyProtocol {
+  public func hash(with hashFunction: inout some HashFunction) {
+    String(reflecting: Self.self).hash(with: &hashFunction)
+    self.string.hash(with: &hashFunction)
+  }
+}
+
+extension FilePath.Component: CacheKeyProtocol {
+  public func hash(with hashFunction: inout some HashFunction) {
+    String(reflecting: Self.self).hash(with: &hashFunction)
+    self.string.hash(with: &hashFunction)
+  }
+}
+
+extension URL: CacheKeyProtocol {
+  public func hash(with hashFunction: inout some HashFunction) {
+    String(reflecting: Self.self).hash(with: &hashFunction)
+    self.description.hash(with: &hashFunction)
+  }
+}
+
+extension Optional: CacheKeyProtocol where Wrapped: CacheKeyProtocol {
+  public func hash(with hashFunction: inout some HashFunction) {
+    String(reflecting: Self.self).hash(with: &hashFunction)
+    if let self {
+      true.hash(with: &hashFunction)
+      self.hash(with: &hashFunction)
+    } else {
+      false.hash(with: &hashFunction)
+    }
+  }
+}
+
+extension Array: CacheKeyProtocol where Element: CacheKeyProtocol {
+  public func hash(with hashFunction: inout some HashFunction) {
+    String(reflecting: Self.self).hash(with: &hashFunction)
+    for element in self {
+      element.hash(with: &hashFunction)
+    }
+  }
+}
+
+@attached(extension, conformances: CacheKeyProtocol, names: named(hash(with:)))
+public macro CacheKey() = #externalMacro(module: "Macros", type: "CacheKeyMacro")

Sources/GeneratorEngine/Cache/FileCacheRecord.swift

@@ -0,0 +1,38 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift open source project
+//
+// Copyright (c) 2023 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+import struct SystemPackage.FilePath
+
+public struct FileCacheRecord: Sendable {
+  public let path: FilePath
+  public let hash: String
+}
+
+extension FileCacheRecord: Codable {
+  enum CodingKeys: CodingKey {
+    case path
+    case hash
+  }
+
+  // FIXME: `Codable` on `FilePath` is broken, thus all `Codable` types with `FilePath` properties need a custom impl.
+  public init(from decoder: any Decoder) throws {
+    let container = try decoder.container(keyedBy: CodingKeys.self)
+    self.path = try FilePath(container.decode(String.self, forKey: .path))
+    self.hash = try container.decode(String.self, forKey: .hash)
+  }
+
+  public func encode(to encoder: any Encoder) throws {
+    var container = encoder.container(keyedBy: CodingKeys.self)
+    try container.encode(self.path.string, forKey: .path)
+    try container.encode(self.hash, forKey: .hash)
+  }
+}