Cache DownloadableArtifacts with GeneratorEngine (#34)

New `GeneratorEngine` allows us to cache all items in `DownloadableArtifacts` in a generic way and to allow redundant re-downloads for artifacts that don't have their checksums hardcoded (e.g. LLVM sources for `lld`).

This now allows us to delete all hardcoded checksums and also to cache other artifacts in the future, like `lld` binary and even whole artifact bundles.

Author: MaxDesiatov

Package.resolved

@@ -113,8 +113,8 @@
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/apple/swift-syntax.git",
       "state" : {
-        "revision" : "ffa3cd6fc2aa62adbedd31d3efaf7c0d86a9f029",
-        "version" : "509.0.1"
+        "revision" : "6ad4ea24b01559dde0773e3d091f1b9e36175036",
+        "version" : "509.0.2"
       }
     },
     {

Package.swift

@@ -26,7 +26,7 @@ let package = Package(
     .package(url: "https://github.com/apple/swift-nio.git", from: "2.58.0"),
     .package(url: "https://github.com/apple/swift-nio-extras.git", from: "1.19.0"),
     .package(url: "https://github.com/apple/swift-log.git", from: "1.5.3"),
-    .package(url: "https://github.com/apple/swift-syntax.git", from: "509.0.1"),
+    .package(url: "https://github.com/apple/swift-syntax.git", from: "509.0.2"),
   ],
   targets: [
     // Targets are the basic building blocks of a package. A target can define a module or a test suite.

Sources/GeneratorCLI/GeneratorCLI.swift

@@ -88,7 +88,7 @@ struct GeneratorCLI: AsyncParsableCommand {
     let linuxDistribution = try LinuxDistribution(name: linuxDistributionName, version: linuxDistributionVersion)
 
     let elapsed = try await ContinuousClock().measure {
-      try await SwiftSDKGenerator(
+      let generator = try await SwiftSDKGenerator(
         hostCPUArchitecture: self.hostArch,
         targetCPUArchitecture: self.targetArch,
         swiftVersion: self.swiftVersion,
@@ -98,7 +98,13 @@ struct GeneratorCLI: AsyncParsableCommand {
         shouldUseDocker: self.withDocker,
         isVerbose: self.verbose
       )
-      .generateBundle(shouldGenerateFromScratch: !self.incremental)
+      do {
+        try await generator.generateBundle(shouldGenerateFromScratch: !self.incremental)
+        try await generator.shutDown()
+      } catch {
+        try await generator.shutDown()
+        throw error
+      }
     }
 
     print("\nTime taken for this generator run: \(elapsed.intervalString).")

Sources/GeneratorEngine/Cache/FileCacheRecord.swift

@@ -12,7 +12,7 @@
 
 import struct SystemPackage.FilePath
 
-struct FileCacheRecord {
+public struct FileCacheRecord: Sendable {
   let path: FilePath
   let hash: String
 }
@@ -23,14 +23,14 @@ extension FileCacheRecord: Codable {
     case hash
   }
 
-  // FIXME: `Codable` on `FilePath` is broken
-  init(from decoder: any Decoder) throws {
+  // FIXME: `Codable` on `FilePath` is broken, thus all `Codable` types with `FilePath` properties need a custom impl.
+  public init(from decoder: any Decoder) throws {
     let container = try decoder.container(keyedBy: CodingKeys.self)
     self.path = try FilePath(container.decode(String.self, forKey: .path))
     self.hash = try container.decode(String.self, forKey: .hash)
   }
 
-  func encode(to encoder: any Encoder) throws {
+  public func encode(to encoder: any Encoder) throws {
     var container = encoder.container(keyedBy: CodingKeys.self)
     try container.encode(self.path.string, forKey: .path)
     try container.encode(self.hash, forKey: .hash)

Sources/GeneratorEngine/Engine.swift

@@ -32,7 +32,6 @@ public actor Engine {
   /// `defer { engine.shutDown }` on the line that follows this initializer call.
   /// - Parameter fileSystem: Implementation of a file system this engine should use.
   /// - Parameter cacheLocation: Location of cache storage used by the engine.
-  /// - Parameter httpClient: HTTP client to use in queries that need it.
   /// - Parameter logger: Logger to use during queries execution.
   public init(
     _ fileSystem: any FileSystem,
@@ -63,7 +62,7 @@ public actor Engine {
   /// Executes a given query if no cached result of it is available. Otherwise fetches the result from engine's cache.
   /// - Parameter query: A query value to execute.
   /// - Returns: A file path to query's result recorded in a file.
-  public subscript(_ query: some QueryProtocol) -> FilePath {
+  public subscript(_ query: some QueryProtocol) -> FileCacheRecord {
     get async throws {
       var hashFunction = SHA512()
       query.hash(with: &hashFunction)
@@ -78,7 +77,7 @@ public actor Engine {
 
         if fileHash == fileRecord.hash {
           self.cacheHits += 1
-          return fileRecord.path
+          return fileRecord
         }
       }
 
@@ -90,11 +89,12 @@ public actor Engine {
         try await $0.hash(with: &hashFunction)
       }
       let resultHash = hashFunction.finalize()
+      let result = FileCacheRecord(path: resultPath, hash: resultHash.description)
 
       // FIXME: update `SQLiteBackedCache` to store `resultHash` directly instead of relying on string conversions
-      try self.resultsCache.set(key, to: FileCacheRecord(path: resultPath, hash: resultHash.description))
+      try self.resultsCache.set(key, to: result)
 
-      return resultPath
+      return result
     }
   }
 }

Sources/SwiftSDKGenerator/Artifacts/DownloadableArtifacts.swift

@@ -11,6 +11,7 @@
 //===----------------------------------------------------------------------===//
 
 import struct Foundation.URL
+import GeneratorEngine
 import struct SystemPackage.FilePath
 
 /// Information about the OS for which the artifact is built, if it's downloaded as prebuilt.
@@ -45,74 +46,26 @@ enum ArtifactOS: Hashable {
 
 typealias CPUMapping = [Triple.CPU: String]
 
-/// SHA256 hashes of binary LLVM artifacts known to the generator.
-private let knownLLVMBinariesVersions: [ArtifactOS: [String: CPUMapping]] = [
-  .macOS: [
-    "15.0.7": [
-      Triple.CPU.arm64: "867c6afd41158c132ef05a8f1ddaecf476a26b91c85def8e124414f9a9ba188d",
-    ],
-    "16.0.0": [
-      Triple.CPU.arm64: "2041587b90626a4a87f0de14a5842c14c6c3374f42c8ed12726ef017416409d9",
-    ],
-    "16.0.1": [
-      Triple.CPU.arm64: "cb487fa991f047dc79ae36430cbb9ef14621c1262075373955b1d97215c75879",
-    ],
-    "16.0.4": [
-      Triple.CPU.arm64: "429b8061d620108fee636313df55a0602ea0d14458c6d3873989e6b130a074bd",
-    ],
-    "16.0.5": [
-      Triple.CPU.arm64: "1aed0787417dd915f0101503ce1d2719c8820a2c92d4a517bfc4044f72035bcc",
-    ],
-  ],
-]
-
-/// SHA256 hashes of binary Swift artifacts known to the generator.
-private let knownSwiftBinariesVersions: [ArtifactOS: [String: CPUMapping]] = [
-  .linux(.ubuntu(.jammy)): [
-    "5.7.3-RELEASE": [
-      .arm64: "75003d5a995292ae3f858b767fbb89bc3edee99488f4574468a0e44341aec55b",
-    ],
-    "5.8-RELEASE": [
-      .arm64: "12ea2df36f9af0aefa74f0989009683600978f62223e7dd73b627c90c7fe9273",
-    ],
-    "5.9-RELEASE": [
-      .arm64: "30b289e02f7e03c380744ea97fdf0e96985dff504b0f09de23e098fdaf6513f3",
-      .x86_64: "bca015e9d727ca39385d7e5b5399f46302d54a02218d40d1c3063662ffc6b42f",
-    ],
-  ],
-  .macOS: [
-    "5.7.3-RELEASE": [
-      .arm64: "ba3516845eb8f4469a8bb06a273687f05791187324a3843996af32a73a2a687d",
-      .x86_64: "ba3516845eb8f4469a8bb06a273687f05791187324a3843996af32a73a2a687d",
-    ],
-    "5.8-RELEASE": [
-      .arm64: "9b6cc56993652ca222c86a2d6b7b66abbd50bb92cc526efc2b23d47d40002097",
-      .x86_64: "9b6cc56993652ca222c86a2d6b7b66abbd50bb92cc526efc2b23d47d40002097",
-    ],
-    "5.9-RELEASE": [
-      .arm64: "3cf7a4b2f3efcfcb4fef42b6588a7b1c54f7b0f2d0a479f41c3e1620b045f48e",
-      .x86_64: "3cf7a4b2f3efcfcb4fef42b6588a7b1c54f7b0f2d0a479f41c3e1620b045f48e",
-    ],
-  ],
-]
-
-private let knownLLVMSourcesVersions: [String: String] = [
-  "16.0.5": "37f540124b9cfd4680666e649f557077f9937c9178489cea285a672e714b2863",
-]
-
-public struct DownloadableArtifacts: Sendable {
-  public struct Item: Sendable {
+struct DownloadableArtifacts: Sendable {
+  @CacheKey
+  struct Item: Sendable {
     let remoteURL: URL
     var localPath: FilePath
-    let checksum: String?
     let isPrebuilt: Bool
   }
 
   let hostSwift: Item
   private(set) var hostLLVM: Item
   let targetSwift: Item
 
-  let allItems: [Item]
+  private let shouldUseDocker: Bool
+  var allItems: [Item] {
+    if self.shouldUseDocker {
+      [self.hostSwift, self.hostLLVM]
+    } else {
+      [self.hostSwift, self.hostLLVM, self.targetSwift]
+    }
+  }
 
   private let versions: VersionsConfiguration
   private let paths: PathsConfiguration
@@ -136,7 +89,6 @@ public struct DownloadableArtifacts: Sendable {
       ),
       localPath: paths.artifactsCachePath
         .appending("host_swift_\(versions.swiftVersion)_\(hostTriple).pkg"),
-      checksum: knownSwiftBinariesVersions[hostArtifactsOS]?[versions.swiftVersion]?[hostTriple.cpu],
       isPrebuilt: true
     )
 
@@ -152,7 +104,6 @@ public struct DownloadableArtifacts: Sendable {
       )!,
       localPath: paths.artifactsCachePath
         .appending("host_llvm_\(versions.lldVersion)_\(hostTriple).tar.xz"),
-      checksum: knownLLVMBinariesVersions[hostArtifactsOS]?[versions.lldVersion]?[hostTriple.cpu],
       isPrebuilt: true
     )
 
@@ -161,15 +112,10 @@ public struct DownloadableArtifacts: Sendable {
       remoteURL: versions.swiftDownloadURL(),
       localPath: paths.artifactsCachePath
         .appending("target_swift_\(versions.swiftVersion)_\(targetTriple).tar.gz"),
-      checksum: knownSwiftBinariesVersions[targetArtifactsOS]?[versions.swiftVersion]?[targetTriple.cpu],
       isPrebuilt: true
     )
 
-    self.allItems = if shouldUseDocker {
-      [self.hostSwift, self.hostLLVM]
-    } else {
-      [self.hostSwift, self.hostLLVM, self.targetSwift]
-    }
+    self.shouldUseDocker = shouldUseDocker
   }
 
   mutating func useLLVMSources() {
@@ -185,7 +131,6 @@ public struct DownloadableArtifacts: Sendable {
       )!,
       localPath: self.paths.artifactsCachePath
         .appending("llvm_\(self.versions.lldVersion).src.tar.xz"),
-      checksum: knownLLVMSourcesVersions[self.versions.lldVersion],
       isPrebuilt: false
     )
   }

Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Download.swift

@@ -12,16 +12,36 @@
 
 import AsyncAlgorithms
 import AsyncHTTPClient
+import GeneratorEngine
 import RegexBuilder
 
 import class Foundation.ByteCountFormatter
 import struct Foundation.URL
 
+import struct SystemPackage.FilePath
+
 private let ubuntuAMD64Mirror = "http://gb.archive.ubuntu.com/ubuntu"
 private let ubuntuARM64Mirror = "http://ports.ubuntu.com/ubuntu-ports"
 
 private let byteCountFormatter = ByteCountFormatter()
 
+@Query
+struct DownloadQuery {
+  let artifact: DownloadableArtifacts.Item
+
+  func run(engine: Engine) async throws -> FilePath {
+    print("Downloading remote artifact not available in local cache: \(self.artifact.remoteURL)")
+    let stream = await engine.httpClient.streamDownloadProgress(for: self.artifact)
+      .removeDuplicates(by: didProgressChangeSignificantly)
+      .throttle(for: .seconds(1))
+
+    for try await item in stream {
+      report(progress: item.progress, for: item.artifact)
+    }
+    return self.artifact.localPath
+  }
+}
+
 extension SwiftSDKGenerator {
   func downloadArtifacts(_ client: HTTPClient) async throws {
     logGenerationStep("Downloading required toolchain packages...")
@@ -35,42 +55,19 @@ extension SwiftSDKGenerator {
       downloadableArtifacts.useLLVMSources()
     }
 
-    let hostSwiftProgressStream = client.streamDownloadProgress(for: downloadableArtifacts.hostSwift)
-      .removeDuplicates(by: didProgressChangeSignificantly)
-    let hostLLVMProgressStream = client.streamDownloadProgress(for: downloadableArtifacts.hostLLVM)
-      .removeDuplicates(by: didProgressChangeSignificantly)
-
-    print("Using these URLs for downloads:")
-
-    for artifact in downloadableArtifacts.allItems {
-      print(artifact.remoteURL)
-    }
-
-    // FIXME: some code duplication is necessary due to https://github.com/apple/swift-async-algorithms/issues/226
-    if shouldUseDocker {
-      let stream = combineLatest(hostSwiftProgressStream, hostLLVMProgressStream)
-        .throttle(for: .seconds(1))
-
-      for try await (swiftProgress, llvmProgress) in stream {
-        report(progress: swiftProgress, for: downloadableArtifacts.hostSwift)
-        report(progress: llvmProgress, for: downloadableArtifacts.hostLLVM)
+    let results = try await withThrowingTaskGroup(of: FileCacheRecord.self) { group in
+      for item in self.downloadableArtifacts.allItems {
+        print(item.remoteURL)
+        group.addTask {
+          try await self.engine[DownloadQuery(artifact: item)]
+        }
       }
-    } else {
-      let targetSwiftProgressStream = client.streamDownloadProgress(for: downloadableArtifacts.targetSwift)
-        .removeDuplicates(by: didProgressChangeSignificantly)
-
-      let stream = combineLatest(
-        hostSwiftProgressStream,
-        hostLLVMProgressStream,
-        targetSwiftProgressStream
-      )
-      .throttle(for: .seconds(1))
 
-      for try await (hostSwiftProgress, hostLLVMProgress, targetSwiftProgress) in stream {
-        report(progress: hostSwiftProgress, for: downloadableArtifacts.hostSwift)
-        report(progress: hostLLVMProgress, for: downloadableArtifacts.hostLLVM)
-        report(progress: targetSwiftProgress, for: downloadableArtifacts.targetSwift)
+      var result = [FileCacheRecord]()
+      for try await file in group {
+        result.append(file)
       }
+      return result
     }
   }
 
@@ -224,14 +221,14 @@ extension HTTPClient {
 /// larger than 1MiB. Returns `false` otherwise.
 @Sendable
 private func didProgressChangeSignificantly(
-  previous: FileDownloadDelegate.Progress,
-  current: FileDownloadDelegate.Progress
+  previous: ArtifactDownloadProgress,
+  current: ArtifactDownloadProgress
 ) -> Bool {
-  guard previous.totalBytes == current.totalBytes else {
+  guard previous.progress.totalBytes == current.progress.totalBytes else {
     return true
   }
 
-  return current.receivedBytes - previous.receivedBytes > 1024 * 1024 * 1024
+  return current.progress.receivedBytes - previous.progress.receivedBytes > 1024 * 1024 * 1024
 }
 
 private func report(progress: FileDownloadDelegate.Progress, for artifact: DownloadableArtifacts.Item) {