Cache lld binaries to avoid redundant rebuilds (#35)

This proceeds with converting more of the generation steps to a query-based architecture. Now `lld` doesn't need to be rebuilt if its source directory and build options don't change. This is implemented with a new `CMakeBuildQuery`, which takes source directory, and build options as arguments, and returns a freshly-built binary that can be cached by the generator engine.

Also renamed `DownloadQuery` to `DownloadArtifactQuery` and moved it to a separate file.

Author: MaxDesiatov

Sources/GeneratorEngine/Cache/CacheKeyProtocol.swift

@@ -24,14 +24,14 @@ public protocol CacheKeyProtocol {
 
 extension Bool: CacheKeyProtocol {
   public func hash(with hashFunction: inout some HashFunction) {
-    "Swift.Bool".hash(with: &hashFunction)
+    String(reflecting: Self.self).hash(with: &hashFunction)
     hashFunction.update(data: self ? [1] : [0])
   }
 }
 
 extension Int: CacheKeyProtocol {
   public func hash(with hashFunction: inout some HashFunction) {
-    "Swift.Int".hash(with: &hashFunction)
+    String(reflecting: Self.self).hash(with: &hashFunction)
     withUnsafeBytes(of: self) {
       hashFunction.update(bufferPointer: $0)
     }
@@ -40,7 +40,7 @@ extension Int: CacheKeyProtocol {
 
 extension String: CacheKeyProtocol {
   public func hash(with hashFunction: inout some HashFunction) {
-    var t = "Swift.String"
+    var t = String(reflecting: Self.self)
     t.withUTF8 {
       hashFunction.update(bufferPointer: .init($0))
     }
@@ -53,21 +53,28 @@ extension String: CacheKeyProtocol {
 
 extension FilePath: CacheKeyProtocol {
   public func hash(with hashFunction: inout some HashFunction) {
-    "SystemPackage.FilePath".hash(with: &hashFunction)
+    String(reflecting: Self.self).hash(with: &hashFunction)
+    self.string.hash(with: &hashFunction)
+  }
+}
+
+extension FilePath.Component: CacheKeyProtocol {
+  public func hash(with hashFunction: inout some HashFunction) {
+    String(reflecting: Self.self).hash(with: &hashFunction)
     self.string.hash(with: &hashFunction)
   }
 }
 
 extension URL: CacheKeyProtocol {
   public func hash(with hashFunction: inout some HashFunction) {
-    "Foundation.URL".hash(with: &hashFunction)
+    String(reflecting: Self.self).hash(with: &hashFunction)
     self.description.hash(with: &hashFunction)
   }
 }
 
 extension Optional: CacheKeyProtocol where Wrapped: CacheKeyProtocol {
   public func hash(with hashFunction: inout some HashFunction) {
-    "Swift.Optional".hash(with: &hashFunction)
+    String(reflecting: Self.self).hash(with: &hashFunction)
     if let self {
       true.hash(with: &hashFunction)
       self.hash(with: &hashFunction)
@@ -77,5 +84,14 @@ extension Optional: CacheKeyProtocol where Wrapped: CacheKeyProtocol {
   }
 }
 
+extension Array: CacheKeyProtocol where Element: CacheKeyProtocol {
+  public func hash(with hashFunction: inout some HashFunction) {
+    String(reflecting: Self.self).hash(with: &hashFunction)
+    for element in self {
+      element.hash(with: &hashFunction)
+    }
+  }
+}
+
 @attached(extension, conformances: CacheKeyProtocol, names: named(hash(with:)))
 public macro CacheKey() = #externalMacro(module: "Macros", type: "CacheKeyMacro")

Sources/GeneratorEngine/Cache/FileCacheRecord.swift

@@ -13,8 +13,8 @@
 import struct SystemPackage.FilePath
 
 public struct FileCacheRecord: Sendable {
-  let path: FilePath
-  let hash: String
+  public let path: FilePath
+  public let hash: String
 }
 
 extension FileCacheRecord: Codable {

Sources/GeneratorEngine/Cache/SQLite.swift

@@ -15,6 +15,7 @@ import SystemPackage
 import SystemSQLite
 
 extension FilePath: @unchecked Sendable {}
+extension FilePath.Component: @unchecked Sendable {}
 
 /// A minimal SQLite wrapper.
 public final class SQLite {

Sources/SwiftSDKGenerator/Artifacts/DownloadableArtifacts.swift

@@ -107,7 +107,6 @@ struct DownloadableArtifacts: Sendable {
       isPrebuilt: true
     )
 
-    let targetArtifactsOS = ArtifactOS(targetTriple.os, versions)
     self.targetSwift = .init(
       remoteURL: versions.swiftDownloadURL(),
       localPath: paths.artifactsCachePath

Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Build.swift

@@ -23,24 +23,7 @@ import struct SystemPackage.FilePath
 private let ubuntuAMD64Mirror = "http://gb.archive.ubuntu.com/ubuntu"
 private let ubuntuARM64Mirror = "http://ports.ubuntu.com/ubuntu-ports"
 
-private let byteCountFormatter = ByteCountFormatter()
-
-@Query
-struct DownloadQuery {
-  let artifact: DownloadableArtifacts.Item
-
-  func run(engine: Engine) async throws -> FilePath {
-    print("Downloading remote artifact not available in local cache: \(self.artifact.remoteURL)")
-    let stream = await engine.httpClient.streamDownloadProgress(for: self.artifact)
-      .removeDuplicates(by: didProgressChangeSignificantly)
-      .throttle(for: .seconds(1))
-
-    for try await item in stream {
-      report(progress: item.progress, for: item.artifact)
-    }
-    return self.artifact.localPath
-  }
-}
+let byteCountFormatter = ByteCountFormatter()
 
 extension SwiftSDKGenerator {
   func downloadArtifacts(_ client: HTTPClient) async throws {
@@ -57,9 +40,8 @@ extension SwiftSDKGenerator {
 
     let results = try await withThrowingTaskGroup(of: FileCacheRecord.self) { group in
       for item in self.downloadableArtifacts.allItems {
-        print(item.remoteURL)
         group.addTask {
-          try await self.engine[DownloadQuery(artifact: item)]
+          try await self.engine[DownloadArtifactQuery(artifact: item)]
         }
       }
 
@@ -69,6 +51,11 @@ extension SwiftSDKGenerator {
       }
       return result
     }
+
+    print("Using downloaded artifacts in these locations:")
+    for path in results.map(\.path) {
+      print(path)
+    }
   }
 
   func downloadUbuntuPackages(_ client: HTTPClient, requiredPackages: [String]) async throws {
@@ -211,40 +198,3 @@ extension HTTPClient {
     return result
   }
 }
-
-/// Checks whether two given progress value are different enough from each other. Used for filtering out progress
-/// values in async streams with `removeDuplicates` operator.
-/// - Parameters:
-///   - previous: Preceding progress value in the stream.
-///   - current: Currently processed progress value in the stream.
-/// - Returns: `true` if `totalBytes` value is different by any amount or if `receivedBytes` is different by amount
-/// larger than 1MiB. Returns `false` otherwise.
-@Sendable
-private func didProgressChangeSignificantly(
-  previous: ArtifactDownloadProgress,
-  current: ArtifactDownloadProgress
-) -> Bool {
-  guard previous.progress.totalBytes == current.progress.totalBytes else {
-    return true
-  }
-
-  return current.progress.receivedBytes - previous.progress.receivedBytes > 1024 * 1024 * 1024
-}
-
-private func report(progress: FileDownloadDelegate.Progress, for artifact: DownloadableArtifacts.Item) {
-  if let total = progress.totalBytes {
-    print("""
-    \(artifact.remoteURL.lastPathComponent) \(
-      byteCountFormatter
-        .string(fromByteCount: Int64(progress.receivedBytes))
-    )/\(
-      byteCountFormatter
-        .string(fromByteCount: Int64(total))
-    )
-    """)
-  } else {
-    print(
-      "\(artifact.remoteURL.lastPathComponent) \(byteCountFormatter.string(fromByteCount: Int64(progress.receivedBytes)))"
-    )
-  }
-}

Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Download.swift

@@ -67,7 +67,7 @@ public extension SwiftSDKGenerator {
       try await self.unpackTargetSwiftPackage()
     }
 
-    try await self.unpackLLDLinker()
+    try await self.prepareLLDLinker()
 
     try self.fixAbsoluteSymlinks()
 

Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Entrypoint.swift

@@ -10,6 +10,8 @@
 //
 //===----------------------------------------------------------------------===//
 
+import struct SystemPackage.FilePath
+
 private let unusedDarwinPlatforms = [
   "watchsimulator",
   "iphonesimulator",
@@ -67,36 +69,43 @@ extension SwiftSDKGenerator {
     }
   }
 
-  func unpackLLDLinker() async throws {
+  func prepareLLDLinker() async throws {
     logGenerationStep("Unpacking and copying `lld` linker...")
     let downloadableArtifacts = self.downloadableArtifacts
     let pathsConfiguration = self.pathsConfiguration
     let targetOS = self.targetTriple.os
 
-    try await inTemporaryDirectory { fileSystem, tmpDir in
-      let llvmArtifact = downloadableArtifacts.hostLLVM
-      try await fileSystem.untar(
-        file: llvmArtifact.localPath,
-        into: tmpDir,
-        stripComponents: 1
-      )
+    let llvmArtifact = downloadableArtifacts.hostLLVM
 
-      let unpackedLLDPath = if llvmArtifact.isPrebuilt {
-        tmpDir.appending("bin/lld")
-      } else {
-        try await self.buildLLD(llvmSourcesDirectory: tmpDir)
-      }
+    let untarDestination = pathsConfiguration.artifactsCachePath.appending(
+      FilePath.Component(llvmArtifact.localPath.stem!)!.stem
+    )
+    try self.createDirectoryIfNeeded(at: untarDestination)
+    try await self.untar(
+      file: llvmArtifact.localPath,
+      into: untarDestination,
+      stripComponents: 1
+    )
 
-      let toolchainLLDPath = switch targetOS {
-      case .linux:
-        pathsConfiguration.toolchainBinDirPath.appending("ld.lld")
-      case .wasi:
-        pathsConfiguration.toolchainBinDirPath.appending("wasm-ld")
-      default:
-        fatalError()
-      }
+    let unpackedLLDPath = if llvmArtifact.isPrebuilt {
+      untarDestination.appending("bin/lld")
+    } else {
+      try await self.engine[CMakeBuildQuery(
+        sourcesDirectory: untarDestination,
+        outputBinarySubpath: ["bin", "lld"],
+        options: "-DLLVM_ENABLE_PROJECTS=lld -DLLVM_TARGETS_TO_BUILD=\(self.targetTriple.cpu.llvmTargetConventionName)"
+      )].path
+    }
 
-      try await fileSystem.copy(from: unpackedLLDPath, to: toolchainLLDPath)
+    let toolchainLLDPath = switch targetOS {
+    case .linux:
+      pathsConfiguration.toolchainBinDirPath.appending("ld.lld")
+    case .wasi:
+      pathsConfiguration.toolchainBinDirPath.appending("wasm-ld")
+    default:
+      fatalError()
     }
+
+    try self.copy(from: unpackedLLDPath, to: toolchainLLDPath)
   }
 }

Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Unpack.swift

@@ -0,0 +1,36 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift open source project
+//
+// Copyright (c) 2022-2023 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+import GeneratorEngine
+import struct SystemPackage.FilePath
+
+@Query
+struct CMakeBuildQuery {
+  let sourcesDirectory: FilePath
+  /// Path to the output binary relative to the CMake build directory.
+  let outputBinarySubpath: [FilePath.Component]
+  let options: String
+
+  func run(engine: Engine) async throws -> FilePath {
+    try await Shell.run(
+      """
+      cmake -B build -G Ninja -S llvm -DCMAKE_BUILD_TYPE=Release \(self.options)
+      """,
+      currentDirectory: self.sourcesDirectory
+    )
+
+    let buildDirectory = self.sourcesDirectory.appending("build")
+    try await Shell.run("ninja", currentDirectory: buildDirectory)
+
+    return self.outputBinarySubpath.reduce(into: buildDirectory) { $0.append($1) }
+  }
+}