diff --git a/Sources/GeneratorEngine/Cache/CacheKeyProtocol.swift b/Sources/GeneratorEngine/Cache/CacheKeyProtocol.swift index 7879f96..ff3cd9c 100644 --- a/Sources/GeneratorEngine/Cache/CacheKeyProtocol.swift +++ b/Sources/GeneratorEngine/Cache/CacheKeyProtocol.swift @@ -24,14 +24,14 @@ public protocol CacheKeyProtocol { extension Bool: CacheKeyProtocol { public func hash(with hashFunction: inout some HashFunction) { - "Swift.Bool".hash(with: &hashFunction) + String(reflecting: Self.self).hash(with: &hashFunction) hashFunction.update(data: self ? [1] : [0]) } } extension Int: CacheKeyProtocol { public func hash(with hashFunction: inout some HashFunction) { - "Swift.Int".hash(with: &hashFunction) + String(reflecting: Self.self).hash(with: &hashFunction) withUnsafeBytes(of: self) { hashFunction.update(bufferPointer: $0) } @@ -40,7 +40,7 @@ extension Int: CacheKeyProtocol { extension String: CacheKeyProtocol { public func hash(with hashFunction: inout some HashFunction) { - var t = "Swift.String" + var t = String(reflecting: Self.self) t.withUTF8 { hashFunction.update(bufferPointer: .init($0)) } @@ -53,21 +53,28 @@ extension String: CacheKeyProtocol { extension FilePath: CacheKeyProtocol { public func hash(with hashFunction: inout some HashFunction) { - "SystemPackage.FilePath".hash(with: &hashFunction) + String(reflecting: Self.self).hash(with: &hashFunction) + self.string.hash(with: &hashFunction) + } +} + +extension FilePath.Component: CacheKeyProtocol { + public func hash(with hashFunction: inout some HashFunction) { + String(reflecting: Self.self).hash(with: &hashFunction) self.string.hash(with: &hashFunction) } } extension URL: CacheKeyProtocol { public func hash(with hashFunction: inout some HashFunction) { - "Foundation.URL".hash(with: &hashFunction) + String(reflecting: Self.self).hash(with: &hashFunction) self.description.hash(with: &hashFunction) } } extension Optional: CacheKeyProtocol where Wrapped: CacheKeyProtocol { public func hash(with hashFunction: inout some HashFunction) { - "Swift.Optional".hash(with: &hashFunction) + String(reflecting: Self.self).hash(with: &hashFunction) if let self { true.hash(with: &hashFunction) self.hash(with: &hashFunction) @@ -77,5 +84,14 @@ extension Optional: CacheKeyProtocol where Wrapped: CacheKeyProtocol { } } +extension Array: CacheKeyProtocol where Element: CacheKeyProtocol { + public func hash(with hashFunction: inout some HashFunction) { + String(reflecting: Self.self).hash(with: &hashFunction) + for element in self { + element.hash(with: &hashFunction) + } + } +} + @attached(extension, conformances: CacheKeyProtocol, names: named(hash(with:))) public macro CacheKey() = #externalMacro(module: "Macros", type: "CacheKeyMacro") diff --git a/Sources/GeneratorEngine/Cache/FileCacheRecord.swift b/Sources/GeneratorEngine/Cache/FileCacheRecord.swift index 6e0492b..fdef551 100644 --- a/Sources/GeneratorEngine/Cache/FileCacheRecord.swift +++ b/Sources/GeneratorEngine/Cache/FileCacheRecord.swift @@ -13,8 +13,8 @@ import struct SystemPackage.FilePath public struct FileCacheRecord: Sendable { - let path: FilePath - let hash: String + public let path: FilePath + public let hash: String } extension FileCacheRecord: Codable { diff --git a/Sources/GeneratorEngine/Cache/SQLite.swift b/Sources/GeneratorEngine/Cache/SQLite.swift index d7cfeec..eecf17b 100644 --- a/Sources/GeneratorEngine/Cache/SQLite.swift +++ b/Sources/GeneratorEngine/Cache/SQLite.swift @@ -15,6 +15,7 @@ import SystemPackage import SystemSQLite extension FilePath: @unchecked Sendable {} +extension FilePath.Component: @unchecked Sendable {} /// A minimal SQLite wrapper. public final class SQLite { diff --git a/Sources/SwiftSDKGenerator/Artifacts/DownloadableArtifacts.swift b/Sources/SwiftSDKGenerator/Artifacts/DownloadableArtifacts.swift index 5608c0b..4749e68 100644 --- a/Sources/SwiftSDKGenerator/Artifacts/DownloadableArtifacts.swift +++ b/Sources/SwiftSDKGenerator/Artifacts/DownloadableArtifacts.swift @@ -107,7 +107,6 @@ struct DownloadableArtifacts: Sendable { isPrebuilt: true ) - let targetArtifactsOS = ArtifactOS(targetTriple.os, versions) self.targetSwift = .init( remoteURL: versions.swiftDownloadURL(), localPath: paths.artifactsCachePath diff --git a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Build.swift b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Build.swift deleted file mode 100644 index f3f9a3f..0000000 --- a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Build.swift +++ /dev/null @@ -1,24 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the Swift open source project -// -// Copyright (c) 2022-2023 Apple Inc. and the Swift project authors -// Licensed under Apache License v2.0 with Runtime Library Exception -// -// See https://swift.org/LICENSE.txt for license information -// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors -// -//===----------------------------------------------------------------------===// - -import SystemPackage - -extension SwiftSDKGenerator { - func buildLLD(llvmSourcesDirectory: FilePath) async throws -> FilePath { - let buildDirectory = try await self.buildCMakeProject( - llvmSourcesDirectory, - options: "-DLLVM_ENABLE_PROJECTS=lld -DLLVM_TARGETS_TO_BUILD=\(self.targetTriple.cpu.llvmTargetConventionName)" - ) - - return buildDirectory.appending("bin").appending("lld") - } -} diff --git a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Download.swift b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Download.swift index 84524e9..f051bd0 100644 --- a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Download.swift +++ b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Download.swift @@ -23,24 +23,7 @@ import struct SystemPackage.FilePath private let ubuntuAMD64Mirror = "http://gb.archive.ubuntu.com/ubuntu" private let ubuntuARM64Mirror = "http://ports.ubuntu.com/ubuntu-ports" -private let byteCountFormatter = ByteCountFormatter() - -@Query -struct DownloadQuery { - let artifact: DownloadableArtifacts.Item - - func run(engine: Engine) async throws -> FilePath { - print("Downloading remote artifact not available in local cache: \(self.artifact.remoteURL)") - let stream = await engine.httpClient.streamDownloadProgress(for: self.artifact) - .removeDuplicates(by: didProgressChangeSignificantly) - .throttle(for: .seconds(1)) - - for try await item in stream { - report(progress: item.progress, for: item.artifact) - } - return self.artifact.localPath - } -} +let byteCountFormatter = ByteCountFormatter() extension SwiftSDKGenerator { func downloadArtifacts(_ client: HTTPClient) async throws { @@ -57,9 +40,8 @@ extension SwiftSDKGenerator { let results = try await withThrowingTaskGroup(of: FileCacheRecord.self) { group in for item in self.downloadableArtifacts.allItems { - print(item.remoteURL) group.addTask { - try await self.engine[DownloadQuery(artifact: item)] + try await self.engine[DownloadArtifactQuery(artifact: item)] } } @@ -69,6 +51,11 @@ extension SwiftSDKGenerator { } return result } + + print("Using downloaded artifacts in these locations:") + for path in results.map(\.path) { + print(path) + } } func downloadUbuntuPackages(_ client: HTTPClient, requiredPackages: [String]) async throws { @@ -211,40 +198,3 @@ extension HTTPClient { return result } } - -/// Checks whether two given progress value are different enough from each other. Used for filtering out progress -/// values in async streams with `removeDuplicates` operator. -/// - Parameters: -/// - previous: Preceding progress value in the stream. -/// - current: Currently processed progress value in the stream. -/// - Returns: `true` if `totalBytes` value is different by any amount or if `receivedBytes` is different by amount -/// larger than 1MiB. Returns `false` otherwise. -@Sendable -private func didProgressChangeSignificantly( - previous: ArtifactDownloadProgress, - current: ArtifactDownloadProgress -) -> Bool { - guard previous.progress.totalBytes == current.progress.totalBytes else { - return true - } - - return current.progress.receivedBytes - previous.progress.receivedBytes > 1024 * 1024 * 1024 -} - -private func report(progress: FileDownloadDelegate.Progress, for artifact: DownloadableArtifacts.Item) { - if let total = progress.totalBytes { - print(""" - \(artifact.remoteURL.lastPathComponent) \( - byteCountFormatter - .string(fromByteCount: Int64(progress.receivedBytes)) - )/\( - byteCountFormatter - .string(fromByteCount: Int64(total)) - ) - """) - } else { - print( - "\(artifact.remoteURL.lastPathComponent) \(byteCountFormatter.string(fromByteCount: Int64(progress.receivedBytes)))" - ) - } -} diff --git a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Entrypoint.swift b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Entrypoint.swift index 5634e04..91f4f57 100644 --- a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Entrypoint.swift +++ b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Entrypoint.swift @@ -67,7 +67,7 @@ public extension SwiftSDKGenerator { try await self.unpackTargetSwiftPackage() } - try await self.unpackLLDLinker() + try await self.prepareLLDLinker() try self.fixAbsoluteSymlinks() diff --git a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Unpack.swift b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Unpack.swift index 077c962..a214672 100644 --- a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Unpack.swift +++ b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Unpack.swift @@ -10,6 +10,8 @@ // //===----------------------------------------------------------------------===// +import struct SystemPackage.FilePath + private let unusedDarwinPlatforms = [ "watchsimulator", "iphonesimulator", @@ -67,36 +69,43 @@ extension SwiftSDKGenerator { } } - func unpackLLDLinker() async throws { + func prepareLLDLinker() async throws { logGenerationStep("Unpacking and copying `lld` linker...") let downloadableArtifacts = self.downloadableArtifacts let pathsConfiguration = self.pathsConfiguration let targetOS = self.targetTriple.os - try await inTemporaryDirectory { fileSystem, tmpDir in - let llvmArtifact = downloadableArtifacts.hostLLVM - try await fileSystem.untar( - file: llvmArtifact.localPath, - into: tmpDir, - stripComponents: 1 - ) + let llvmArtifact = downloadableArtifacts.hostLLVM - let unpackedLLDPath = if llvmArtifact.isPrebuilt { - tmpDir.appending("bin/lld") - } else { - try await self.buildLLD(llvmSourcesDirectory: tmpDir) - } + let untarDestination = pathsConfiguration.artifactsCachePath.appending( + FilePath.Component(llvmArtifact.localPath.stem!)!.stem + ) + try self.createDirectoryIfNeeded(at: untarDestination) + try await self.untar( + file: llvmArtifact.localPath, + into: untarDestination, + stripComponents: 1 + ) - let toolchainLLDPath = switch targetOS { - case .linux: - pathsConfiguration.toolchainBinDirPath.appending("ld.lld") - case .wasi: - pathsConfiguration.toolchainBinDirPath.appending("wasm-ld") - default: - fatalError() - } + let unpackedLLDPath = if llvmArtifact.isPrebuilt { + untarDestination.appending("bin/lld") + } else { + try await self.engine[CMakeBuildQuery( + sourcesDirectory: untarDestination, + outputBinarySubpath: ["bin", "lld"], + options: "-DLLVM_ENABLE_PROJECTS=lld -DLLVM_TARGETS_TO_BUILD=\(self.targetTriple.cpu.llvmTargetConventionName)" + )].path + } - try await fileSystem.copy(from: unpackedLLDPath, to: toolchainLLDPath) + let toolchainLLDPath = switch targetOS { + case .linux: + pathsConfiguration.toolchainBinDirPath.appending("ld.lld") + case .wasi: + pathsConfiguration.toolchainBinDirPath.appending("wasm-ld") + default: + fatalError() } + + try self.copy(from: unpackedLLDPath, to: toolchainLLDPath) } } diff --git a/Sources/SwiftSDKGenerator/Queries/CMakeBuildQuery.swift b/Sources/SwiftSDKGenerator/Queries/CMakeBuildQuery.swift new file mode 100644 index 0000000..31c0a0e --- /dev/null +++ b/Sources/SwiftSDKGenerator/Queries/CMakeBuildQuery.swift @@ -0,0 +1,36 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2022-2023 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import GeneratorEngine +import struct SystemPackage.FilePath + +@Query +struct CMakeBuildQuery { + let sourcesDirectory: FilePath + /// Path to the output binary relative to the CMake build directory. + let outputBinarySubpath: [FilePath.Component] + let options: String + + func run(engine: Engine) async throws -> FilePath { + try await Shell.run( + """ + cmake -B build -G Ninja -S llvm -DCMAKE_BUILD_TYPE=Release \(self.options) + """, + currentDirectory: self.sourcesDirectory + ) + + let buildDirectory = self.sourcesDirectory.appending("build") + try await Shell.run("ninja", currentDirectory: buildDirectory) + + return self.outputBinarySubpath.reduce(into: buildDirectory) { $0.append($1) } + } +} diff --git a/Sources/SwiftSDKGenerator/Queries/DownloadArtifactQuery.swift b/Sources/SwiftSDKGenerator/Queries/DownloadArtifactQuery.swift new file mode 100644 index 0000000..2d9708a --- /dev/null +++ b/Sources/SwiftSDKGenerator/Queries/DownloadArtifactQuery.swift @@ -0,0 +1,69 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2022-2023 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import class AsyncHTTPClient.FileDownloadDelegate +import GeneratorEngine +import struct SystemPackage.FilePath + +@Query +struct DownloadArtifactQuery { + let artifact: DownloadableArtifacts.Item + + func run(engine: Engine) async throws -> FilePath { + print("Downloading remote artifact not available in local cache: \(self.artifact.remoteURL)") + let stream = await engine.httpClient.streamDownloadProgress(for: self.artifact) + .removeDuplicates(by: didProgressChangeSignificantly) + .throttle(for: .seconds(1)) + + for try await item in stream { + report(progress: item.progress, for: item.artifact) + } + return self.artifact.localPath + } +} + +/// Checks whether two given progress value are different enough from each other. Used for filtering out progress +/// values in async streams with `removeDuplicates` operator. +/// - Parameters: +/// - previous: Preceding progress value in the stream. +/// - current: Currently processed progress value in the stream. +/// - Returns: `true` if `totalBytes` value is different by any amount or if `receivedBytes` is different by amount +/// larger than 1MiB. Returns `false` otherwise. +@Sendable +private func didProgressChangeSignificantly( + previous: ArtifactDownloadProgress, + current: ArtifactDownloadProgress +) -> Bool { + guard previous.progress.totalBytes == current.progress.totalBytes else { + return true + } + + return current.progress.receivedBytes - previous.progress.receivedBytes > 1024 * 1024 * 1024 +} + +private func report(progress: FileDownloadDelegate.Progress, for artifact: DownloadableArtifacts.Item) { + if let total = progress.totalBytes { + print(""" + \(artifact.remoteURL.lastPathComponent) \( + byteCountFormatter + .string(fromByteCount: Int64(progress.receivedBytes)) + )/\( + byteCountFormatter + .string(fromByteCount: Int64(total)) + ) + """) + } else { + print( + "\(artifact.remoteURL.lastPathComponent) \(byteCountFormatter.string(fromByteCount: Int64(progress.receivedBytes)))" + ) + } +}