From 4db7eaf827d20ee1a9793607326d91fa91911be4 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Thu, 26 Oct 2023 00:16:39 +0200 Subject: [PATCH 01/15] Add `SystemSQLite` target for future use in caching In subsequent PRs I'd like to introduce a SQLite-backed cache for storing hashes of generated artifacts. That will help us in avoiding redundant work and will make generator significantly quicker for re-runs that change only a few arguments. --- Package.swift | 1 + Sources/SystemSQLite/module.modulemap | 5 +++++ Sources/SystemSQLite/sqlite.h | 14 ++++++++++++++ 3 files changed, 20 insertions(+) create mode 100644 Sources/SystemSQLite/module.modulemap create mode 100644 Sources/SystemSQLite/sqlite.h diff --git a/Package.swift b/Package.swift index b92f94f..e1a2d5e 100644 --- a/Package.swift +++ b/Package.swift @@ -60,6 +60,7 @@ let package = Package( .enableExperimentalFeature("StrictConcurrency=complete"), ] ), + .systemLibrary(name: "SystemSQLite", pkgConfig: "sqlite3"), .target( name: "AsyncProcess", dependencies: [ diff --git a/Sources/SystemSQLite/module.modulemap b/Sources/SystemSQLite/module.modulemap new file mode 100644 index 0000000..819e61f --- /dev/null +++ b/Sources/SystemSQLite/module.modulemap @@ -0,0 +1,5 @@ +module SystemSQLite [system] { + header "sqlite.h" + link "sqlite3" + export * +} diff --git a/Sources/SystemSQLite/sqlite.h b/Sources/SystemSQLite/sqlite.h new file mode 100644 index 0000000..85b59cf --- /dev/null +++ b/Sources/SystemSQLite/sqlite.h @@ -0,0 +1,14 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2014-2020 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +#pragma once +#include From 3ea95f4eae32299766ed32e061c96d987e5ad746 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Thu, 26 Oct 2023 00:26:43 +0200 Subject: [PATCH 02/15] Add `GeneratorEngine` module with tests and macros --- Package.resolved | 18 + Package.swift | 46 ++- .../Cache/CacheKeyProtocol.swift | 69 ++++ .../Cache/FileCacheRecord.swift | 38 +++ Sources/GeneratorEngine/Cache/SQLite.swift | 310 ++++++++++++++++++ .../Cache/SQLiteBackedCache.swift | 253 ++++++++++++++ Sources/GeneratorEngine/Engine.swift | 81 +++++ .../GeneratorEngine/FileSystem/FileLock.swift | 240 ++++++++++++++ .../FileSystem/FileSystem.swift | 24 ++ .../FileSystem/LocalFileSystem.swift | 62 ++++ .../FileSystem/ReadableFileStream.swift | 106 ++++++ .../FileSystem/VirtualFileSystem.swift | 41 +++ Sources/GeneratorEngine/Query.swift | 20 ++ Sources/Macros/CacheKeyMacro.swift | 62 ++++ Sources/Macros/Macros.swift | 19 ++ Sources/Macros/QueryMacro.swift | 43 +++ Sources/Macros/SimpleDiagnosticMessage.swift | 23 ++ Tests/GeneratorEngineTests/EngineTests.swift | 141 ++++++++ Tests/MacrosTests/MacrosTests.swift | 69 ++++ 19 files changed, 1662 insertions(+), 3 deletions(-) create mode 100644 Sources/GeneratorEngine/Cache/CacheKeyProtocol.swift create mode 100644 Sources/GeneratorEngine/Cache/FileCacheRecord.swift create mode 100644 Sources/GeneratorEngine/Cache/SQLite.swift create mode 100644 Sources/GeneratorEngine/Cache/SQLiteBackedCache.swift create mode 100644 Sources/GeneratorEngine/Engine.swift create mode 100644 Sources/GeneratorEngine/FileSystem/FileLock.swift create mode 100644 Sources/GeneratorEngine/FileSystem/FileSystem.swift create mode 100644 Sources/GeneratorEngine/FileSystem/LocalFileSystem.swift create mode 100644 Sources/GeneratorEngine/FileSystem/ReadableFileStream.swift create mode 100644 Sources/GeneratorEngine/FileSystem/VirtualFileSystem.swift create mode 100644 Sources/GeneratorEngine/Query.swift create mode 100644 Sources/Macros/CacheKeyMacro.swift create mode 100644 Sources/Macros/Macros.swift create mode 100644 Sources/Macros/QueryMacro.swift create mode 100644 Sources/Macros/SimpleDiagnosticMessage.swift create mode 100644 Tests/GeneratorEngineTests/EngineTests.swift create mode 100644 Tests/MacrosTests/MacrosTests.swift diff --git a/Package.resolved b/Package.resolved index 0036bd3..6d1c3b4 100644 --- a/Package.resolved +++ b/Package.resolved @@ -45,6 +45,15 @@ "version" : "1.0.4" } }, + { + "identity" : "swift-crypto", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-crypto.git", + "state" : { + "revision" : "b51f1d6845b353a2121de1c6a670738ec33561a6", + "version" : "3.1.0" + } + }, { "identity" : "swift-log", "kind" : "remoteSourceControl", @@ -99,6 +108,15 @@ "version" : "1.19.0" } }, + { + "identity" : "swift-syntax", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-syntax.git", + "state" : { + "revision" : "ffa3cd6fc2aa62adbedd31d3efaf7c0d86a9f029", + "version" : "509.0.1" + } + }, { "identity" : "swift-system", "kind" : "remoteSourceControl", diff --git a/Package.swift b/Package.swift index e1a2d5e..bdbe54c 100644 --- a/Package.swift +++ b/Package.swift @@ -1,6 +1,7 @@ // swift-tools-version: 5.9 // The swift-tools-version declares the minimum version of Swift required to build this package. +import CompilerPluginSupport import PackageDescription let package = Package( @@ -20,10 +21,12 @@ let package = Package( .package(url: "https://github.com/apple/swift-argument-parser", from: "1.2.2"), .package(url: "https://github.com/apple/swift-async-algorithms.git", exact: "1.0.0-alpha"), .package(url: "https://github.com/apple/swift-atomics.git", from: "1.1.0"), + .package(url: "https://github.com/apple/swift-collections.git", from: "1.0.4"), + .package(url: "https://github.com/apple/swift-crypto.git", from: "3.1.0"), .package(url: "https://github.com/apple/swift-nio.git", from: "2.58.0"), .package(url: "https://github.com/apple/swift-nio-extras.git", from: "1.19.0"), .package(url: "https://github.com/apple/swift-log.git", from: "1.5.3"), - .package(url: "https://github.com/apple/swift-collections.git", from: "1.0.4"), + .package(url: "https://github.com/apple/swift-syntax.git", from: "509.0.1"), ], targets: [ // Targets are the basic building blocks of a package. A target can define a module or a test suite. @@ -33,7 +36,7 @@ let package = Package( dependencies: [ "SwiftSDKGenerator", .product(name: "ArgumentParser", package: "swift-argument-parser"), - ], + ], swiftSettings: [ .enableExperimentalFeature("StrictConcurrency=complete"), ] @@ -44,7 +47,9 @@ let package = Package( .target(name: "AsyncProcess"), .product(name: "AsyncAlgorithms", package: "swift-async-algorithms"), .product(name: "AsyncHTTPClient", package: "async-http-client"), + .product(name: "Logging", package: "swift-log"), .product(name: "SystemPackage", package: "swift-system"), + "GeneratorEngine", ], exclude: ["Dockerfiles"], swiftSettings: [ @@ -54,12 +59,47 @@ let package = Package( .testTarget( name: "SwiftSDKGeneratorTests", dependencies: [ - .target(name: "SwiftSDKGenerator"), + "SwiftSDKGenerator", ], swiftSettings: [ .enableExperimentalFeature("StrictConcurrency=complete"), ] ), + .target( + name: "GeneratorEngine", + dependencies: [ + .product(name: "AsyncHTTPClient", package: "async-http-client"), + .product(name: "Crypto", package: "swift-crypto"), + .product(name: "Logging", package: "swift-log"), + .product(name: "SystemPackage", package: "swift-system"), + "Macros", + "SystemSQLite", + ] + ), + .testTarget( + name: "GeneratorEngineTests", + dependencies: [ + "GeneratorEngine", + ] + ), + .macro( + name: "Macros", + dependencies: [ + .product(name: "SwiftCompilerPlugin", package: "swift-syntax"), + .product(name: "SwiftSyntax", package: "swift-syntax"), + .product(name: "SwiftSyntaxBuilder", package: "swift-syntax"), + .product(name: "SwiftSyntaxMacros", package: "swift-syntax"), + .product(name: "SwiftDiagnostics", package: "swift-syntax"), + ] + ), + .testTarget( + name: "MacrosTests", + dependencies: [ + "Macros", + .product(name: "SwiftSyntaxMacros", package: "swift-syntax"), + .product(name: "SwiftSyntaxMacrosTestSupport", package: "swift-syntax"), + ] + ), .systemLibrary(name: "SystemSQLite", pkgConfig: "sqlite3"), .target( name: "AsyncProcess", diff --git a/Sources/GeneratorEngine/Cache/CacheKeyProtocol.swift b/Sources/GeneratorEngine/Cache/CacheKeyProtocol.swift new file mode 100644 index 0000000..d97b004 --- /dev/null +++ b/Sources/GeneratorEngine/Cache/CacheKeyProtocol.swift @@ -0,0 +1,69 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2023 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Macros + +@_exported import protocol Crypto.HashFunction +import struct Foundation.URL +import struct SystemPackage.FilePath + +/// Indicates that values of a conforming type can be hashed with an arbitrary hashing function. Unlike `Hashable`, +/// this protocol doesn't utilize random seed values and produces consistent hash values across process launches. +public protocol CacheKeyProtocol { + func hash(with hashFunction: inout some HashFunction) +} + +extension Bool: CacheKeyProtocol { + public func hash(with hashFunction: inout some HashFunction) { + hashFunction.update(data: self ? [1] : [0]) + } +} + +extension Int: CacheKeyProtocol { + public func hash(with hashFunction: inout some HashFunction) { + withUnsafeBytes(of: self) { + hashFunction.update(bufferPointer: $0) + } + } +} + +extension String: CacheKeyProtocol { + public func hash(with hashFunction: inout some HashFunction) { + var x = self + x.withUTF8 { + hashFunction.update(bufferPointer: .init($0)) + } + } +} + +extension FilePath: CacheKeyProtocol { + public func hash(with hashFunction: inout some HashFunction) { + self.description.hash(with: &hashFunction) + } +} + +extension URL: CacheKeyProtocol { + public func hash(with hashFunction: inout some HashFunction) { + self.description.hash(with: &hashFunction) + } +} + +extension Optional: CacheKeyProtocol where Wrapped: CacheKeyProtocol { + public func hash(with hashFunction: inout some HashFunction) { + if let self { + self.hash(with: &hashFunction) + } + } +} + +@attached(extension, conformances: CacheKeyProtocol, names: named(hash(with:))) +public macro CacheKey() = #externalMacro(module: "Macros", type: "CacheKeyMacro") diff --git a/Sources/GeneratorEngine/Cache/FileCacheRecord.swift b/Sources/GeneratorEngine/Cache/FileCacheRecord.swift new file mode 100644 index 0000000..cedfd3a --- /dev/null +++ b/Sources/GeneratorEngine/Cache/FileCacheRecord.swift @@ -0,0 +1,38 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2023 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import struct SystemPackage.FilePath + +struct FileCacheRecord { + let path: FilePath + let hash: String +} + +extension FileCacheRecord: Codable { + enum CodingKeys: CodingKey { + case path + case hash + } + + // FIXME: `Codable` on `FilePath` is broken + init(from decoder: any Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + self.path = try FilePath(container.decode(String.self, forKey: .path)) + self.hash = try container.decode(String.self, forKey: .hash) + } + + func encode(to encoder: any Encoder) throws { + var container = encoder.container(keyedBy: CodingKeys.self) + try container.encode(self.path.string, forKey: .path) + try container.encode(self.hash, forKey: .hash) + } +} diff --git a/Sources/GeneratorEngine/Cache/SQLite.swift b/Sources/GeneratorEngine/Cache/SQLite.swift new file mode 100644 index 0000000..e323b29 --- /dev/null +++ b/Sources/GeneratorEngine/Cache/SQLite.swift @@ -0,0 +1,310 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2014-2023 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import struct Foundation.Data +import SystemPackage +import SystemSQLite + +extension FilePath: @unchecked Sendable {} + +/// A minimal SQLite wrapper. +public final class SQLite { + enum Error: Swift.Error, Equatable { + case databaseFull + case message(String) + } + + /// The location of the database. + let location: Location + + /// The configuration for the database. + let configuration: Configuration + + /// Pointer to the database. + let db: OpaquePointer + + /// Create or open the database at the given path. + /// + /// The database is opened in serialized mode. + init(location: Location, configuration: Configuration = Configuration()) throws { + self.location = location + self.configuration = configuration + + var handle: OpaquePointer? + try Self.checkError( + { + sqlite3_open_v2( + location.pathString, + &handle, + SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_FULLMUTEX, + nil + ) + }, + description: "Unable to open database at \(self.location)" + ) + + guard let db = handle else { + throw Error.message("Unable to open database at \(self.location)") + } + self.db = db + try Self.checkError({ sqlite3_extended_result_codes(db, 1) }, description: "Unable to configure database") + try Self.checkError( + { sqlite3_busy_timeout(db, self.configuration.busyTimeoutMilliseconds) }, + description: "Unable to configure database busy timeout" + ) + if let maxPageCount = self.configuration.maxPageCount { + try self.exec(query: "PRAGMA max_page_count=\(maxPageCount);") + } + } + + /// Prepare the given query. + func prepare(query: String) throws -> PreparedStatement { + try PreparedStatement(db: self.db, query: query) + } + + /// Directly execute the given query. + /// + /// Note: Use withCString for string arguments. + func exec(query queryString: String, args: [CVarArg] = [], _ callback: SQLiteExecCallback? = nil) throws { + let query = withVaList(args) { ptr in + sqlite3_vmprintf(queryString, ptr) + } + + let wcb = callback.map { CallbackWrapper($0) } + let callbackCtx = wcb.map { Unmanaged.passUnretained($0).toOpaque() } + + var err: UnsafeMutablePointer? + try Self.checkError { sqlite3_exec(self.db, query, sqlite_callback, callbackCtx, &err) } + + sqlite3_free(query) + + if let err { + let errorString = String(cString: err) + sqlite3_free(err) + throw Error.message(errorString) + } + } + + func close() throws { + try Self.checkError { sqlite3_close(self.db) } + } + + typealias SQLiteExecCallback = ([Column]) -> () + + struct Configuration { + var busyTimeoutMilliseconds: Int32 + var maxSizeInBytes: Int? + + // https://www.sqlite.org/pgszchng2016.html + private let defaultPageSizeInBytes = 1024 + + init() { + self.busyTimeoutMilliseconds = 5000 + self.maxSizeInBytes = .none + } + + public var maxSizeInMegabytes: Int? { + get { + self.maxSizeInBytes.map { $0 / (1024 * 1024) } + } + set { + self.maxSizeInBytes = newValue.map { $0 * 1024 * 1024 } + } + } + + public var maxPageCount: Int? { + self.maxSizeInBytes.map { $0 / self.defaultPageSizeInBytes } + } + } + + public enum Location: Sendable { + case path(FilePath) + case memory + case temporary + + var pathString: String { + switch self { + case let .path(path): + path.string + case .memory: + ":memory:" + case .temporary: + "" + } + } + } + + /// Represents an sqlite value. + enum SQLiteValue { + case null + case string(String) + case int(Int) + case blob(Data) + } + + /// Represents a row returned by called step() on a prepared statement. + struct Row { + /// The pointer to the prepared statement. + let stmt: OpaquePointer + + /// Get integer at the given column index. + func int(at index: Int32) -> Int { + Int(sqlite3_column_int64(self.stmt, index)) + } + + /// Get blob data at the given column index. + func blob(at index: Int32) -> Data { + let bytes = sqlite3_column_blob(stmt, index)! + let count = sqlite3_column_bytes(stmt, index) + return Data(bytes: bytes, count: Int(count)) + } + + /// Get string at the given column index. + func string(at index: Int32) -> String { + String(cString: sqlite3_column_text(self.stmt, index)) + } + } + + struct Column { + var name: String + var value: String + } + + /// Represents a prepared statement. + struct PreparedStatement { + typealias sqlite3_destructor_type = @convention(c) (UnsafeMutableRawPointer?) -> () + static let SQLITE_STATIC = unsafeBitCast(0, to: sqlite3_destructor_type.self) + static let SQLITE_TRANSIENT = unsafeBitCast(-1, to: sqlite3_destructor_type.self) + + /// The pointer to the prepared statement. + let stmt: OpaquePointer + + init(db: OpaquePointer, query: String) throws { + var stmt: OpaquePointer? + try SQLite.checkError { sqlite3_prepare_v2(db, query, -1, &stmt, nil) } + self.stmt = stmt! + } + + /// Evaluate the prepared statement. + @discardableResult + func step() throws -> Row? { + let result = sqlite3_step(stmt) + + switch result { + case SQLITE_DONE: + return nil + case SQLITE_ROW: + return Row(stmt: self.stmt) + default: + throw Error.message(String(cString: sqlite3_errstr(result))) + } + } + + /// Bind the given arguments to the statement. + func bind(_ arguments: [SQLiteValue]) throws { + for (idx, argument) in arguments.enumerated() { + let idx = Int32(idx) + 1 + switch argument { + case .null: + try checkError { sqlite3_bind_null(self.stmt, idx) } + case let .int(int): + try checkError { sqlite3_bind_int64(self.stmt, idx, Int64(int)) } + case let .string(str): + try checkError { sqlite3_bind_text(self.stmt, idx, str, -1, Self.SQLITE_TRANSIENT) } + case let .blob(blob): + try checkError { + blob.withUnsafeBytes { ptr in + sqlite3_bind_blob( + self.stmt, + idx, + ptr.baseAddress, + Int32(blob.count), + Self.SQLITE_TRANSIENT + ) + } + } + } + } + } + + /// Reset the prepared statement. + func reset() throws { + try SQLite.checkError { sqlite3_reset(self.stmt) } + } + + /// Clear bindings from the prepared statment. + func clearBindings() throws { + try SQLite.checkError { sqlite3_clear_bindings(self.stmt) } + } + + /// Finalize the statement and free up resources. + func finalize() throws { + try SQLite.checkError { sqlite3_finalize(self.stmt) } + } + } + + fileprivate class CallbackWrapper { + var callback: SQLiteExecCallback + init(_ callback: @escaping SQLiteExecCallback) { + self.callback = callback + } + } + + private static func checkError(_ fn: () -> Int32, description prefix: String? = .none) throws { + let result = fn() + if result != SQLITE_OK { + var description = String(cString: sqlite3_errstr(result)) + switch description.lowercased() { + case "database or disk is full": + throw Error.databaseFull + default: + if let prefix { + description = "\(prefix): \(description)" + } + throw Error.message(description) + } + } + } +} + +// Explicitly mark this class as non-Sendable +@available(*, unavailable) +extension SQLite: Sendable {} + +private func sqlite_callback( + _ ctx: UnsafeMutableRawPointer?, + _ numColumns: Int32, + _ columns: UnsafeMutablePointer?>?, + _ columnNames: UnsafeMutablePointer?>? +) -> Int32 { + guard let ctx else { return 0 } + guard let columnNames, let columns else { return 0 } + let numColumns = Int(numColumns) + var result: [SQLite.Column] = [] + + for idx in 0...fromOpaque(ctx).takeUnretainedValue() + wcb.callback(result) + + return 0 +} diff --git a/Sources/GeneratorEngine/Cache/SQLiteBackedCache.swift b/Sources/GeneratorEngine/Cache/SQLiteBackedCache.swift new file mode 100644 index 0000000..5849ac2 --- /dev/null +++ b/Sources/GeneratorEngine/Cache/SQLiteBackedCache.swift @@ -0,0 +1,253 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2021-2023 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Foundation +import Logging +import SystemPackage + +/// SQLite backed persistent cache. +final class SQLiteBackedCache { + typealias Key = String + + let tableName: String + let location: SQLite.Location + let configuration: Configuration + private let logger: Logger + + private var state = State.idle + + private let jsonEncoder: JSONEncoder + private let jsonDecoder: JSONDecoder + + /// Creates a SQLite-backed cache. + /// + /// - Parameters: + /// - tableName: The SQLite table name. Must follow SQLite naming rules (e.g., no spaces). + /// - location: SQLite.Location + /// - configuration: Optional. Configuration for the cache. + init( + tableName: String, + location: SQLite.Location, + configuration: Configuration = .init(), + _ logger: Logger + ) { + self.tableName = tableName + self.location = location + self.logger = logger + self.configuration = configuration + self.jsonEncoder = JSONEncoder() + self.jsonDecoder = JSONDecoder() + } + + deinit { + try? self.withStateLock { + if case let .connected(db) = self.state { + // TODO: we could wrap the failure here with diagnostics if it was available + assertionFailure("db should be closed") + try db.close() + } + } + } + + public func close() throws { + try self.withStateLock { + if case let .connected(db) = self.state { + try db.close() + } + self.state = .disconnected + } + } + + private func put( + key: Key, + value: some Codable, + replace: Bool = false + ) throws { + do { + let query = "INSERT OR \(replace ? "REPLACE" : "IGNORE") INTO \(self.tableName) VALUES (?, ?);" + try self.executeStatement(query) { statement in + let data = try self.jsonEncoder.encode(value) + let bindings: [SQLite.SQLiteValue] = [ + .string(key), + .blob(data), + ] + try statement.bind(bindings) + try statement.step() + } + } catch let error as SQLite.Error where error == .databaseFull { + if !self.configuration.truncateWhenFull { + throw error + } + self.logger.warning( + "truncating \(self.tableName) cache database since it reached max size of \(self.configuration.maxSizeInBytes ?? 0) bytes" + ) + try self.executeStatement("DELETE FROM \(self.tableName);") { statement in + try statement.step() + } + try self.put(key: key, value: value, replace: replace) + } catch { + throw error + } + } + + func get(_ key: String, as: Value.Type) throws -> Value? { + let query = "SELECT value FROM \(self.tableName) WHERE key = ? LIMIT 1;" + return try self.executeStatement(query) { statement -> Value? in + try statement.bind([.string(key)]) + let data = try statement.step()?.blob(at: 0) + return try data.flatMap { + try self.jsonDecoder.decode(Value.self, from: $0) + } + } + } + + func set(_ key: String, to value: some Codable) throws { + try self.put(key: key, value: value, replace: true) + } + + func remove(key: Key) throws { + let query = "DELETE FROM \(self.tableName) WHERE key = ?;" + try self.executeStatement(query) { statement in + try statement.bind([.string(key)]) + try statement.step() + } + } + + @discardableResult + private func executeStatement(_ query: String, _ body: (SQLite.PreparedStatement) throws -> T) throws -> T { + try self.withDB { db in + let result: Result + let statement = try db.prepare(query: query) + do { + result = try .success(body(statement)) + } catch { + result = .failure(error) + } + try statement.finalize() + switch result { + case let .failure(error): + throw error + case let .success(value): + return value + } + } + } + + private func withDB(_ body: (SQLite) throws -> T) throws -> T { + let createDB = { () throws -> SQLite in + let db = try SQLite(location: self.location, configuration: self.configuration.underlying) + try self.createSchemaIfNecessary(db: db) + return db + } + + let db: SQLite + let fm = FileManager.default + switch (self.location, self.state) { + case let (.path(path), .connected(database)): + if fm.fileExists(atPath: path.string) { + db = database + } else { + try database.close() + try fm.createDirectory(atPath: path.removingLastComponent().string, withIntermediateDirectories: true) + db = try createDB() + } + case let (.path(path), _): + if !fm.fileExists(atPath: path.string) { + try fm.createDirectory(atPath: path.removingLastComponent().string, withIntermediateDirectories: true) + } + db = try createDB() + case let (_, .connected(database)): + db = database + case (_, _): + db = try createDB() + } + self.state = .connected(db) + return try body(db) + } + + private func createSchemaIfNecessary(db: SQLite) throws { + let table = """ + CREATE TABLE IF NOT EXISTS \(self.tableName) ( + key STRING PRIMARY KEY NOT NULL, + value BLOB NOT NULL + ); + """ + + try db.exec(query: table) + try db.exec(query: "PRAGMA journal_mode=WAL;") + } + + private func withStateLock(_ body: () throws -> T) throws -> T { + switch self.location { + case let .path(path): + let fm = FileManager.default + if !fm.fileExists(atPath: path.string) { + try fm.createDirectory(atPath: path.removingLastComponent().string, withIntermediateDirectories: true) + } + + return try FileLock.withLock(fileToLock: path, body: body) + case .memory, .temporary: + return try body() + } + } + + private enum State { + case idle + case connected(SQLite) + case disconnected + } + + public struct Configuration { + var truncateWhenFull: Bool + + fileprivate var underlying: SQLite.Configuration + + init() { + self.underlying = .init() + self.truncateWhenFull = true + self.maxSizeInMegabytes = 100 + // see https://www.sqlite.org/c3ref/busy_timeout.html + self.busyTimeoutMilliseconds = 1000 + } + + var maxSizeInMegabytes: Int? { + get { + self.underlying.maxSizeInMegabytes + } + set { + self.underlying.maxSizeInMegabytes = newValue + } + } + + var maxSizeInBytes: Int? { + get { + self.underlying.maxSizeInBytes + } + set { + self.underlying.maxSizeInBytes = newValue + } + } + + var busyTimeoutMilliseconds: Int32 { + get { + self.underlying.busyTimeoutMilliseconds + } + set { + self.underlying.busyTimeoutMilliseconds = newValue + } + } + } +} + +// Explicitly mark this class as non-Sendable +@available(*, unavailable) +extension SQLiteBackedCache: Sendable {} diff --git a/Sources/GeneratorEngine/Engine.swift b/Sources/GeneratorEngine/Engine.swift new file mode 100644 index 0000000..37a056e --- /dev/null +++ b/Sources/GeneratorEngine/Engine.swift @@ -0,0 +1,81 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2023 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import class AsyncHTTPClient.HTTPClient +@_exported import Crypto +import struct Logging.Logger +@_exported import struct SystemPackage.FilePath + +/// Cacheable computations engine. Currently the engine makes an assumption that computations produce same results for +/// the same query values and write results to a single file path. +public actor Engine { + private(set) var cacheHits = 0 + private(set) var cacheMisses = 0 + + public let fileSystem: any FileSystem + public let httpClient: HTTPClient + public let logger: Logger + private let resultsCache: SQLiteBackedCache + + /// Creates a new instance of the ``Engine`` actor. + /// - Parameter fileSystem: Implementation of a file system this engine should use. + /// - Parameter cacheLocation: Location of cache storage used by the engine. + /// - Parameter httpClient: HTTP client to use in queries that need it. + /// - Parameter logger: Logger to use during queries execution. + public init( + _ fileSystem: any FileSystem, + _ httpClient: HTTPClient, + _ logger: Logger, + cacheLocation: SQLite.Location + ) { + self.fileSystem = fileSystem + self.httpClient = httpClient + self.logger = logger + self.resultsCache = SQLiteBackedCache(tableName: "cache_table", location: cacheLocation, logger) + } + + deinit { + try! resultsCache.close() + } + + /// Executes a given query if no cached result of it is available. Otherwise fetches the result from engine's cache. + /// - Parameter query: A query value to execute. + /// - Returns: A file path to query's result recorded in a file. + public subscript(_ query: some QueryProtocol) -> FilePath { + get async throws { + var hashFunction = SHA512() + query.hash(with: &hashFunction) + let key = hashFunction.finalize().description + + if let fileRecord = try resultsCache.get(key, as: FileCacheRecord.self) { + hashFunction = SHA512() + try await self.fileSystem.hash(fileRecord.path, with: &hashFunction) + let fileHash = hashFunction.finalize().description + + if fileHash == fileRecord.hash { + self.cacheHits += 1 + return fileRecord.path + } + } + + self.cacheMisses += 1 + let resultPath = try await query.run(engine: self) + hashFunction = SHA512() + try await self.fileSystem.hash(resultPath, with: &hashFunction) + let resultHash = hashFunction.finalize().description + + try self.resultsCache.set(key, to: FileCacheRecord(path: resultPath, hash: resultHash)) + + return resultPath + } + } +} diff --git a/Sources/GeneratorEngine/FileSystem/FileLock.swift b/Sources/GeneratorEngine/FileSystem/FileLock.swift new file mode 100644 index 0000000..b446544 --- /dev/null +++ b/Sources/GeneratorEngine/FileSystem/FileLock.swift @@ -0,0 +1,240 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2014-2023 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Foundation +import SystemPackage + +enum ProcessLockError: Error { + case unableToAquireLock(errno: Int32) +} + +extension ProcessLockError: CustomNSError { + public var errorUserInfo: [String: Any] { + [NSLocalizedDescriptionKey: "\(self)"] + } +} + +/// Provides functionality to acquire a lock on a file via POSIX's flock() method. +/// It can be used for things like serializing concurrent mutations on a shared resource +/// by multiple instances of a process. The `FileLock` is not thread-safe. +final class FileLock { + enum LockType { + case exclusive + case shared + } + + enum Error: Swift.Error { + case noEntry(FilePath) + case notDirectory(FilePath) + case errno(Int32, FilePath) + } + + /// File descriptor to the lock file. + #if os(Windows) + private var handle: HANDLE? + #else + private var fileDescriptor: CInt? + #endif + + /// Path to the lock file. + private let lockFile: FilePath + + /// Create an instance of FileLock at the path specified + /// + /// Note: The parent directory path should be a valid directory. + init(at lockFile: FilePath) { + self.lockFile = lockFile + } + + /// Try to acquire a lock. This method will block until lock the already aquired by other process. + /// + /// Note: This method can throw if underlying POSIX methods fail. + func lock(type: LockType = .exclusive) throws { + #if os(Windows) + if self.handle == nil { + let h: HANDLE = self.lockFile.pathString.withCString(encodedAs: UTF16.self) { + CreateFileW( + $0, + UInt32(GENERIC_READ) | UInt32(GENERIC_WRITE), + UInt32(FILE_SHARE_READ) | UInt32(FILE_SHARE_WRITE), + nil, + DWORD(OPEN_ALWAYS), + DWORD(FILE_ATTRIBUTE_NORMAL), + nil + ) + } + if h == INVALID_HANDLE_VALUE { + throw FileSystemError(errno: Int32(GetLastError()), self.lockFile) + } + self.handle = h + } + var overlapped = OVERLAPPED() + overlapped.Offset = 0 + overlapped.OffsetHigh = 0 + overlapped.hEvent = nil + switch type { + case .exclusive: + if !LockFileEx( + self.handle, + DWORD(LOCKFILE_EXCLUSIVE_LOCK), + 0, + UInt32.max, + UInt32.max, + &overlapped + ) { + throw ProcessLockError.unableToAquireLock(errno: Int32(GetLastError())) + } + case .shared: + if !LockFileEx( + self.handle, + 0, + 0, + UInt32.max, + UInt32.max, + &overlapped + ) { + throw ProcessLockError.unableToAquireLock(errno: Int32(GetLastError())) + } + } + #else + // Open the lock file. + if self.fileDescriptor == nil { + let fd = try FileDescriptor.open( + self.lockFile, + .writeOnly, + options: [.create, .closeOnExec], + permissions: [.groupReadWrite, .ownerReadWrite, .otherReadWrite] + ).rawValue + if fd == -1 { + throw Error.errno(errno, self.lockFile) + } + self.fileDescriptor = fd + } + // Aquire lock on the file. + while true { + if type == .exclusive && flock(self.fileDescriptor!, LOCK_EX) == 0 { + break + } else if type == .shared && flock(self.fileDescriptor!, LOCK_SH) == 0 { + break + } + // Retry if interrupted. + if errno == EINTR { continue } + throw ProcessLockError.unableToAquireLock(errno: errno) + } + #endif + } + + /// Unlock the held lock. + public func unlock() { + #if os(Windows) + var overlapped = OVERLAPPED() + overlapped.Offset = 0 + overlapped.OffsetHigh = 0 + overlapped.hEvent = nil + UnlockFileEx(self.handle, 0, UInt32.max, UInt32.max, &overlapped) + #else + guard let fd = fileDescriptor else { return } + flock(fd, LOCK_UN) + #endif + } + + deinit { + #if os(Windows) + guard let handle else { return } + CloseHandle(handle) + #else + guard let fd = fileDescriptor else { return } + close(fd) + #endif + } + + /// Execute the given block while holding the lock. + private func withLock(type: LockType = .exclusive, _ body: () throws -> T) throws -> T { + try self.lock(type: type) + defer { unlock() } + return try body() + } + + /// Execute the given block while holding the lock. + private func withLock(type: LockType = .exclusive, _ body: () async throws -> T) async throws -> T { + try self.lock(type: type) + defer { unlock() } + return try await body() + } + + private static func prepareLock( + fileToLock: FilePath, + at lockFilesDirectory: FilePath? = nil, + _ type: LockType = .exclusive + ) throws -> FileLock { + let fm = FileManager.default + + // unless specified, we use the tempDirectory to store lock files + let lockFilesDirectory = lockFilesDirectory ?? FilePath(fm.temporaryDirectory.path()) + var isDirectory: ObjCBool = false + if !fm.fileExists(atPath: lockFilesDirectory.string, isDirectory: &isDirectory) { + throw Error.noEntry(lockFilesDirectory) + } + if !isDirectory.boolValue { + throw Error.notDirectory(lockFilesDirectory) + } + // use the parent path to generate unique filename in temp + var lockFileName = + ( + FilePath(URL(string: fileToLock.removingLastComponent().string)!.resolvingSymlinksInPath().path()) + .appending(fileToLock.lastComponent!) + ) + .components.map(\.string).joined(separator: "_") + .replacingOccurrences(of: ":", with: "_") + ".lock" + #if os(Windows) + // NTFS has an ARC limit of 255 codepoints + var lockFileUTF16 = lockFileName.utf16.suffix(255) + while String(lockFileUTF16) == nil { + lockFileUTF16 = lockFileUTF16.dropFirst() + } + lockFileName = String(lockFileUTF16) ?? lockFileName + #else + // back off until it occupies at most `NAME_MAX` UTF-8 bytes but without splitting scalars + // (we might split clusters but it's not worth the effort to keep them together as long as we get a valid file name) + var lockFileUTF8 = lockFileName.utf8.suffix(Int(NAME_MAX)) + while String(lockFileUTF8) == nil { + // in practice this will only be a few iterations + lockFileUTF8 = lockFileUTF8.dropFirst() + } + // we will never end up with nil since we have ASCII characters at the end + lockFileName = String(lockFileUTF8) ?? lockFileName + #endif + let lockFilePath = lockFilesDirectory.appending(lockFileName) + + return FileLock(at: lockFilePath) + } + + static func withLock( + fileToLock: FilePath, + lockFilesDirectory: FilePath? = nil, + type: LockType = .exclusive, + body: () throws -> T + ) throws -> T { + let lock = try Self.prepareLock(fileToLock: fileToLock, at: lockFilesDirectory, type) + return try lock.withLock(type: type, body) + } + + static func withLock( + fileToLock: FilePath, + lockFilesDirectory: FilePath? = nil, + type: LockType = .exclusive, + body: () async throws -> T + ) async throws -> T { + let lock = try Self.prepareLock(fileToLock: fileToLock, at: lockFilesDirectory, type) + return try await lock.withLock(type: type, body) + } +} diff --git a/Sources/GeneratorEngine/FileSystem/FileSystem.swift b/Sources/GeneratorEngine/FileSystem/FileSystem.swift new file mode 100644 index 0000000..404d314 --- /dev/null +++ b/Sources/GeneratorEngine/FileSystem/FileSystem.swift @@ -0,0 +1,24 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2023 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import protocol Crypto.HashFunction +import struct SystemPackage.FilePath + +public protocol FileSystem: Actor { + func read(_ path: FilePath) async throws -> ReadableFileStream + func write(_ path: FilePath, _ bytes: [UInt8]) async throws + func hash(_ path: FilePath, with hashFunction: inout some HashFunction) async throws +} + +enum FileSystemError: Error { + case fileDoesNotExist(FilePath) +} diff --git a/Sources/GeneratorEngine/FileSystem/LocalFileSystem.swift b/Sources/GeneratorEngine/FileSystem/LocalFileSystem.swift new file mode 100644 index 0000000..b92bb99 --- /dev/null +++ b/Sources/GeneratorEngine/FileSystem/LocalFileSystem.swift @@ -0,0 +1,62 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2023 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import SystemPackage + +public actor LocalFileSystem: FileSystem { + public static let defaultChunkSize = 128 * 1024 + + let readChunkSize: Int + + public init(readChunkSize: Int = defaultChunkSize) { + self.readChunkSize = readChunkSize + } + + public func read(_ path: FilePath) throws -> ReadableFileStream { + try .local( + LocalReadableFileStream( + fileDescriptor: FileDescriptor.open(path, .readOnly), + readChunkSize: self.readChunkSize + ) + ) + } + + public func write(_ path: FilePath, _ bytes: [UInt8]) throws { + let fd = try FileDescriptor.open(path, .writeOnly) + + try fd.closeAfter { + _ = try fd.writeAll(bytes) + } + } + + public func hash( + _ path: FilePath, + with hashFunction: inout some HashFunction + ) throws { + let fd = try FileDescriptor.open(path, .readOnly) + + try fd.closeAfter { + var buffer = [UInt8](repeating: 0, count: readChunkSize) + var bytesRead = 0 + repeat { + bytesRead = try buffer.withUnsafeMutableBytes { + try fd.read(into: $0) + } + + if bytesRead > 0 { + hashFunction.update(data: buffer[0.. 0 + } + } +} diff --git a/Sources/GeneratorEngine/FileSystem/ReadableFileStream.swift b/Sources/GeneratorEngine/FileSystem/ReadableFileStream.swift new file mode 100644 index 0000000..88b3e48 --- /dev/null +++ b/Sources/GeneratorEngine/FileSystem/ReadableFileStream.swift @@ -0,0 +1,106 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2023 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import SystemPackage + +public enum ReadableFileStream: AsyncSequence { + public typealias Element = [UInt8] + + case local(LocalReadableFileStream) + case virtual(VirtualReadableFileStream) + + public enum Iterator: AsyncIteratorProtocol { + case local(LocalReadableFileStream.Iterator) + case virtual(VirtualReadableFileStream.Iterator) + + public func next() async throws -> [UInt8]? { + switch self { + case let .local(local): + try await local.next() + case let .virtual(virtual): + try await virtual.next() + } + } + } + + public func makeAsyncIterator() -> Iterator { + switch self { + case let .local(local): + .local(local.makeAsyncIterator()) + case let .virtual(virtual): + .virtual(virtual.makeAsyncIterator()) + } + } +} + +public struct LocalReadableFileStream: AsyncSequence { + public typealias Element = [UInt8] + + let fileDescriptor: FileDescriptor + let readChunkSize: Int + + public final class Iterator: AsyncIteratorProtocol { + init(_ fileDescriptor: FileDescriptor, readChunkSize: Int) { + self.fileDescriptor = fileDescriptor + self.readChunkSize = readChunkSize + } + + private let fileDescriptor: FileDescriptor + private let readChunkSize: Int + + public func next() async throws -> [UInt8]? { + var buffer = [UInt8](repeating: 0, count: readChunkSize) + + let bytesRead = try buffer.withUnsafeMutableBytes { + try self.fileDescriptor.read(into: $0) + } + + guard bytesRead > 0 else { + return nil + } + + return .init(buffer[0.. Iterator { + Iterator(self.fileDescriptor, readChunkSize: self.readChunkSize) + } +} + +public struct VirtualReadableFileStream: AsyncSequence { + public typealias Element = [UInt8] + + public final class Iterator: AsyncIteratorProtocol { + init(bytes: [UInt8]? = nil) { + self.bytes = bytes + } + + var bytes: [UInt8]? + + public func next() async throws -> [UInt8]? { + defer { bytes = nil } + + return self.bytes + } + } + + let bytes: [UInt8] + + public func makeAsyncIterator() -> Iterator { + Iterator(bytes: self.bytes) + } +} diff --git a/Sources/GeneratorEngine/FileSystem/VirtualFileSystem.swift b/Sources/GeneratorEngine/FileSystem/VirtualFileSystem.swift new file mode 100644 index 0000000..1d40a3d --- /dev/null +++ b/Sources/GeneratorEngine/FileSystem/VirtualFileSystem.swift @@ -0,0 +1,41 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2023 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import struct SystemPackage.FilePath + +actor VirtualFileSystem: FileSystem { + private var content: [FilePath: [UInt8]] + + init(content: [FilePath: [UInt8]] = [:]) { + self.content = content + } + + func read(_ path: FilePath) throws -> ReadableFileStream { + guard let bytes = self.content[path] else { + throw FileSystemError.fileDoesNotExist(path) + } + + return .virtual(VirtualReadableFileStream(bytes: bytes)) + } + + func write(_ path: FilePath, _ bytes: [UInt8]) throws { + self.content[path] = bytes + } + + func hash(_ path: FilePath, with hashFunction: inout some HashFunction) throws { + guard let bytes = self.content[path] else { + throw FileSystemError.fileDoesNotExist(path) + } + + hashFunction.update(data: bytes) + } +} diff --git a/Sources/GeneratorEngine/Query.swift b/Sources/GeneratorEngine/Query.swift new file mode 100644 index 0000000..79a4067 --- /dev/null +++ b/Sources/GeneratorEngine/Query.swift @@ -0,0 +1,20 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2023 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import struct SystemPackage.FilePath + +@attached(extension, conformances: QueryProtocol, CacheKeyProtocol, names: named(hash(with:))) +public macro Query() = #externalMacro(module: "Macros", type: "QueryMacro") + +public protocol QueryProtocol: CacheKeyProtocol { + func run(engine: Engine) async throws -> FilePath +} diff --git a/Sources/Macros/CacheKeyMacro.swift b/Sources/Macros/CacheKeyMacro.swift new file mode 100644 index 0000000..0c6537c --- /dev/null +++ b/Sources/Macros/CacheKeyMacro.swift @@ -0,0 +1,62 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2023 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import SwiftDiagnostics +import SwiftSyntax +import SwiftSyntaxBuilder +import SwiftSyntaxMacros + +public enum CacheKeyMacro: ExtensionMacro { + /// Unique identifier for messages related to this macro. + private static let messageID = MessageID(domain: "Macros", id: "CacheKeyMacro") + + public static func expansion( + of node: AttributeSyntax, + attachedTo declaration: some DeclGroupSyntax, + providingExtensionsOf type: some TypeSyntaxProtocol, + conformingTo protocols: [TypeSyntax], + in context: some MacroExpansionContext + ) throws -> [ExtensionDeclSyntax] { + guard let structDecl = declaration.as(StructDeclSyntax.self) else { + throw SimpleDiagnosticMessage( + message: "Macro `CacheKey` can only be applied to a struct", + diagnosticID: self.messageID, + severity: .error + ) + } + + let expressions = structDecl.memberBlock.members.map(\.decl).compactMap { declaration -> CodeBlockItemSyntax? in + guard let storedPropertyIdentifier = declaration.as( + VariableDeclSyntax.self + )?.bindings.first?.pattern.as(IdentifierPatternSyntax.self) else { + return nil + } + + return "\(storedPropertyIdentifier.identifier).hash(with: &hashFunction)" + } + + let inheritanceClause = InheritanceClauseSyntax(inheritedTypesBuilder: { + InheritedTypeSyntax(type: IdentifierTypeSyntax(name: "CacheKeyProtocol")) + }) + + let cacheKeyExtension = try ExtensionDeclSyntax(extendedType: type, inheritanceClause: inheritanceClause) { + try FunctionDeclSyntax("func hash(with hashFunction: inout some HashFunction)") { + "String(reflecting: Self.self).hash(with: &hashFunction)" + for expression in expressions { + expression + } + } + } + + return [cacheKeyExtension] + } +} diff --git a/Sources/Macros/Macros.swift b/Sources/Macros/Macros.swift new file mode 100644 index 0000000..231534e --- /dev/null +++ b/Sources/Macros/Macros.swift @@ -0,0 +1,19 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2023 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import SwiftCompilerPlugin +import SwiftSyntaxMacros + +@main +struct Macros: CompilerPlugin { + var providingMacros: [Macro.Type] = [QueryMacro.self, CacheKeyMacro.self] +} diff --git a/Sources/Macros/QueryMacro.swift b/Sources/Macros/QueryMacro.swift new file mode 100644 index 0000000..9cc0dcf --- /dev/null +++ b/Sources/Macros/QueryMacro.swift @@ -0,0 +1,43 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2023 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import SwiftDiagnostics +import SwiftSyntax +import SwiftSyntaxBuilder +import SwiftSyntaxMacros + +public enum QueryMacro: ExtensionMacro { + /// Unique identifier for messages related to this macro. + private static let messageID = MessageID(domain: "Macros", id: "QueryMacro") + + public static func expansion( + of node: AttributeSyntax, + attachedTo declaration: some DeclGroupSyntax, + providingExtensionsOf type: some TypeSyntaxProtocol, + conformingTo protocols: [TypeSyntax], + in context: some MacroExpansionContext + ) throws -> [ExtensionDeclSyntax] { + let inheritanceClause = InheritanceClauseSyntax(inheritedTypesBuilder: { + InheritedTypeSyntax(type: IdentifierTypeSyntax(name: "QueryProtocol")) + }) + + let queryExtension = ExtensionDeclSyntax(extendedType: type, inheritanceClause: inheritanceClause) {} + + return try [queryExtension] + CacheKeyMacro.expansion( + of: node, + attachedTo: declaration, + providingExtensionsOf: type, + conformingTo: protocols, + in: context + ) + } +} diff --git a/Sources/Macros/SimpleDiagnosticMessage.swift b/Sources/Macros/SimpleDiagnosticMessage.swift new file mode 100644 index 0000000..e12983d --- /dev/null +++ b/Sources/Macros/SimpleDiagnosticMessage.swift @@ -0,0 +1,23 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2023 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import SwiftDiagnostics + +struct SimpleDiagnosticMessage: DiagnosticMessage, Error { + let message: String + let diagnosticID: MessageID + let severity: DiagnosticSeverity +} + +extension SimpleDiagnosticMessage: FixItMessage { + var fixItID: MessageID { self.diagnosticID } +} diff --git a/Tests/GeneratorEngineTests/EngineTests.swift b/Tests/GeneratorEngineTests/EngineTests.swift new file mode 100644 index 0000000..73db77f --- /dev/null +++ b/Tests/GeneratorEngineTests/EngineTests.swift @@ -0,0 +1,141 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2023 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import protocol Crypto.HashFunction +@testable import GeneratorEngine +import struct SystemPackage.FilePath +import XCTest + +private let encoder = JSONEncoder() +private let decoder = JSONDecoder() + +private extension FileSystem { + func read(_ path: FilePath, as: V.Type) async throws -> V { + let fileStream = try await self.read(path) + var bytes = [UInt8]() + for try await chunk in fileStream { + bytes += chunk + } + + return try decoder.decode(V.self, from: .init(bytes)) + } + + func write(_ path: FilePath, _ value: some Encodable) async throws { + let data = try encoder.encode(value) + try await self.write(path, .init(data)) + } +} + +@Query +struct Const { + let x: Int + + func run(engine: Engine) async throws -> FilePath { + let resultPath = FilePath("/Const-\(x)") + try await engine.fileSystem.write(resultPath, self.x) + return resultPath + } +} + +@Query +struct MultiplyByTwo { + let x: Int + + func run(engine: Engine) async throws -> FilePath { + let constPath = try await engine[Const(x: self.x)] + + let constResult = try await engine.fileSystem.read(constPath, as: Int.self) + + let resultPath = FilePath("/MultiplyByTwo-\(constResult)") + try await engine.fileSystem.write(resultPath, constResult * 2) + return resultPath + } +} + +@Query +struct AddThirty { + let x: Int + + func run(engine: Engine) async throws -> FilePath { + let constPath = try await engine[Const(x: self.x)] + let constResult = try await engine.fileSystem.read(constPath, as: Int.self) + + let resultPath = FilePath("/AddThirty-\(constResult)") + try await engine.fileSystem.write(resultPath, constResult + 30) + return resultPath + } +} + +@Query +struct Expression { + let x: Int + let y: Int + + func run(engine: Engine) async throws -> FilePath { + let multiplyPath = try await engine[MultiplyByTwo(x: self.x)] + let addThirtyPath = try await engine[AddThirty(x: self.y)] + + let multiplyResult = try await engine.fileSystem.read(multiplyPath, as: Int.self) + let addThirtyResult = try await engine.fileSystem.read(addThirtyPath, as: Int.self) + + let resultPath = FilePath("/Expression-\(multiplyResult)-\(addThirtyResult)") + try await engine.fileSystem.write(resultPath, multiplyResult + addThirtyResult) + return resultPath + } +} + +final class EngineTests: XCTestCase { + func testSimpleCaching() async throws { + let engine = Engine(VirtualFileSystem(), cacheLocation: .memory) + + var resultPath = try await engine[Expression(x: 1, y: 2)] + var result = try await engine.fileSystem.read(resultPath, as: Int.self) + + XCTAssertEqual(result, 34) + + var cacheMisses = await engine.cacheMisses + XCTAssertEqual(cacheMisses, 5) + + var cacheHits = await engine.cacheHits + XCTAssertEqual(cacheHits, 0) + + resultPath = try await engine[Expression(x: 1, y: 2)] + result = try await engine.fileSystem.read(resultPath, as: Int.self) + XCTAssertEqual(result, 34) + + cacheMisses = await engine.cacheMisses + XCTAssertEqual(cacheMisses, 5) + + cacheHits = await engine.cacheHits + XCTAssertEqual(cacheHits, 1) + + resultPath = try await engine[Expression(x: 2, y: 1)] + result = try await engine.fileSystem.read(resultPath, as: Int.self) + XCTAssertEqual(result, 35) + + cacheMisses = await engine.cacheMisses + XCTAssertEqual(cacheMisses, 8) + + cacheHits = await engine.cacheHits + XCTAssertEqual(cacheHits, 3) + + resultPath = try await engine[Expression(x: 2, y: 1)] + result = try await engine.fileSystem.read(resultPath, as: Int.self) + XCTAssertEqual(result, 35) + + cacheMisses = await engine.cacheMisses + XCTAssertEqual(cacheMisses, 8) + + cacheHits = await engine.cacheHits + XCTAssertEqual(cacheHits, 4) + } +} diff --git a/Tests/MacrosTests/MacrosTests.swift b/Tests/MacrosTests/MacrosTests.swift new file mode 100644 index 0000000..f9a0b46 --- /dev/null +++ b/Tests/MacrosTests/MacrosTests.swift @@ -0,0 +1,69 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2023 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Macros +import SwiftSyntaxMacros +import SwiftSyntaxMacrosTestSupport +import XCTest + +final class MacrosTests: XCTestCase { + private let macros: [String: Macro.Type] = ["CacheKey": CacheKeyMacro.self, "Query": QueryMacro.self] + + func testCacheKeyDerived() { + assertMacroExpansion( + """ + @CacheKey + struct Message { + let text: String + let sender: String + } + + @Query + struct Q { + let number: Int + let text: String + } + """, + expandedSource: """ + struct Message { + let text: String + let sender: String + } + struct Q { + let number: Int + let text: String + } + + extension Message: CacheKeyProtocol { + func hash(with hashFunction: inout some HashFunction) { + String(reflecting: Self.self).hash(with: &hashFunction) + text.hash(with: &hashFunction) + sender.hash(with: &hashFunction) + } + } + + extension Q: QueryProtocol { + } + + extension Q: CacheKeyProtocol { + func hash(with hashFunction: inout some HashFunction) { + String(reflecting: Self.self).hash(with: &hashFunction) + number.hash(with: &hashFunction) + text.hash(with: &hashFunction) + } + } + """, + macros: self.macros, + indentationWidth: .spaces(2) + ) + } +} From 73c1f4e4f390ef1473f7121cf5f6f40ac15967b3 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Thu, 26 Oct 2023 00:36:24 +0200 Subject: [PATCH 03/15] Fix EngineTests after API updates --- Tests/GeneratorEngineTests/EngineTests.swift | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/Tests/GeneratorEngineTests/EngineTests.swift b/Tests/GeneratorEngineTests/EngineTests.swift index 73db77f..a68c637 100644 --- a/Tests/GeneratorEngineTests/EngineTests.swift +++ b/Tests/GeneratorEngineTests/EngineTests.swift @@ -10,8 +10,9 @@ // //===----------------------------------------------------------------------===// -import protocol Crypto.HashFunction +import class AsyncHTTPClient.HTTPClient @testable import GeneratorEngine +import struct Logging.Logger import struct SystemPackage.FilePath import XCTest @@ -95,7 +96,17 @@ struct Expression { final class EngineTests: XCTestCase { func testSimpleCaching() async throws { - let engine = Engine(VirtualFileSystem(), cacheLocation: .memory) + let httpClient = HTTPClient() + let engine = Engine( + VirtualFileSystem(), + httpClient, + Logger(label: "engine-tests"), + cacheLocation: .memory + ) + + defer { + try! httpClient.syncShutdown() + } var resultPath = try await engine[Expression(x: 1, y: 2)] var result = try await engine.fileSystem.read(resultPath, as: Int.self) From c525ec928c749239476e5589ea5e45e98777a64b Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Thu, 26 Oct 2023 16:42:52 +0200 Subject: [PATCH 04/15] Fix duplicate `SystemSQLite` target --- Package.swift | 1 - 1 file changed, 1 deletion(-) diff --git a/Package.swift b/Package.swift index 1c0baab..bdbe54c 100644 --- a/Package.swift +++ b/Package.swift @@ -65,7 +65,6 @@ let package = Package( .enableExperimentalFeature("StrictConcurrency=complete"), ] ), - .systemLibrary(name: "SystemSQLite", pkgConfig: "sqlite3"), .target( name: "GeneratorEngine", dependencies: [ From 09201eadde6be45cada6d7a52aede2735330fab0 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Thu, 26 Oct 2023 16:57:10 +0200 Subject: [PATCH 05/15] Add `libsqlite-dev` dependency to `Dockerfile` --- Docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Docker/Dockerfile b/Docker/Dockerfile index daba577..c682f9d 100644 --- a/Docker/Dockerfile +++ b/Docker/Dockerfile @@ -7,7 +7,7 @@ ARG swift_version ARG ubuntu_version # set as UTF-8 -RUN apt-get update && apt-get install -y locales locales-all +RUN apt-get update && apt-get install -y locales locales-all libsqlite-dev ENV LC_ALL en_US.UTF-8 ENV LANG en_US.UTF-8 ENV LANGUAGE en_US.UTF-8 From 80a417395be70bd9a611b902af2edff6166561fc Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Thu, 26 Oct 2023 16:57:38 +0200 Subject: [PATCH 06/15] Bump default version to 5.9 in `Docker/Dockerfile` --- Docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Docker/Dockerfile b/Docker/Dockerfile index c682f9d..41d2c29 100644 --- a/Docker/Dockerfile +++ b/Docker/Dockerfile @@ -1,4 +1,4 @@ -ARG swift_version=5.7 +ARG swift_version=5.9 ARG ubuntu_version=jammy ARG base_image=swift:$swift_version-$ubuntu_version FROM $base_image From b5304e8c71209742ad515d10757bb498cc2d3d35 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Thu, 26 Oct 2023 17:35:55 +0200 Subject: [PATCH 07/15] Use `libsqlite3-dev` instead of `libsqlite-dev` in `Dockerfile` --- Docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Docker/Dockerfile b/Docker/Dockerfile index 41d2c29..ba606a4 100644 --- a/Docker/Dockerfile +++ b/Docker/Dockerfile @@ -7,7 +7,7 @@ ARG swift_version ARG ubuntu_version # set as UTF-8 -RUN apt-get update && apt-get install -y locales locales-all libsqlite-dev +RUN apt-get update && apt-get install -y locales locales-all libsqlite3-dev ENV LC_ALL en_US.UTF-8 ENV LANG en_US.UTF-8 ENV LANGUAGE en_US.UTF-8 From c96bda68c721d398bfd8a968fe6fc11c39e82b13 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Thu, 26 Oct 2023 17:39:24 +0200 Subject: [PATCH 08/15] Fix incompatibility with swift-corelibs-foundation --- Sources/GeneratorEngine/FileSystem/FileLock.swift | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Sources/GeneratorEngine/FileSystem/FileLock.swift b/Sources/GeneratorEngine/FileSystem/FileLock.swift index b446544..0652ebf 100644 --- a/Sources/GeneratorEngine/FileSystem/FileLock.swift +++ b/Sources/GeneratorEngine/FileSystem/FileLock.swift @@ -179,7 +179,7 @@ final class FileLock { let fm = FileManager.default // unless specified, we use the tempDirectory to store lock files - let lockFilesDirectory = lockFilesDirectory ?? FilePath(fm.temporaryDirectory.path()) + let lockFilesDirectory = lockFilesDirectory ?? FilePath(fm.temporaryDirectory.path) var isDirectory: ObjCBool = false if !fm.fileExists(atPath: lockFilesDirectory.string, isDirectory: &isDirectory) { throw Error.noEntry(lockFilesDirectory) @@ -190,7 +190,7 @@ final class FileLock { // use the parent path to generate unique filename in temp var lockFileName = ( - FilePath(URL(string: fileToLock.removingLastComponent().string)!.resolvingSymlinksInPath().path()) + FilePath(URL(string: fileToLock.removingLastComponent().string)!.resolvingSymlinksInPath().path) .appending(fileToLock.lastComponent!) ) .components.map(\.string).joined(separator: "_") From 2d677b5eb296f93ab0f2aee0d0514d14b7200852 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Mon, 30 Oct 2023 14:52:00 +0000 Subject: [PATCH 09/15] Address PR feedback --- .../Cache/CacheKeyProtocol.swift | 14 ++++- Sources/GeneratorEngine/Engine.swift | 32 +++++++--- .../FileSystem/FileSystem.swift | 6 +- .../FileSystem/LocalFileSystem.swift | 59 ++++++++----------- .../FileSystem/OpenReadableFile.swift | 53 +++++++++++++++++ .../FileSystem/OpenWritableFile.swift | 31 ++++++++++ .../FileSystem/ReadableFileStream.swift | 7 +-- .../FileSystem/VirtualFileSystem.swift | 30 ++++++---- Tests/GeneratorEngineTests/EngineTests.swift | 31 ++++++---- 9 files changed, 187 insertions(+), 76 deletions(-) create mode 100644 Sources/GeneratorEngine/FileSystem/OpenReadableFile.swift create mode 100644 Sources/GeneratorEngine/FileSystem/OpenWritableFile.swift diff --git a/Sources/GeneratorEngine/Cache/CacheKeyProtocol.swift b/Sources/GeneratorEngine/Cache/CacheKeyProtocol.swift index d97b004..7879f96 100644 --- a/Sources/GeneratorEngine/Cache/CacheKeyProtocol.swift +++ b/Sources/GeneratorEngine/Cache/CacheKeyProtocol.swift @@ -24,12 +24,14 @@ public protocol CacheKeyProtocol { extension Bool: CacheKeyProtocol { public func hash(with hashFunction: inout some HashFunction) { + "Swift.Bool".hash(with: &hashFunction) hashFunction.update(data: self ? [1] : [0]) } } extension Int: CacheKeyProtocol { public func hash(with hashFunction: inout some HashFunction) { + "Swift.Int".hash(with: &hashFunction) withUnsafeBytes(of: self) { hashFunction.update(bufferPointer: $0) } @@ -38,6 +40,10 @@ extension Int: CacheKeyProtocol { extension String: CacheKeyProtocol { public func hash(with hashFunction: inout some HashFunction) { + var t = "Swift.String" + t.withUTF8 { + hashFunction.update(bufferPointer: .init($0)) + } var x = self x.withUTF8 { hashFunction.update(bufferPointer: .init($0)) @@ -47,20 +53,26 @@ extension String: CacheKeyProtocol { extension FilePath: CacheKeyProtocol { public func hash(with hashFunction: inout some HashFunction) { - self.description.hash(with: &hashFunction) + "SystemPackage.FilePath".hash(with: &hashFunction) + self.string.hash(with: &hashFunction) } } extension URL: CacheKeyProtocol { public func hash(with hashFunction: inout some HashFunction) { + "Foundation.URL".hash(with: &hashFunction) self.description.hash(with: &hashFunction) } } extension Optional: CacheKeyProtocol where Wrapped: CacheKeyProtocol { public func hash(with hashFunction: inout some HashFunction) { + "Swift.Optional".hash(with: &hashFunction) if let self { + true.hash(with: &hashFunction) self.hash(with: &hashFunction) + } else { + false.hash(with: &hashFunction) } } } diff --git a/Sources/GeneratorEngine/Engine.swift b/Sources/GeneratorEngine/Engine.swift index 37a056e..1b5bde8 100644 --- a/Sources/GeneratorEngine/Engine.swift +++ b/Sources/GeneratorEngine/Engine.swift @@ -22,29 +22,42 @@ public actor Engine { private(set) var cacheMisses = 0 public let fileSystem: any FileSystem - public let httpClient: HTTPClient + public let httpClient = HTTPClient() public let logger: Logger private let resultsCache: SQLiteBackedCache + private var isShutDown = false - /// Creates a new instance of the ``Engine`` actor. + /// Creates a new instance of the ``Engine`` actor. Requires an explicit call to ``Engine//shutdown`` before the + /// instance is deinitialized. The recommended approach to resource management is to place + /// `defer { engine.shutDown }` on the line that follows this initializer call. /// - Parameter fileSystem: Implementation of a file system this engine should use. /// - Parameter cacheLocation: Location of cache storage used by the engine. /// - Parameter httpClient: HTTP client to use in queries that need it. /// - Parameter logger: Logger to use during queries execution. public init( _ fileSystem: any FileSystem, - _ httpClient: HTTPClient, _ logger: Logger, cacheLocation: SQLite.Location ) { self.fileSystem = fileSystem - self.httpClient = httpClient self.logger = logger self.resultsCache = SQLiteBackedCache(tableName: "cache_table", location: cacheLocation, logger) } + public func shutDown() async throws { + precondition(!self.isShutDown, "`Engine/shutDown` should be called only once") + try self.resultsCache.close() + try await self.httpClient.shutdown() + + self.isShutDown = true + } + deinit { - try! resultsCache.close() + let isShutDown = self.isShutDown + precondition( + isShutDown, + "`Engine/shutDown` should be called explicitly on instances of `Engine` before deinitialization" + ) } /// Executes a given query if no cached result of it is available. Otherwise fetches the result from engine's cache. @@ -58,7 +71,9 @@ public actor Engine { if let fileRecord = try resultsCache.get(key, as: FileCacheRecord.self) { hashFunction = SHA512() - try await self.fileSystem.hash(fileRecord.path, with: &hashFunction) + try await self.fileSystem.withOpenReadableFile(fileRecord.path) { + try await $0.hash(with: &hashFunction) + } let fileHash = hashFunction.finalize().description if fileHash == fileRecord.hash { @@ -70,7 +85,10 @@ public actor Engine { self.cacheMisses += 1 let resultPath = try await query.run(engine: self) hashFunction = SHA512() - try await self.fileSystem.hash(resultPath, with: &hashFunction) + + try await self.fileSystem.withOpenReadableFile(resultPath) { + try await $0.hash(with: &hashFunction) + } let resultHash = hashFunction.finalize().description try self.resultsCache.set(key, to: FileCacheRecord(path: resultPath, hash: resultHash)) diff --git a/Sources/GeneratorEngine/FileSystem/FileSystem.swift b/Sources/GeneratorEngine/FileSystem/FileSystem.swift index 404d314..3e5d298 100644 --- a/Sources/GeneratorEngine/FileSystem/FileSystem.swift +++ b/Sources/GeneratorEngine/FileSystem/FileSystem.swift @@ -14,11 +14,11 @@ import protocol Crypto.HashFunction import struct SystemPackage.FilePath public protocol FileSystem: Actor { - func read(_ path: FilePath) async throws -> ReadableFileStream - func write(_ path: FilePath, _ bytes: [UInt8]) async throws - func hash(_ path: FilePath, with hashFunction: inout some HashFunction) async throws + func withOpenReadableFile(_ path: FilePath, _ body: (OpenReadableFile) async throws -> T) async throws -> T + func withOpenWritableFile(_ path: FilePath, _ body: (OpenWritableFile) async throws -> T) async throws -> T } enum FileSystemError: Error { case fileDoesNotExist(FilePath) + case bufferLimitExceeded(FilePath) } diff --git a/Sources/GeneratorEngine/FileSystem/LocalFileSystem.swift b/Sources/GeneratorEngine/FileSystem/LocalFileSystem.swift index b92bb99..d54692e 100644 --- a/Sources/GeneratorEngine/FileSystem/LocalFileSystem.swift +++ b/Sources/GeneratorEngine/FileSystem/LocalFileSystem.swift @@ -13,7 +13,7 @@ import SystemPackage public actor LocalFileSystem: FileSystem { - public static let defaultChunkSize = 128 * 1024 + public static let defaultChunkSize = 512 * 1024 let readChunkSize: Int @@ -21,42 +21,33 @@ public actor LocalFileSystem: FileSystem { self.readChunkSize = readChunkSize } - public func read(_ path: FilePath) throws -> ReadableFileStream { - try .local( - LocalReadableFileStream( - fileDescriptor: FileDescriptor.open(path, .readOnly), - readChunkSize: self.readChunkSize - ) - ) - } - - public func write(_ path: FilePath, _ bytes: [UInt8]) throws { - let fd = try FileDescriptor.open(path, .writeOnly) - - try fd.closeAfter { - _ = try fd.writeAll(bytes) - } - } - - public func hash( + public func withOpenReadableFile( _ path: FilePath, - with hashFunction: inout some HashFunction - ) throws { + _ body: (OpenReadableFile) async throws -> T + ) async throws -> T { let fd = try FileDescriptor.open(path, .readOnly) + do { + let result = try await body(.init(readChunkSize: readChunkSize, fileHandle: .local(fd))) + try fd.close() + return result + } catch { + try fd.close() + throw error + } + } - try fd.closeAfter { - var buffer = [UInt8](repeating: 0, count: readChunkSize) - var bytesRead = 0 - repeat { - bytesRead = try buffer.withUnsafeMutableBytes { - try fd.read(into: $0) - } - - if bytesRead > 0 { - hashFunction.update(data: buffer[0.. 0 + public func withOpenWritableFile( + _ path: SystemPackage.FilePath, + _ body: (OpenWritableFile) async throws -> T + ) async throws -> T { + let fd = try FileDescriptor.open(path, .writeOnly) + do { + let result = try await body(.init(fileHandle: .local(fd))) + try fd.close() + return result + } catch { + try fd.close() + throw error } } } diff --git a/Sources/GeneratorEngine/FileSystem/OpenReadableFile.swift b/Sources/GeneratorEngine/FileSystem/OpenReadableFile.swift new file mode 100644 index 0000000..31d1d58 --- /dev/null +++ b/Sources/GeneratorEngine/FileSystem/OpenReadableFile.swift @@ -0,0 +1,53 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2023 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import struct SystemPackage.FileDescriptor + +public struct OpenReadableFile { + let readChunkSize: Int + + enum FileHandle { + case local(FileDescriptor) + case virtual([UInt8]) + } + + let fileHandle: FileHandle + + func read() async throws -> ReadableFileStream { + switch self.fileHandle { + case let .local(fileDescriptor): + ReadableFileStream.local(.init(fileDescriptor: fileDescriptor, readChunkSize: self.readChunkSize)) + case let .virtual(array): + ReadableFileStream.virtual(.init(bytes: array)) + } + } + + func hash(with hashFunction: inout some HashFunction) async throws { + switch self.fileHandle { + case let .local(fileDescriptor): + var buffer = [UInt8](repeating: 0, count: readChunkSize) + var bytesRead = 0 + repeat { + bytesRead = try buffer.withUnsafeMutableBytes { + try fileDescriptor.read(into: $0) + } + + if bytesRead > 0 { + hashFunction.update(data: buffer[0.. 0 + case let .virtual(array): + hashFunction.update(data: array) + } + } +} diff --git a/Sources/GeneratorEngine/FileSystem/OpenWritableFile.swift b/Sources/GeneratorEngine/FileSystem/OpenWritableFile.swift new file mode 100644 index 0000000..0131baa --- /dev/null +++ b/Sources/GeneratorEngine/FileSystem/OpenWritableFile.swift @@ -0,0 +1,31 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2023 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import struct SystemPackage.FileDescriptor + +public struct OpenWritableFile { + enum FileHandle { + case local(FileDescriptor) + case virtual(VirtualFileSystem.Storage, FilePath) + } + + let fileHandle: FileHandle + + func write(_ bytes: some Sequence) async throws { + switch self.fileHandle { + case let .local(fileDescriptor): + _ = try fileDescriptor.writeAll(bytes) + case let .virtual(storage, path): + storage.content[path, default: []].append(contentsOf: bytes) + } + } +} diff --git a/Sources/GeneratorEngine/FileSystem/ReadableFileStream.swift b/Sources/GeneratorEngine/FileSystem/ReadableFileStream.swift index 88b3e48..31ef14e 100644 --- a/Sources/GeneratorEngine/FileSystem/ReadableFileStream.swift +++ b/Sources/GeneratorEngine/FileSystem/ReadableFileStream.swift @@ -68,11 +68,8 @@ public struct LocalReadableFileStream: AsyncSequence { return nil } - return .init(buffer[0.. ReadableFileStream { - guard let bytes = self.content[path] else { - throw FileSystemError.fileDoesNotExist(path) + final class Storage { + init(_ content: [FilePath: [UInt8]]) { + self.content = content } - return .virtual(VirtualReadableFileStream(bytes: bytes)) + var content: [FilePath: [UInt8]] } - func write(_ path: FilePath, _ bytes: [UInt8]) throws { - self.content[path] = bytes + private let storage: Storage + + init(content: [FilePath: [UInt8]] = [:], readChunkSize: Int = defaultChunkSize) { + self.storage = .init(content) + self.readChunkSize = readChunkSize } - func hash(_ path: FilePath, with hashFunction: inout some HashFunction) throws { - guard let bytes = self.content[path] else { + func withOpenReadableFile(_ path: FilePath, _ body: (OpenReadableFile) async throws -> T) async throws -> T { + guard let bytes = storage.content[path] else { throw FileSystemError.fileDoesNotExist(path) } + return try await body(.init(readChunkSize: self.readChunkSize, fileHandle: .virtual(bytes))) + } - hashFunction.update(data: bytes) + func withOpenWritableFile(_ path: FilePath, _ body: (OpenWritableFile) async throws -> T) async throws -> T { + try await body(.init(fileHandle: .virtual(self.storage, path))) } } diff --git a/Tests/GeneratorEngineTests/EngineTests.swift b/Tests/GeneratorEngineTests/EngineTests.swift index a68c637..66ca93f 100644 --- a/Tests/GeneratorEngineTests/EngineTests.swift +++ b/Tests/GeneratorEngineTests/EngineTests.swift @@ -11,6 +11,7 @@ //===----------------------------------------------------------------------===// import class AsyncHTTPClient.HTTPClient +import struct Foundation.Data @testable import GeneratorEngine import struct Logging.Logger import struct SystemPackage.FilePath @@ -20,19 +21,27 @@ private let encoder = JSONEncoder() private let decoder = JSONDecoder() private extension FileSystem { - func read(_ path: FilePath, as: V.Type) async throws -> V { - let fileStream = try await self.read(path) - var bytes = [UInt8]() - for try await chunk in fileStream { - bytes += chunk + func read(_ path: FilePath, bufferLimit: Int = 10 * 1024 * 1024, as: V.Type) async throws -> V { + let data = try await self.withOpenReadableFile(path) { + var data = Data() + for try await chunk in try await $0.read() { + data.append(contentsOf: chunk) + + guard data.count < bufferLimit else { + throw FileSystemError.bufferLimitExceeded(path) + } + } + return data } - return try decoder.decode(V.self, from: .init(bytes)) + return try decoder.decode(V.self, from: data) } func write(_ path: FilePath, _ value: some Encodable) async throws { let data = try encoder.encode(value) - try await self.write(path, .init(data)) + try await self.withOpenWritableFile(path) { fileHandle in + try await fileHandle.write(data) + } } } @@ -96,18 +105,12 @@ struct Expression { final class EngineTests: XCTestCase { func testSimpleCaching() async throws { - let httpClient = HTTPClient() let engine = Engine( VirtualFileSystem(), - httpClient, Logger(label: "engine-tests"), cacheLocation: .memory ) - defer { - try! httpClient.syncShutdown() - } - var resultPath = try await engine[Expression(x: 1, y: 2)] var result = try await engine.fileSystem.read(resultPath, as: Int.self) @@ -148,5 +151,7 @@ final class EngineTests: XCTestCase { cacheHits = await engine.cacheHits XCTAssertEqual(cacheHits, 4) + + try await engine.shutDown() } } From 5ccecd4cb5fada3e0926b92ca28f234acd544f67 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Mon, 30 Oct 2023 15:14:31 +0000 Subject: [PATCH 10/15] Avoid passing keys as strings in most places --- Sources/GeneratorEngine/Cache/SQLite.swift | 2 +- Sources/GeneratorEngine/Cache/SQLiteBackedCache.swift | 10 +++++----- Sources/GeneratorEngine/Engine.swift | 7 ++++--- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/Sources/GeneratorEngine/Cache/SQLite.swift b/Sources/GeneratorEngine/Cache/SQLite.swift index e323b29..d7cfeec 100644 --- a/Sources/GeneratorEngine/Cache/SQLite.swift +++ b/Sources/GeneratorEngine/Cache/SQLite.swift @@ -241,7 +241,7 @@ public final class SQLite { try SQLite.checkError { sqlite3_reset(self.stmt) } } - /// Clear bindings from the prepared statment. + /// Clear bindings from the prepared statement. func clearBindings() throws { try SQLite.checkError { sqlite3_clear_bindings(self.stmt) } } diff --git a/Sources/GeneratorEngine/Cache/SQLiteBackedCache.swift b/Sources/GeneratorEngine/Cache/SQLiteBackedCache.swift index 5849ac2..50a53c1 100644 --- a/Sources/GeneratorEngine/Cache/SQLiteBackedCache.swift +++ b/Sources/GeneratorEngine/Cache/SQLiteBackedCache.swift @@ -68,7 +68,7 @@ final class SQLiteBackedCache { } private func put( - key: Key, + key: some Sequence, value: some Codable, replace: Bool = false ) throws { @@ -77,7 +77,7 @@ final class SQLiteBackedCache { try self.executeStatement(query) { statement in let data = try self.jsonEncoder.encode(value) let bindings: [SQLite.SQLiteValue] = [ - .string(key), + .blob(Data(key)), .blob(data), ] try statement.bind(bindings) @@ -99,10 +99,10 @@ final class SQLiteBackedCache { } } - func get(_ key: String, as: Value.Type) throws -> Value? { + func get(_ key: some Sequence, as: Value.Type) throws -> Value? { let query = "SELECT value FROM \(self.tableName) WHERE key = ? LIMIT 1;" return try self.executeStatement(query) { statement -> Value? in - try statement.bind([.string(key)]) + try statement.bind([.blob(Data(key))]) let data = try statement.step()?.blob(at: 0) return try data.flatMap { try self.jsonDecoder.decode(Value.self, from: $0) @@ -110,7 +110,7 @@ final class SQLiteBackedCache { } } - func set(_ key: String, to value: some Codable) throws { + func set(_ key: some Sequence, to value: some Codable) throws { try self.put(key: key, value: value, replace: true) } diff --git a/Sources/GeneratorEngine/Engine.swift b/Sources/GeneratorEngine/Engine.swift index 1b5bde8..56954b7 100644 --- a/Sources/GeneratorEngine/Engine.swift +++ b/Sources/GeneratorEngine/Engine.swift @@ -67,7 +67,7 @@ public actor Engine { get async throws { var hashFunction = SHA512() query.hash(with: &hashFunction) - let key = hashFunction.finalize().description + let key = hashFunction.finalize() if let fileRecord = try resultsCache.get(key, as: FileCacheRecord.self) { hashFunction = SHA512() @@ -89,9 +89,10 @@ public actor Engine { try await self.fileSystem.withOpenReadableFile(resultPath) { try await $0.hash(with: &hashFunction) } - let resultHash = hashFunction.finalize().description + let resultHash = hashFunction.finalize() - try self.resultsCache.set(key, to: FileCacheRecord(path: resultPath, hash: resultHash)) + // FIXME: update `SQLiteBackedCache` to store `resultHash` directly instead of relying on string conversions + try self.resultsCache.set(key, to: FileCacheRecord(path: resultPath, hash: resultHash.description)) return resultPath } From e477ac509122233f18fa66f1dd270540dbffd268 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Mon, 30 Oct 2023 18:53:09 +0000 Subject: [PATCH 11/15] Cache `DownloadableArtifacts` with `GenratorEngine` --- Sources/GeneratorCLI/GeneratorCLI.swift | 10 ++- .../Cache/FileCacheRecord.swift | 8 +-- Sources/GeneratorEngine/Engine.swift | 10 +-- .../Artifacts/DownloadableArtifacts.swift | 21 +++--- .../SwiftSDKGenerator+Download.swift | 71 +++++++++---------- .../Generator/SwiftSDKGenerator.swift | 29 +++++++- .../SystemUtils/HTTPClient+Download.swift | 11 ++- Tests/GeneratorEngineTests/EngineTests.swift | 16 ++--- 8 files changed, 108 insertions(+), 68 deletions(-) diff --git a/Sources/GeneratorCLI/GeneratorCLI.swift b/Sources/GeneratorCLI/GeneratorCLI.swift index 552b78c..402ef76 100644 --- a/Sources/GeneratorCLI/GeneratorCLI.swift +++ b/Sources/GeneratorCLI/GeneratorCLI.swift @@ -88,7 +88,7 @@ struct GeneratorCLI: AsyncParsableCommand { let linuxDistribution = try LinuxDistribution(name: linuxDistributionName, version: linuxDistributionVersion) let elapsed = try await ContinuousClock().measure { - try await SwiftSDKGenerator( + let generator = try await SwiftSDKGenerator( hostCPUArchitecture: self.hostArch, targetCPUArchitecture: self.targetArch, swiftVersion: self.swiftVersion, @@ -98,7 +98,13 @@ struct GeneratorCLI: AsyncParsableCommand { shouldUseDocker: self.withDocker, isVerbose: self.verbose ) - .generateBundle(shouldGenerateFromScratch: !self.incremental) + do { + try await generator.generateBundle(shouldGenerateFromScratch: !self.incremental) + try await generator.shutDown() + } catch { + try await generator.shutDown() + throw error + } } print("\nTime taken for this generator run: \(elapsed.intervalString).") diff --git a/Sources/GeneratorEngine/Cache/FileCacheRecord.swift b/Sources/GeneratorEngine/Cache/FileCacheRecord.swift index cedfd3a..6e0492b 100644 --- a/Sources/GeneratorEngine/Cache/FileCacheRecord.swift +++ b/Sources/GeneratorEngine/Cache/FileCacheRecord.swift @@ -12,7 +12,7 @@ import struct SystemPackage.FilePath -struct FileCacheRecord { +public struct FileCacheRecord: Sendable { let path: FilePath let hash: String } @@ -23,14 +23,14 @@ extension FileCacheRecord: Codable { case hash } - // FIXME: `Codable` on `FilePath` is broken - init(from decoder: any Decoder) throws { + // FIXME: `Codable` on `FilePath` is broken, thus all `Codable` types with `FilePath` properties need a custom impl. + public init(from decoder: any Decoder) throws { let container = try decoder.container(keyedBy: CodingKeys.self) self.path = try FilePath(container.decode(String.self, forKey: .path)) self.hash = try container.decode(String.self, forKey: .hash) } - func encode(to encoder: any Encoder) throws { + public func encode(to encoder: any Encoder) throws { var container = encoder.container(keyedBy: CodingKeys.self) try container.encode(self.path.string, forKey: .path) try container.encode(self.hash, forKey: .hash) diff --git a/Sources/GeneratorEngine/Engine.swift b/Sources/GeneratorEngine/Engine.swift index 56954b7..549d2f5 100644 --- a/Sources/GeneratorEngine/Engine.swift +++ b/Sources/GeneratorEngine/Engine.swift @@ -32,7 +32,6 @@ public actor Engine { /// `defer { engine.shutDown }` on the line that follows this initializer call. /// - Parameter fileSystem: Implementation of a file system this engine should use. /// - Parameter cacheLocation: Location of cache storage used by the engine. - /// - Parameter httpClient: HTTP client to use in queries that need it. /// - Parameter logger: Logger to use during queries execution. public init( _ fileSystem: any FileSystem, @@ -63,7 +62,7 @@ public actor Engine { /// Executes a given query if no cached result of it is available. Otherwise fetches the result from engine's cache. /// - Parameter query: A query value to execute. /// - Returns: A file path to query's result recorded in a file. - public subscript(_ query: some QueryProtocol) -> FilePath { + public subscript(_ query: some QueryProtocol) -> FileCacheRecord { get async throws { var hashFunction = SHA512() query.hash(with: &hashFunction) @@ -78,7 +77,7 @@ public actor Engine { if fileHash == fileRecord.hash { self.cacheHits += 1 - return fileRecord.path + return fileRecord } } @@ -90,11 +89,12 @@ public actor Engine { try await $0.hash(with: &hashFunction) } let resultHash = hashFunction.finalize() + let result = FileCacheRecord(path: resultPath, hash: resultHash.description) // FIXME: update `SQLiteBackedCache` to store `resultHash` directly instead of relying on string conversions - try self.resultsCache.set(key, to: FileCacheRecord(path: resultPath, hash: resultHash.description)) + try self.resultsCache.set(key, to: result) - return resultPath + return result } } } diff --git a/Sources/SwiftSDKGenerator/Artifacts/DownloadableArtifacts.swift b/Sources/SwiftSDKGenerator/Artifacts/DownloadableArtifacts.swift index b88e10a..e0cd3b3 100644 --- a/Sources/SwiftSDKGenerator/Artifacts/DownloadableArtifacts.swift +++ b/Sources/SwiftSDKGenerator/Artifacts/DownloadableArtifacts.swift @@ -11,6 +11,7 @@ //===----------------------------------------------------------------------===// import struct Foundation.URL +import GeneratorEngine import struct SystemPackage.FilePath /// Information about the OS for which the artifact is built, if it's downloaded as prebuilt. @@ -100,8 +101,9 @@ private let knownLLVMSourcesVersions: [String: String] = [ "16.0.5": "37f540124b9cfd4680666e649f557077f9937c9178489cea285a672e714b2863", ] -public struct DownloadableArtifacts: Sendable { - public struct Item: Sendable { +struct DownloadableArtifacts: Sendable { + @CacheKey + struct Item: Sendable { let remoteURL: URL var localPath: FilePath let checksum: String? @@ -112,7 +114,14 @@ public struct DownloadableArtifacts: Sendable { private(set) var hostLLVM: Item let targetSwift: Item - let allItems: [Item] + private let shouldUseDocker: Bool + var allItems: [Item] { + if self.shouldUseDocker { + [self.hostSwift, self.hostLLVM] + } else { + [self.hostSwift, self.hostLLVM, self.targetSwift] + } + } private let versions: VersionsConfiguration private let paths: PathsConfiguration @@ -165,11 +174,7 @@ public struct DownloadableArtifacts: Sendable { isPrebuilt: true ) - self.allItems = if shouldUseDocker { - [self.hostSwift, self.hostLLVM] - } else { - [self.hostSwift, self.hostLLVM, self.targetSwift] - } + self.shouldUseDocker = shouldUseDocker } mutating func useLLVMSources() { diff --git a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Download.swift b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Download.swift index e69907f..84524e9 100644 --- a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Download.swift +++ b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Download.swift @@ -12,16 +12,36 @@ import AsyncAlgorithms import AsyncHTTPClient +import GeneratorEngine import RegexBuilder import class Foundation.ByteCountFormatter import struct Foundation.URL +import struct SystemPackage.FilePath + private let ubuntuAMD64Mirror = "http://gb.archive.ubuntu.com/ubuntu" private let ubuntuARM64Mirror = "http://ports.ubuntu.com/ubuntu-ports" private let byteCountFormatter = ByteCountFormatter() +@Query +struct DownloadQuery { + let artifact: DownloadableArtifacts.Item + + func run(engine: Engine) async throws -> FilePath { + print("Downloading remote artifact not available in local cache: \(self.artifact.remoteURL)") + let stream = await engine.httpClient.streamDownloadProgress(for: self.artifact) + .removeDuplicates(by: didProgressChangeSignificantly) + .throttle(for: .seconds(1)) + + for try await item in stream { + report(progress: item.progress, for: item.artifact) + } + return self.artifact.localPath + } +} + extension SwiftSDKGenerator { func downloadArtifacts(_ client: HTTPClient) async throws { logGenerationStep("Downloading required toolchain packages...") @@ -35,42 +55,19 @@ extension SwiftSDKGenerator { downloadableArtifacts.useLLVMSources() } - let hostSwiftProgressStream = client.streamDownloadProgress(for: downloadableArtifacts.hostSwift) - .removeDuplicates(by: didProgressChangeSignificantly) - let hostLLVMProgressStream = client.streamDownloadProgress(for: downloadableArtifacts.hostLLVM) - .removeDuplicates(by: didProgressChangeSignificantly) - - print("Using these URLs for downloads:") - - for artifact in downloadableArtifacts.allItems { - print(artifact.remoteURL) - } - - // FIXME: some code duplication is necessary due to https://github.com/apple/swift-async-algorithms/issues/226 - if shouldUseDocker { - let stream = combineLatest(hostSwiftProgressStream, hostLLVMProgressStream) - .throttle(for: .seconds(1)) - - for try await (swiftProgress, llvmProgress) in stream { - report(progress: swiftProgress, for: downloadableArtifacts.hostSwift) - report(progress: llvmProgress, for: downloadableArtifacts.hostLLVM) + let results = try await withThrowingTaskGroup(of: FileCacheRecord.self) { group in + for item in self.downloadableArtifacts.allItems { + print(item.remoteURL) + group.addTask { + try await self.engine[DownloadQuery(artifact: item)] + } } - } else { - let targetSwiftProgressStream = client.streamDownloadProgress(for: downloadableArtifacts.targetSwift) - .removeDuplicates(by: didProgressChangeSignificantly) - - let stream = combineLatest( - hostSwiftProgressStream, - hostLLVMProgressStream, - targetSwiftProgressStream - ) - .throttle(for: .seconds(1)) - for try await (hostSwiftProgress, hostLLVMProgress, targetSwiftProgress) in stream { - report(progress: hostSwiftProgress, for: downloadableArtifacts.hostSwift) - report(progress: hostLLVMProgress, for: downloadableArtifacts.hostLLVM) - report(progress: targetSwiftProgress, for: downloadableArtifacts.targetSwift) + var result = [FileCacheRecord]() + for try await file in group { + result.append(file) } + return result } } @@ -224,14 +221,14 @@ extension HTTPClient { /// larger than 1MiB. Returns `false` otherwise. @Sendable private func didProgressChangeSignificantly( - previous: FileDownloadDelegate.Progress, - current: FileDownloadDelegate.Progress + previous: ArtifactDownloadProgress, + current: ArtifactDownloadProgress ) -> Bool { - guard previous.totalBytes == current.totalBytes else { + guard previous.progress.totalBytes == current.progress.totalBytes else { return true } - return current.receivedBytes - previous.receivedBytes > 1024 * 1024 * 1024 + return current.progress.receivedBytes - previous.progress.receivedBytes > 1024 * 1024 * 1024 } private func report(progress: FileDownloadDelegate.Progress, for artifact: DownloadableArtifacts.Item) { diff --git a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator.swift b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator.swift index a0bfdbb..0872855 100644 --- a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator.swift +++ b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator.swift @@ -11,9 +11,11 @@ //===----------------------------------------------------------------------===// import Foundation +import GeneratorEngine +import Logging import SystemPackage -/// Implementation of ``SwiftSDKGenerator`` for the local file system. +/// Top-level actor that sequences all of the required SDK generation steps. public actor SwiftSDKGenerator { let hostTriple: Triple let targetTriple: Triple @@ -24,6 +26,9 @@ public actor SwiftSDKGenerator { let shouldUseDocker: Bool let isVerbose: Bool + let engine: Engine + private var isShutDown = false + public init( hostCPUArchitecture: Triple.CPU?, targetCPUArchitecture: Triple.CPU?, @@ -83,6 +88,28 @@ public actor SwiftSDKGenerator { ) self.shouldUseDocker = shouldUseDocker self.isVerbose = isVerbose + + let engineCachePath = self.pathsConfiguration.artifactsCachePath.appending("cache.db") + self.engine = .init( + LocalFileSystem(), + Logger(label: "org.swift.swift-sdk-generator"), + cacheLocation: .path(engineCachePath) + ) + } + + public func shutDown() async throws { + precondition(!self.isShutDown, "`SwiftSDKGenerator/shutDown` should be called only once") + try await self.engine.shutDown() + + self.isShutDown = true + } + + deinit { + let isShutDown = self.isShutDown + precondition( + isShutDown, + "`Engine/shutDown` should be called explicitly on instances of `Engine` before deinitialization" + ) } private let fileManager = FileManager.default diff --git a/Sources/SwiftSDKGenerator/SystemUtils/HTTPClient+Download.swift b/Sources/SwiftSDKGenerator/SystemUtils/HTTPClient+Download.swift index 5ade8dd..1aa30b1 100644 --- a/Sources/SwiftSDKGenerator/SystemUtils/HTTPClient+Download.swift +++ b/Sources/SwiftSDKGenerator/SystemUtils/HTTPClient+Download.swift @@ -18,6 +18,11 @@ extension FileDownloadDelegate.Progress: @unchecked Sendable {} extension FilePath: @unchecked Sendable {} +struct ArtifactDownloadProgress { + let artifact: DownloadableArtifacts.Item + let progress: FileDownloadDelegate.Progress +} + extension HTTPClient { func downloadFile( from url: URL, @@ -54,7 +59,7 @@ extension HTTPClient { func streamDownloadProgress( for artifact: DownloadableArtifacts.Item - ) -> AsyncThrowingStream { + ) -> AsyncThrowingStream { .init { continuation in do { let delegate = try FileDownloadDelegate( @@ -66,7 +71,7 @@ extension HTTPClient { } }, reportProgress: { - continuation.yield($0) + continuation.yield(ArtifactDownloadProgress(artifact: artifact, progress: $0)) } ) let request = try HTTPClient.Request(url: artifact.remoteURL) @@ -76,7 +81,7 @@ extension HTTPClient { case let .failure(error): continuation.finish(throwing: error) case let .success(finalProgress): - continuation.yield(finalProgress) + continuation.yield(ArtifactDownloadProgress(artifact: artifact, progress: finalProgress)) continuation.finish() } } diff --git a/Tests/GeneratorEngineTests/EngineTests.swift b/Tests/GeneratorEngineTests/EngineTests.swift index 66ca93f..fdeb7e5 100644 --- a/Tests/GeneratorEngineTests/EngineTests.swift +++ b/Tests/GeneratorEngineTests/EngineTests.swift @@ -61,7 +61,7 @@ struct MultiplyByTwo { let x: Int func run(engine: Engine) async throws -> FilePath { - let constPath = try await engine[Const(x: self.x)] + let constPath = try await engine[Const(x: self.x)].path let constResult = try await engine.fileSystem.read(constPath, as: Int.self) @@ -76,7 +76,7 @@ struct AddThirty { let x: Int func run(engine: Engine) async throws -> FilePath { - let constPath = try await engine[Const(x: self.x)] + let constPath = try await engine[Const(x: self.x)].path let constResult = try await engine.fileSystem.read(constPath, as: Int.self) let resultPath = FilePath("/AddThirty-\(constResult)") @@ -91,8 +91,8 @@ struct Expression { let y: Int func run(engine: Engine) async throws -> FilePath { - let multiplyPath = try await engine[MultiplyByTwo(x: self.x)] - let addThirtyPath = try await engine[AddThirty(x: self.y)] + let multiplyPath = try await engine[MultiplyByTwo(x: self.x)].path + let addThirtyPath = try await engine[AddThirty(x: self.y)].path let multiplyResult = try await engine.fileSystem.read(multiplyPath, as: Int.self) let addThirtyResult = try await engine.fileSystem.read(addThirtyPath, as: Int.self) @@ -111,7 +111,7 @@ final class EngineTests: XCTestCase { cacheLocation: .memory ) - var resultPath = try await engine[Expression(x: 1, y: 2)] + var resultPath = try await engine[Expression(x: 1, y: 2)].path var result = try await engine.fileSystem.read(resultPath, as: Int.self) XCTAssertEqual(result, 34) @@ -122,7 +122,7 @@ final class EngineTests: XCTestCase { var cacheHits = await engine.cacheHits XCTAssertEqual(cacheHits, 0) - resultPath = try await engine[Expression(x: 1, y: 2)] + resultPath = try await engine[Expression(x: 1, y: 2)].path result = try await engine.fileSystem.read(resultPath, as: Int.self) XCTAssertEqual(result, 34) @@ -132,7 +132,7 @@ final class EngineTests: XCTestCase { cacheHits = await engine.cacheHits XCTAssertEqual(cacheHits, 1) - resultPath = try await engine[Expression(x: 2, y: 1)] + resultPath = try await engine[Expression(x: 2, y: 1)].path result = try await engine.fileSystem.read(resultPath, as: Int.self) XCTAssertEqual(result, 35) @@ -142,7 +142,7 @@ final class EngineTests: XCTestCase { cacheHits = await engine.cacheHits XCTAssertEqual(cacheHits, 3) - resultPath = try await engine[Expression(x: 2, y: 1)] + resultPath = try await engine[Expression(x: 2, y: 1)].path result = try await engine.fileSystem.read(resultPath, as: Int.self) XCTAssertEqual(result, 35) From 6ca73f11a648d4c0faad6a05d514648cc60ca04e Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Mon, 30 Oct 2023 18:58:23 +0000 Subject: [PATCH 12/15] Remove redundant checkum validation --- .../Artifacts/DownloadableArtifacts.swift | 60 ------------------- .../SwiftSDKGenerator+Entrypoint.swift | 37 +----------- .../Generator/SwiftSDKGenerator.swift | 18 ------ 3 files changed, 3 insertions(+), 112 deletions(-) diff --git a/Sources/SwiftSDKGenerator/Artifacts/DownloadableArtifacts.swift b/Sources/SwiftSDKGenerator/Artifacts/DownloadableArtifacts.swift index e0cd3b3..5608c0b 100644 --- a/Sources/SwiftSDKGenerator/Artifacts/DownloadableArtifacts.swift +++ b/Sources/SwiftSDKGenerator/Artifacts/DownloadableArtifacts.swift @@ -46,67 +46,11 @@ enum ArtifactOS: Hashable { typealias CPUMapping = [Triple.CPU: String] -/// SHA256 hashes of binary LLVM artifacts known to the generator. -private let knownLLVMBinariesVersions: [ArtifactOS: [String: CPUMapping]] = [ - .macOS: [ - "15.0.7": [ - Triple.CPU.arm64: "867c6afd41158c132ef05a8f1ddaecf476a26b91c85def8e124414f9a9ba188d", - ], - "16.0.0": [ - Triple.CPU.arm64: "2041587b90626a4a87f0de14a5842c14c6c3374f42c8ed12726ef017416409d9", - ], - "16.0.1": [ - Triple.CPU.arm64: "cb487fa991f047dc79ae36430cbb9ef14621c1262075373955b1d97215c75879", - ], - "16.0.4": [ - Triple.CPU.arm64: "429b8061d620108fee636313df55a0602ea0d14458c6d3873989e6b130a074bd", - ], - "16.0.5": [ - Triple.CPU.arm64: "1aed0787417dd915f0101503ce1d2719c8820a2c92d4a517bfc4044f72035bcc", - ], - ], -] - -/// SHA256 hashes of binary Swift artifacts known to the generator. -private let knownSwiftBinariesVersions: [ArtifactOS: [String: CPUMapping]] = [ - .linux(.ubuntu(.jammy)): [ - "5.7.3-RELEASE": [ - .arm64: "75003d5a995292ae3f858b767fbb89bc3edee99488f4574468a0e44341aec55b", - ], - "5.8-RELEASE": [ - .arm64: "12ea2df36f9af0aefa74f0989009683600978f62223e7dd73b627c90c7fe9273", - ], - "5.9-RELEASE": [ - .arm64: "30b289e02f7e03c380744ea97fdf0e96985dff504b0f09de23e098fdaf6513f3", - .x86_64: "bca015e9d727ca39385d7e5b5399f46302d54a02218d40d1c3063662ffc6b42f", - ], - ], - .macOS: [ - "5.7.3-RELEASE": [ - .arm64: "ba3516845eb8f4469a8bb06a273687f05791187324a3843996af32a73a2a687d", - .x86_64: "ba3516845eb8f4469a8bb06a273687f05791187324a3843996af32a73a2a687d", - ], - "5.8-RELEASE": [ - .arm64: "9b6cc56993652ca222c86a2d6b7b66abbd50bb92cc526efc2b23d47d40002097", - .x86_64: "9b6cc56993652ca222c86a2d6b7b66abbd50bb92cc526efc2b23d47d40002097", - ], - "5.9-RELEASE": [ - .arm64: "3cf7a4b2f3efcfcb4fef42b6588a7b1c54f7b0f2d0a479f41c3e1620b045f48e", - .x86_64: "3cf7a4b2f3efcfcb4fef42b6588a7b1c54f7b0f2d0a479f41c3e1620b045f48e", - ], - ], -] - -private let knownLLVMSourcesVersions: [String: String] = [ - "16.0.5": "37f540124b9cfd4680666e649f557077f9937c9178489cea285a672e714b2863", -] - struct DownloadableArtifacts: Sendable { @CacheKey struct Item: Sendable { let remoteURL: URL var localPath: FilePath - let checksum: String? let isPrebuilt: Bool } @@ -145,7 +89,6 @@ struct DownloadableArtifacts: Sendable { ), localPath: paths.artifactsCachePath .appending("host_swift_\(versions.swiftVersion)_\(hostTriple).pkg"), - checksum: knownSwiftBinariesVersions[hostArtifactsOS]?[versions.swiftVersion]?[hostTriple.cpu], isPrebuilt: true ) @@ -161,7 +104,6 @@ struct DownloadableArtifacts: Sendable { )!, localPath: paths.artifactsCachePath .appending("host_llvm_\(versions.lldVersion)_\(hostTriple).tar.xz"), - checksum: knownLLVMBinariesVersions[hostArtifactsOS]?[versions.lldVersion]?[hostTriple.cpu], isPrebuilt: true ) @@ -170,7 +112,6 @@ struct DownloadableArtifacts: Sendable { remoteURL: versions.swiftDownloadURL(), localPath: paths.artifactsCachePath .appending("target_swift_\(versions.swiftVersion)_\(targetTriple).tar.gz"), - checksum: knownSwiftBinariesVersions[targetArtifactsOS]?[versions.swiftVersion]?[targetTriple.cpu], isPrebuilt: true ) @@ -190,7 +131,6 @@ struct DownloadableArtifacts: Sendable { )!, localPath: self.paths.artifactsCachePath .appending("llvm_\(self.versions.lldVersion).src.tar.xz"), - checksum: knownLLVMSourcesVersions[self.versions.lldVersion], isPrebuilt: false ) } diff --git a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Entrypoint.swift b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Entrypoint.swift index e65ecda..b885a63 100644 --- a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Entrypoint.swift +++ b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Entrypoint.swift @@ -26,8 +26,8 @@ public extension Triple.CPU { } } -extension SwiftSDKGenerator { - public func generateBundle(shouldGenerateFromScratch: Bool) async throws { +public extension SwiftSDKGenerator { + func generateBundle(shouldGenerateFromScratch: Bool) async throws { var configuration = HTTPClient.Configuration(redirectConfiguration: .follow(max: 5, allowCycles: false)) // Workaround an issue with github.com returning 400 instead of 404 status to HEAD requests from AHC. configuration.httpVersion = .http1Only @@ -49,9 +49,7 @@ extension SwiftSDKGenerator { try createDirectoryIfNeeded(at: pathsConfiguration.sdkDirPath) try createDirectoryIfNeeded(at: pathsConfiguration.toolchainDirPath) - if try await !self.isCacheValid { - try await self.downloadArtifacts(client) - } + try await self.downloadArtifacts(client) if !shouldUseDocker { guard case let .ubuntu(version) = versionsConfiguration.linuxDistribution else { @@ -102,35 +100,6 @@ extension SwiftSDKGenerator { """ ) } - - /// Check whether cached downloads for required `DownloadArtifacts.Item` values can be reused instead of downloading - /// them each time the generator is running. - /// - Returns: `true` if artifacts are valid, `false` otherwise. - private var isCacheValid: Bool { - get async throws { - logGenerationStep("Checking packages cache...") - - guard downloadableArtifacts.allItems.map(\.localPath).allSatisfy(doesFileExist(at:)) else { - return false - } - - return try await withThrowingTaskGroup(of: Bool.self) { taskGroup in - for artifact in downloadableArtifacts.allItems { - taskGroup.addTask { - try await Self.isChecksumValid(artifact: artifact, isVerbose: self.isVerbose) - } - } - - for try await isValid in taskGroup { - guard isValid else { - return false - } - } - - return true - } - } - } } func logGenerationStep(_ message: String) { diff --git a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator.swift b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator.swift index 0872855..48d7768 100644 --- a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator.swift +++ b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator.swift @@ -136,24 +136,6 @@ public actor SwiftSDKGenerator { #endif } - static func isChecksumValid(artifact: DownloadableArtifacts.Item, isVerbose: Bool) async throws -> Bool { - guard let expectedChecksum = artifact.checksum else { return false } - - let computedChecksum = try await String( - Shell.readStdout("openssl dgst -sha256 \(artifact.localPath)", shouldLogCommands: isVerbose) - .split(separator: "= ")[1] - // drop the trailing newline - .dropLast() - ) - - guard computedChecksum == expectedChecksum else { - print("SHA256 digest of file at `\(artifact.localPath)` does not match expected value: \(expectedChecksum)") - return false - } - - return true - } - private func buildDockerImage(name: String, dockerfileDirectory: FilePath) async throws { try await Shell.run( "\(Self.dockerCommand) build . -t \(name)", From acd4973026e458e17cdd56b93a9ddae3307484b4 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Tue, 31 Oct 2023 17:23:10 +0000 Subject: [PATCH 13/15] Cache `lld` binaries to avoid redundant rebuilds This proceeds with converting more of the generation steps to a query-based architecture. Now `lld` doesn't need to be rebuilt if its source directory and build options don't change. Also renamed `DownloadQuery` to `DownloadArtifactQuery` and moved it to a separate file. --- .../Cache/CacheKeyProtocol.swift | 28 ++++++-- .../Cache/FileCacheRecord.swift | 4 +- Sources/GeneratorEngine/Cache/SQLite.swift | 1 + .../Artifacts/DownloadableArtifacts.swift | 1 - .../Generator/SwiftSDKGenerator+Build.swift | 24 ------- .../SwiftSDKGenerator+Download.swift | 64 ++--------------- .../SwiftSDKGenerator+Entrypoint.swift | 2 +- .../Generator/SwiftSDKGenerator+Unpack.swift | 53 ++++++++------ .../Queries/CMakeBuildQuery.swift | 35 ++++++++++ .../Queries/DownloadArtifactQuery.swift | 69 +++++++++++++++++++ 10 files changed, 168 insertions(+), 113 deletions(-) delete mode 100644 Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Build.swift create mode 100644 Sources/SwiftSDKGenerator/Queries/CMakeBuildQuery.swift create mode 100644 Sources/SwiftSDKGenerator/Queries/DownloadArtifactQuery.swift diff --git a/Sources/GeneratorEngine/Cache/CacheKeyProtocol.swift b/Sources/GeneratorEngine/Cache/CacheKeyProtocol.swift index 7879f96..ff3cd9c 100644 --- a/Sources/GeneratorEngine/Cache/CacheKeyProtocol.swift +++ b/Sources/GeneratorEngine/Cache/CacheKeyProtocol.swift @@ -24,14 +24,14 @@ public protocol CacheKeyProtocol { extension Bool: CacheKeyProtocol { public func hash(with hashFunction: inout some HashFunction) { - "Swift.Bool".hash(with: &hashFunction) + String(reflecting: Self.self).hash(with: &hashFunction) hashFunction.update(data: self ? [1] : [0]) } } extension Int: CacheKeyProtocol { public func hash(with hashFunction: inout some HashFunction) { - "Swift.Int".hash(with: &hashFunction) + String(reflecting: Self.self).hash(with: &hashFunction) withUnsafeBytes(of: self) { hashFunction.update(bufferPointer: $0) } @@ -40,7 +40,7 @@ extension Int: CacheKeyProtocol { extension String: CacheKeyProtocol { public func hash(with hashFunction: inout some HashFunction) { - var t = "Swift.String" + var t = String(reflecting: Self.self) t.withUTF8 { hashFunction.update(bufferPointer: .init($0)) } @@ -53,21 +53,28 @@ extension String: CacheKeyProtocol { extension FilePath: CacheKeyProtocol { public func hash(with hashFunction: inout some HashFunction) { - "SystemPackage.FilePath".hash(with: &hashFunction) + String(reflecting: Self.self).hash(with: &hashFunction) + self.string.hash(with: &hashFunction) + } +} + +extension FilePath.Component: CacheKeyProtocol { + public func hash(with hashFunction: inout some HashFunction) { + String(reflecting: Self.self).hash(with: &hashFunction) self.string.hash(with: &hashFunction) } } extension URL: CacheKeyProtocol { public func hash(with hashFunction: inout some HashFunction) { - "Foundation.URL".hash(with: &hashFunction) + String(reflecting: Self.self).hash(with: &hashFunction) self.description.hash(with: &hashFunction) } } extension Optional: CacheKeyProtocol where Wrapped: CacheKeyProtocol { public func hash(with hashFunction: inout some HashFunction) { - "Swift.Optional".hash(with: &hashFunction) + String(reflecting: Self.self).hash(with: &hashFunction) if let self { true.hash(with: &hashFunction) self.hash(with: &hashFunction) @@ -77,5 +84,14 @@ extension Optional: CacheKeyProtocol where Wrapped: CacheKeyProtocol { } } +extension Array: CacheKeyProtocol where Element: CacheKeyProtocol { + public func hash(with hashFunction: inout some HashFunction) { + String(reflecting: Self.self).hash(with: &hashFunction) + for element in self { + element.hash(with: &hashFunction) + } + } +} + @attached(extension, conformances: CacheKeyProtocol, names: named(hash(with:))) public macro CacheKey() = #externalMacro(module: "Macros", type: "CacheKeyMacro") diff --git a/Sources/GeneratorEngine/Cache/FileCacheRecord.swift b/Sources/GeneratorEngine/Cache/FileCacheRecord.swift index 6e0492b..fdef551 100644 --- a/Sources/GeneratorEngine/Cache/FileCacheRecord.swift +++ b/Sources/GeneratorEngine/Cache/FileCacheRecord.swift @@ -13,8 +13,8 @@ import struct SystemPackage.FilePath public struct FileCacheRecord: Sendable { - let path: FilePath - let hash: String + public let path: FilePath + public let hash: String } extension FileCacheRecord: Codable { diff --git a/Sources/GeneratorEngine/Cache/SQLite.swift b/Sources/GeneratorEngine/Cache/SQLite.swift index d7cfeec..eecf17b 100644 --- a/Sources/GeneratorEngine/Cache/SQLite.swift +++ b/Sources/GeneratorEngine/Cache/SQLite.swift @@ -15,6 +15,7 @@ import SystemPackage import SystemSQLite extension FilePath: @unchecked Sendable {} +extension FilePath.Component: @unchecked Sendable {} /// A minimal SQLite wrapper. public final class SQLite { diff --git a/Sources/SwiftSDKGenerator/Artifacts/DownloadableArtifacts.swift b/Sources/SwiftSDKGenerator/Artifacts/DownloadableArtifacts.swift index 5608c0b..4749e68 100644 --- a/Sources/SwiftSDKGenerator/Artifacts/DownloadableArtifacts.swift +++ b/Sources/SwiftSDKGenerator/Artifacts/DownloadableArtifacts.swift @@ -107,7 +107,6 @@ struct DownloadableArtifacts: Sendable { isPrebuilt: true ) - let targetArtifactsOS = ArtifactOS(targetTriple.os, versions) self.targetSwift = .init( remoteURL: versions.swiftDownloadURL(), localPath: paths.artifactsCachePath diff --git a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Build.swift b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Build.swift deleted file mode 100644 index f3f9a3f..0000000 --- a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Build.swift +++ /dev/null @@ -1,24 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the Swift open source project -// -// Copyright (c) 2022-2023 Apple Inc. and the Swift project authors -// Licensed under Apache License v2.0 with Runtime Library Exception -// -// See https://swift.org/LICENSE.txt for license information -// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors -// -//===----------------------------------------------------------------------===// - -import SystemPackage - -extension SwiftSDKGenerator { - func buildLLD(llvmSourcesDirectory: FilePath) async throws -> FilePath { - let buildDirectory = try await self.buildCMakeProject( - llvmSourcesDirectory, - options: "-DLLVM_ENABLE_PROJECTS=lld -DLLVM_TARGETS_TO_BUILD=\(self.targetTriple.cpu.llvmTargetConventionName)" - ) - - return buildDirectory.appending("bin").appending("lld") - } -} diff --git a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Download.swift b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Download.swift index 84524e9..f051bd0 100644 --- a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Download.swift +++ b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Download.swift @@ -23,24 +23,7 @@ import struct SystemPackage.FilePath private let ubuntuAMD64Mirror = "http://gb.archive.ubuntu.com/ubuntu" private let ubuntuARM64Mirror = "http://ports.ubuntu.com/ubuntu-ports" -private let byteCountFormatter = ByteCountFormatter() - -@Query -struct DownloadQuery { - let artifact: DownloadableArtifacts.Item - - func run(engine: Engine) async throws -> FilePath { - print("Downloading remote artifact not available in local cache: \(self.artifact.remoteURL)") - let stream = await engine.httpClient.streamDownloadProgress(for: self.artifact) - .removeDuplicates(by: didProgressChangeSignificantly) - .throttle(for: .seconds(1)) - - for try await item in stream { - report(progress: item.progress, for: item.artifact) - } - return self.artifact.localPath - } -} +let byteCountFormatter = ByteCountFormatter() extension SwiftSDKGenerator { func downloadArtifacts(_ client: HTTPClient) async throws { @@ -57,9 +40,8 @@ extension SwiftSDKGenerator { let results = try await withThrowingTaskGroup(of: FileCacheRecord.self) { group in for item in self.downloadableArtifacts.allItems { - print(item.remoteURL) group.addTask { - try await self.engine[DownloadQuery(artifact: item)] + try await self.engine[DownloadArtifactQuery(artifact: item)] } } @@ -69,6 +51,11 @@ extension SwiftSDKGenerator { } return result } + + print("Using downloaded artifacts in these locations:") + for path in results.map(\.path) { + print(path) + } } func downloadUbuntuPackages(_ client: HTTPClient, requiredPackages: [String]) async throws { @@ -211,40 +198,3 @@ extension HTTPClient { return result } } - -/// Checks whether two given progress value are different enough from each other. Used for filtering out progress -/// values in async streams with `removeDuplicates` operator. -/// - Parameters: -/// - previous: Preceding progress value in the stream. -/// - current: Currently processed progress value in the stream. -/// - Returns: `true` if `totalBytes` value is different by any amount or if `receivedBytes` is different by amount -/// larger than 1MiB. Returns `false` otherwise. -@Sendable -private func didProgressChangeSignificantly( - previous: ArtifactDownloadProgress, - current: ArtifactDownloadProgress -) -> Bool { - guard previous.progress.totalBytes == current.progress.totalBytes else { - return true - } - - return current.progress.receivedBytes - previous.progress.receivedBytes > 1024 * 1024 * 1024 -} - -private func report(progress: FileDownloadDelegate.Progress, for artifact: DownloadableArtifacts.Item) { - if let total = progress.totalBytes { - print(""" - \(artifact.remoteURL.lastPathComponent) \( - byteCountFormatter - .string(fromByteCount: Int64(progress.receivedBytes)) - )/\( - byteCountFormatter - .string(fromByteCount: Int64(total)) - ) - """) - } else { - print( - "\(artifact.remoteURL.lastPathComponent) \(byteCountFormatter.string(fromByteCount: Int64(progress.receivedBytes)))" - ) - } -} diff --git a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Entrypoint.swift b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Entrypoint.swift index b885a63..e20f7cf 100644 --- a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Entrypoint.swift +++ b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Entrypoint.swift @@ -67,7 +67,7 @@ public extension SwiftSDKGenerator { try await self.unpackTargetSwiftPackage() } - try await self.unpackLLDLinker() + try await self.prepareLLDLinker() try self.fixAbsoluteSymlinks() diff --git a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Unpack.swift b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Unpack.swift index 48b89ee..1c55d85 100644 --- a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Unpack.swift +++ b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Unpack.swift @@ -10,6 +10,8 @@ // //===----------------------------------------------------------------------===// +import struct SystemPackage.FilePath + private let unusedDarwinPlatforms = [ "watchsimulator", "iphonesimulator", @@ -66,36 +68,43 @@ extension SwiftSDKGenerator { } } - func unpackLLDLinker() async throws { + func prepareLLDLinker() async throws { logGenerationStep("Unpacking and copying `lld` linker...") let downloadableArtifacts = self.downloadableArtifacts let pathsConfiguration = self.pathsConfiguration let targetOS = self.targetTriple.os - try await inTemporaryDirectory { fileSystem, tmpDir in - let llvmArtifact = downloadableArtifacts.hostLLVM - try await fileSystem.untar( - file: llvmArtifact.localPath, - into: tmpDir, - stripComponents: 1 - ) + let llvmArtifact = downloadableArtifacts.hostLLVM - let unpackedLLDPath = if llvmArtifact.isPrebuilt { - tmpDir.appending("bin/lld") - } else { - try await self.buildLLD(llvmSourcesDirectory: tmpDir) - } + let untarDestination = pathsConfiguration.artifactsCachePath.appending( + FilePath.Component(llvmArtifact.localPath.stem!)!.stem + ) + try self.createDirectoryIfNeeded(at: untarDestination) + try await self.untar( + file: llvmArtifact.localPath, + into: untarDestination, + stripComponents: 1 + ) - let toolchainLLDPath = switch targetOS { - case .linux: - pathsConfiguration.toolchainBinDirPath.appending("ld.lld") - case .wasi: - pathsConfiguration.toolchainBinDirPath.appending("wasm-ld") - default: - fatalError() - } + let unpackedLLDPath = if llvmArtifact.isPrebuilt { + untarDestination.appending("bin/lld") + } else { + try await self.engine[CMakeBuildQuery( + sourcesDirectory: untarDestination, + outputBinarySubpath: ["bin", "lld"], + options: "-DLLVM_ENABLE_PROJECTS=lld -DLLVM_TARGETS_TO_BUILD=\(self.targetTriple.cpu.llvmTargetConventionName)" + )].path + } - try await fileSystem.copy(from: unpackedLLDPath, to: toolchainLLDPath) + let toolchainLLDPath = switch targetOS { + case .linux: + pathsConfiguration.toolchainBinDirPath.appending("ld.lld") + case .wasi: + pathsConfiguration.toolchainBinDirPath.appending("wasm-ld") + default: + fatalError() } + + try self.copy(from: unpackedLLDPath, to: toolchainLLDPath) } } diff --git a/Sources/SwiftSDKGenerator/Queries/CMakeBuildQuery.swift b/Sources/SwiftSDKGenerator/Queries/CMakeBuildQuery.swift new file mode 100644 index 0000000..8025160 --- /dev/null +++ b/Sources/SwiftSDKGenerator/Queries/CMakeBuildQuery.swift @@ -0,0 +1,35 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2022-2023 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import GeneratorEngine +import struct SystemPackage.FilePath + +@Query +struct CMakeBuildQuery { + let sourcesDirectory: FilePath + let outputBinarySubpath: [FilePath.Component] + let options: String + + func run(engine: Engine) async throws -> FilePath { + try await Shell.run( + """ + cmake -B build -G Ninja -S llvm -DCMAKE_BUILD_TYPE=Release \(self.options) + """, + currentDirectory: self.sourcesDirectory + ) + + let buildDirectory = self.sourcesDirectory.appending("build") + try await Shell.run("ninja", currentDirectory: buildDirectory) + + return self.outputBinarySubpath.reduce(into: buildDirectory) { $0.append($1) } + } +} diff --git a/Sources/SwiftSDKGenerator/Queries/DownloadArtifactQuery.swift b/Sources/SwiftSDKGenerator/Queries/DownloadArtifactQuery.swift new file mode 100644 index 0000000..2d9708a --- /dev/null +++ b/Sources/SwiftSDKGenerator/Queries/DownloadArtifactQuery.swift @@ -0,0 +1,69 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2022-2023 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import class AsyncHTTPClient.FileDownloadDelegate +import GeneratorEngine +import struct SystemPackage.FilePath + +@Query +struct DownloadArtifactQuery { + let artifact: DownloadableArtifacts.Item + + func run(engine: Engine) async throws -> FilePath { + print("Downloading remote artifact not available in local cache: \(self.artifact.remoteURL)") + let stream = await engine.httpClient.streamDownloadProgress(for: self.artifact) + .removeDuplicates(by: didProgressChangeSignificantly) + .throttle(for: .seconds(1)) + + for try await item in stream { + report(progress: item.progress, for: item.artifact) + } + return self.artifact.localPath + } +} + +/// Checks whether two given progress value are different enough from each other. Used for filtering out progress +/// values in async streams with `removeDuplicates` operator. +/// - Parameters: +/// - previous: Preceding progress value in the stream. +/// - current: Currently processed progress value in the stream. +/// - Returns: `true` if `totalBytes` value is different by any amount or if `receivedBytes` is different by amount +/// larger than 1MiB. Returns `false` otherwise. +@Sendable +private func didProgressChangeSignificantly( + previous: ArtifactDownloadProgress, + current: ArtifactDownloadProgress +) -> Bool { + guard previous.progress.totalBytes == current.progress.totalBytes else { + return true + } + + return current.progress.receivedBytes - previous.progress.receivedBytes > 1024 * 1024 * 1024 +} + +private func report(progress: FileDownloadDelegate.Progress, for artifact: DownloadableArtifacts.Item) { + if let total = progress.totalBytes { + print(""" + \(artifact.remoteURL.lastPathComponent) \( + byteCountFormatter + .string(fromByteCount: Int64(progress.receivedBytes)) + )/\( + byteCountFormatter + .string(fromByteCount: Int64(total)) + ) + """) + } else { + print( + "\(artifact.remoteURL.lastPathComponent) \(byteCountFormatter.string(fromByteCount: Int64(progress.receivedBytes)))" + ) + } +} From 5ca817cde85d3ac6a670a026c5af62570b83d25c Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Wed, 1 Nov 2023 15:42:47 +0000 Subject: [PATCH 14/15] Add a relative path doc string to `CMakeBuildQuery` --- Sources/SwiftSDKGenerator/Queries/CMakeBuildQuery.swift | 1 + 1 file changed, 1 insertion(+) diff --git a/Sources/SwiftSDKGenerator/Queries/CMakeBuildQuery.swift b/Sources/SwiftSDKGenerator/Queries/CMakeBuildQuery.swift index 8025160..31c0a0e 100644 --- a/Sources/SwiftSDKGenerator/Queries/CMakeBuildQuery.swift +++ b/Sources/SwiftSDKGenerator/Queries/CMakeBuildQuery.swift @@ -16,6 +16,7 @@ import struct SystemPackage.FilePath @Query struct CMakeBuildQuery { let sourcesDirectory: FilePath + /// Path to the output binary relative to the CMake build directory. let outputBinarySubpath: [FilePath.Component] let options: String From 45b30b23f6c34f333a49a611891d9f6de05c5d63 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Fri, 3 Nov 2023 14:37:37 +0000 Subject: [PATCH 15/15] Fix conflict resolution error --- .../Generator/SwiftSDKGenerator+Download.swift | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Download.swift b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Download.swift index 0509df5..f051bd0 100644 --- a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Download.swift +++ b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Download.swift @@ -25,23 +25,6 @@ private let ubuntuARM64Mirror = "http://ports.ubuntu.com/ubuntu-ports" let byteCountFormatter = ByteCountFormatter() -@Query -struct DownloadQuery { - let artifact: DownloadableArtifacts.Item - - func run(engine: Engine) async throws -> FilePath { - print("Downloading remote artifact not available in local cache: \(self.artifact.remoteURL)") - let stream = await engine.httpClient.streamDownloadProgress(for: self.artifact) - .removeDuplicates(by: didProgressChangeSignificantly) - .throttle(for: .seconds(1)) - - for try await item in stream { - report(progress: item.progress, for: item.artifact) - } - return self.artifact.localPath - } -} - extension SwiftSDKGenerator { func downloadArtifacts(_ client: HTTPClient) async throws { logGenerationStep("Downloading required toolchain packages...")