From c4b3edd6dfccc4e858c9328e3e65b8f91a51e181 Mon Sep 17 00:00:00 2001 From: Rintaro Ishizaki Date: Mon, 13 Mar 2023 12:17:24 -0700 Subject: [PATCH 1/6] [Macros] Add swift-plugin-server executable This executable is intended to be installed in the toolchain and act as an executable compiler plugin just like other 'macro' plugins. This plugin server has an optional method 'loadPluginLibrary' that dynamically loads dylib plugins. The compiler has a newly added option '-external-plugin-path'. This option receives a pair of the plugin library search path (just like '-plugin-path') and the corresponding "plugin server" path, separated by '#'. i.e. -external-plugin-path # For exmaple, when there's a macro decl: @freestanding(expression) macro stringify(T) -> (T, String) = #externalMacro(module: "BasicMacro", type: "StringifyMacro") The compiler look for 'libBasicMacro.dylib' in '-plugin-path' paths, if not found, it falls back to '-external-plugin-path' and tries to find 'libBasicMacro.dylib' in them. If it's found, the "plugin server" path is launched just like an executable plugin, then 'loadPluginLibrary' method is invoked via IPC, which 'dlopen' the library path in the plugin server. At the actual macro expansion, the mangled name for 'BasicMacro.StringifyMacro' is used to resolve the macro just like dylib plugins in the compiler. This is useful for * Isolating the plugin process, so the plugin crashes doesn't result the compiler crash * Being able to use library plugins linked with other `swift-syntax` versions rdar://105104850 --- include/swift/AST/ASTContext.h | 20 +- include/swift/AST/SearchPathOptions.h | 5 + include/swift/Demangling/Demangle.h | 6 + include/swift/Option/Options.td | 5 + lib/AST/ASTContext.cpp | 28 ++- lib/ASTGen/Package.swift | 3 +- lib/ASTGen/Sources/ASTGen/Macros.swift | 3 + lib/ASTGen/Sources/ASTGen/PluginHost.swift | 37 ++++ .../Sources/ASTGen/PluginMessages.swift | 28 ++- lib/ASTGen/Sources/LLVMJSON/LLVMJSON.swift | 4 +- lib/CMakeLists.txt | 1 + lib/Demangling/Demangler.cpp | 35 +++ lib/Driver/ToolChains.cpp | 1 + lib/Frontend/CompilerInvocation.cpp | 9 + lib/Sema/TypeCheckMacros.cpp | 77 +++---- test/Macros/macro_plugin_server.swift | 39 ++++ test/lit.cfg | 2 + tools/CMakeLists.txt | 1 + tools/swift-plugin-server/CMakeLists.txt | 32 +++ tools/swift-plugin-server/Package.swift | 39 ++++ .../CSwiftPluginServer/PluginServer.cpp | 123 +++++++++++ .../CSwiftPluginServer/include/PluginServer.h | 57 +++++ .../include/module.modulemap | 4 + .../swift-plugin-server.swift | 206 ++++++++++++++++++ 24 files changed, 710 insertions(+), 55 deletions(-) create mode 100644 test/Macros/macro_plugin_server.swift create mode 100644 tools/swift-plugin-server/CMakeLists.txt create mode 100644 tools/swift-plugin-server/Package.swift create mode 100644 tools/swift-plugin-server/Sources/CSwiftPluginServer/PluginServer.cpp create mode 100644 tools/swift-plugin-server/Sources/CSwiftPluginServer/include/PluginServer.h create mode 100644 tools/swift-plugin-server/Sources/CSwiftPluginServer/include/module.modulemap create mode 100644 tools/swift-plugin-server/Sources/swift-plugin-server/swift-plugin-server.swift diff --git a/include/swift/AST/ASTContext.h b/include/swift/AST/ASTContext.h index c62ff6aec2c76..1ea0b421d9ea1 100644 --- a/include/swift/AST/ASTContext.h +++ b/include/swift/AST/ASTContext.h @@ -1467,8 +1467,24 @@ class ASTContext final { Type getNamedSwiftType(ModuleDecl *module, StringRef name); - LoadedExecutablePlugin * - lookupExecutablePluginByModuleName(Identifier moduleName); + /// Lookup an executable plugin that is declared to handle \p moduleName + /// module by '-load-plugin-executable'. Note that the returned path might be + /// in the current VFS. i.e. use FS.getRealPath() to get the real path. + Optional lookupExecutablePluginByModuleName(Identifier moduleName); + + /// From paths '-external-plugin-path', look for dylib file that has + /// 'lib${moduleName}.dylib' (or equialent depending on the platform) and + /// return the found dylib path and the path to the "plugin server" for that. + /// Note that the returned path might be in the current VFS. + Optional> + lookupExternalLibraryPluginByModuleName(Identifier moduleName); + + /// Launch the specified executable plugin path resolving the path with the + /// current VFS. If it fails to load the plugin, a diagnostic is emitted, and + /// returns a nullptr. + /// NOTE: This method is idempotent. If the plugin is already loaded, the same + /// instance is simply returned. + LoadedExecutablePlugin *loadExecutablePlugin(StringRef path); /// Get the plugin registry this ASTContext is using. PluginRegistry *getPluginRegistry() const; diff --git a/include/swift/AST/SearchPathOptions.h b/include/swift/AST/SearchPathOptions.h index 300054d341639..e480cec3ce4a6 100644 --- a/include/swift/AST/SearchPathOptions.h +++ b/include/swift/AST/SearchPathOptions.h @@ -382,6 +382,11 @@ class SearchPathOptions { /// macro implementations. std::vector PluginSearchPaths; + /// Paths that contain compiler plugins and the path to the plugin server + /// executable. + /// e.g. '/path/to/usr/lib/swift/host/plugins#/path/to/usr/bin/plugin-server'. + std::vector ExternalPluginSearchPaths; + /// Don't look in for compiler-provided modules. bool SkipRuntimeLibraryImportPaths = false; diff --git a/include/swift/Demangling/Demangle.h b/include/swift/Demangling/Demangle.h index c29871226711f..8b1094837fad4 100644 --- a/include/swift/Demangling/Demangle.h +++ b/include/swift/Demangling/Demangle.h @@ -725,6 +725,12 @@ bool isFunctionAttr(Node::Kind kind); /// contain symbolic references. llvm::StringRef makeSymbolicMangledNameStringRef(const char *base); +/// Produce the mangled name for the nominal type descriptor of a type +/// referenced by its module and type name. +std::string mangledNameForTypeMetadataAccessor(llvm::StringRef moduleName, + llvm::StringRef typeName, + Node::Kind typeKind); + SWIFT_END_INLINE_NAMESPACE } // end namespace Demangle } // end namespace swift diff --git a/include/swift/Option/Options.td b/include/swift/Option/Options.td index 35ff52533ea88..d83364e155a7c 100644 --- a/include/swift/Option/Options.td +++ b/include/swift/Option/Options.td @@ -303,6 +303,11 @@ def plugin_path : Separate<["-"], "plugin-path">, Flags<[FrontendOption, ArgumentIsPath, SwiftAPIExtractOption, SwiftSymbolGraphExtractOption, SwiftAPIDigesterOption]>, HelpText<"Add directory to the plugin search path">; +def external_plugin_path : Separate<["-"], "external-plugin-path">, + Flags<[FrontendOption, ArgumentIsPath, SwiftAPIExtractOption, SwiftSymbolGraphExtractOption, SwiftAPIDigesterOption]>, + HelpText<"Add directory to the plugin search path with a plugin server executable">, + MetaVarName<"#">; + def import_underlying_module : Flag<["-"], "import-underlying-module">, Flags<[FrontendOption, NoInteractiveOption]>, HelpText<"Implicitly imports the Objective-C half of a module">; diff --git a/lib/AST/ASTContext.cpp b/lib/AST/ASTContext.cpp index a0a204e646f2c..557ff11a63f52 100644 --- a/lib/AST/ASTContext.cpp +++ b/lib/AST/ASTContext.cpp @@ -67,6 +67,7 @@ #include "llvm/ADT/Statistic.h" #include "llvm/ADT/StringMap.h" #include "llvm/ADT/StringSwitch.h" +#include "llvm/Config/config.h" #include "llvm/IR/LLVMContext.h" #include "llvm/Support/Allocator.h" #include "llvm/Support/Compiler.h" @@ -6344,15 +6345,34 @@ Type ASTContext::getNamedSwiftType(ModuleDecl *module, StringRef name) { return decl->getDeclaredInterfaceType(); } -LoadedExecutablePlugin * +Optional ASTContext::lookupExecutablePluginByModuleName(Identifier moduleName) { auto &execPluginPaths = getImpl().ExecutablePluginPaths; auto found = execPluginPaths.find(moduleName); if (found == execPluginPaths.end()) - return nullptr; + return None; + return found->second; +} + +Optional> +ASTContext::lookupExternalLibraryPluginByModuleName(Identifier moduleName) { + auto fs = this->SourceMgr.getFileSystem(); + for (auto &pair : SearchPathOpts.ExternalPluginSearchPaths) { + StringRef searchPath; + StringRef serverPath; + std::tie(searchPath, serverPath) = StringRef(pair).split('#'); + + SmallString<128> fullPath(searchPath); + llvm::sys::path::append(fullPath, "lib" + moduleName.str() + LTDL_SHLIB_EXT); + + if (fs->exists(fullPath)) { + return {{std::string(fullPath), serverPath.str()}}; + } + } + return None; +} - // Let the VFS to map the path. - auto &path = found->second; +LoadedExecutablePlugin *ASTContext::loadExecutablePlugin(StringRef path) { SmallString<128> resolvedPath; auto fs = this->SourceMgr.getFileSystem(); if (auto err = fs->getRealPath(path, resolvedPath)) { diff --git a/lib/ASTGen/Package.swift b/lib/ASTGen/Package.swift index 5e9c09b4432a3..683feef602934 100644 --- a/lib/ASTGen/Package.swift +++ b/lib/ASTGen/Package.swift @@ -22,7 +22,8 @@ let package = Package( .macOS(.v10_15) ], products: [ - .library(name: "swiftASTGen", targets: ["swiftASTGen"]) + .library(name: "swiftASTGen", targets: ["swiftASTGen"]), + .library(name: "swiftLLVMJSON", targets: ["swiftLLVMJSON"]), ], dependencies: [ .package(path: "../../../swift-syntax") diff --git a/lib/ASTGen/Sources/ASTGen/Macros.swift b/lib/ASTGen/Sources/ASTGen/Macros.swift index bcb3cf9e53a58..d65ea1adf0dcf 100644 --- a/lib/ASTGen/Sources/ASTGen/Macros.swift +++ b/lib/ASTGen/Sources/ASTGen/Macros.swift @@ -112,6 +112,9 @@ public func resolveExecutableMacro( typeNameLength: Int, pluginOpaqueHandle: UnsafeMutableRawPointer ) -> UnsafeRawPointer { + // NOTE: This doesn't actually resolve anything. + // Executable plugins is "trusted" to have the macro implementation. If not, + // the actual expansion fails. let exportedPtr = UnsafeMutablePointer.allocate(capacity: 1) exportedPtr.initialize(to: .init( moduleName: String(bufferStart: moduleName, count: moduleNameLength), diff --git a/lib/ASTGen/Sources/ASTGen/PluginHost.swift b/lib/ASTGen/Sources/ASTGen/PluginHost.swift index a28926aaf042f..aed5ec040d7a5 100644 --- a/lib/ASTGen/Sources/ASTGen/PluginHost.swift +++ b/lib/ASTGen/Sources/ASTGen/PluginHost.swift @@ -37,6 +37,35 @@ public func _deinitializePlugin( plugin.deinitialize() } +/// Load the library plugin in the plugin server. +@_cdecl("swift_ASTGen_pluginServerLoadLibraryPlugin") +func swift_ASTGen_pluginServerLoadLibraryPlugin( + opaqueHandle: UnsafeMutableRawPointer, + libraryPath: UnsafePointer, + moduleName: UnsafePointer, + cxxDiagnosticEngine: UnsafeMutablePointer +) -> Bool { + let plugin = CompilerPlugin(opaqueHandle: opaqueHandle) + assert(plugin.capability.features?.contains("loadPluginLibrary") == true) + let libraryPath = String(cString: libraryPath) + let moduleName = String(cString: moduleName) + let diagEngine = PluginDiagnosticsEngine(cxxDiagnosticEngine: cxxDiagnosticEngine) + + do { + let result = try plugin.sendMessageAndWait( + .loadPluginLibrary(libraryPath: libraryPath, moduleName: moduleName) + ) + guard case .loadPluginLibraryResult(let loaded, let diagnostics) = result else { + throw PluginError.invalidReponseKind + } + diagEngine.emit(diagnostics); + return loaded + } catch { + diagEngine.diagnose(error: error) + return false + } +} + struct CompilerPlugin { let opaqueHandle: UnsafeMutableRawPointer @@ -224,6 +253,14 @@ class PluginDiagnosticsEngine { } } + func diagnose(error: Error) { + self.emitSingle( + message: String(describing: error), + severity: .error, + position: .invalid + ) + } + /// Produce the C++ source location for a given position based on a /// syntax node. private func cxxSourceLocation( diff --git a/lib/ASTGen/Sources/ASTGen/PluginMessages.swift b/lib/ASTGen/Sources/ASTGen/PluginMessages.swift index de5e44f0cfa74..76ffae944ca34 100644 --- a/lib/ASTGen/Sources/ASTGen/PluginMessages.swift +++ b/lib/ASTGen/Sources/ASTGen/PluginMessages.swift @@ -14,14 +14,17 @@ // NOTE: Types in this file should be self-contained and should not depend on any non-stdlib types. internal enum HostToPluginMessage: Codable { + /// Get capability of this plugin. case getCapability + /// Expand a '@freestanding' macro. case expandFreestandingMacro( macro: PluginMessage.MacroReference, discriminator: String, syntax: PluginMessage.Syntax ) + /// Expand an '@attached' macro. case expandAttachedMacro( macro: PluginMessage.MacroReference, macroRole: PluginMessage.MacroRole, @@ -30,9 +33,21 @@ internal enum HostToPluginMessage: Codable { declSyntax: PluginMessage.Syntax, parentDeclSyntax: PluginMessage.Syntax? ) + + /// Optionally implemented message to load a dynamic link library. + /// 'moduleName' can be used as a hint indicating that the library + /// provides the specified module. + case loadPluginLibrary( + libraryPath: String, + moduleName: String + ) } internal enum PluginToHostMessage: Codable { + case getCapabilityResult( + capability: PluginMessage.PluginCapability + ) + case expandFreestandingMacroResult( expandedSource: String?, diagnostics: [PluginMessage.Diagnostic] @@ -43,18 +58,21 @@ internal enum PluginToHostMessage: Codable { diagnostics: [PluginMessage.Diagnostic] ) - case getCapabilityResult(capability: PluginMessage.PluginCapability) + case loadPluginLibraryResult( + loaded: Bool, + diagnostics: [PluginMessage.Diagnostic] + ) } /*namespace*/ internal enum PluginMessage { - static var PROTOCOL_VERSION_NUMBER: Int { 3 } // Renamed 'customAttributeSyntax' to 'attributeSyntax'. + static var PROTOCOL_VERSION_NUMBER: Int { 4 } // Added 'loadPluginLibrary'. struct PluginCapability: Codable { var protocolVersion: Int - } - static var capability: PluginCapability { - PluginCapability(protocolVersion: PluginMessage.PROTOCOL_VERSION_NUMBER) + /// Optional features this plugin provides. + /// * resolveLibraryMacro: 'resolveLibraryMacro' message is implemented. + var features: [String]? } struct MacroReference: Codable { diff --git a/lib/ASTGen/Sources/LLVMJSON/LLVMJSON.swift b/lib/ASTGen/Sources/LLVMJSON/LLVMJSON.swift index b6a209e260f82..cee714e080c00 100644 --- a/lib/ASTGen/Sources/LLVMJSON/LLVMJSON.swift +++ b/lib/ASTGen/Sources/LLVMJSON/LLVMJSON.swift @@ -24,7 +24,7 @@ extension String { public struct LLVMJSON { /// Encode an `Encodable` value to JSON data, and call `body` is the buffer. /// Note that the buffer is valid onlu in `body`. - public static func encoding(_ value: T, body: (UnsafeBufferPointer) -> R) throws -> R { + public static func encoding(_ value: T, body: (UnsafeBufferPointer) throws -> R) throws -> R { let valuePtr = JSON_newValue() defer { JSON_value_delete(valuePtr) } @@ -36,7 +36,7 @@ public struct LLVMJSON { assert(data.baseAddress != nil) defer { BridgedData_free(data) } let buffer = UnsafeBufferPointer(start: data.baseAddress, count: data.size) - return body(buffer) + return try body(buffer) } /// Decode a JSON data to a Swift value. diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index a518c068ee9a3..612c03aed535d 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -28,6 +28,7 @@ if (SWIFT_SWIFT_PARSER) SwiftOperators SwiftSyntaxBuilder SwiftSyntaxMacros + SwiftCompilerPluginMessageHandling ) # Compute the list of SwiftSyntax targets that we will link against. diff --git a/lib/Demangling/Demangler.cpp b/lib/Demangling/Demangler.cpp index 5c4a6ca4abcf5..65108b974e3df 100644 --- a/lib/Demangling/Demangler.cpp +++ b/lib/Demangling/Demangler.cpp @@ -306,6 +306,41 @@ bool swift::Demangle::isStruct(llvm::StringRef mangledName) { return isStructNode(Dem.demangleType(mangledName)); } +std::string swift::Demangle::mangledNameForTypeMetadataAccessor( + StringRef moduleName, StringRef typeName, Node::Kind typeKind) { + using namespace Demangle; + + // kind=Global + // kind=NominalTypeDescriptor + // kind=Type + // kind=Structure|Enum|Class + // kind=Module, text=moduleName + // kind=Identifier, text=typeName + Demangle::Demangler D; + auto *global = D.createNode(Node::Kind::Global); + { + auto *nominalDescriptor = + D.createNode(Node::Kind::TypeMetadataAccessFunction); + { + auto *type = D.createNode(Node::Kind::Type); + { + auto *module = D.createNode(Node::Kind::Module, moduleName); + auto *identifier = D.createNode(Node::Kind::Identifier, typeName); + auto *structNode = D.createNode(typeKind); + structNode->addChild(module, D); + structNode->addChild(identifier, D); + type->addChild(structNode, D); + } + nominalDescriptor->addChild(type, D); + } + global->addChild(nominalDescriptor, D); + } + + auto mangleResult = mangleNode(global); + assert(mangleResult.isSuccess()); + return mangleResult.result(); +} + using namespace swift; using namespace Demangle; diff --git a/lib/Driver/ToolChains.cpp b/lib/Driver/ToolChains.cpp index b7b8685f857fb..2f4442ae650ba 100644 --- a/lib/Driver/ToolChains.cpp +++ b/lib/Driver/ToolChains.cpp @@ -237,6 +237,7 @@ void ToolChain::addCommonFrontendArgs(const OutputInfo &OI, inputArgs.AddAllArgs(arguments, options::OPT_F, options::OPT_Fsystem); inputArgs.AddAllArgs(arguments, options::OPT_vfsoverlay); inputArgs.AddAllArgs(arguments, options::OPT_plugin_path); + inputArgs.AddAllArgs(arguments, options::OPT_external_plugin_path); inputArgs.AddLastArg(arguments, options::OPT_AssertConfig); inputArgs.AddLastArg(arguments, options::OPT_autolink_force_load); diff --git a/lib/Frontend/CompilerInvocation.cpp b/lib/Frontend/CompilerInvocation.cpp index 74803044912b7..7233a0521c69b 100644 --- a/lib/Frontend/CompilerInvocation.cpp +++ b/lib/Frontend/CompilerInvocation.cpp @@ -1492,6 +1492,15 @@ static bool ParseSearchPathArgs(SearchPathOptions &Opts, Opts.PluginSearchPaths.push_back(resolveSearchPath(A->getValue())); } + for (const Arg *A : Args.filtered(OPT_external_plugin_path)) { + // '#'. + StringRef dylibPath; + StringRef serverPath; + std::tie(dylibPath, serverPath) = StringRef(A->getValue()).split('#'); + Opts.ExternalPluginSearchPaths.push_back( + resolveSearchPath(dylibPath) + "#" + resolveSearchPath(serverPath)); + } + for (const Arg *A : Args.filtered(OPT_L)) { Opts.LibrarySearchPaths.push_back(resolveSearchPath(A->getValue())); } diff --git a/lib/Sema/TypeCheckMacros.cpp b/lib/Sema/TypeCheckMacros.cpp index be7e153eb081f..8529f1176d946 100644 --- a/lib/Sema/TypeCheckMacros.cpp +++ b/lib/Sema/TypeCheckMacros.cpp @@ -72,43 +72,9 @@ extern "C" ptrdiff_t swift_ASTGen_expandAttachedMacro( extern "C" void swift_ASTGen_initializePlugin(void *handle); extern "C" void swift_ASTGen_deinitializePlugin(void *handle); - -/// Produce the mangled name for the nominal type descriptor of a type -/// referenced by its module and type name. -static std::string mangledNameForTypeMetadataAccessor( - StringRef moduleName, StringRef typeName, Node::Kind typeKind) { - using namespace Demangle; - - // kind=Global - // kind=NominalTypeDescriptor - // kind=Type - // kind=Structure|Enum|Class - // kind=Module, text=moduleName - // kind=Identifier, text=typeName - Demangle::Demangler D; - auto *global = D.createNode(Node::Kind::Global); - { - auto *nominalDescriptor = - D.createNode(Node::Kind::TypeMetadataAccessFunction); - { - auto *type = D.createNode(Node::Kind::Type); - { - auto *module = D.createNode(Node::Kind::Module, moduleName); - auto *identifier = D.createNode(Node::Kind::Identifier, typeName); - auto *structNode = D.createNode(typeKind); - structNode->addChild(module, D); - structNode->addChild(identifier, D); - type->addChild(structNode, D); - } - nominalDescriptor->addChild(type, D); - } - global->addChild(nominalDescriptor, D); - } - - auto mangleResult = mangleNode(global); - assert(mangleResult.isSuccess()); - return mangleResult.result(); -} +extern "C" bool swift_ASTGen_pluginServerLoadLibraryPlugin( + void *handle, const char *libraryPath, const char *moduleName, + void *diagEngine); #if SWIFT_SWIFT_PARSER /// Look for macro's type metadata given its external module and type name. @@ -125,7 +91,7 @@ static void const *lookupMacroTypeMetadataByExternalName( void *accessorAddr = nullptr; for (auto typeKind : typeKinds) { - auto symbolName = mangledNameForTypeMetadataAccessor( + auto symbolName = Demangle::mangledNameForTypeMetadataAccessor( moduleName, typeName, typeKind); accessorAddr = ctx.getAddressOfSymbol(symbolName.c_str(), libraryHint); if (accessorAddr) @@ -375,9 +341,23 @@ static Optional resolveExecutableMacro(ASTContext &ctx, Identifier moduleName, Identifier typeName) { #if SWIFT_SWIFT_PARSER - // Find macros in exectuable plugins. - auto *executablePlugin = - ctx.lookupExecutablePluginByModuleName(moduleName); + std::string executablePluginPath; + std::string libraryPath; + + // Find macros in executable plugins. + if (auto found = ctx.lookupExternalLibraryPluginByModuleName(moduleName)) { + // Found in '-external-plugin-path'. + std::tie(libraryPath, executablePluginPath) = found.value(); + } else if (auto found = ctx.lookupExecutablePluginByModuleName(moduleName)) { + // Found in '-load-plugin-executable'. + executablePluginPath = found->str(); + } + if (executablePluginPath.empty()) + return None; + + // Launch the plugin. + LoadedExecutablePlugin *executablePlugin = + ctx.loadExecutablePlugin(executablePluginPath); if (!executablePlugin) return None; @@ -390,6 +370,21 @@ resolveExecutableMacro(ASTContext &ctx, Identifier moduleName, }); } + // If this is a plugin server. Load the library in that process before + // resolving the macro. + if (!libraryPath.empty()) { + llvm::SmallString<128> resolvedLibraryPath; + auto fs = ctx.SourceMgr.getFileSystem(); + if (fs->getRealPath(libraryPath, resolvedLibraryPath)) { + return None; + } + bool loaded = swift_ASTGen_pluginServerLoadLibraryPlugin( + executablePlugin, resolvedLibraryPath.c_str(), moduleName.str().data(), + &ctx.Diags); + if (!loaded) + return None; + } + if (auto *execMacro = swift_ASTGen_resolveExecutableMacro( moduleName.str().data(), moduleName.str().size(), typeName.str().data(), typeName.str().size(), executablePlugin)) { diff --git a/test/Macros/macro_plugin_server.swift b/test/Macros/macro_plugin_server.swift new file mode 100644 index 0000000000000..bed9378c4e300 --- /dev/null +++ b/test/Macros/macro_plugin_server.swift @@ -0,0 +1,39 @@ +// FIXME: Swift parser is not enabled on Linux CI yet. +// REQUIRES: OS=macosx + +// RUN: %empty-directory(%t) +// RUN: %empty-directory(%t/plugins) +// +//== Build the plugin library +// RUN: %target-build-swift \ +// RUN: -swift-version 5 \ +// RUN: -I %swift-host-lib-dir \ +// RUN: -L %swift-host-lib-dir \ +// RUN: -emit-library \ +// RUN: -o %t/plugins/%target-library-name(MacroDefinition) \ +// RUN: -module-name=MacroDefinition \ +// RUN: %S/Inputs/syntax_macro_definitions.swift \ +// RUN: -g -no-toolchain-stdlib-rpath + +// RUN: %swift-target-frontend \ +// RUN: -typecheck -verify \ +// RUN: -swift-version 5 \ +// RUN: -external-plugin-path %t/plugins#%swift-plugin-server \ +// RUN: -dump-macro-expansions \ +// RUN: %s \ +// RUN: 2>&1 | tee %t/macro-expansions.txt + +// RUN: %FileCheck -strict-whitespace %s < %t/macro-expansions.txt + + +@freestanding(expression) macro stringify(_ value: T) -> (T, String) = #externalMacro(module: "MacroDefinition", type: "StringifyMacro") + +func testStringify(a: Int, b: Int) { + let s: String = #stringify(a + b).1 + print(s) +} + +// CHECK: {{^}}------------------------------ +// CHECK-NEXT: {{^}}(a + b, "a + b") +// CHECK-NEXT: {{^}}------------------------------ + diff --git a/test/lit.cfg b/test/lit.cfg index c0356c0d6ed9a..7249609851870 100644 --- a/test/lit.cfg +++ b/test/lit.cfg @@ -343,6 +343,7 @@ config.benchmark_o = inferSwiftBinary('Benchmark_O') config.benchmark_driver = inferSwiftBinary('Benchmark_Driver') config.wasmer = inferSwiftBinary('wasmer') config.wasm_ld = inferSwiftBinary('wasm-ld') +config.swift_plugin_server = inferSwiftBinary('swift-plugin-server') config.swift_utils = make_path(config.swift_src_root, 'utils') config.line_directive = make_path(config.swift_utils, 'line-directive') @@ -573,6 +574,7 @@ config.substitutions.append( ('%Benchmark_Driver', config.benchmark_driver) ) config.substitutions.append( ('%llvm-strings', config.llvm_strings) ) config.substitutions.append( ('%target-ptrauth', run_ptrauth ) ) config.substitutions.append( ('%swift-path', config.swift) ) +config.substitutions.append( ('%swift-plugin-server', config.swift_plugin_server) ) # This must come after all substitutions containing "%swift". config.substitutions.append( diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 5ae2380b82241..2aa93906f1c37 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -34,6 +34,7 @@ add_swift_tool_subdirectory(swift-refactor) add_swift_tool_subdirectory(libSwiftScan) add_swift_tool_subdirectory(libStaticMirror) add_swift_tool_subdirectory(libMockPlugin) +add_swift_tool_subdirectory(swift-plugin-server) if(SWIFT_INCLUDE_TESTS OR SWIFT_INCLUDE_TEST_BINARIES) add_swift_tool_subdirectory(swift-ide-test) diff --git a/tools/swift-plugin-server/CMakeLists.txt b/tools/swift-plugin-server/CMakeLists.txt new file mode 100644 index 0000000000000..319d665681e44 --- /dev/null +++ b/tools/swift-plugin-server/CMakeLists.txt @@ -0,0 +1,32 @@ +if (SWIFT_SWIFT_PARSER) + # _swiftCSwiftPluginServer is just a C support library for wift-plugin-server + # Don't bother to create '.a' for that. + add_swift_host_library(_swiftCSwiftPluginServer OBJECT + Sources/CSwiftPluginServer/PluginServer.cpp + ) + target_link_libraries(_swiftCSwiftPluginServer + swiftDemangling + ) + target_include_directories(_swiftCSwiftPluginServer PUBLIC + Sources/CSwiftPluginServer/include + ) + + add_pure_swift_host_tool(swift-plugin-server + Sources/swift-plugin-server/swift-plugin-server.swift + DEPENDENCIES + swiftDemangling + $ + SWIFT_DEPENDENCIES + SwiftSyntax::SwiftSyntaxMacros + SwiftSyntax::SwiftCompilerPluginMessageHandling + swiftLLVMJSON + ) + target_include_directories(swift-plugin-server PRIVATE + Sources/CSwiftPluginServer/include + ) + swift_install_in_component(TARGETS swift-plugin-server + RUNTIME + DESTINATION bin + COMPONENT compiler + ) +endif() diff --git a/tools/swift-plugin-server/Package.swift b/tools/swift-plugin-server/Package.swift new file mode 100644 index 0000000000000..641cd5696070d --- /dev/null +++ b/tools/swift-plugin-server/Package.swift @@ -0,0 +1,39 @@ +// swift-tools-version: 5.6 + +import PackageDescription + +let package = Package( + name: "swift-plugin-server", + platforms: [ + .macOS(.v10_15) + ], + dependencies: [ + .package(path: "../../../swift-syntax"), + .package(path: "../../lib/ASTGen"), + ], + targets: [ + .target( + name: "CSwiftPluginServer", + cxxSettings: [ + .unsafeFlags([ + "-I", "../../include", + "-I", "../../../llvm-project/llvm/include", + ]) + ] + ), + .executableTarget( + name: "swift-plugin-server", + dependencies: [ + .product(name: "swiftLLVMJSON", package: "ASTGen"), + .product(name: "SwiftCompilerPluginMessageHandling", package: "swift-syntax"), + .product(name: "SwiftDiagnostics", package: "swift-syntax"), + .product(name: "SwiftSyntax", package: "swift-syntax"), + .product(name: "SwiftOperators", package: "swift-syntax"), + .product(name: "SwiftParser", package: "swift-syntax"), + .product(name: "SwiftSyntaxMacros", package: "swift-syntax"), + "CSwiftPluginServer" + ] + ), + + ] +) diff --git a/tools/swift-plugin-server/Sources/CSwiftPluginServer/PluginServer.cpp b/tools/swift-plugin-server/Sources/CSwiftPluginServer/PluginServer.cpp new file mode 100644 index 0000000000000..4566b21be2373 --- /dev/null +++ b/tools/swift-plugin-server/Sources/CSwiftPluginServer/PluginServer.cpp @@ -0,0 +1,123 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2023 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See http://swift.org/LICENSE.txt for license information +// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +#include "PluginServer.h" +#include "swift/ABI/MetadataValues.h" +#include "swift/Demangling/Demangle.h" + +#include +#include +#include +#include + +using namespace swift; + +namespace { +struct ConnectionHandle { + int inputFD; + int outputFD; + + ConnectionHandle(int inputFD, int outputFD) + : inputFD(inputFD), outputFD(outputFD) {} +}; +} // namespace + +const void *PluginServer_createConnection(const char **errorMessage) { + // Duplicate the `stdin` file descriptor, which we will then use for + // receiving messages from the plugin host. + auto inputFD = dup(STDIN_FILENO); + if (inputFD < 0) { + *errorMessage = strerror(errno); + return nullptr; + } + + // Having duplicated the original standard-input descriptor, we close + // `stdin` so that attempts by the plugin to read console input (which + // are usually a mistake) return errors instead of blocking. + if (close(STDIN_FILENO) < 0) { + *errorMessage = strerror(errno); + return nullptr; + } + + // Duplicate the `stdout` file descriptor, which we will then use for + // sending messages to the plugin host. + auto outputFD = dup(STDOUT_FILENO); + if (outputFD < 0) { + *errorMessage = strerror(errno); + return nullptr; + } + + // Having duplicated the original standard-output descriptor, redirect + // `stdout` to `stderr` so that all free-form text output goes there. + if (dup2(STDERR_FILENO, STDOUT_FILENO) < 0) { + *errorMessage = strerror(errno); + return nullptr; + } + + // Open a message channel for communicating with the plugin host. + return new ConnectionHandle(inputFD, outputFD); +} + +void PluginServer_destroyConnection(const void *connHandle) { + const auto *conn = static_cast(connHandle); + delete conn; +} + +ssize_t PluginServer_read(const void *connHandle, void *data, size_t nbyte) { + const auto *conn = static_cast(connHandle); + return ::read(conn->inputFD, data, nbyte); +} + +ssize_t PluginServer_write(const void *connHandle, const void *data, + size_t nbyte) { + const auto *conn = static_cast(connHandle); + return ::write(conn->outputFD, data, nbyte); +} + +void *PluginServer_dlopen(const char *filename, const char **errorMessage) { + auto *handle = ::dlopen(filename, RTLD_LAZY | RTLD_LOCAL); + if (!handle) { + *errorMessage = dlerror(); + } + return handle; +} + +const void *PluginServer_lookupMacroTypeMetadataByExternalName( + const char *moduleName, const char *typeName, void *libraryHint, + const char **errorMessage) { + + // Look up the type metadata accessor as a struct, enum, or class. + const Demangle::Node::Kind typeKinds[] = { + Demangle::Node::Kind::Structure, + Demangle::Node::Kind::Enum, + Demangle::Node::Kind::Class, + }; + + void *accessorAddr = nullptr; + for (auto typeKind : typeKinds) { + auto symbolName = + mangledNameForTypeMetadataAccessor(moduleName, typeName, typeKind); + + auto *handle = libraryHint ? libraryHint : RTLD_DEFAULT; + accessorAddr = ::dlsym(handle, symbolName.c_str()); + if (accessorAddr) + break; + } + + if (!accessorAddr) + return nullptr; + + // Call the accessor to form type metadata. + using MetadataAccessFunc = const void *(MetadataRequest); + auto accessor = reinterpret_cast(accessorAddr); + return accessor(MetadataRequest(MetadataState::Complete)); +} diff --git a/tools/swift-plugin-server/Sources/CSwiftPluginServer/include/PluginServer.h b/tools/swift-plugin-server/Sources/CSwiftPluginServer/include/PluginServer.h new file mode 100644 index 0000000000000..ddbcb56599268 --- /dev/null +++ b/tools/swift-plugin-server/Sources/CSwiftPluginServer/include/PluginServer.h @@ -0,0 +1,57 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2023 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See http://swift.org/LICENSE.txt for license information +// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +#ifndef SWIFT_PLUGINSERVER_PLUGINSERVER_H +#define SWIFT_PLUGINSERVER_PLUGINSERVER_H + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +//===----------------------------------------------------------------------===// +// Inter-process communication. +//===----------------------------------------------------------------------===// + +/// Create an IPC communication handle. +const void *PluginServer_createConnection(const char **errorMessage); + +/// Destroy an IPC communication handle created by +/// 'PluginServer_createConnection'. +void PluginServer_destroyConnection(const void *connHandle); + +/// Read bytes from the IPC communication handle. +ssize_t PluginServer_read(const void *connHandle, void *data, size_t nbyte); + +/// Write bytes to the IPC communication handle. +ssize_t PluginServer_write(const void *connHandle, const void *data, + size_t nbyte); + +//===----------------------------------------------------------------------===// +// Dynamic link +//===----------------------------------------------------------------------===// + +/// Load a dynamic link library, and return the handle. +void *PluginServer_dlopen(const char *filename, const char **errorMessage); + +/// Resolve a type metadata by a pair of the module name and the type name. +/// 'libraryHint' is a +const void *PluginServer_lookupMacroTypeMetadataByExternalName( + const char *moduleName, const char *typeName, void *libraryHint, + const char **errorMessage); + +#ifdef __cplusplus +} +#endif + +#endif /* SWIFT_PLUGINSERVER_PLUGINSERVER_H */ diff --git a/tools/swift-plugin-server/Sources/CSwiftPluginServer/include/module.modulemap b/tools/swift-plugin-server/Sources/CSwiftPluginServer/include/module.modulemap new file mode 100644 index 0000000000000..0f3ca3686e5b4 --- /dev/null +++ b/tools/swift-plugin-server/Sources/CSwiftPluginServer/include/module.modulemap @@ -0,0 +1,4 @@ +module CSwiftPluginServer { + header "PluginServer.h" + export * +} diff --git a/tools/swift-plugin-server/Sources/swift-plugin-server/swift-plugin-server.swift b/tools/swift-plugin-server/Sources/swift-plugin-server/swift-plugin-server.swift new file mode 100644 index 0000000000000..0eb59355b8be2 --- /dev/null +++ b/tools/swift-plugin-server/Sources/swift-plugin-server/swift-plugin-server.swift @@ -0,0 +1,206 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2023 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See http://swift.org/LICENSE.txt for license information +// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import SwiftCompilerPluginMessageHandling +import SwiftSyntaxMacros +import swiftLLVMJSON +import CSwiftPluginServer + +@main +final class SwiftPluginServer { + struct MacroRef: Hashable { + var moduleName: String + var typeName: String + init(_ moduleName: String, _ typeName: String) { + self.moduleName = moduleName + self.typeName = typeName + } + } + + /// Loaded dylib handles associated with the module name. + var loadedLibraryPlugins: [String: UnsafeMutableRawPointer] = [:] + + /// Resolved cached macros. + var resolvedMacros: [MacroRef: Macro.Type] = [:] + + /// @main entry point. + static func main() throws { + let connection = try PluginHostConnection() + let messageHandler = CompilerPluginMessageHandler( + connection: connection, + provider: self.init() + ) + try messageHandler.main() + } +} + +extension SwiftPluginServer: PluginProvider { + /// Load a macro implementation from the dynamic link library. + func loadPluginLibrary(libraryPath: String, moduleName: String) throws { + var errorMessage: UnsafePointer? + guard let dlHandle = PluginServer_dlopen(libraryPath, &errorMessage) else { + throw PluginServerError(message: String(cString: errorMessage!)) + } + loadedLibraryPlugins[moduleName] = dlHandle + } + + /// Lookup a loaded macro by a pair of module name and type name. + func resolveMacro(moduleName: String, typeName: String) -> Macro.Type? { + if let resolved = resolvedMacros[.init(moduleName, typeName)] { + return resolved + } + + // Find 'dlopen'ed library for the module name. + guard let dlHandle = loadedLibraryPlugins[moduleName] else { + return nil + } + + // Lookup the type metadata. + var errorMessage: UnsafePointer? + guard let macroTypePtr = PluginServer_lookupMacroTypeMetadataByExternalName( + moduleName, typeName, dlHandle, &errorMessage + ) else { + // FIXME: Propagate error message? + return nil + } + + // THe type must be a 'Macro' type. + let macroType = unsafeBitCast(macroTypePtr, to: Any.Type.self) + guard let macro = macroType as? Macro.Type else { + return nil + } + + // Cache the resolved type. + resolvedMacros[.init(moduleName, typeName)] = macro + return macro + } + + /// This 'PluginProvider' implements 'loadLibraryMacro()'. + var features: [SwiftCompilerPluginMessageHandling.PluginFeature] { + [.loadPluginLibrary] + } +} + +final class PluginHostConnection: MessageConnection { + let handle: UnsafeRawPointer + init() throws { + var errorMessage: UnsafePointer? = nil + guard let handle = PluginServer_createConnection(&errorMessage) else { + throw PluginServerError(message: String(cString: errorMessage!)) + } + self.handle = handle + } + + deinit { + PluginServer_destroyConnection(self.handle) + } + + func sendMessage(_ message: TX) throws { + try LLVMJSON.encoding(message) { buffer in + try self.sendMessageData(buffer) + } + } + + func waitForNextMessage(_ type: RX.Type) throws -> RX? { + return try self.withReadingMessageData { jsonData in + try LLVMJSON.decode(RX.self, from: jsonData) + } + } + + /// Send a serialized message data to the message channel. + private func sendMessageData(_ data: UnsafeBufferPointer) throws { + // Write the header (a 64-bit length field in little endian byte order). + var header: UInt64 = UInt64(data.count).littleEndian + let writtenSize = try Swift.withUnsafeBytes(of: &header) { buffer in + try self.write(data: UnsafeRawBufferPointer(buffer)) + } + guard writtenSize == 8 else { + throw PluginServerError(message: "failed to write message header") + } + + // Write the body. + guard try self.write(data: UnsafeRawBufferPointer(data)) == data.count else { + throw PluginServerError(message: "failed to write message data") + } + } + + /// Read a serialized message data from the message channel and call the 'body' + /// with the data. + private func withReadingMessageData(_ body: (UnsafeBufferPointer) throws -> R) throws -> R? { + // Read the header. + var header: UInt64 = 0 + let readSize = try Swift.withUnsafeMutableBytes(of: &header) { buffer in + try self.read(into: UnsafeMutableRawBufferPointer(buffer)) + } + if readSize == 0 { + // The host closed the pipe. + return nil + } + guard readSize == 8 else { + throw PluginServerError(message: "failed to read message header") + } + + // Read the body. + let count = Int(UInt64(littleEndian: header)) + let data = UnsafeMutableBufferPointer.allocate(capacity: count) + defer { data.deallocate() } + guard try self.read(into: UnsafeMutableRawBufferPointer(data)) == count else { + throw PluginServerError(message: "failed to read message data") + } + + // Invoke the handler. + return try body(UnsafeBufferPointer(data)) + } + + /// Write the 'data' to the message channel. + /// Returns the number of bytes succeeded to write. + private func write(data: UnsafeRawBufferPointer) throws -> Int { + var bytesToWrite = data.count + var ptr = data.baseAddress! + + while (bytesToWrite > 0) { + let writtenSize = PluginServer_write(handle, ptr, bytesToWrite) + if (writtenSize <= 0) { + // error e.g. broken pipe. + break + } + ptr = ptr.advanced(by: writtenSize) + bytesToWrite -= writtenSize + } + return data.count - bytesToWrite + } + + /// Read data from the message channel into the 'buffer' up to 'buffer.count' bytes. + /// Returns the number of bytes succeeded to read. + private func read(into buffer: UnsafeMutableRawBufferPointer) throws -> Int { + var bytesToRead = buffer.count + var ptr = buffer.baseAddress! + + while bytesToRead > 0 { + let readSize = PluginServer_read(handle, ptr, bytesToRead) + if (readSize <= 0) { + // 0: EOF (the host closed), -1: Broken pipe (the host crashed?) + break; + } + ptr = ptr.advanced(by: readSize) + bytesToRead -= readSize + } + return buffer.count - bytesToRead + } +} + +struct PluginServerError: Error, CustomStringConvertible { + var description: String + init(message: String) { + self.description = message + } +} From 1851ba2d8fd17e566b113cf1e2d02095429f97fa Mon Sep 17 00:00:00 2001 From: Rintaro Ishizaki Date: Wed, 15 Mar 2023 10:02:50 -0700 Subject: [PATCH 2/6] [Macros] swift-plugin-server cosmetic tweaks --- .../Sources/ASTGen/PluginMessages.swift | 2 +- .../swift-plugin-server.swift | 43 +++++++++++-------- 2 files changed, 26 insertions(+), 19 deletions(-) diff --git a/lib/ASTGen/Sources/ASTGen/PluginMessages.swift b/lib/ASTGen/Sources/ASTGen/PluginMessages.swift index 76ffae944ca34..9e25623b95d26 100644 --- a/lib/ASTGen/Sources/ASTGen/PluginMessages.swift +++ b/lib/ASTGen/Sources/ASTGen/PluginMessages.swift @@ -71,7 +71,7 @@ internal enum PluginToHostMessage: Codable { var protocolVersion: Int /// Optional features this plugin provides. - /// * resolveLibraryMacro: 'resolveLibraryMacro' message is implemented. + /// * loadPluginLibrary: 'loadPluginLibrary' message is implemented. var features: [String]? } diff --git a/tools/swift-plugin-server/Sources/swift-plugin-server/swift-plugin-server.swift b/tools/swift-plugin-server/Sources/swift-plugin-server/swift-plugin-server.swift index 0eb59355b8be2..9d0f4cda301b0 100644 --- a/tools/swift-plugin-server/Sources/swift-plugin-server/swift-plugin-server.swift +++ b/tools/swift-plugin-server/Sources/swift-plugin-server/swift-plugin-server.swift @@ -116,36 +116,37 @@ final class PluginHostConnection: MessageConnection { } } - /// Send a serialized message data to the message channel. + /// Send a serialized message to the message channel. private func sendMessageData(_ data: UnsafeBufferPointer) throws { // Write the header (a 64-bit length field in little endian byte order). var header: UInt64 = UInt64(data.count).littleEndian let writtenSize = try Swift.withUnsafeBytes(of: &header) { buffer in - try self.write(data: UnsafeRawBufferPointer(buffer)) + try self.write(buffer: UnsafeRawBufferPointer(buffer)) } - guard writtenSize == 8 else { + guard writtenSize == MemoryLayout.size(ofValue: header) else { throw PluginServerError(message: "failed to write message header") } // Write the body. - guard try self.write(data: UnsafeRawBufferPointer(data)) == data.count else { - throw PluginServerError(message: "failed to write message data") + guard try self.write(buffer: UnsafeRawBufferPointer(data)) == data.count else { + throw PluginServerError(message: "failed to write message body") } } - /// Read a serialized message data from the message channel and call the 'body' + /// Read a serialized message from the message channel and call the 'body' /// with the data. private func withReadingMessageData(_ body: (UnsafeBufferPointer) throws -> R) throws -> R? { - // Read the header. + // Read the header (a 64-bit length field in little endian byte order). var header: UInt64 = 0 let readSize = try Swift.withUnsafeMutableBytes(of: &header) { buffer in try self.read(into: UnsafeMutableRawBufferPointer(buffer)) } - if readSize == 0 { - // The host closed the pipe. - return nil - } - guard readSize == 8 else { + guard readSize == MemoryLayout.size(ofValue: header) else { + if readSize == 0 { + // The host closed the pipe. + return nil + } + // Otherwise, some error happened. throw PluginServerError(message: "failed to read message header") } @@ -154,18 +155,21 @@ final class PluginHostConnection: MessageConnection { let data = UnsafeMutableBufferPointer.allocate(capacity: count) defer { data.deallocate() } guard try self.read(into: UnsafeMutableRawBufferPointer(data)) == count else { - throw PluginServerError(message: "failed to read message data") + throw PluginServerError(message: "failed to read message body") } // Invoke the handler. return try body(UnsafeBufferPointer(data)) } - /// Write the 'data' to the message channel. + /// Write the 'buffer' to the message channel. /// Returns the number of bytes succeeded to write. - private func write(data: UnsafeRawBufferPointer) throws -> Int { - var bytesToWrite = data.count - var ptr = data.baseAddress! + private func write(buffer: UnsafeRawBufferPointer) throws -> Int { + var bytesToWrite = buffer.count + guard bytesToWrite > 0 else { + return 0 + } + var ptr = buffer.baseAddress! while (bytesToWrite > 0) { let writtenSize = PluginServer_write(handle, ptr, bytesToWrite) @@ -176,13 +180,16 @@ final class PluginHostConnection: MessageConnection { ptr = ptr.advanced(by: writtenSize) bytesToWrite -= writtenSize } - return data.count - bytesToWrite + return buffer.count - bytesToWrite } /// Read data from the message channel into the 'buffer' up to 'buffer.count' bytes. /// Returns the number of bytes succeeded to read. private func read(into buffer: UnsafeMutableRawBufferPointer) throws -> Int { var bytesToRead = buffer.count + guard bytesToRead > 0 else { + return 0 + } var ptr = buffer.baseAddress! while bytesToRead > 0 { From c675ecf184cff0418ad6b83ec712c3581cf377cb Mon Sep 17 00:00:00 2001 From: Rintaro Ishizaki Date: Wed, 15 Mar 2023 13:47:20 -0700 Subject: [PATCH 3/6] [Macros] Move 'loadExecutablePlugin' logic to 'CompilerPluginLoadRequest' Separate "load plugin" and "resolve macro" phases. So that the loaded executable plugin is now cached in ASTContext and reused. This saves a 'loadPluginLibrary' IPC messaging when the library provides multiple macros. --- include/swift/AST/TypeCheckRequests.h | 45 ++++++- include/swift/AST/TypeCheckerTypeIDZone.def | 2 +- lib/Sema/TypeCheckMacros.cpp | 141 +++++++++++--------- 3 files changed, 120 insertions(+), 68 deletions(-) diff --git a/include/swift/AST/TypeCheckRequests.h b/include/swift/AST/TypeCheckRequests.h index 6d8115377b117..217ab8cfcd8f1 100644 --- a/include/swift/AST/TypeCheckRequests.h +++ b/include/swift/AST/TypeCheckRequests.h @@ -49,6 +49,7 @@ struct ExternalMacroDefinition; class ClosureExpr; class GenericParamList; class LabeledStmt; +class LoadedExecutablePlugin; class MacroDefinition; class PrecedenceGroupDecl; class PropertyWrapperInitializerInfo; @@ -4000,19 +4001,51 @@ class ExpandSynthesizedMemberMacroRequest /// Load a plugin module with the given name. /// /// +class LoadedCompilerPlugin { + enum class PluginKind : uint8_t { + None, + InProcess, + Executable, + }; + PluginKind kind; + void *ptr; + + LoadedCompilerPlugin(PluginKind kind, void *ptr) : kind(kind), ptr(ptr) { + assert(ptr != nullptr || kind == PluginKind::None); + } + +public: + LoadedCompilerPlugin(std::nullptr_t) : kind(PluginKind::None), ptr(nullptr) {} + + static LoadedCompilerPlugin inProcess(void *ptr) { + return {PluginKind::InProcess, ptr}; + } + static LoadedCompilerPlugin executable(LoadedExecutablePlugin *ptr) { + return {PluginKind::Executable, ptr}; + } + + void *getAsInProcessPlugin() const { + return kind == PluginKind::InProcess ? static_cast(ptr) : nullptr; + } + LoadedExecutablePlugin *getAsExecutablePlugin() const { + return kind == PluginKind::Executable + ? static_cast(ptr) + : nullptr; + } +}; + class CompilerPluginLoadRequest - : public SimpleRequest { + : public SimpleRequest { public: using SimpleRequest::SimpleRequest; private: friend SimpleRequest; - void *evaluate( - Evaluator &evaluator, ASTContext *ctx, Identifier moduleName - ) const; + LoadedCompilerPlugin evaluate(Evaluator &evaluator, ASTContext *ctx, + Identifier moduleName) const; public: // Source location diff --git a/include/swift/AST/TypeCheckerTypeIDZone.def b/include/swift/AST/TypeCheckerTypeIDZone.def index 465b5e53f7cd0..f1c7ad395ba60 100644 --- a/include/swift/AST/TypeCheckerTypeIDZone.def +++ b/include/swift/AST/TypeCheckerTypeIDZone.def @@ -431,7 +431,7 @@ SWIFT_REQUEST(TypeChecker, MacroDefinitionRequest, MacroDefinition(MacroDecl *), Cached, NoLocationInfo) SWIFT_REQUEST(TypeChecker, CompilerPluginLoadRequest, - void *(ASTContext *, Identifier), + LoadedCompilerPlugin(ASTContext *, Identifier), Cached, NoLocationInfo) SWIFT_REQUEST(TypeChecker, ExternalMacroDefinitionRequest, Optional(ASTContext *, Identifier, Identifier), diff --git a/lib/Sema/TypeCheckMacros.cpp b/lib/Sema/TypeCheckMacros.cpp index 8529f1176d946..e81144792b634 100644 --- a/lib/Sema/TypeCheckMacros.cpp +++ b/lib/Sema/TypeCheckMacros.cpp @@ -286,10 +286,9 @@ MacroDefinition MacroDefinitionRequest::evaluate( } /// Load a plugin library based on a module name. -static void *loadPluginByName(StringRef searchPath, - StringRef moduleName, - llvm::vfs::FileSystem &fs, - PluginRegistry *registry) { +static void *loadLibraryPluginByName(StringRef searchPath, StringRef moduleName, + llvm::vfs::FileSystem &fs, + PluginRegistry *registry) { SmallString<128> fullPath(searchPath); llvm::sys::path::append(fullPath, "lib" + moduleName + LTDL_SHLIB_EXT); if (fs.getRealPath(fullPath, fullPath)) @@ -298,53 +297,12 @@ static void *loadPluginByName(StringRef searchPath, return loadResult ? *loadResult : nullptr; } -void *CompilerPluginLoadRequest::evaluate( - Evaluator &evaluator, ASTContext *ctx, Identifier moduleName -) const { - auto fs = ctx->SourceMgr.getFileSystem(); - auto &searchPathOpts = ctx->SearchPathOpts; - auto *registry = ctx->getPluginRegistry(); - for (const auto &path : searchPathOpts.PluginSearchPaths) { - if (auto found = loadPluginByName(path, moduleName.str(), *fs, registry)) - return found; - } - - return nullptr; -} - -static Optional -resolveInProcessMacro( - ASTContext &ctx, Identifier moduleName, Identifier typeName, - void *libraryHint = nullptr -) { -#if SWIFT_SWIFT_PARSER - /// Look for the type metadata given the external module and type names. - auto macroMetatype = lookupMacroTypeMetadataByExternalName( - ctx, moduleName.str(), typeName.str(), libraryHint); - if (macroMetatype) { - // Check whether the macro metatype is in-process. - if (auto inProcess = swift_ASTGen_resolveMacroType(macroMetatype)) { - // Make sure we clean up after the macro. - ctx.addCleanup([inProcess]() { - swift_ASTGen_destroyMacro(inProcess); - }); - - return ExternalMacroDefinition{ - ExternalMacroDefinition::PluginKind::InProcess, inProcess}; - } - } -#endif - return None; -} - -static Optional -resolveExecutableMacro(ASTContext &ctx, Identifier moduleName, - Identifier typeName) { -#if SWIFT_SWIFT_PARSER - std::string executablePluginPath; +static LoadedExecutablePlugin * +loadExecutablePluginByName(ASTContext &ctx, Identifier moduleName) { + // Find an executable plugin. std::string libraryPath; + std::string executablePluginPath; - // Find macros in executable plugins. if (auto found = ctx.lookupExternalLibraryPluginByModuleName(moduleName)) { // Found in '-external-plugin-path'. std::tie(libraryPath, executablePluginPath) = found.value(); @@ -353,38 +311,97 @@ resolveExecutableMacro(ASTContext &ctx, Identifier moduleName, executablePluginPath = found->str(); } if (executablePluginPath.empty()) - return None; + return nullptr; // Launch the plugin. LoadedExecutablePlugin *executablePlugin = ctx.loadExecutablePlugin(executablePluginPath); if (!executablePlugin) - return None; + return nullptr; // FIXME: Ideally this should be done right after invoking the plugin. // But plugin loading is in libAST and it can't link ASTGen symbols. if (!executablePlugin->isInitialized()) { +#if SWIFT_SWIFT_PARSER swift_ASTGen_initializePlugin(executablePlugin); executablePlugin->setCleanup([executablePlugin] { swift_ASTGen_deinitializePlugin(executablePlugin); }); +#endif } - // If this is a plugin server. Load the library in that process before - // resolving the macro. + // If this is a plugin server, load the library. if (!libraryPath.empty()) { +#if SWIFT_SWIFT_PARSER llvm::SmallString<128> resolvedLibraryPath; auto fs = ctx.SourceMgr.getFileSystem(); if (fs->getRealPath(libraryPath, resolvedLibraryPath)) { - return None; + return nullptr; } bool loaded = swift_ASTGen_pluginServerLoadLibraryPlugin( executablePlugin, resolvedLibraryPath.c_str(), moduleName.str().data(), &ctx.Diags); if (!loaded) - return None; + return nullptr; +#endif + } + + return executablePlugin; +} + +LoadedCompilerPlugin +CompilerPluginLoadRequest::evaluate(Evaluator &evaluator, ASTContext *ctx, + Identifier moduleName) const { + auto fs = ctx->SourceMgr.getFileSystem(); + auto &searchPathOpts = ctx->SearchPathOpts; + auto *registry = ctx->getPluginRegistry(); + + // First, check '-plugin-path' paths. + for (const auto &path : searchPathOpts.PluginSearchPaths) { + if (auto found = + loadLibraryPluginByName(path, moduleName.str(), *fs, registry)) + return LoadedCompilerPlugin::inProcess(found); + } + + // Fall back to executable plugins. + // i.e. '-external-plugin-path', and '-load-plugin-executable'. + if (auto *found = loadExecutablePluginByName(*ctx, moduleName)) { + return LoadedCompilerPlugin::executable(found); } + return nullptr; +} + +static Optional +resolveInProcessMacro( + ASTContext &ctx, Identifier moduleName, Identifier typeName, + void *libraryHint = nullptr +) { +#if SWIFT_SWIFT_PARSER + /// Look for the type metadata given the external module and type names. + auto macroMetatype = lookupMacroTypeMetadataByExternalName( + ctx, moduleName.str(), typeName.str(), libraryHint); + if (macroMetatype) { + // Check whether the macro metatype is in-process. + if (auto inProcess = swift_ASTGen_resolveMacroType(macroMetatype)) { + // Make sure we clean up after the macro. + ctx.addCleanup([inProcess]() { + swift_ASTGen_destroyMacro(inProcess); + }); + + return ExternalMacroDefinition{ + ExternalMacroDefinition::PluginKind::InProcess, inProcess}; + } + } +#endif + return None; +} + +static Optional +resolveExecutableMacro(ASTContext &ctx, + LoadedExecutablePlugin *executablePlugin, + Identifier moduleName, Identifier typeName) { +#if SWIFT_SWIFT_PARSER if (auto *execMacro = swift_ASTGen_resolveExecutableMacro( moduleName.str().data(), moduleName.str().size(), typeName.str().data(), typeName.str().size(), executablePlugin)) { @@ -395,7 +412,6 @@ resolveExecutableMacro(ASTContext &ctx, Identifier moduleName, ExternalMacroDefinition::PluginKind::Executable, execMacro}; } #endif - return None; } @@ -406,8 +422,9 @@ ExternalMacroDefinitionRequest::evaluate(Evaluator &evaluator, ASTContext *ctx, // Try to load a plugin module from the plugin search paths. If it // succeeds, resolve in-process from that plugin CompilerPluginLoadRequest loadRequest{ctx, moduleName}; - if (auto loadedLibrary = evaluateOrDefault( - evaluator, loadRequest, nullptr)) { + LoadedCompilerPlugin loaded = + evaluateOrDefault(evaluator, loadRequest, nullptr); + if (auto loadedLibrary = loaded.getAsInProcessPlugin()) { if (auto inProcess = resolveInProcessMacro( *ctx, moduleName, typeName, loadedLibrary)) return *inProcess; @@ -418,9 +435,11 @@ ExternalMacroDefinitionRequest::evaluate(Evaluator &evaluator, ASTContext *ctx, return *inProcess; // Try executable plugins. - if (auto executableMacro = - resolveExecutableMacro(*ctx, moduleName, typeName)) { - return executableMacro; + if (auto *executablePlugin = loaded.getAsExecutablePlugin()) { + if (auto executableMacro = resolveExecutableMacro(*ctx, executablePlugin, + moduleName, typeName)) { + return executableMacro; + } } return None; From 025d8746fe9e0f841bfbc3832054bad1db6fc8d3 Mon Sep 17 00:00:00 2001 From: Rintaro Ishizaki Date: Thu, 16 Mar 2023 12:06:39 -0700 Subject: [PATCH 4/6] [CMake] Set correct RPATH in add_pure_swift_host_tool Also, build pure swift library/tool with install RPATH like non Swift things. --- cmake/modules/AddPureSwift.cmake | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/cmake/modules/AddPureSwift.cmake b/cmake/modules/AddPureSwift.cmake index 87f79460163f1..b558a06afbf29 100644 --- a/cmake/modules/AddPureSwift.cmake +++ b/cmake/modules/AddPureSwift.cmake @@ -137,6 +137,9 @@ function(add_pure_swift_host_library name) add_library(${name} ${libkind} ${APSHL_SOURCES}) _add_host_swift_compile_options(${name}) + set_property(TARGET ${name} + PROPERTY BUILD_WITH_INSTALL_RPATH YES) + # Respect LLVM_COMMON_DEPENDS if it is set. # # LLVM_COMMON_DEPENDS if a global variable set in ./lib that provides targets @@ -257,6 +260,13 @@ function(add_pure_swift_host_tool name) add_executable(${name} ${APSHT_SOURCES}) _add_host_swift_compile_options(${name}) + set_property(TARGET ${name} + APPEND PROPERTY INSTALL_RPATH + "@executable_path/../lib/swift/host") + + set_property(TARGET ${name} + PROPERTY BUILD_WITH_INSTALL_RPATH YES) + # Respect LLVM_COMMON_DEPENDS if it is set. # # LLVM_COMMON_DEPENDS if a global variable set in ./lib that provides targets From 54884f05e5baa0e0d3c9dfc2e4f14907233d5bfb Mon Sep 17 00:00:00 2001 From: Rintaro Ishizaki Date: Fri, 17 Mar 2023 10:00:46 -0700 Subject: [PATCH 5/6] [Macros/Plugin] Make 'features' a set of enum values --- lib/ASTGen/Sources/ASTGen/PluginHost.swift | 56 ++++++++++++------- .../Sources/ASTGen/PluginMessages.swift | 2 +- 2 files changed, 38 insertions(+), 20 deletions(-) diff --git a/lib/ASTGen/Sources/ASTGen/PluginHost.swift b/lib/ASTGen/Sources/ASTGen/PluginHost.swift index aed5ec040d7a5..1cbd3a53943a0 100644 --- a/lib/ASTGen/Sources/ASTGen/PluginHost.swift +++ b/lib/ASTGen/Sources/ASTGen/PluginHost.swift @@ -46,7 +46,7 @@ func swift_ASTGen_pluginServerLoadLibraryPlugin( cxxDiagnosticEngine: UnsafeMutablePointer ) -> Bool { let plugin = CompilerPlugin(opaqueHandle: opaqueHandle) - assert(plugin.capability.features?.contains("loadPluginLibrary") == true) + assert(plugin.capability?.features.contains(.loadPluginLibrary) == true) let libraryPath = String(cString: libraryPath) let moduleName = String(cString: moduleName) let diagEngine = PluginDiagnosticsEngine(cxxDiagnosticEngine: cxxDiagnosticEngine) @@ -67,6 +67,24 @@ func swift_ASTGen_pluginServerLoadLibraryPlugin( } struct CompilerPlugin { + struct Capability { + enum Feature: String { + case loadPluginLibrary = "load-plugin-library" + } + + var protocolVersion: Int + var features: Set + + init(_ message: PluginMessage.PluginCapability) { + self.protocolVersion = message.protocolVersion + if let features = message.features { + self.features = Set(features.compactMap(Feature.init(rawValue:))) + } else { + self.features = [] + } + } + } + let opaqueHandle: UnsafeMutableRawPointer private func withLock(_ body: () throws -> R) rethrows -> R { @@ -111,26 +129,26 @@ struct CompilerPlugin { } func initialize() { - self.withLock { - // Get capability. - let response: PluginToHostMessage - do { + // Don't use `sendMessageAndWait` because we want to keep the lock until + // setting the returned value. + do { + try self.withLock { + // Get capability. try self.sendMessage(.getCapability) - response = try self.waitForNextMessage() - } catch { - assertionFailure(String(describing: error)) - return - } - switch response { - case .getCapabilityResult(capability: let capability): - let ptr = UnsafeMutablePointer.allocate(capacity: 1) - ptr.initialize(to: capability) + let response = try self.waitForNextMessage() + guard case .getCapabilityResult(let capability) = response else { + throw PluginError.invalidReponseKind + } + let ptr = UnsafeMutablePointer.allocate(capacity: 1) + ptr.initialize(to: .init(capability)) Plugin_setCapability(opaqueHandle, UnsafeRawPointer(ptr)) - default: - assertionFailure("invalid response") } + } catch { + assertionFailure(String(describing: error)) + return } } + func deinitialize() { self.withLock { if let ptr = Plugin_getCapability(opaqueHandle) { @@ -142,11 +160,11 @@ struct CompilerPlugin { } } - var capability: PluginMessage.PluginCapability { + var capability: Capability? { if let ptr = Plugin_getCapability(opaqueHandle) { - return ptr.assumingMemoryBound(to: PluginMessage.PluginCapability.self).pointee + return ptr.assumingMemoryBound(to: Capability.self).pointee } - return PluginMessage.PluginCapability(protocolVersion: 0) + return nil } } diff --git a/lib/ASTGen/Sources/ASTGen/PluginMessages.swift b/lib/ASTGen/Sources/ASTGen/PluginMessages.swift index 9e25623b95d26..df67a178f43c3 100644 --- a/lib/ASTGen/Sources/ASTGen/PluginMessages.swift +++ b/lib/ASTGen/Sources/ASTGen/PluginMessages.swift @@ -71,7 +71,7 @@ internal enum PluginToHostMessage: Codable { var protocolVersion: Int /// Optional features this plugin provides. - /// * loadPluginLibrary: 'loadPluginLibrary' message is implemented. + /// * 'load-plugin-library': 'loadPluginLibrary' message is implemented. var features: [String]? } From b2542a7ce92b415d67663d6c6f1ec51c8057a52b Mon Sep 17 00:00:00 2001 From: Rintaro Ishizaki Date: Fri, 17 Mar 2023 14:21:27 -0700 Subject: [PATCH 6/6] [Macros] Small review changes * New struct to store a pair of plugin search path and the server path * typo --- include/swift/AST/ASTContext.h | 13 +++++++------ include/swift/AST/SearchPathOptions.h | 16 ++++++++++++---- include/swift/AST/TypeCheckRequests.h | 2 +- lib/AST/ASTContext.cpp | 8 ++------ lib/Frontend/CompilerInvocation.cpp | 3 ++- tools/swift-plugin-server/CMakeLists.txt | 2 +- 6 files changed, 25 insertions(+), 19 deletions(-) diff --git a/include/swift/AST/ASTContext.h b/include/swift/AST/ASTContext.h index 1ea0b421d9ea1..2241add60ecc1 100644 --- a/include/swift/AST/ASTContext.h +++ b/include/swift/AST/ASTContext.h @@ -1468,14 +1468,15 @@ class ASTContext final { Type getNamedSwiftType(ModuleDecl *module, StringRef name); /// Lookup an executable plugin that is declared to handle \p moduleName - /// module by '-load-plugin-executable'. Note that the returned path might be - /// in the current VFS. i.e. use FS.getRealPath() to get the real path. + /// module by '-load-plugin-executable'. + /// The path is valid within the VFS, use `FS.getRealPath()` for the + /// underlying path. Optional lookupExecutablePluginByModuleName(Identifier moduleName); - /// From paths '-external-plugin-path', look for dylib file that has - /// 'lib${moduleName}.dylib' (or equialent depending on the platform) and - /// return the found dylib path and the path to the "plugin server" for that. - /// Note that the returned path might be in the current VFS. + /// Look for dynamic libraries in paths from `-external-plugin-path` and + /// return a pair of `(library path, plugin server executable)` if found. + /// These paths are valid within the VFS, use `FS.getRealPath()` for their + /// underlying path. Optional> lookupExternalLibraryPluginByModuleName(Identifier moduleName); diff --git a/include/swift/AST/SearchPathOptions.h b/include/swift/AST/SearchPathOptions.h index e480cec3ce4a6..f060f6a396169 100644 --- a/include/swift/AST/SearchPathOptions.h +++ b/include/swift/AST/SearchPathOptions.h @@ -174,6 +174,13 @@ class ModuleSearchPathLookup { llvm::vfs::FileSystem *FS, bool IsOSDarwin); }; +/// Pair of a plugin search path and the corresponding plugin server executable +/// path. +struct ExternalPluginSearchPathAndServerPath { + std::string SearchPath; + std::string ServerPath; +}; + /// Options for controlling search path behavior. class SearchPathOptions { /// To call \c addImportSearchPath and \c addFrameworkSearchPath from @@ -382,10 +389,11 @@ class SearchPathOptions { /// macro implementations. std::vector PluginSearchPaths; - /// Paths that contain compiler plugins and the path to the plugin server - /// executable. - /// e.g. '/path/to/usr/lib/swift/host/plugins#/path/to/usr/bin/plugin-server'. - std::vector ExternalPluginSearchPaths; + /// Pairs of external compiler plugin search paths and the corresponding + /// plugin server executables. + /// e.g. {"/path/to/usr/lib/swift/host/plugins", + /// "/path/to/usr/bin/plugin-server"} + std::vector ExternalPluginSearchPaths; /// Don't look in for compiler-provided modules. bool SkipRuntimeLibraryImportPaths = false; diff --git a/include/swift/AST/TypeCheckRequests.h b/include/swift/AST/TypeCheckRequests.h index 217ab8cfcd8f1..c92b4309a9f1c 100644 --- a/include/swift/AST/TypeCheckRequests.h +++ b/include/swift/AST/TypeCheckRequests.h @@ -4025,7 +4025,7 @@ class LoadedCompilerPlugin { } void *getAsInProcessPlugin() const { - return kind == PluginKind::InProcess ? static_cast(ptr) : nullptr; + return kind == PluginKind::InProcess ? ptr : nullptr; } LoadedExecutablePlugin *getAsExecutablePlugin() const { return kind == PluginKind::Executable diff --git a/lib/AST/ASTContext.cpp b/lib/AST/ASTContext.cpp index 557ff11a63f52..8a7ca22d9bb53 100644 --- a/lib/AST/ASTContext.cpp +++ b/lib/AST/ASTContext.cpp @@ -6358,15 +6358,11 @@ Optional> ASTContext::lookupExternalLibraryPluginByModuleName(Identifier moduleName) { auto fs = this->SourceMgr.getFileSystem(); for (auto &pair : SearchPathOpts.ExternalPluginSearchPaths) { - StringRef searchPath; - StringRef serverPath; - std::tie(searchPath, serverPath) = StringRef(pair).split('#'); - - SmallString<128> fullPath(searchPath); + SmallString<128> fullPath(pair.SearchPath); llvm::sys::path::append(fullPath, "lib" + moduleName.str() + LTDL_SHLIB_EXT); if (fs->exists(fullPath)) { - return {{std::string(fullPath), serverPath.str()}}; + return {{std::string(fullPath), pair.ServerPath}}; } } return None; diff --git a/lib/Frontend/CompilerInvocation.cpp b/lib/Frontend/CompilerInvocation.cpp index 7233a0521c69b..0f5cc08dcb4a1 100644 --- a/lib/Frontend/CompilerInvocation.cpp +++ b/lib/Frontend/CompilerInvocation.cpp @@ -1494,11 +1494,12 @@ static bool ParseSearchPathArgs(SearchPathOptions &Opts, for (const Arg *A : Args.filtered(OPT_external_plugin_path)) { // '#'. + // FIXME: '#' can be used in the paths. StringRef dylibPath; StringRef serverPath; std::tie(dylibPath, serverPath) = StringRef(A->getValue()).split('#'); Opts.ExternalPluginSearchPaths.push_back( - resolveSearchPath(dylibPath) + "#" + resolveSearchPath(serverPath)); + {resolveSearchPath(dylibPath), resolveSearchPath(serverPath)}); } for (const Arg *A : Args.filtered(OPT_L)) { diff --git a/tools/swift-plugin-server/CMakeLists.txt b/tools/swift-plugin-server/CMakeLists.txt index 319d665681e44..254cf3361d0a2 100644 --- a/tools/swift-plugin-server/CMakeLists.txt +++ b/tools/swift-plugin-server/CMakeLists.txt @@ -1,5 +1,5 @@ if (SWIFT_SWIFT_PARSER) - # _swiftCSwiftPluginServer is just a C support library for wift-plugin-server + # _swiftCSwiftPluginServer is just a C support library for swift-plugin-server # Don't bother to create '.a' for that. add_swift_host_library(_swiftCSwiftPluginServer OBJECT Sources/CSwiftPluginServer/PluginServer.cpp