[Macros] Add swift-plugin-server executable

cmake/modules/AddPureSwift.cmake

@@ -137,6 +137,9 @@ function(add_pure_swift_host_library name)
   add_library(${name} ${libkind} ${APSHL_SOURCES})
   _add_host_swift_compile_options(${name})
 
+  set_property(TARGET ${name}
+    PROPERTY BUILD_WITH_INSTALL_RPATH YES)
+
   # Respect LLVM_COMMON_DEPENDS if it is set.
   #
   # LLVM_COMMON_DEPENDS if a global variable set in ./lib that provides targets
@@ -257,6 +260,13 @@ function(add_pure_swift_host_tool name)
   add_executable(${name} ${APSHT_SOURCES})
   _add_host_swift_compile_options(${name})
 
+  set_property(TARGET ${name}
+    APPEND PROPERTY INSTALL_RPATH
+      "@executable_path/../lib/swift/host")
+
+  set_property(TARGET ${name}
+    PROPERTY BUILD_WITH_INSTALL_RPATH YES)
+
   # Respect LLVM_COMMON_DEPENDS if it is set.
   #
   # LLVM_COMMON_DEPENDS if a global variable set in ./lib that provides targets

include/swift/AST/ASTContext.h

@@ -1467,8 +1467,25 @@ class ASTContext final {
 
   Type getNamedSwiftType(ModuleDecl *module, StringRef name);
 
-  LoadedExecutablePlugin *
-  lookupExecutablePluginByModuleName(Identifier moduleName);
+  /// Lookup an executable plugin that is declared to handle \p moduleName
+  /// module by '-load-plugin-executable'.
+  /// The path is valid within the VFS, use `FS.getRealPath()` for the
+  /// underlying path.
+  Optional<StringRef> lookupExecutablePluginByModuleName(Identifier moduleName);
+
+  /// Look for dynamic libraries in paths from `-external-plugin-path` and
+  /// return a pair of `(library path, plugin server executable)` if found.
+  /// These paths are valid within the VFS, use `FS.getRealPath()` for their
+  /// underlying path.
+  Optional<std::pair<std::string, std::string>>
+  lookupExternalLibraryPluginByModuleName(Identifier moduleName);
+
+  /// Launch the specified executable plugin path resolving the path with the
+  /// current VFS. If it fails to load the plugin, a diagnostic is emitted, and
+  /// returns a nullptr.
+  /// NOTE: This method is idempotent. If the plugin is already loaded, the same
+  /// instance is simply returned.
+  LoadedExecutablePlugin *loadExecutablePlugin(StringRef path);
 
   /// Get the plugin registry this ASTContext is using.
   PluginRegistry *getPluginRegistry() const;

include/swift/AST/SearchPathOptions.h

@@ -174,6 +174,13 @@ class ModuleSearchPathLookup {
                             llvm::vfs::FileSystem *FS, bool IsOSDarwin);
 };
 
+/// Pair of a plugin search path and the corresponding plugin server executable
+/// path.
+struct ExternalPluginSearchPathAndServerPath {
+  std::string SearchPath;
+  std::string ServerPath;
+};
+
 /// Options for controlling search path behavior.
 class SearchPathOptions {
   /// To call \c addImportSearchPath and \c addFrameworkSearchPath from
@@ -382,6 +389,12 @@ class SearchPathOptions {
   /// macro implementations.
   std::vector<std::string> PluginSearchPaths;
 
+  /// Pairs of external compiler plugin search paths and the corresponding
+  /// plugin server executables.
+  /// e.g. {"/path/to/usr/lib/swift/host/plugins",
+  ///       "/path/to/usr/bin/plugin-server"}
+  std::vector<ExternalPluginSearchPathAndServerPath> ExternalPluginSearchPaths;
+
   /// Don't look in for compiler-provided modules.
   bool SkipRuntimeLibraryImportPaths = false;
 

include/swift/AST/TypeCheckRequests.h

@@ -49,6 +49,7 @@ struct ExternalMacroDefinition;
 class ClosureExpr;
 class GenericParamList;
 class LabeledStmt;
+class LoadedExecutablePlugin;
 class MacroDefinition;
 class PrecedenceGroupDecl;
 class PropertyWrapperInitializerInfo;
@@ -4000,19 +4001,51 @@ class ExpandSynthesizedMemberMacroRequest
 /// Load a plugin module with the given name.
 ///
 ///
+class LoadedCompilerPlugin {
+  enum class PluginKind : uint8_t {
+    None,
+    InProcess,
+    Executable,
+  };
+  PluginKind kind;
+  void *ptr;
+
+  LoadedCompilerPlugin(PluginKind kind, void *ptr) : kind(kind), ptr(ptr) {
+    assert(ptr != nullptr || kind == PluginKind::None);
+  }
+
+public:
+  LoadedCompilerPlugin(std::nullptr_t) : kind(PluginKind::None), ptr(nullptr) {}
+
+  static LoadedCompilerPlugin inProcess(void *ptr) {
+    return {PluginKind::InProcess, ptr};
+  }
+  static LoadedCompilerPlugin executable(LoadedExecutablePlugin *ptr) {
+    return {PluginKind::Executable, ptr};
+  }
+
+  void *getAsInProcessPlugin() const {
+    return kind == PluginKind::InProcess ? ptr : nullptr;
+  }
+  LoadedExecutablePlugin *getAsExecutablePlugin() const {
+    return kind == PluginKind::Executable
+               ? static_cast<LoadedExecutablePlugin *>(ptr)
+               : nullptr;
+  }
+};
+
 class CompilerPluginLoadRequest
-  : public SimpleRequest<CompilerPluginLoadRequest,
-                         void *(ASTContext *, Identifier),
-                         RequestFlags::Cached> {
+    : public SimpleRequest<CompilerPluginLoadRequest,
+                           LoadedCompilerPlugin(ASTContext *, Identifier),
+                           RequestFlags::Cached> {
 public:
   using SimpleRequest::SimpleRequest;
 
 private:
   friend SimpleRequest;
 
-  void *evaluate(
-      Evaluator &evaluator, ASTContext *ctx, Identifier moduleName
-  ) const;
+  LoadedCompilerPlugin evaluate(Evaluator &evaluator, ASTContext *ctx,
+                                Identifier moduleName) const;
 
 public:
   // Source location

include/swift/AST/TypeCheckerTypeIDZone.def

@@ -431,7 +431,7 @@ SWIFT_REQUEST(TypeChecker, MacroDefinitionRequest,
               MacroDefinition(MacroDecl *),
               Cached, NoLocationInfo)
 SWIFT_REQUEST(TypeChecker, CompilerPluginLoadRequest,
-              void *(ASTContext *, Identifier),
+              LoadedCompilerPlugin(ASTContext *, Identifier),
               Cached, NoLocationInfo)
 SWIFT_REQUEST(TypeChecker, ExternalMacroDefinitionRequest,
               Optional<ExternalMacroDefinition>(ASTContext *, Identifier, Identifier),

include/swift/Demangling/Demangle.h

@@ -725,6 +725,12 @@ bool isFunctionAttr(Node::Kind kind);
 /// contain symbolic references.
 llvm::StringRef makeSymbolicMangledNameStringRef(const char *base);
 
+/// Produce the mangled name for the nominal type descriptor of a type
+/// referenced by its module and type name.
+std::string mangledNameForTypeMetadataAccessor(llvm::StringRef moduleName,
+                                               llvm::StringRef typeName,
+                                               Node::Kind typeKind);
+
 SWIFT_END_INLINE_NAMESPACE
 } // end namespace Demangle
 } // end namespace swift

include/swift/Option/Options.td

@@ -303,6 +303,11 @@ def plugin_path : Separate<["-"], "plugin-path">,
   Flags<[FrontendOption, ArgumentIsPath, SwiftAPIExtractOption, SwiftSymbolGraphExtractOption, SwiftAPIDigesterOption]>,
   HelpText<"Add directory to the plugin search path">;
 
+def external_plugin_path : Separate<["-"], "external-plugin-path">,
+  Flags<[FrontendOption, ArgumentIsPath, SwiftAPIExtractOption, SwiftSymbolGraphExtractOption, SwiftAPIDigesterOption]>,
+  HelpText<"Add directory to the plugin search path with a plugin server executable">,
+  MetaVarName<"<path>#<plugin-server-path>">;
+
 def import_underlying_module : Flag<["-"], "import-underlying-module">,
   Flags<[FrontendOption, NoInteractiveOption]>,
   HelpText<"Implicitly imports the Objective-C half of a module">;

lib/AST/ASTContext.cpp

@@ -67,6 +67,7 @@
 #include "llvm/ADT/Statistic.h"
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringSwitch.h"
+#include "llvm/Config/config.h"
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/Support/Allocator.h"
 #include "llvm/Support/Compiler.h"
@@ -6344,15 +6345,30 @@ Type ASTContext::getNamedSwiftType(ModuleDecl *module, StringRef name) {
   return decl->getDeclaredInterfaceType();
 }
 
-LoadedExecutablePlugin *
+Optional<StringRef>
 ASTContext::lookupExecutablePluginByModuleName(Identifier moduleName) {
   auto &execPluginPaths = getImpl().ExecutablePluginPaths;
   auto found = execPluginPaths.find(moduleName);
   if (found == execPluginPaths.end())
-    return nullptr;
+    return None;
+  return found->second;
+}
+
+Optional<std::pair<std::string, std::string>>
+ASTContext::lookupExternalLibraryPluginByModuleName(Identifier moduleName) {
+  auto fs = this->SourceMgr.getFileSystem();
+  for (auto &pair : SearchPathOpts.ExternalPluginSearchPaths) {
+    SmallString<128> fullPath(pair.SearchPath);
+    llvm::sys::path::append(fullPath, "lib" + moduleName.str() + LTDL_SHLIB_EXT);
+
+    if (fs->exists(fullPath)) {
+      return {{std::string(fullPath), pair.ServerPath}};
+    }
+  }
+  return None;
+}
 
-  // Let the VFS to map the path.
-  auto &path = found->second;
+LoadedExecutablePlugin *ASTContext::loadExecutablePlugin(StringRef path) {
   SmallString<128> resolvedPath;
   auto fs = this->SourceMgr.getFileSystem();
   if (auto err = fs->getRealPath(path, resolvedPath)) {

lib/ASTGen/Package.swift

@@ -22,7 +22,8 @@ let package = Package(
     .macOS(.v10_15)
   ],
   products: [
-    .library(name: "swiftASTGen", targets: ["swiftASTGen"])
+    .library(name: "swiftASTGen", targets: ["swiftASTGen"]),
+    .library(name: "swiftLLVMJSON", targets: ["swiftLLVMJSON"]),
   ],
   dependencies: [
     .package(path: "../../../swift-syntax")

lib/ASTGen/Sources/ASTGen/Macros.swift

@@ -112,6 +112,9 @@ public func resolveExecutableMacro(
   typeNameLength: Int,
   pluginOpaqueHandle: UnsafeMutableRawPointer
 ) -> UnsafeRawPointer {
+  // NOTE: This doesn't actually resolve anything.
+  // Executable plugins is "trusted" to have the macro implementation. If not,
+  // the actual expansion fails.
   let exportedPtr = UnsafeMutablePointer<ExportedExecutableMacro>.allocate(capacity: 1)
   exportedPtr.initialize(to: .init(
     moduleName: String(bufferStart: moduleName, count: moduleNameLength),

lib/ASTGen/Sources/ASTGen/PluginHost.swift

@@ -37,7 +37,54 @@ public func _deinitializePlugin(
   plugin.deinitialize()
 }
 
+/// Load the library plugin in the plugin server.
+@_cdecl("swift_ASTGen_pluginServerLoadLibraryPlugin")
+func swift_ASTGen_pluginServerLoadLibraryPlugin(
+  opaqueHandle: UnsafeMutableRawPointer,
+  libraryPath: UnsafePointer<Int8>,
+  moduleName: UnsafePointer<Int8>,
+  cxxDiagnosticEngine: UnsafeMutablePointer<UInt8>
+) -> Bool {
+  let plugin =  CompilerPlugin(opaqueHandle: opaqueHandle)
+  assert(plugin.capability?.features.contains(.loadPluginLibrary) == true)
+  let libraryPath = String(cString: libraryPath)
+  let moduleName = String(cString: moduleName)
+  let diagEngine = PluginDiagnosticsEngine(cxxDiagnosticEngine: cxxDiagnosticEngine)
+
+  do {
+    let result = try plugin.sendMessageAndWait(
+      .loadPluginLibrary(libraryPath: libraryPath, moduleName: moduleName)
+    )
+    guard case .loadPluginLibraryResult(let loaded, let diagnostics) = result else {
+      throw PluginError.invalidReponseKind
+    }
+    diagEngine.emit(diagnostics);
+    return loaded
+  } catch {
+    diagEngine.diagnose(error: error)
+    return false
+  }
+}
+
 struct CompilerPlugin {
+  struct Capability {
+    enum Feature: String {
+      case loadPluginLibrary = "load-plugin-library"
+    }
+
+    var protocolVersion: Int
+    var features: Set<Feature>
+
+    init(_ message: PluginMessage.PluginCapability) {
+      self.protocolVersion = message.protocolVersion
+      if let features = message.features {
+        self.features = Set(features.compactMap(Feature.init(rawValue:)))
+      } else {
+        self.features = []
+      }
+    }
+  }
+
   let opaqueHandle: UnsafeMutableRawPointer
 
   private func withLock<R>(_ body: () throws -> R) rethrows -> R {
@@ -82,26 +129,26 @@ struct CompilerPlugin {
   }
 
   func initialize() {
-    self.withLock {
-      // Get capability.
-      let response: PluginToHostMessage
-      do {
+    // Don't use `sendMessageAndWait` because we want to keep the lock until
+    // setting the returned value.
+    do {
+      try self.withLock {
+        // Get capability.
         try self.sendMessage(.getCapability)
-        response = try self.waitForNextMessage()
-      } catch {
-        assertionFailure(String(describing: error))
-        return
-      }
-      switch response {
-      case .getCapabilityResult(capability: let capability):
-        let ptr = UnsafeMutablePointer<PluginMessage.PluginCapability>.allocate(capacity: 1)
-        ptr.initialize(to: capability)
+        let response = try self.waitForNextMessage()
+        guard case .getCapabilityResult(let capability) = response else {
+          throw PluginError.invalidReponseKind
+        }
+        let ptr = UnsafeMutablePointer<Capability>.allocate(capacity: 1)
+        ptr.initialize(to: .init(capability))
         Plugin_setCapability(opaqueHandle, UnsafeRawPointer(ptr))
-      default:
-        assertionFailure("invalid response")
       }
+    } catch {
+      assertionFailure(String(describing: error))
+      return
     }
   }
+
   func deinitialize() {
     self.withLock {
       if let ptr = Plugin_getCapability(opaqueHandle) {
@@ -113,11 +160,11 @@ struct CompilerPlugin {
     }
   }
 
-  var capability: PluginMessage.PluginCapability {
+  var capability: Capability? {
     if let ptr = Plugin_getCapability(opaqueHandle) {
-      return ptr.assumingMemoryBound(to: PluginMessage.PluginCapability.self).pointee
+      return ptr.assumingMemoryBound(to: Capability.self).pointee
     }
-    return PluginMessage.PluginCapability(protocolVersion: 0)
+    return nil
   }
 }
 
@@ -224,6 +271,14 @@ class PluginDiagnosticsEngine {
     }
   }
 
+  func diagnose(error: Error) {
+    self.emitSingle(
+      message: String(describing: error),
+      severity: .error,
+      position: .invalid
+    )
+  }
+
   /// Produce the C++ source location for a given position based on a
   /// syntax node.
   private func cxxSourceLocation(