diff --git a/include/swift/AST/DiagnosticsSema.def b/include/swift/AST/DiagnosticsSema.def index 06186f62e83a2..b8b5fea22f11d 100644 --- a/include/swift/AST/DiagnosticsSema.def +++ b/include/swift/AST/DiagnosticsSema.def @@ -1265,6 +1265,10 @@ REMARK(macro_loaded,none, "compiler plugin server '%2' with shared library '%3'}1", (Identifier, unsigned, StringRef, StringRef)) +ERROR(resolved_macro_changed,none, + "resolved macro library '%0' failed verification: %1", + (StringRef, StringRef)) + REMARK(transitive_dependency_behavior,none, "%1 has %select{a required|an optional|an ignored}2 " "transitive dependency on '%0'", diff --git a/include/swift/AST/PluginLoader.h b/include/swift/AST/PluginLoader.h index fdcd73f13ba97..576d88e5b36b1 100644 --- a/include/swift/AST/PluginLoader.h +++ b/include/swift/AST/PluginLoader.h @@ -52,10 +52,17 @@ class PluginLoader { /// Get or lazily create and populate 'PluginMap'. llvm::DenseMap &getPluginMap(); + /// Resolved plugin path remappings. + std::vector PathRemap; + public: PluginLoader(ASTContext &Ctx, DependencyTracker *DepTracker, + std::optional> Remap = std::nullopt, bool disableSandbox = false) - : Ctx(Ctx), DepTracker(DepTracker), disableSandbox(disableSandbox) {} + : Ctx(Ctx), DepTracker(DepTracker), disableSandbox(disableSandbox) { + if (Remap) + PathRemap = std::move(*Remap); + } void setRegistry(PluginRegistry *newValue); PluginRegistry *getRegistry(); diff --git a/include/swift/AST/SearchPathOptions.h b/include/swift/AST/SearchPathOptions.h index ff697f5ef49b4..1d32b1625bd91 100644 --- a/include/swift/AST/SearchPathOptions.h +++ b/include/swift/AST/SearchPathOptions.h @@ -510,6 +510,9 @@ class SearchPathOptions { /// Scanner Prefix Mapper. std::vector ScannerPrefixMapper; + /// Verify resolved plugin is not changed. + bool ResolvedPluginVerification = false; + /// When set, don't validate module system dependencies. /// /// If a system header is modified and this is not set, the compiler will diff --git a/include/swift/Option/Options.td b/include/swift/Option/Options.td index 664572632985d..c04a95a55484b 100644 --- a/include/swift/Option/Options.td +++ b/include/swift/Option/Options.td @@ -2263,6 +2263,10 @@ def load_resolved_plugin: "and exectuable path can be empty if not used">, MetaVarName<"##">; +def resolved_plugin_verification : Flag<["-"], "resolved-plugin-verification">, + Flags<[FrontendOption, NoDriverOption]>, + HelpText<"verify resolved plugins">; + def in_process_plugin_server_path : Separate<["-"], "in-process-plugin-server-path">, Flags<[FrontendOption, ArgumentIsPath]>, HelpText<"Path to dynamic library plugin server">; diff --git a/lib/AST/PluginLoader.cpp b/lib/AST/PluginLoader.cpp index 59646d7721192..271c484fc99b8 100644 --- a/lib/AST/PluginLoader.cpp +++ b/lib/AST/PluginLoader.cpp @@ -13,11 +13,12 @@ #include "swift/AST/PluginLoader.h" #include "swift/AST/ASTContext.h" #include "swift/AST/DiagnosticEngine.h" -#include "swift/AST/DiagnosticsFrontend.h" +#include "swift/AST/DiagnosticsSema.h" #include "swift/Basic/Assertions.h" #include "swift/Basic/SourceManager.h" #include "swift/Parse/Lexer.h" #include "llvm/Config/config.h" +#include "llvm/Support/PrefixMapper.h" #include "llvm/Support/VirtualFileSystem.h" using namespace swift; @@ -96,9 +97,45 @@ PluginLoader::getPluginMap() { map[moduleNameIdentifier] = {libPath, execPath}; }; + std::optional mapper; + if (!PathRemap.empty()) { + SmallVector prefixes; + llvm::MappedPrefix::transformJoinedIfValid(PathRemap, prefixes); + mapper.emplace(); + mapper->addRange(prefixes); + mapper->sort(); + } + auto remapPath = [&mapper](StringRef path) { + if (!mapper) + return path.str(); + return mapper->mapToString(path); + }; + auto fs = getPluginLoadingFS(Ctx); std::error_code ec; + auto validateLibrary = [&](StringRef path) -> llvm::Expected { + auto remappedPath = remapPath(path); + if (!Ctx.SearchPathOpts.ResolvedPluginVerification || path.empty()) + return remappedPath; + + auto currentStat = fs->status(remappedPath); + if (!currentStat) + return llvm::createFileError(remappedPath, currentStat.getError()); + + auto goldStat = Ctx.SourceMgr.getFileSystem()->status(path); + if (!goldStat) + return llvm::createStringError( + "cannot open gold reference library to compare"); + + // Compare the size for difference for now. + if (currentStat->getSize() != goldStat->getSize()) + return llvm::createStringError( + "plugin has changed since dependency scanning"); + + return remappedPath; + }; + for (auto &entry : Ctx.SearchPathOpts.PluginSearchOpts) { switch (entry.getKind()) { @@ -156,9 +193,17 @@ PluginLoader::getPluginMap() { // Respect resolved plugin config above other search path, and it can // overwrite plugins found by other options or previous resolved // configuration. - for (auto &moduleName : val.ModuleNames) - try_emplace(moduleName, val.LibraryPath, val.ExecutablePath, + for (auto &moduleName : val.ModuleNames) { + auto libPath = validateLibrary(val.LibraryPath); + if (!libPath) { + Ctx.Diags.diagnose(SourceLoc(), diag::resolved_macro_changed, + remapPath(val.LibraryPath), + toString(libPath.takeError())); + continue; + } + try_emplace(moduleName, *libPath, remapPath(val.ExecutablePath), /*overwrite*/ true); + } continue; } } diff --git a/lib/DependencyScan/ModuleDependencyScanner.cpp b/lib/DependencyScan/ModuleDependencyScanner.cpp index f032321f77371..dbe75b944e1c6 100644 --- a/lib/DependencyScan/ModuleDependencyScanner.cpp +++ b/lib/DependencyScan/ModuleDependencyScanner.cpp @@ -201,6 +201,7 @@ ModuleDependencyScanningWorker::ModuleDependencyScanningWorker( ScanASTContext.SourceMgr, Diagnostics)); auto loader = std::make_unique( *workerASTContext, /*DepTracker=*/nullptr, + workerCompilerInvocation->getFrontendOptions().CacheReplayPrefixMap, workerCompilerInvocation->getFrontendOptions().DisableSandbox); workerASTContext->setPluginLoader(std::move(loader)); diff --git a/lib/DependencyScan/ScanDependencies.cpp b/lib/DependencyScan/ScanDependencies.cpp index 12d900ffd7c54..d13e926e8a86a 100644 --- a/lib/DependencyScan/ScanDependencies.cpp +++ b/lib/DependencyScan/ScanDependencies.cpp @@ -214,9 +214,23 @@ class ExplicitModuleDependencyResolver { depInfo.addMacroDependency(macro.first(), macro.second.LibraryPath, macro.second.ExecutablePath); + bool needPathRemapping = instance.getInvocation() + .getSearchPathOptions() + .ResolvedPluginVerification && + cache.getScanService().hasPathMapping(); + auto mapPath = [&](StringRef path) { + if (!needPathRemapping) + return path.str(); + + return cache.getScanService().remapPath(path); + }; + if (needPathRemapping) + commandline.push_back("-resolved-plugin-verification"); + for (auto ¯o : depInfo.getMacroDependencies()) { - std::string arg = macro.second.LibraryPath + "#" + - macro.second.ExecutablePath + "#" + macro.first; + std::string arg = mapPath(macro.second.LibraryPath) + "#" + + mapPath(macro.second.ExecutablePath) + "#" + + macro.first; commandline.push_back("-load-resolved-plugin"); commandline.push_back(arg); } @@ -480,9 +494,10 @@ class ExplicitModuleDependencyResolver { llvm::for_each( sourceDep->auxiliaryFiles, [this](const std::string &file) { tracker->trackFile(file); }); - llvm::for_each(sourceDep->macroDependencies, [this](const auto &entry) { - tracker->trackFile(entry.second.LibraryPath); - }); + llvm::for_each(dependencyInfoCopy.getMacroDependencies(), + [this](const auto &entry) { + tracker->trackFile(entry.second.LibraryPath); + }); auto root = tracker->createTreeFromDependencies(); if (!root) return root.takeError(); @@ -496,7 +511,7 @@ class ExplicitModuleDependencyResolver { llvm::for_each( textualDep->auxiliaryFiles, [this](const std::string &file) { tracker->trackFile(file); }); - llvm::for_each(textualDep->macroDependencies, + llvm::for_each(dependencyInfoCopy.getMacroDependencies(), [this](const auto &entry) { tracker->trackFile(entry.second.LibraryPath); }); diff --git a/lib/Frontend/CachedDiagnostics.cpp b/lib/Frontend/CachedDiagnostics.cpp index b1b9c48780d63..3c2975c5add05 100644 --- a/lib/Frontend/CachedDiagnostics.cpp +++ b/lib/Frontend/CachedDiagnostics.cpp @@ -18,7 +18,9 @@ #include "swift/AST/DiagnosticBridge.h" #include "swift/AST/DiagnosticConsumer.h" +#include "swift/AST/DiagnosticsCommon.h" #include "swift/AST/DiagnosticsFrontend.h" +#include "swift/AST/DiagnosticsSema.h" #include "swift/Basic/Assertions.h" #include "swift/Basic/SourceManager.h" #include "swift/Frontend/Frontend.h" @@ -755,6 +757,13 @@ class CachingDiagnosticsProcessor::Implementation auto &Serializer = getSerializer(); assert(SM.getFileSystem() == Serializer.getSourceMgr().getFileSystem() && "Caching for a different file system"); + + // Bypass the caching. + if (BypassDiagIDs.count(Info.ID)) { + for (auto *Diag : OrigConsumers) + Diag->handleDiagnostic(Serializer.getSourceMgr(), Info); + return; + } Serializer.handleDiagnostic(SM, Info, [&](const DiagnosticInfo &Info) { for (auto *Diag : OrigConsumers) Diag->handleDiagnostic(Serializer.getSourceMgr(), Info); @@ -809,6 +818,8 @@ class CachingDiagnosticsProcessor::Implementation // Processor/Serializer alive until then. std::unique_ptr Serializer; + const llvm::SmallDenseSet BypassDiagIDs = {diag::macro_loaded.ID}; + SourceManager &InstanceSourceMgr; const FrontendInputsAndOutputs &InAndOut; DiagnosticEngine &Diags; diff --git a/lib/Frontend/CompilerInvocation.cpp b/lib/Frontend/CompilerInvocation.cpp index 32ff6a863f126..8fe58738fb7dc 100644 --- a/lib/Frontend/CompilerInvocation.cpp +++ b/lib/Frontend/CompilerInvocation.cpp @@ -2417,6 +2417,9 @@ static bool ParseSearchPathArgs(SearchPathOptions &Opts, ArgList &Args, Opts.ScannerPrefixMapper.push_back(Opt.str()); } + Opts.ResolvedPluginVerification |= + Args.hasArg(OPT_resolved_plugin_verification); + // rdar://132340493 disable scanner-side validation for non-caching builds Opts.ScannerModuleValidation |= Args.hasFlag(OPT_scanner_module_validation, OPT_no_scanner_module_validation, diff --git a/lib/Frontend/Frontend.cpp b/lib/Frontend/Frontend.cpp index c33c3d9812782..3d02ccca65824 100644 --- a/lib/Frontend/Frontend.cpp +++ b/lib/Frontend/Frontend.cpp @@ -878,6 +878,7 @@ bool CompilerInstance::setUpPluginLoader() { /// FIXME: If Invocation has 'PluginRegistry', we can set it. But should we? auto loader = std::make_unique( *Context, getDependencyTracker(), + Invocation.getFrontendOptions().CacheReplayPrefixMap, Invocation.getFrontendOptions().DisableSandbox); Context->setPluginLoader(std::move(loader)); return false; diff --git a/lib/Frontend/ModuleInterfaceLoader.cpp b/lib/Frontend/ModuleInterfaceLoader.cpp index 33d8d7ed084f6..98a9addcf21ec 100644 --- a/lib/Frontend/ModuleInterfaceLoader.cpp +++ b/lib/Frontend/ModuleInterfaceLoader.cpp @@ -1932,6 +1932,8 @@ InterfaceSubContextDelegateImpl::InterfaceSubContextDelegateImpl( // Load plugin libraries for macro expression as default arguments genericSubInvocation.getSearchPathOptions().PluginSearchOpts = searchPathOpts.PluginSearchOpts; + genericSubInvocation.getSearchPathOptions().ResolvedPluginVerification = + searchPathOpts.ResolvedPluginVerification; // Get module loading behavior options. genericSubInvocation.getSearchPathOptions().ScannerModuleValidation = searchPathOpts.ScannerModuleValidation; diff --git a/test/CAS/macro_plugin_external.swift b/test/CAS/macro_plugin_external.swift index f4688ba9f1904..7bed89ac31e4e 100644 --- a/test/CAS/macro_plugin_external.swift +++ b/test/CAS/macro_plugin_external.swift @@ -43,11 +43,78 @@ // RUN: %target-swift-frontend \ // RUN: -typecheck -verify -cache-compile-job -cas-path %t/cas \ // RUN: -swift-version 5 -disable-implicit-swift-modules \ -// RUN: -external-plugin-path %t/plugins/#%swift-plugin-server \ // RUN: -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import \ // RUN: -module-name MyApp -explicit-swift-module-map-file @%t/map.casid \ // RUN: %t/macro.swift @%t/MyApp.cmd +// RUN: %target-swift-frontend -scan-dependencies -module-load-mode prefer-serialized -module-name MyApp -module-cache-path %t/clang-module-cache -O \ +// RUN: -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import \ +// RUN: %t/macro.swift -o %t/deps2.json -swift-version 5 -cache-compile-job -cas-path %t/cas -external-plugin-path %t/plugins#%swift-plugin-server \ +// RUN: -scanner-prefix-map %t=/^test -scanner-prefix-map %swift-bin-dir=/^bin -resolved-plugin-verification + +// RUN: %S/Inputs/SwiftDepsExtractor.py %t/deps2.json MyApp casFSRootID > %t/fs.casid +// RUN: %cache-tool -cas-path %t/cas -cache-tool-action print-include-tree-list @%t/fs.casid | %FileCheck %s --check-prefix=FS-REMAP -DLIB=%target-library-name(MacroDefinition) + +/// CASFS is remapped. +// FS-REMAP: /^test/plugins/[[LIB]] + +// RUN: %S/Inputs/BuildCommandExtractor.py %t/deps2.json clang:SwiftShims > %t/SwiftShims2.cmd +// RUN: %swift_frontend_plain @%t/SwiftShims2.cmd + +// RUN: %S/Inputs/BuildCommandExtractor.py %t/deps2.json MyApp > %t/MyApp2.cmd +// RUN: %{python} %S/Inputs/GenerateExplicitModuleMap.py %t/deps2.json > %t/map2.json +// RUN: llvm-cas --cas %t/cas --make-blob --data %t/map2.json > %t/map2.casid + +/// Command-line is remapped. +// RUN: %FileCheck %s --check-prefix=CMD-REMAP --input-file=%t/MyApp2.cmd -DLIB=%target-library-name(MacroDefinition) + +// CMD-REMAP: -resolved-plugin-verification +// CMD-REMAP-NEXT: -load-resolved-plugin +// CMD-REMAP-NEXT: /^test/plugins/[[LIB]]#/^bin/swift-plugin-server#MacroDefinition + +// RUN: %target-swift-frontend \ +// RUN: -emit-module -o %t/Macro.swiftmodule -cache-compile-job -cas-path %t/cas \ +// RUN: -swift-version 5 -disable-implicit-swift-modules -O \ +// RUN: -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import \ +// RUN: -module-name MyApp -explicit-swift-module-map-file @%t/map2.casid -Rmacro-loading -Rcache-compile-job \ +// RUN: /^test/macro.swift @%t/MyApp2.cmd -cache-replay-prefix-map /^test=%t -cache-replay-prefix-map /^bin=%swift-bin-dir 2>&1 | %FileCheck %s --check-prefix=REMARK +// REMAKR: remark: cache miss +// REMARK: remark: loaded macro implementation module 'MacroDefinition' from compiler plugin server + +/// Encoded PLUGIN_SEARCH_OPTION is remapped. +// RUN: llvm-bcanalyzer -dump %t/Macro.swiftmodule | %FileCheck %s --check-prefix=MOD -DLIB=%target-library-name(MacroDefinition) + +// MOD: blob data = '/^test/plugins/[[LIB]]#/^bin/swift-plugin-server#MacroDefinition' + +/// Cache hit has no macro-loading remarks because no macro is actually loaded and the path might not be correct due to different mapping. +// RUN: %target-swift-frontend \ +// RUN: -emit-module -o %t/Macro.swiftmodule -cache-compile-job -cas-path %t/cas \ +// RUN: -swift-version 5 -disable-implicit-swift-modules -O \ +// RUN: -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import \ +// RUN: -module-name MyApp -explicit-swift-module-map-file @%t/map2.casid -Rmacro-loading -Rcache-compile-job \ +// RUN: /^test/macro.swift @%t/MyApp2.cmd -cache-replay-prefix-map /^test=%t -cache-replay-prefix-map /^bin=%swift-bin-dir 2>&1 | %FileCheck %s --check-prefix=NO-REMARK +// NO-REMARK: remark: replay output file +// NO-REMARK-NOT: remark: loaded macro implementation module 'MacroDefinition' from compiler plugin server + +/// Update timestamp, the build should still work. +// RUN: touch %t/plugins/%target-library-name(MacroDefinition) +// RUN: %target-swift-frontend \ +// RUN: -emit-module -o %t/Macro.swiftmodule -cache-compile-job -cas-path %t/cas \ +// RUN: -swift-version 5 -disable-implicit-swift-modules -O \ +// RUN: -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import \ +// RUN: -module-name MyApp -explicit-swift-module-map-file @%t/map2.casid -Rmacro-loading -Rcache-compile-job -cache-disable-replay \ +// RUN: /^test/macro.swift @%t/MyApp2.cmd -cache-replay-prefix-map /^test=%t -cache-replay-prefix-map /^bin=%swift-bin-dir + +/// Change the dylib content, this will fail the build. +// RUN: echo " " >> %t/plugins/%target-library-name(MacroDefinition) +// RUN: not %target-swift-frontend \ +// RUN: -emit-module -o %t/Macro.swiftmodule -cache-compile-job -cas-path %t/cas \ +// RUN: -swift-version 5 -disable-implicit-swift-modules -O \ +// RUN: -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import \ +// RUN: -module-name MyApp -explicit-swift-module-map-file @%t/map2.casid -Rmacro-loading -Rcache-compile-job -cache-disable-replay \ +// RUN: /^test/macro.swift @%t/MyApp2.cmd -cache-replay-prefix-map /^test=%t -cache-replay-prefix-map /^bin=%swift-bin-dir 2>&1 | %FileCheck %s --check-prefix=FAILED +// FAILED: plugin has changed since dependency scanning + //--- nomacro.swift func test() {}