diff --git a/clang-tools-extra/clangd/ModulesBuilder.cpp b/clang-tools-extra/clangd/ModulesBuilder.cpp index 97f67ddf5495a..2bce3a2082561 100644 --- a/clang-tools-extra/clangd/ModulesBuilder.cpp +++ b/clang-tools-extra/clangd/ModulesBuilder.cpp @@ -13,6 +13,8 @@ #include "clang/Frontend/FrontendActions.h" #include "clang/Serialization/ASTReader.h" #include "clang/Serialization/InMemoryModuleCache.h" +#include "llvm/ADT/ScopeExit.h" +#include namespace clang { namespace clangd { @@ -124,10 +126,58 @@ struct ModuleFile { llvm::sys::fs::remove(ModuleFilePath); } + StringRef getModuleName() const { return ModuleName; } + + StringRef getModuleFilePath() const { return ModuleFilePath; } + +private: std::string ModuleName; std::string ModuleFilePath; }; +// ReusablePrerequisiteModules - stands for PrerequisiteModules for which all +// the required modules are built successfully. All the module files +// are owned by the modules builder. +class ReusablePrerequisiteModules : public PrerequisiteModules { +public: + ReusablePrerequisiteModules() = default; + + ReusablePrerequisiteModules(const ReusablePrerequisiteModules &Other) = + default; + ReusablePrerequisiteModules & + operator=(const ReusablePrerequisiteModules &) = default; + ReusablePrerequisiteModules(ReusablePrerequisiteModules &&) = delete; + ReusablePrerequisiteModules + operator=(ReusablePrerequisiteModules &&) = delete; + + ~ReusablePrerequisiteModules() override = default; + + void adjustHeaderSearchOptions(HeaderSearchOptions &Options) const override { + // Appending all built module files. + for (const auto &RequiredModule : RequiredModules) + Options.PrebuiltModuleFiles.insert_or_assign( + RequiredModule->getModuleName().str(), + RequiredModule->getModuleFilePath().str()); + } + + bool canReuse(const CompilerInvocation &CI, + llvm::IntrusiveRefCntPtr) const override; + + bool isModuleUnitBuilt(llvm::StringRef ModuleName) const { + return BuiltModuleNames.contains(ModuleName); + } + + void addModuleFile(std::shared_ptr ModuleFile) { + BuiltModuleNames.insert(ModuleFile->getModuleName()); + RequiredModules.emplace_back(std::move(ModuleFile)); + } + +private: + llvm::SmallVector, 8> RequiredModules; + // A helper class to speedup the query if a module is built. + llvm::StringSet<> BuiltModuleNames; +}; + bool IsModuleFileUpToDate(PathRef ModuleFilePath, const PrerequisiteModules &RequisiteModules, llvm::IntrusiveRefCntPtr VFS) { @@ -192,89 +242,20 @@ bool IsModuleFilesUpToDate( }); } -// StandalonePrerequisiteModules - stands for PrerequisiteModules for which all -// the required modules are built successfully. All the module files -// are owned by the StandalonePrerequisiteModules class. -// -// Any of the built module files won't be shared with other instances of the -// class. So that we can avoid worrying thread safety. -// -// We don't need to worry about duplicated module names here since the standard -// guarantees the module names should be unique to a program. -class StandalonePrerequisiteModules : public PrerequisiteModules { -public: - StandalonePrerequisiteModules() = default; - - StandalonePrerequisiteModules(const StandalonePrerequisiteModules &) = delete; - StandalonePrerequisiteModules - operator=(const StandalonePrerequisiteModules &) = delete; - StandalonePrerequisiteModules(StandalonePrerequisiteModules &&) = delete; - StandalonePrerequisiteModules - operator=(StandalonePrerequisiteModules &&) = delete; - - ~StandalonePrerequisiteModules() override = default; - - void adjustHeaderSearchOptions(HeaderSearchOptions &Options) const override { - // Appending all built module files. - for (auto &RequiredModule : RequiredModules) - Options.PrebuiltModuleFiles.insert_or_assign( - RequiredModule.ModuleName, RequiredModule.ModuleFilePath); - } - - bool canReuse(const CompilerInvocation &CI, - llvm::IntrusiveRefCntPtr) const override; - - bool isModuleUnitBuilt(llvm::StringRef ModuleName) const { - return BuiltModuleNames.contains(ModuleName); - } - - void addModuleFile(llvm::StringRef ModuleName, - llvm::StringRef ModuleFilePath) { - RequiredModules.emplace_back(ModuleName, ModuleFilePath); - BuiltModuleNames.insert(ModuleName); - } - -private: - llvm::SmallVector RequiredModules; - // A helper class to speedup the query if a module is built. - llvm::StringSet<> BuiltModuleNames; -}; - -// Build a module file for module with `ModuleName`. The information of built -// module file are stored in \param BuiltModuleFiles. -llvm::Error buildModuleFile(llvm::StringRef ModuleName, - const GlobalCompilationDatabase &CDB, - const ThreadsafeFS &TFS, ProjectModules &MDB, - PathRef ModuleFilesPrefix, - StandalonePrerequisiteModules &BuiltModuleFiles) { - if (BuiltModuleFiles.isModuleUnitBuilt(ModuleName)) - return llvm::Error::success(); - - PathRef ModuleUnitFileName = MDB.getSourceForModuleName(ModuleName); - // It is possible that we're meeting third party modules (modules whose - // source are not in the project. e.g, the std module may be a third-party - // module for most projects) or something wrong with the implementation of - // ProjectModules. - // FIXME: How should we treat third party modules here? If we want to ignore - // third party modules, we should return true instead of false here. - // Currently we simply bail out. - if (ModuleUnitFileName.empty()) - return llvm::createStringError("Failed to get the primary source"); - +/// Build a module file for module with `ModuleName`. The information of built +/// module file are stored in \param BuiltModuleFiles. +llvm::Expected +buildModuleFile(llvm::StringRef ModuleName, PathRef ModuleUnitFileName, + const GlobalCompilationDatabase &CDB, const ThreadsafeFS &TFS, + const ReusablePrerequisiteModules &BuiltModuleFiles) { // Try cheap operation earlier to boil-out cheaply if there are problems. auto Cmd = CDB.getCompileCommand(ModuleUnitFileName); if (!Cmd) return llvm::createStringError( llvm::formatv("No compile command for {0}", ModuleUnitFileName)); - for (auto &RequiredModuleName : MDB.getRequiredModules(ModuleUnitFileName)) { - // Return early if there are errors building the module file. - if (llvm::Error Err = buildModuleFile(RequiredModuleName, CDB, TFS, MDB, - ModuleFilesPrefix, BuiltModuleFiles)) - return llvm::createStringError( - llvm::formatv("Failed to build dependency {0}: {1}", - RequiredModuleName, llvm::toString(std::move(Err)))); - } + llvm::SmallString<256> ModuleFilesPrefix = + getUniqueModuleFilesPath(ModuleUnitFileName); Cmd->Output = getModuleFilePath(ModuleName, ModuleFilesPrefix); @@ -316,15 +297,158 @@ llvm::Error buildModuleFile(llvm::StringRef ModuleName, if (Clang->getDiagnostics().hasErrorOccurred()) return llvm::createStringError("Compilation failed"); - BuiltModuleFiles.addModuleFile(ModuleName, Inputs.CompileCommand.Output); - return llvm::Error::success(); + return ModuleFile{ModuleName, Inputs.CompileCommand.Output}; } + +bool ReusablePrerequisiteModules::canReuse( + const CompilerInvocation &CI, + llvm::IntrusiveRefCntPtr VFS) const { + if (RequiredModules.empty()) + return true; + + llvm::SmallVector BMIPaths; + for (auto &MF : RequiredModules) + BMIPaths.push_back(MF->getModuleFilePath()); + return IsModuleFilesUpToDate(BMIPaths, *this, VFS); +} + +class ModuleFileCache { +public: + ModuleFileCache(const GlobalCompilationDatabase &CDB) : CDB(CDB) {} + const GlobalCompilationDatabase &getCDB() const { return CDB; } + + std::shared_ptr getModule(StringRef ModuleName); + + void add(StringRef ModuleName, std::shared_ptr ModuleFile) { + std::lock_guard Lock(ModuleFilesMutex); + + ModuleFiles[ModuleName] = ModuleFile; + } + + void remove(StringRef ModuleName); + +private: + const GlobalCompilationDatabase &CDB; + + llvm::StringMap> ModuleFiles; + // Mutex to guard accesses to ModuleFiles. + std::mutex ModuleFilesMutex; +}; + +std::shared_ptr +ModuleFileCache::getModule(StringRef ModuleName) { + std::lock_guard Lock(ModuleFilesMutex); + + auto Iter = ModuleFiles.find(ModuleName); + if (Iter == ModuleFiles.end()) + return nullptr; + + if (auto Res = Iter->second.lock()) + return Res; + + ModuleFiles.erase(Iter); + return nullptr; +} + +void ModuleFileCache::remove(StringRef ModuleName) { + std::lock_guard Lock(ModuleFilesMutex); + + ModuleFiles.erase(ModuleName); +} + +/// Collect the directly and indirectly required module names for \param +/// ModuleName in topological order. The \param ModuleName is guaranteed to +/// be the last element in \param ModuleNames. +llvm::SmallVector getAllRequiredModules(ProjectModules &MDB, + StringRef ModuleName) { + llvm::SmallVector ModuleNames; + llvm::StringSet<> ModuleNamesSet; + + auto VisitDeps = [&](StringRef ModuleName, auto Visitor) -> void { + ModuleNamesSet.insert(ModuleName); + + for (StringRef RequiredModuleName : + MDB.getRequiredModules(MDB.getSourceForModuleName(ModuleName))) + if (ModuleNamesSet.insert(RequiredModuleName).second) + Visitor(RequiredModuleName, Visitor); + + ModuleNames.push_back(ModuleName); + }; + VisitDeps(ModuleName, VisitDeps); + + return ModuleNames; +} + } // namespace +class ModulesBuilder::ModulesBuilderImpl { +public: + ModulesBuilderImpl(const GlobalCompilationDatabase &CDB) : Cache(CDB) {} + + const GlobalCompilationDatabase &getCDB() const { return Cache.getCDB(); } + + llvm::Error + getOrBuildModuleFile(StringRef ModuleName, const ThreadsafeFS &TFS, + ProjectModules &MDB, + ReusablePrerequisiteModules &BuiltModuleFiles); + +private: + ModuleFileCache Cache; +}; + +llvm::Error ModulesBuilder::ModulesBuilderImpl::getOrBuildModuleFile( + StringRef ModuleName, const ThreadsafeFS &TFS, ProjectModules &MDB, + ReusablePrerequisiteModules &BuiltModuleFiles) { + if (BuiltModuleFiles.isModuleUnitBuilt(ModuleName)) + return llvm::Error::success(); + + PathRef ModuleUnitFileName = MDB.getSourceForModuleName(ModuleName); + /// It is possible that we're meeting third party modules (modules whose + /// source are not in the project. e.g, the std module may be a third-party + /// module for most project) or something wrong with the implementation of + /// ProjectModules. + /// FIXME: How should we treat third party modules here? If we want to ignore + /// third party modules, we should return true instead of false here. + /// Currently we simply bail out. + if (ModuleUnitFileName.empty()) + return llvm::createStringError( + llvm::formatv("Don't get the module unit for module {0}", ModuleName)); + + // Get Required modules in topological order. + auto ReqModuleNames = getAllRequiredModules(MDB, ModuleName); + for (llvm::StringRef ReqModuleName : ReqModuleNames) { + if (BuiltModuleFiles.isModuleUnitBuilt(ModuleName)) + continue; + + if (auto Cached = Cache.getModule(ReqModuleName)) { + if (IsModuleFileUpToDate(Cached->getModuleFilePath(), BuiltModuleFiles, + TFS.view(std::nullopt))) { + log("Reusing module {0} from {1}", ModuleName, + Cached->getModuleFilePath()); + BuiltModuleFiles.addModuleFile(std::move(Cached)); + continue; + } + Cache.remove(ReqModuleName); + } + + llvm::Expected MF = buildModuleFile( + ModuleName, ModuleUnitFileName, getCDB(), TFS, BuiltModuleFiles); + if (llvm::Error Err = MF.takeError()) + return Err; + + log("Built module {0} to {1}", ModuleName, MF->getModuleFilePath()); + auto BuiltModuleFile = std::make_shared(std::move(*MF)); + Cache.add(ModuleName, BuiltModuleFile); + BuiltModuleFiles.addModuleFile(std::move(BuiltModuleFile)); + } + + return llvm::Error::success(); +} + std::unique_ptr ModulesBuilder::buildPrerequisiteModulesFor(PathRef File, - const ThreadsafeFS &TFS) const { - std::unique_ptr MDB = CDB.getProjectModules(File); + const ThreadsafeFS &TFS) { + std::unique_ptr MDB = Impl->getCDB().getProjectModules(File); if (!MDB) { elog("Failed to get Project Modules information for {0}", File); return std::make_unique(); @@ -332,42 +456,27 @@ ModulesBuilder::buildPrerequisiteModulesFor(PathRef File, std::vector RequiredModuleNames = MDB->getRequiredModules(File); if (RequiredModuleNames.empty()) - return std::make_unique(); - - llvm::SmallString<256> ModuleFilesPrefix = getUniqueModuleFilesPath(File); - - log("Trying to build required modules for {0} in {1}", File, - ModuleFilesPrefix); - - auto RequiredModules = std::make_unique(); + return std::make_unique(); + auto RequiredModules = std::make_unique(); for (llvm::StringRef RequiredModuleName : RequiredModuleNames) { // Return early if there is any error. - if (llvm::Error Err = - buildModuleFile(RequiredModuleName, CDB, TFS, *MDB.get(), - ModuleFilesPrefix, *RequiredModules.get())) { + if (llvm::Error Err = Impl->getOrBuildModuleFile( + RequiredModuleName, TFS, *MDB.get(), *RequiredModules.get())) { elog("Failed to build module {0}; due to {1}", RequiredModuleName, toString(std::move(Err))); return std::make_unique(); } } - log("Built required modules for {0} in {1}", File, ModuleFilesPrefix); - return std::move(RequiredModules); } -bool StandalonePrerequisiteModules::canReuse( - const CompilerInvocation &CI, - llvm::IntrusiveRefCntPtr VFS) const { - if (RequiredModules.empty()) - return true; - - SmallVector BMIPaths; - for (auto &MF : RequiredModules) - BMIPaths.push_back(MF.ModuleFilePath); - return IsModuleFilesUpToDate(BMIPaths, *this, VFS); +ModulesBuilder::ModulesBuilder(const GlobalCompilationDatabase &CDB) { + Impl = std::make_unique(CDB); } +ModulesBuilder::~ModulesBuilder() {} + } // namespace clangd } // namespace clang diff --git a/clang-tools-extra/clangd/ModulesBuilder.h b/clang-tools-extra/clangd/ModulesBuilder.h index 0514e7486475d..f40a9006e9169 100644 --- a/clang-tools-extra/clangd/ModulesBuilder.h +++ b/clang-tools-extra/clangd/ModulesBuilder.h @@ -85,7 +85,8 @@ class PrerequisiteModules { /// different versions and different source files. class ModulesBuilder { public: - ModulesBuilder(const GlobalCompilationDatabase &CDB) : CDB(CDB) {} + ModulesBuilder(const GlobalCompilationDatabase &CDB); + ~ModulesBuilder(); ModulesBuilder(const ModulesBuilder &) = delete; ModulesBuilder(ModulesBuilder &&) = delete; @@ -94,10 +95,11 @@ class ModulesBuilder { ModulesBuilder &operator=(ModulesBuilder &&) = delete; std::unique_ptr - buildPrerequisiteModulesFor(PathRef File, const ThreadsafeFS &TFS) const; + buildPrerequisiteModulesFor(PathRef File, const ThreadsafeFS &TFS); private: - const GlobalCompilationDatabase &CDB; + class ModulesBuilderImpl; + std::unique_ptr Impl; }; } // namespace clangd diff --git a/clang-tools-extra/clangd/unittests/PrerequisiteModulesTest.cpp b/clang-tools-extra/clangd/unittests/PrerequisiteModulesTest.cpp index 691a93e7acd0a..1bb8e19cce23e 100644 --- a/clang-tools-extra/clangd/unittests/PrerequisiteModulesTest.cpp +++ b/clang-tools-extra/clangd/unittests/PrerequisiteModulesTest.cpp @@ -482,6 +482,78 @@ void func() { EXPECT_EQ(Result.signatures[0].parameters[0].labelString, "int a"); } +TEST_F(PrerequisiteModulesTests, ReusablePrerequisiteModulesTest) { + MockDirectoryCompilationDatabase CDB(TestDir, FS); + + CDB.addFile("M.cppm", R"cpp( +export module M; +export int M = 43; + )cpp"); + CDB.addFile("A.cppm", R"cpp( +export module A; +import M; +export int A = 43 + M; + )cpp"); + CDB.addFile("B.cppm", R"cpp( +export module B; +import M; +export int B = 44 + M; + )cpp"); + + ModulesBuilder Builder(CDB); + + auto AInfo = Builder.buildPrerequisiteModulesFor(getFullPath("A.cppm"), FS); + EXPECT_TRUE(AInfo); + auto BInfo = Builder.buildPrerequisiteModulesFor(getFullPath("B.cppm"), FS); + EXPECT_TRUE(BInfo); + HeaderSearchOptions HSOptsA(TestDir); + HeaderSearchOptions HSOptsB(TestDir); + AInfo->adjustHeaderSearchOptions(HSOptsA); + BInfo->adjustHeaderSearchOptions(HSOptsB); + + EXPECT_FALSE(HSOptsA.PrebuiltModuleFiles.empty()); + EXPECT_FALSE(HSOptsB.PrebuiltModuleFiles.empty()); + + // Check that we're reusing the module files. + EXPECT_EQ(HSOptsA.PrebuiltModuleFiles, HSOptsB.PrebuiltModuleFiles); + + // Update M.cppm to check if the modules builder can update correctly. + CDB.addFile("M.cppm", R"cpp( +export module M; +export constexpr int M = 43; + )cpp"); + + ParseInputs AUse = getInputs("A.cppm", CDB); + AUse.ModulesManager = &Builder; + std::unique_ptr AInvocation = + buildCompilerInvocation(AUse, DiagConsumer); + EXPECT_FALSE(AInfo->canReuse(*AInvocation, FS.view(TestDir))); + + ParseInputs BUse = getInputs("B.cppm", CDB); + AUse.ModulesManager = &Builder; + std::unique_ptr BInvocation = + buildCompilerInvocation(BUse, DiagConsumer); + EXPECT_FALSE(BInfo->canReuse(*BInvocation, FS.view(TestDir))); + + auto NewAInfo = + Builder.buildPrerequisiteModulesFor(getFullPath("A.cppm"), FS); + auto NewBInfo = + Builder.buildPrerequisiteModulesFor(getFullPath("B.cppm"), FS); + EXPECT_TRUE(NewAInfo); + EXPECT_TRUE(NewBInfo); + HeaderSearchOptions NewHSOptsA(TestDir); + HeaderSearchOptions NewHSOptsB(TestDir); + NewAInfo->adjustHeaderSearchOptions(NewHSOptsA); + NewBInfo->adjustHeaderSearchOptions(NewHSOptsB); + + EXPECT_FALSE(NewHSOptsA.PrebuiltModuleFiles.empty()); + EXPECT_FALSE(NewHSOptsB.PrebuiltModuleFiles.empty()); + + EXPECT_EQ(NewHSOptsA.PrebuiltModuleFiles, NewHSOptsB.PrebuiltModuleFiles); + // Check that we didn't reuse the old and stale module files. + EXPECT_NE(NewHSOptsA.PrebuiltModuleFiles, HSOptsA.PrebuiltModuleFiles); +} + } // namespace } // namespace clang::clangd