From 925fc1ba1ea4980066b1dfb72311a5f9eb322ef9 Mon Sep 17 00:00:00 2001 From: Kyungwoo Lee Date: Sun, 29 Sep 2024 22:58:52 -0700 Subject: [PATCH 1/2] [NFC] Refactor FileCache - Turn it into a type from a function. - Store the cache directory for the future use. --- llvm/include/llvm/LTO/LTO.h | 2 +- llvm/include/llvm/Support/Caching.h | 22 +++++++++++++++++++++- llvm/lib/LTO/LTO.cpp | 2 +- llvm/lib/Support/Caching.cpp | 5 +++-- 4 files changed, 26 insertions(+), 5 deletions(-) diff --git a/llvm/include/llvm/LTO/LTO.h b/llvm/include/llvm/LTO/LTO.h index 214aa4e1c562d..a281c377f2601 100644 --- a/llvm/include/llvm/LTO/LTO.h +++ b/llvm/include/llvm/LTO/LTO.h @@ -298,7 +298,7 @@ class LTO { /// /// The client will receive at most one callback (via either AddStream or /// Cache) for each task identifier. - Error run(AddStreamFn AddStream, FileCache Cache = nullptr); + Error run(AddStreamFn AddStream, FileCache Cache = {}); /// Static method that returns a list of libcall symbols that can be generated /// by LTO but might not be visible from bitcode symbol table. diff --git a/llvm/include/llvm/Support/Caching.h b/llvm/include/llvm/Support/Caching.h index 4fa57cc92e51f..cc86d1583fd6e 100644 --- a/llvm/include/llvm/Support/Caching.h +++ b/llvm/include/llvm/Support/Caching.h @@ -54,9 +54,29 @@ using AddStreamFn = std::function>( /// /// if (AddStreamFn AddStream = Cache(Task, Key, ModuleName)) /// ProduceContent(AddStream); -using FileCache = std::function( +using FileCacheFunction = std::function( unsigned Task, StringRef Key, const Twine &ModuleName)>; +struct FileCache { + FileCache(FileCacheFunction CacheFn, const std::string &DirectoryPath) + : CacheFunction(std::move(CacheFn)), CacheDirectoryPath(DirectoryPath) {} + FileCache() = default; + + Expected operator()(unsigned Task, StringRef Key, + const Twine &ModuleName) { + assert(isValid() && "Invalid cache function"); + return CacheFunction(Task, Key, ModuleName); + } + const std::string &getCacheDirectoryPath() const { + return CacheDirectoryPath; + } + bool isValid() const { return static_cast(CacheFunction); } + +private: + FileCacheFunction CacheFunction = nullptr; + std::string CacheDirectoryPath; +}; + /// This type defines the callback to add a pre-existing file (e.g. in a cache). /// /// Buffer callbacks must be thread safe. diff --git a/llvm/lib/LTO/LTO.cpp b/llvm/lib/LTO/LTO.cpp index a88124dacfaef..be49b447f7dcf 100644 --- a/llvm/lib/LTO/LTO.cpp +++ b/llvm/lib/LTO/LTO.cpp @@ -1483,7 +1483,7 @@ class InProcessThinBackend : public ThinBackendProc { return E; } - if (!Cache || !CombinedIndex.modulePaths().count(ModuleID) || + if (!Cache.isValid() || !CombinedIndex.modulePaths().count(ModuleID) || all_of(CombinedIndex.getModuleHash(ModuleID), [](uint32_t V) { return V == 0; })) // Cache disabled or no entry for this module in the combined index or diff --git a/llvm/lib/Support/Caching.cpp b/llvm/lib/Support/Caching.cpp index 1ef51db218e89..66e540efaca97 100644 --- a/llvm/lib/Support/Caching.cpp +++ b/llvm/lib/Support/Caching.cpp @@ -37,8 +37,8 @@ Expected llvm::localCache(const Twine &CacheNameRef, TempFilePrefixRef.toVector(TempFilePrefix); CacheDirectoryPathRef.toVector(CacheDirectoryPath); - return [=](unsigned Task, StringRef Key, - const Twine &ModuleName) -> Expected { + auto Func = [=](unsigned Task, StringRef Key, + const Twine &ModuleName) -> Expected { // This choice of file name allows the cache to be pruned (see pruneCache() // in include/llvm/Support/CachePruning.h). SmallString<64> EntryPath; @@ -167,4 +167,5 @@ Expected llvm::localCache(const Twine &CacheNameRef, Task); }; }; + return FileCache(Func, CacheDirectoryPathRef.str()); } From 85d94e35e57258db37108d7299fbf15659bc3908 Mon Sep 17 00:00:00 2001 From: Kyungwoo Lee Date: Thu, 3 Oct 2024 11:21:42 -0700 Subject: [PATCH 2/2] Address comments from ellishg --- llvm/include/llvm/Support/Caching.h | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/llvm/include/llvm/Support/Caching.h b/llvm/include/llvm/Support/Caching.h index cc86d1583fd6e..cf45145619d95 100644 --- a/llvm/include/llvm/Support/Caching.h +++ b/llvm/include/llvm/Support/Caching.h @@ -41,22 +41,29 @@ class CachedFileStream { using AddStreamFn = std::function>( unsigned Task, const Twine &ModuleName)>; -/// This is the type of a file cache. To request an item from the cache, pass a -/// unique string as the Key. For hits, the cached file will be added to the -/// link and this function will return AddStreamFn(). For misses, the cache will -/// return a stream callback which must be called at most once to produce -/// content for the stream. The file stream produced by the stream callback will -/// add the file to the link after the stream is written to. ModuleName is the -/// unique module identifier for the bitcode module the cache is being checked -/// for. +/// This is a callable that manages file caching operations. It accepts a task +/// ID \p Task, a unique key \p Key, and a module name \p ModuleName, and +/// returns AddStreamFn(). This function determines whether a cache hit or miss +/// occurs and handles the appropriate actions. +using FileCacheFunction = std::function( + unsigned Task, StringRef Key, const Twine &ModuleName)>; + +/// This type represents a file cache system that manages caching of files. +/// It encapsulates a caching function and the directory path where the cache is +/// stored. To request an item from the cache, pass a unique string as the Key. +/// For hits, the cached file will be added to the link and this function will +/// return AddStreamFn(). For misses, the cache will return a stream callback +/// which must be called at most once to produce content for the stream. The +/// file stream produced by the stream callback will add the file to the link +/// after the stream is written to. ModuleName is the unique module identifier +/// for the bitcode module the cache is being checked for. /// /// Clients generally look like this: /// /// if (AddStreamFn AddStream = Cache(Task, Key, ModuleName)) /// ProduceContent(AddStream); -using FileCacheFunction = std::function( - unsigned Task, StringRef Key, const Twine &ModuleName)>; - +/// +/// CacheDirectoryPath stores the directory path where cached files are kept. struct FileCache { FileCache(FileCacheFunction CacheFn, const std::string &DirectoryPath) : CacheFunction(std::move(CacheFn)), CacheDirectoryPath(DirectoryPath) {}