Merge pull request #66940 from cachemeifyoucan/eng/PR-dep-sharing-serivce-issue

[CAS][DependencyScanning] Don't keep a shared state of common file deps

Author: cachemeifyoucan

include/swift/AST/ModuleDependencies.h

@@ -18,8 +18,9 @@
 #ifndef SWIFT_AST_MODULE_DEPENDENCIES_H
 #define SWIFT_AST_MODULE_DEPENDENCIES_H
 
-#include "swift/Basic/LLVM.h"
 #include "swift/AST/Import.h"
+#include "swift/AST/SearchPathOptions.h"
+#include "swift/Basic/LLVM.h"
 #include "clang/CAS/CASOptions.h"
 #include "clang/Tooling/DependencyScanning/DependencyScanningService.h"
 #include "clang/Tooling/DependencyScanning/DependencyScanningTool.h"
@@ -31,12 +32,12 @@
 #include "llvm/CAS/CASReference.h"
 #include "llvm/CAS/CachingOnDiskFileSystem.h"
 #include "llvm/CAS/ObjectStore.h"
-#include "llvm/Support/Mutex.h"
 #include "llvm/Support/Error.h"
+#include "llvm/Support/Mutex.h"
 #include "llvm/Support/VirtualFileSystem.h"
 #include <string>
-#include <vector>
 #include <unordered_map>
+#include <vector>
 
 namespace swift {
 
@@ -733,17 +734,16 @@ using ModuleDependenciesKindRefMap =
 /// Track swift dependency
 class SwiftDependencyTracker {
 public:
-  SwiftDependencyTracker(llvm::cas::CachingOnDiskFileSystem &FS,
-                         const std::vector<std::string> &CommonFiles)
-      : FS(FS.createProxyFS()), Files(CommonFiles) {}
+  SwiftDependencyTracker(llvm::cas::CachingOnDiskFileSystem &FS)
+      : FS(FS.createProxyFS()) {}
 
   void startTracking();
+  void addCommonSearchPathDeps(const SearchPathOptions& Opts);
   void trackFile(const Twine &path) { (void)FS->status(path); }
   llvm::Expected<llvm::cas::ObjectProxy> createTreeFromDependencies();
 
 private:
   llvm::IntrusiveRefCntPtr<llvm::cas::CachingOnDiskFileSystem> FS;
-  const std::vector<std::string> &Files;
 };
 
 // MARK: SwiftDependencyScanningService
@@ -848,7 +848,7 @@ class SwiftDependencyScanningService {
     if (!CacheFS)
       return llvm::None;
 
-    return SwiftDependencyTracker(*CacheFS, CommonDependencyFiles);
+    return SwiftDependencyTracker(*CacheFS);
   }
 
   llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> getClangScanningFS() const {

lib/AST/ModuleDependencies.cpp

@@ -394,11 +394,31 @@ SwiftDependencyScanningService::SwiftDependencyScanningService() {
   SharedFilesystemCache.emplace();
 }
 
+void SwiftDependencyTracker::addCommonSearchPathDeps(
+    const SearchPathOptions &Opts) {
+  // Add SDKSetting file.
+  SmallString<256> SDKSettingPath;
+  llvm::sys::path::append(SDKSettingPath, Opts.getSDKPath(),
+                          "SDKSettings.json");
+  FS->status(SDKSettingPath);
+
+  // Add Legacy layout file.
+  const std::vector<std::string> AllSupportedArches = {
+      "arm64", "arm64e", "x86_64", "i386",
+      "armv7", "armv7s", "armv7k", "arm64_32"};
+
+  for (auto RuntimeLibPath : Opts.RuntimeLibraryPaths) {
+    std::error_code EC;
+    for (auto &Arch : AllSupportedArches) {
+      SmallString<256> LayoutFile(RuntimeLibPath);
+      llvm::sys::path::append(LayoutFile, "layout-" + Arch + ".yaml");
+      FS->status(LayoutFile);
+    }
+  }
+}
+
 void SwiftDependencyTracker::startTracking() {
   FS->trackNewAccesses();
-
-  for (auto &file : Files)
-    (void)FS->status(file);
 }
 
 llvm::Expected<llvm::cas::ObjectProxy>
@@ -459,11 +479,6 @@ bool SwiftDependencyScanningService::setupCachingDependencyScanningService(
     }
   }
 
-  // Fetch some dependency files from clang importer.
-  auto clangImporter = static_cast<ClangImporter *>(
-      Instance.getASTContext().getClangModuleLoader());
-  clangImporter->addClangInvovcationDependencies(CommonDependencyFiles);
-
   auto CachingFS =
       llvm::cas::createCachingOnDiskFileSystem(Instance.getObjectStore());
   if (!CachingFS) {

lib/DependencyScan/ScanDependencies.cpp

@@ -1626,7 +1626,7 @@ forEachBatchEntry(CompilerInstance &invocationInstance,
   return false;
 }
 
-static ModuleDependencyInfo identifyMainModuleDependencies(
+static llvm::ErrorOr<ModuleDependencyInfo> identifyMainModuleDependencies(
     CompilerInstance &instance,
     llvm::Optional<SwiftDependencyTracker> tracker = llvm::None) {
   ModuleDecl *mainModule = instance.getMainModule();
@@ -1659,8 +1659,23 @@ static ModuleDependencyInfo identifyMainModuleDependencies(
         continue;
       tracker->trackFile(sf->getFilename());
     }
-    auto root = cantFail(tracker->createTreeFromDependencies());
-    rootID = root.getID().toString();
+    tracker->addCommonSearchPathDeps(
+        instance.getInvocation().getSearchPathOptions());
+    // Fetch some dependency files from clang importer.
+    std::vector<std::string> clangDependencyFiles;
+    auto clangImporter = static_cast<ClangImporter *>(
+        instance.getASTContext().getClangModuleLoader());
+    clangImporter->addClangInvovcationDependencies(clangDependencyFiles);
+    llvm::for_each(clangDependencyFiles,
+                   [&](std::string &file) { tracker->trackFile(file); });
+
+    auto root = tracker->createTreeFromDependencies();
+    if (!root) {
+      instance.getASTContext().Diags.diagnose(SourceLoc(), diag::error_cas,
+                                              toString(root.takeError()));
+      return std::make_error_code(std::errc::io_error);
+    }
+    rootID = root->getID().toString();
   }
 
   auto mainDependencies =
@@ -1927,11 +1942,14 @@ swift::dependencies::performModuleScan(CompilerInstance &instance,
   // First, identify the dependencies of the main module
   auto mainDependencies = identifyMainModuleDependencies(
       instance, cache.getScanService().createSwiftDependencyTracker());
+  if (!mainDependencies)
+    return mainDependencies.getError();
   auto &ctx = instance.getASTContext();
 
   // Add the main module.
   StringRef mainModuleName = mainModule->getNameStr();
-  auto mainModuleID = ModuleDependencyID{mainModuleName.str(), mainDependencies.getKind()};
+  auto mainModuleID =
+      ModuleDependencyID{mainModuleName.str(), mainDependencies->getKind()};
 
   ModuleDependencyIDSetVector allModules;
   allModules.insert(mainModuleID);
@@ -1943,9 +1961,9 @@ swift::dependencies::performModuleScan(CompilerInstance &instance,
     cache.updateDependency(
         std::make_pair(mainModuleName.str(),
                        ModuleDependencyKind::SwiftSource),
-        std::move(mainDependencies));
+        std::move(*mainDependencies));
   } else {
-    cache.recordDependency(mainModuleName, std::move(mainDependencies));
+    cache.recordDependency(mainModuleName, std::move(*mainDependencies));
   }
 
   auto ModuleCachePath = getModuleCachePathFromClang(
@@ -2051,8 +2069,10 @@ llvm::ErrorOr<swiftscan_import_set_t>
 swift::dependencies::performModulePrescan(CompilerInstance &instance) {
   // Execute import prescan, and write JSON output to the output stream
   auto mainDependencies = identifyMainModuleDependencies(instance);
+  if (!mainDependencies)
+    return mainDependencies.getError();
   auto *importSet = new swiftscan_import_set_s;
-  importSet->imports = create_set(mainDependencies.getModuleImports());
+  importSet->imports = create_set(mainDependencies->getModuleImports());
   return importSet;
 }
 

lib/Serialization/ModuleDependencyScanner.cpp

@@ -174,6 +174,14 @@ ErrorOr<ModuleDependencyInfo> ModuleDependencyScanner::scanInterfaceFile(
     std::string RootID;
     if (dependencyTracker) {
       dependencyTracker->startTracking();
+      dependencyTracker->addCommonSearchPathDeps(Ctx.SearchPathOpts);
+      std::vector<std::string> clangDependencyFiles;
+      auto clangImporter =
+          static_cast<ClangImporter *>(Ctx.getClangModuleLoader());
+      clangImporter->addClangInvovcationDependencies(clangDependencyFiles);
+      llvm::for_each(clangDependencyFiles, [&](std::string &file) {
+        dependencyTracker->trackFile(file);
+      });
       dependencyTracker->trackFile(moduleInterfacePath);
       auto RootOrError = dependencyTracker->createTreeFromDependencies();
       if (!RootOrError)

test/CAS/module_deps_clang_extras.swift

@@ -0,0 +1,48 @@
+// REQUIRES: objc_interop
+
+// RUN: %empty-directory(%t)
+// RUN: mkdir -p %t/clang-module-cache
+// RUN: mkdir -p %t/cas
+// RUN: split-file %s %t
+// RUN: %hmaptool write %t/hmap.json %t/empty.hmap
+
+// RUN: %target-swift-frontend -scan-dependencies -module-cache-path %t/clang-module-cache \
+// RUN:   %t/Test.swift -o %t/deps.json -cache-compile-job -cas-path %t/cas -clang-include-tree \
+// RUN:   -Xcc -fmodule-map-file=%t/module.modulemap -Xcc -ivfsoverlay -Xcc %t/empty.yaml \
+// RUN:   -Xcc -I%t/empty.hmap
+
+// RUN: %S/Inputs/SwiftDepsExtractor.py %t/deps.json deps casFSRootID > %t/fs.casid
+// RUN: llvm-cas --cas %t/cas --ls-tree-recursive @%t/fs.casid | %FileCheck %s -DDIR=%basename_t -check-prefix FS_ROOT
+// RUN: %S/Inputs/SwiftDepsExtractor.py %t/deps.json clang:Dummy clangIncludeTree > %t/tree.casid
+// RUN: clang-cas-test --cas %t/cas --print-include-tree @%t/tree.casid | %FileCheck %s -DDIR=%basename_t -check-prefix INCLUDE_TREE
+
+// FS_ROOT: [[DIR]].tmp/empty.hmap
+// FS_ROOT: [[DIR]].tmp/empty.yaml
+
+// INCLUDE_TREE: [[DIR]].tmp/Dummy.h
+
+//--- Test.swift
+import Dummy
+func test() {}
+
+//--- module.modulemap
+module Dummy {
+ umbrella header "Dummy.h"
+}
+
+//--- Dummy.h
+void dummy(void);
+
+//--- hmap.json
+{
+  "mappings": {} 
+}
+
+//--- empty.yaml
+{
+  "version": 0,
+  "case-sensitive": "false",
+  "use-external-names": true,
+  "roots": []
+}
+

test/lit.cfg

@@ -2640,6 +2640,7 @@ config.substitutions.append(('%target-resilience-test', config.target_resilience
 
 config.substitutions.append(('%llvm-profdata', config.llvm_profdata))
 config.substitutions.append(('%llvm-cov', config.llvm_cov))
+config.substitutions.append(('%hmaptool', os.path.join(config.llvm_src_root, '..', 'clang', 'utils', 'hmaptool', 'hmaptool')))
 
 # Set up the host library environment.
 if hasattr(config, 'target_library_path_var'):