Cherry-pick ([Clang][ScanDeps] Add C API and implement enough to build Foundation.) and follow up fixes into stable 20200108

clang/include/clang-c/Dependencies.h

@@ -0,0 +1,223 @@
+/*==-- clang-c/Dependencies.h - Dependency Discovery C Interface --*- C -*-===*\
+|*                                                                            *|
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM          *|
+|* Exceptions.                                                                *|
+|* See https://llvm.org/LICENSE.txt for license information.                  *|
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception                    *|
+|*                                                                            *|
+|*===----------------------------------------------------------------------===*|
+|*                                                                            *|
+|* This header provides a dependency discovery interface similar to           *|
+|* clang-scan-deps.                                                           *|
+|*                                                                            *|
+|* An example of its usage is available in c-index-test/core_main.cpp.        *|
+|*                                                                            *|
+|* EXPERIMENTAL: These interfaces are experimental and will change. If you    *|
+|* use these be prepared for them to change without notice on any commit.     *|
+|*                                                                            *|
+\*===----------------------------------------------------------------------===*/
+
+#ifndef LLVM_CLANG_C_DEPENDENCIES_H
+#define LLVM_CLANG_C_DEPENDENCIES_H
+
+#include "clang-c/BuildSystem.h"
+#include "clang-c/CXErrorCode.h"
+#include "clang-c/CXString.h"
+#include "clang-c/Platform.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * \defgroup SCAN_DEPS Dependency scanning service.
+ * @{
+ */
+
+typedef struct {
+  CXString Name;
+  /**
+   * The context hash of a module represents the set of compiler options that
+   * may make one version of a module incompatible from another. This includes
+   * things like language mode, predefined macros, header search paths, etc...
+   *
+   * Modules with the same name but a different \c ContextHash should be treated
+   * as separate modules for the purpose of a build.
+   */
+  CXString ContextHash;
+
+  /**
+   * The path to the modulemap file which defines this module.
+   *
+   * This can be used to explicitly build this module. This file will
+   * additionally appear in \c FileDeps as a dependency.
+   */
+  CXString ModuleMapPath;
+
+  /**
+   * The list of files which this module directly depends on.
+   *
+   * If any of these change then the module needs to be rebuilt.
+   */
+  CXStringSet *FileDeps;
+
+  /**
+   * The list of modules which this module direct depends on.
+   *
+   * This does include the context hash. The format is
+   * `<module-name>:<context-hash>`
+   */
+  CXStringSet *ModuleDeps;
+
+  /**
+   * The full command line needed to build this module.
+   *
+   * Not including `-fmodule-file=` or `-o`.
+   */
+  CXStringSet *BuildArguments;
+} CXModuleDependency;
+
+typedef struct {
+  int Count;
+  CXModuleDependency *Modules;
+} CXModuleDependencySet;
+
+/**
+ * See \c CXModuleDependency for the meaning of these fields, with the addition
+ * that they represent only the direct dependencies for \c CXDependencyMode_Full
+ * mode.
+ */
+typedef struct {
+  CXString ContextHash;
+  CXStringSet *FileDeps;
+  CXStringSet *ModuleDeps;
+
+  /**
+   * Additional arguments to append to the build of this file.
+   *
+   * This contains things like disabling implicit modules. This does not include
+   * the `-fmodule-file=` arguments that are needed.
+   */
+  CXStringSet *AdditionalArguments;
+} CXFileDependencies;
+
+CINDEX_LINKAGE void
+clang_experimental_ModuleDependencySet_dispose(CXModuleDependencySet *MD);
+
+CINDEX_LINKAGE void
+clang_experimental_FileDependencies_dispose(CXFileDependencies *ID);
+
+/**
+ * Object encapsulating instance of a dependency scanner service.
+ *
+ * The dependency scanner service is a global instance that owns the
+ * global cache and other global state that's shared between the dependency
+ * scanner workers. The service APIs are thread safe.
+ */
+typedef struct CXOpaqueDependencyScannerService *CXDependencyScannerService;
+
+/**
+ * The mode to report module dependencies in.
+ */
+typedef enum {
+  /**
+   * Flatten all module dependencies. This reports the full transitive set of
+   * header and module map dependencies needed to do an implicit module build.
+   */
+  CXDependencyMode_Flat,
+
+  /**
+   * Report the full module graph. This reports only the direct dependencies of
+   * each file, and calls a callback for each module that is discovered.
+   */
+  CXDependencyMode_Full,
+} CXDependencyMode;
+
+/**
+ * Create a \c CXDependencyScannerService object.
+ * Must be disposed with \c clang_DependencyScannerService_dispose().
+ */
+CINDEX_LINKAGE CXDependencyScannerService
+clang_experimental_DependencyScannerService_create_v0(CXDependencyMode Format);
+
+/**
+ * Dispose of a \c CXDependencyScannerService object.
+ *
+ * The service object must be disposed of after the workers are disposed of.
+ */
+CINDEX_LINKAGE void clang_experimental_DependencyScannerService_dispose_v0(
+    CXDependencyScannerService);
+
+/**
+ * Object encapsulating instance of a dependency scanner worker.
+ *
+ * The dependency scanner workers are expected to be used in separate worker
+ * threads. An individual worker is not thread safe.
+ *
+ * Operations on a worker are not thread-safe and should only be used from a
+ * single thread at a time. They are intended to be used by a single dedicated
+ * thread in a thread pool, but they are not inherently pinned to a thread.
+ */
+typedef struct CXOpaqueDependencyScannerWorker *CXDependencyScannerWorker;
+
+/**
+ * Create a \c CXDependencyScannerWorker object.
+ * Must be disposed with
+ * \c clang_experimental_DependencyScannerWorker_dispose_v0().
+ */
+CINDEX_LINKAGE CXDependencyScannerWorker
+    clang_experimental_DependencyScannerWorker_create_v0(
+        CXDependencyScannerService);
+
+CINDEX_LINKAGE void clang_experimental_DependencyScannerWorker_dispose_v0(
+    CXDependencyScannerWorker);
+
+/**
+ * A callback that is called whenever a module is discovered when in
+ * \c CXDependencyMode_Full mode.
+ *
+ * \param Context the context that was passed to
+ *         \c clang_experimental_DependencyScannerWorker_getFileDependencies_v0.
+ * \param MDS the list of discovered modules. Must be freed by calling
+ *            \c clang_experimental_ModuleDependencySet_dispose.
+ */
+typedef void CXModuleDiscoveredCallback(void *Context,
+                                        CXModuleDependencySet *MDS);
+
+/**
+ * Returns the list of file dependencies for a particular compiler invocation.
+ *
+ * \param argc the number of compiler invocation arguments (including argv[0]).
+ * \param argv the compiler invocation arguments (including argv[0]).
+ *             the invocation may be a -cc1 clang invocation or a driver
+ *             invocation.
+ * \param WorkingDirectory the directory in which the invocation runs.
+ * \param MDC a callback that is called whenever a new module is discovered.
+ *            This may receive the same module on different workers. This should
+ *            be NULL if
+ *            \c clang_experimental_DependencyScannerService_create_v0 was
+ *            called with \c CXDependencyMode_Flat. This callback will be called
+ *            on the same thread that called this function.
+ * \param Context the context that will be passed to \c MDC each time it is
+ *                called.
+ * \param [out] error the error string to pass back to client (if any).
+ *
+ * \returns A pointer to a CXFileDependencies on success, NULL otherwise. The
+ *          CXFileDependencies must be freed by calling
+ *          \c clang_experimental_FileDependencies_dispose.
+ */
+CINDEX_LINKAGE CXFileDependencies *
+clang_experimental_DependencyScannerWorker_getFileDependencies_v0(
+    CXDependencyScannerWorker Worker, int argc, const char *const *argv,
+    const char *WorkingDirectory, CXModuleDiscoveredCallback *MDC,
+    void *Context, CXString *error);
+
+/**
+ * @}
+ */
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif // LLVM_CLANG_C_DEPENDENCIES_H

clang/include/clang-c/Driver.h

@@ -0,0 +1,78 @@
+/*==-- clang-c/Driver.h - A C Interface for the Clang Driver ------*- C -*-===*\
+|*                                                                            *|
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM          *|
+|* Exceptions.                                                                *|
+|* See https://llvm.org/LICENSE.txt for license information.                  *|
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception                    *|
+|*                                                                            *|
+|*===----------------------------------------------------------------------===*|
+|*                                                                            *|
+|* This header provides a C API for extracting information from the clang     *|
+|* driver.                                                                    *|
+|*                                                                            *|
+\*===----------------------------------------------------------------------===*/
+
+
+#ifndef CLANG_CLANG_C_DRIVER
+#define CLANG_CLANG_C_DRIVER
+
+#include "clang-c/Index.h"
+#include "clang-c/Platform.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * Contains the command line arguments for an external action.  Same format as
+ * provided to main.
+ */
+typedef struct {
+  /* Number of arguments in ArgV */
+  int ArgC;
+  /* Null terminated array of pointers to null terminated argument strings */
+  const char **ArgV;
+} CXExternalAction;
+
+/**
+ * Contains the list of external actions clang would invoke.
+ */
+typedef struct {
+  int Count;
+  CXExternalAction **Actions;
+} CXExternalActionList;
+
+/**
+ * Get the external actions that the clang driver will invoke for the given
+ * command line.
+ *
+ * \param ArgC number of arguments in \p ArgV.
+ * \param ArgV array of null terminated arguments.  Doesn't need to be null
+ *   terminated.
+ * \param Environment must be null.
+ * \param WorkingDirectory a null terminated path to the working directory to
+ *   use for this invocation.  `nullptr` to use the current working directory of
+ *   the process.
+ * \param OutDiags will be set to a \c CXDiagnosticSet if there's an error.
+ *   Must be freed by calling \c clang_disposeDiagnosticSet .
+ * \returns A pointer to a \c CXExternalActionList on success, null on failure.
+ *   The returned \c CXExternalActionList must be freed by calling
+ *   \c clang_Driver_ExternalActionList_dispose .
+ */
+CINDEX_LINKAGE CXExternalActionList *
+clang_Driver_getExternalActionsForCommand_v0(int ArgC, const char **ArgV,
+                                             const char **Environment,
+                                             const char *WorkingDirectory,
+                                             CXDiagnosticSet *OutDiags);
+
+/**
+ * Deallocate a \c CXExternalActionList
+ */
+CINDEX_LINKAGE void
+clang_Driver_ExternalActionList_dispose(CXExternalActionList *EAL);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif // CLANG_CLANG_C_DRIVER

clang/include/clang/Basic/Module.h

@@ -109,6 +109,9 @@ class Module {
   /// The name of the umbrella entry, as written in the module map.
   std::string UmbrellaAsWritten;
 
+  // The path to the umbrella entry relative to the root module's \c Directory.
+  std::string UmbrellaRelativeToRootModuleDirectory;
+
   /// The module through which entities defined in this module will
   /// eventually be exposed, for use in "private" modules.
   std::string ExportAsModule;
@@ -156,6 +159,7 @@ class Module {
   /// file.
   struct Header {
     std::string NameAsWritten;
+    std::string PathRelativeToRootModuleDirectory;
     const FileEntry *Entry;
 
     explicit operator bool() { return Entry; }
@@ -165,6 +169,7 @@ class Module {
   /// file.
   struct DirectoryName {
     std::string NameAsWritten;
+    std::string PathRelativeToRootModuleDirectory;
     const DirectoryEntry *Entry;
 
     explicit operator bool() { return Entry; }
@@ -491,7 +496,8 @@ class Module {
   /// module.
   Header getUmbrellaHeader() const {
     if (auto *E = Umbrella.dyn_cast<const FileEntry *>())
-      return Header{UmbrellaAsWritten, E};
+      return Header{UmbrellaAsWritten, UmbrellaRelativeToRootModuleDirectory,
+                    E};
     return Header{};
   }
 

clang/include/clang/Driver/CC1Options.td

@@ -12,6 +12,18 @@
 
 let Flags = [CC1Option, NoDriverOption] in {
 
+//===----------------------------------------------------------------------===//
+// Option Options
+//===----------------------------------------------------------------------===//
+
+def remove_preceeding_explicit_module_build_incompatible_options :
+  Flag<["-"], "remove-preceeding-explicit-module-build-incompatible-options">,
+  HelpText<"Removes any arguments before this one that would be incompatible "
+           "with explicitly building a module. This includes things like -o "
+           "and input files. This option can be used to append arguments to "
+           "convert a build of a translation unit with implicit modules "
+           "into an explicit build of a specific module.">;
+
 //===----------------------------------------------------------------------===//
 // Target Options
 //===----------------------------------------------------------------------===//

clang/include/clang/Driver/Options.td

@@ -1538,7 +1538,7 @@ def fno_merge_all_constants : Flag<["-"], "fno-merge-all-constants">, Group<f_Gr
 def fno_modules : Flag <["-"], "fno-modules">, Group<f_Group>,
   Flags<[DriverOption]>;
 def fno_implicit_module_maps : Flag <["-"], "fno-implicit-module-maps">, Group<f_Group>,
-  Flags<[DriverOption]>;
+  Flags<[DriverOption, CC1Option]>;
 def fno_module_maps : Flag <["-"], "fno-module-maps">, Alias<fno_implicit_module_maps>;
 def fno_modules_decluse : Flag <["-"], "fno-modules-decluse">, Group<f_Group>,
   Flags<[DriverOption]>;

clang/include/clang/Lex/ModuleMap.h

@@ -653,12 +653,14 @@ class ModuleMap {
   /// Sets the umbrella header of the given module to the given
   /// header.
   void setUmbrellaHeader(Module *Mod, const FileEntry *UmbrellaHeader,
-                         Twine NameAsWritten);
+                         Twine NameAsWritten,
+                         Twine PathRelativeToRootModuleDirectory);
 
   /// Sets the umbrella directory of the given module to the given
   /// directory.
   void setUmbrellaDir(Module *Mod, const DirectoryEntry *UmbrellaDir,
-                      Twine NameAsWritten);
+                      Twine NameAsWritten,
+                      Twine PathRelativeToRootModuleDirectory);
 
   /// Adds this header to the given module.
   /// \param Role The role of the header wrt the module.