Merge pull request #2838 from dmichon-msft/project-cache

[rush] Improve startup performance with read cache

Author: dmichon-msft

apps/rush-lib/src/api/RushProjectConfiguration.ts

@@ -126,6 +126,9 @@ export class RushProjectConfiguration {
       }
     });
 
+  private static readonly _configCache: Map<RushConfigurationProject, RushProjectConfiguration | false> =
+    new Map();
+
   public readonly project: RushConfigurationProject;
 
   /**
@@ -177,8 +180,17 @@ export class RushProjectConfiguration {
   public static async tryLoadForProjectAsync(
     project: RushConfigurationProject,
     repoCommandLineConfiguration: CommandLineConfiguration | undefined,
-    terminal: Terminal
+    terminal: Terminal,
+    skipCache?: boolean
   ): Promise<RushProjectConfiguration | undefined> {
+    // false is a signal that the project config does not exist
+    const cacheEntry: RushProjectConfiguration | false | undefined = skipCache
+      ? undefined
+      : RushProjectConfiguration._configCache.get(project);
+    if (cacheEntry !== undefined) {
+      return cacheEntry || undefined;
+    }
+
     const rigConfig: RigConfig = await RigConfig.loadForProjectFolderAsync({
       projectFolderPath: project.projectFolder
     });
@@ -197,8 +209,11 @@ export class RushProjectConfiguration {
         repoCommandLineConfiguration,
         terminal
       );
-      return new RushProjectConfiguration(project, rushProjectJson);
+      const result: RushProjectConfiguration = new RushProjectConfiguration(project, rushProjectJson);
+      RushProjectConfiguration._configCache.set(project, result);
+      return result;
     } else {
+      RushProjectConfiguration._configCache.set(project, false);
       return undefined;
     }
   }

apps/rush-lib/src/logic/ProjectChangeAnalyzer.ts

@@ -184,22 +184,21 @@ export class ProjectChangeAnalyzer {
       return undefined;
     }
 
-    const projectHashDeps: Map<string, Map<string, string>> = new Map<string, Map<string, string>>();
+    const projectHashDeps: Map<string, Map<string, string>> = new Map();
+
+    for (const project of this._rushConfiguration.projects) {
+      projectHashDeps.set(project.packageName, new Map());
+    }
 
     // Sort each project folder into its own package deps hash
     for (const [filePath, fileHash] of repoDeps) {
       // findProjectForPosixRelativePath uses LookupByPath, for which lookups are O(K)
       // K being the maximum folder depth of any project in rush.json (usually on the order of 3)
       const owningProject: RushConfigurationProject | undefined =
         this._rushConfiguration.findProjectForPosixRelativePath(filePath);
+
       if (owningProject) {
-        let owningProjectHashDeps: Map<string, string> | undefined = projectHashDeps.get(
-          owningProject.packageName
-        );
-        if (!owningProjectHashDeps) {
-          owningProjectHashDeps = new Map<string, string>();
-          projectHashDeps.set(owningProject.packageName, owningProjectHashDeps);
-        }
+        const owningProjectHashDeps: Map<string, string> = projectHashDeps.get(owningProject.packageName)!;
         owningProjectHashDeps.set(filePath, fileHash);
       }
     }
@@ -268,16 +267,15 @@ export class ProjectChangeAnalyzer {
   private async _getIgnoreMatcherForProjectAsync(
     project: RushConfigurationProject,
     terminal: Terminal
-  ): Promise<Ignore> {
+  ): Promise<Ignore | undefined> {
     const projectConfiguration: RushProjectConfiguration | undefined =
       await RushProjectConfiguration.tryLoadForProjectAsync(project, undefined, terminal);
-    const ignoreMatcher: Ignore = ignore();
 
     if (projectConfiguration && projectConfiguration.incrementalBuildIgnoredGlobs) {
+      const ignoreMatcher: Ignore = ignore();
       ignoreMatcher.add(projectConfiguration.incrementalBuildIgnoredGlobs);
+      return ignoreMatcher;
     }
-
-    return ignoreMatcher;
   }
 
   private _getRepoDeps(terminal: Terminal): Map<string, string> | undefined {

common/changes/@microsoft/rush/project-cache_2021-08-02-23-04.json

@@ -0,0 +1,11 @@
+{
+  "changes": [
+    {
+      "packageName": "@microsoft/rush",
+      "comment": "Cache rush-project.json reads",
+      "type": "none"
+    }
+  ],
+  "packageName": "@microsoft/rush",
+  "email": "[email protected]"
+}

common/changes/@rushstack/rig-package/project-cache_2021-08-02-23-04.json

@@ -0,0 +1,11 @@
+{
+  "changes": [
+    {
+      "packageName": "@rushstack/rig-package",
+      "comment": "Cache rig.json reads",
+      "type": "minor"
+    }
+  ],
+  "packageName": "@rushstack/rig-package",
+  "email": "[email protected]"
+}

common/reviews/api/rig-package.api.md

@@ -6,6 +6,7 @@
 
 // @public
 export interface ILoadForProjectFolderOptions {
+    bypassCache?: boolean;
     overrideRigJsonObject?: IRigConfigJson;
     projectFolderPath: string;
 }
@@ -33,7 +34,6 @@ export class RigConfig {
     readonly rigProfile: string;
     tryResolveConfigFilePath(configFileRelativePath: string): string | undefined;
     tryResolveConfigFilePathAsync(configFileRelativePath: string): Promise<string | undefined>;
-    }
-
+}
 
 ```

libraries/rig-package/src/RigConfig.ts

@@ -42,7 +42,7 @@ interface IRigConfigOptions {
   rigFound: boolean;
   filePath: string;
   rigPackageName: string;
-  rigProfile: string;
+  rigProfile?: string;
 }
 
 /**
@@ -60,6 +60,11 @@ export interface ILoadForProjectFolderOptions {
    * If specified, instead of loading the `config/rig.json` from disk, this object will be substituted instead.
    */
   overrideRigJsonObject?: IRigConfigJson;
+
+  /**
+   * If specified, force a fresh load instead of returning a cached entry, if one existed.
+   */
+  bypassCache?: boolean;
 }
 
 /**
@@ -91,6 +96,8 @@ export class RigConfig {
   public static jsonSchemaPath: string = path.resolve(__dirname, './schemas/rig.schema.json');
   private static _jsonSchemaObject: object | undefined = undefined;
 
+  private static readonly _configCache: Map<string, RigConfig> = new Map();
+
   /**
    * The project folder path that was passed to {@link RigConfig.loadForProjectFolder},
    * which maybe an absolute or relative path.
@@ -160,13 +167,15 @@ export class RigConfig {
   private _resolvedProfileFolder: string | undefined;
 
   private constructor(options: IRigConfigOptions) {
-    this.projectFolderOriginalPath = options.projectFolderPath;
-    this.projectFolderPath = path.resolve(options.projectFolderPath);
+    const { projectFolderPath, rigFound, filePath, rigPackageName, rigProfile = 'default' } = options;
+
+    this.projectFolderOriginalPath = projectFolderPath;
+    this.projectFolderPath = path.resolve(projectFolderPath);
 
-    this.rigFound = options.rigFound;
-    this.filePath = options.filePath;
-    this.rigPackageName = options.rigPackageName;
-    this.rigProfile = options.rigProfile;
+    this.rigFound = rigFound;
+    this.filePath = filePath;
+    this.rigPackageName = rigPackageName;
+    this.rigProfile = rigProfile;
 
     if (this.rigFound) {
       this.relativeProfileFolderPath = 'profiles/' + this.rigProfile;
@@ -199,80 +208,110 @@ export class RigConfig {
    * equal to `false`.
    */
   public static loadForProjectFolder(options: ILoadForProjectFolderOptions): RigConfig {
-    const rigConfigFilePath: string = path.join(options.projectFolderPath, 'config/rig.json');
+    const { overrideRigJsonObject, projectFolderPath } = options;
 
-    let json: IRigConfigJson;
-    try {
-      if (options.overrideRigJsonObject) {
-        json = options.overrideRigJsonObject;
-      } else {
-        if (!fs.existsSync(rigConfigFilePath)) {
-          return new RigConfig({
-            projectFolderPath: options.projectFolderPath,
-
-            rigFound: false,
-            filePath: '',
-            rigPackageName: '',
-            rigProfile: ''
-          });
-        }
+    const fromCache: RigConfig | undefined =
+      !options.bypassCache && !overrideRigJsonObject
+        ? RigConfig._configCache.get(projectFolderPath)
+        : undefined;
 
+    if (fromCache) {
+      return fromCache;
+    }
+
+    const rigConfigFilePath: string = path.join(projectFolderPath, 'config/rig.json');
+
+    let config: RigConfig | undefined;
+    let json: IRigConfigJson | undefined = overrideRigJsonObject;
+    try {
+      if (!json) {
         const rigConfigFileContent: string = fs.readFileSync(rigConfigFilePath).toString();
-        json = JSON.parse(stripJsonComments(rigConfigFileContent));
+        json = JSON.parse(stripJsonComments(rigConfigFileContent)) as IRigConfigJson;
       }
       RigConfig._validateSchema(json);
     } catch (error) {
-      throw new Error(error.message + '\nError loading config file: ' + rigConfigFilePath);
+      config = RigConfig._handleConfigError(error, projectFolderPath, rigConfigFilePath);
     }
 
-    return new RigConfig({
-      projectFolderPath: options.projectFolderPath,
+    if (!config) {
+      config = new RigConfig({
+        projectFolderPath: projectFolderPath,
 
-      rigFound: true,
-      filePath: rigConfigFilePath,
-      rigPackageName: json.rigPackageName,
-      rigProfile: json.rigProfile || 'default'
-    });
+        rigFound: true,
+        filePath: rigConfigFilePath,
+        rigPackageName: json!.rigPackageName,
+        rigProfile: json!.rigProfile
+      });
+    }
+
+    if (!overrideRigJsonObject) {
+      RigConfig._configCache.set(projectFolderPath, config);
+    }
+    return config;
   }
 
   /**
    * An async variant of {@link RigConfig.loadForProjectFolder}
    */
   public static async loadForProjectFolderAsync(options: ILoadForProjectFolderOptions): Promise<RigConfig> {
-    const rigConfigFilePath: string = path.join(options.projectFolderPath, 'config/rig.json');
+    const { overrideRigJsonObject, projectFolderPath } = options;
 
-    let json: IRigConfigJson;
-    try {
-      if (options.overrideRigJsonObject) {
-        json = options.overrideRigJsonObject;
-      } else {
-        if (!(await Helpers.fsExistsAsync(rigConfigFilePath))) {
-          return new RigConfig({
-            projectFolderPath: options.projectFolderPath,
-
-            rigFound: false,
-            filePath: '',
-            rigPackageName: '',
-            rigProfile: ''
-          });
-        }
+    const fromCache: RigConfig | false | undefined =
+      !options.bypassCache && !overrideRigJsonObject && RigConfig._configCache.get(projectFolderPath);
+
+    if (fromCache) {
+      return fromCache;
+    }
+
+    const rigConfigFilePath: string = path.join(projectFolderPath, 'config/rig.json');
 
+    let config: RigConfig | undefined;
+    let json: IRigConfigJson | undefined = overrideRigJsonObject;
+    try {
+      if (!json) {
         const rigConfigFileContent: string = (await fs.promises.readFile(rigConfigFilePath)).toString();
-        json = JSON.parse(stripJsonComments(rigConfigFileContent));
+        json = JSON.parse(stripJsonComments(rigConfigFileContent)) as IRigConfigJson;
       }
 
       RigConfig._validateSchema(json);
     } catch (error) {
+      config = RigConfig._handleConfigError(error, projectFolderPath, rigConfigFilePath);
+    }
+
+    if (!config) {
+      config = new RigConfig({
+        projectFolderPath: projectFolderPath,
+
+        rigFound: true,
+        filePath: rigConfigFilePath,
+        rigPackageName: json!.rigPackageName,
+        rigProfile: json!.rigProfile
+      });
+    }
+
+    if (!overrideRigJsonObject) {
+      RigConfig._configCache.set(projectFolderPath, config);
+    }
+    return config;
+  }
+
+  private static _handleConfigError(
+    error: NodeJS.ErrnoException,
+    projectFolderPath: string,
+    rigConfigFilePath: string
+  ): RigConfig {
+    if (error.code !== 'ENOENT' && error.code !== 'ENOTDIR') {
       throw new Error(error.message + '\nError loading config file: ' + rigConfigFilePath);
     }
 
+    // File not found, i.e. no rig config
     return new RigConfig({
-      projectFolderPath: options.projectFolderPath,
+      projectFolderPath,
 
-      rigFound: true,
-      filePath: rigConfigFilePath,
-      rigPackageName: json.rigPackageName,
-      rigProfile: json.rigProfile || 'default'
+      rigFound: false,
+      filePath: '',
+      rigPackageName: '',
+      rigProfile: ''
     });
   }