feat: support for nested node_modules in linker

This PR adds support for nested node_modules folders in the linker, which allows targets to depend on npm packages from multiple yarn_install & npm_install repository rules (max one per directory in the workspace).

This is an opt-in feature which is enabled by specifying package_path in yarn_install or npm_install for nested package.json files. Setting package_path informs the linker to link the external npm repository to a node_modules folder in the package_path sub-directory specified.

For example, given two yarn_install rules:

```
yarn_install(
    name = "npm",
    package_json = "//:package.json",
    yarn_lock = "//:yarn.lock",
)

yarn_install(
    name = "npm_subdir",
    package_json = "//subdir:package.json",
    package_path = "subdir",
    yarn_lock = "//subdir:yarn.lock",
)
```

a target may depend on npm packages from both,

```
jasmine_node_test(
    name = "test",
    srcs = ["test.js"],
    deps = [
        "@npm//foo",
        "@npm_subdir//bar",
    ],
)
```

and the linker will link multiple 3rd party node_modules folders,

```
/node_modules => @npm//:node_modules
/subdir/node_modules => @npm_subdir/:node_modules
```

making the 3rd party npm deps foo & bar available at

```
/node_modules/foo
/subdir/node_modules/bar
```

This feature is opt-in as package_path currently defaults to "". In a future major release (4.0.0), package_path will default to the directory the package.json file is in, which will turn it on by default.

This feature is an alternative to yarn workspaces & npm workspaces, which also lay out node_modules in multiple directories from multiple package.json files.

To date, targets have been limited to depending on and resolving form a single yarn_install or npm_install workspace which was always linked to a node_modules folder at the root of the workspace. With this change, the linker is able to link to both the root of the workspace as well as sub-directories in the workspace so targets may now resolve 3rd party npm deps from multiple package.json files which correspond to multiple node_modules folders in the tree.

1st party deps are still always linked to the root node_modules folder as they were before this change.

NB: depending on multiple npm workspaces will not be supported by ts_library as its resolution logic is limited to a single node_modules root

NB: pre-linker node require patches (deprecated as of 3.0.0) will only include root node_modules; no nested node_modules shall included in require patch resolution

Author: Greg Magolan

WORKSPACE

@@ -19,6 +19,7 @@ workspace(
         # cypress_deps must be a managed directory to ensure it is downloaded before cypress_repository is run.
         "@cypress_deps": ["packages/cypress/test/node_modules"],
         "@npm": ["node_modules"],
+        "@npm_internal_linker_test_multi_linker": ["internal/linker/test/multi_linker/node_modules"],
         "@npm_node_patches": ["packages/node-patches/node_modules"],
     },
 )
@@ -57,6 +58,21 @@ yarn_install(
     yarn_lock = "//:yarn.lock",
 )
 
+yarn_install(
+    name = "npm_internal_linker_test_multi_linker",
+    package_json = "//internal/linker/test/multi_linker:package.json",
+    package_path = "internal/linker/test/multi_linker",
+    yarn_lock = "//internal/linker/test/multi_linker:yarn.lock",
+)
+
+yarn_install(
+    name = "onepa_npm_deps",
+    package_json = "//internal/linker/test/multi_linker/onepa:package.json",
+    package_path = "internal/linker/test/multi_linker/onepa",
+    symlink_node_modules = False,
+    yarn_lock = "//internal/linker/test/multi_linker/onepa:yarn.lock",
+)
+
 npm_install(
     name = "npm_node_patches",
     package_json = "//packages/node-patches:package.json",

internal/bazel_integration_test/test_runner.js

@@ -358,11 +358,10 @@ for (const bazelCommand of config.bazelCommands) {
     'BASH_FUNC_is_absolute%%',
     'BASH_FUNC_rlocation%%',
     'BASH_FUNC_runfiles_export_envvars%%',
-    'BAZEL_NODE_MODULES_ROOT',
+    'BAZEL_NODE_MODULES_ROOTS',
     'BAZEL_NODE_PATCH_REQUIRE',
     'BAZEL_NODE_RUNFILES_HELPER',
-    'BAZEL_PATCH_GUARDS',
-    'BAZEL_PATCH_ROOT',
+    'BAZEL_PATCH_ROOTS',
     'BAZEL_TARGET',
     'BAZEL_WORKSPACE',
     'BAZELISK_SKIP_WRAPPER',

internal/js_library/js_library.bzl

@@ -38,7 +38,7 @@ _ATTRS = {
     ),
     "deps": attr.label_list(),
     "external_npm_package": attr.bool(
-        doc = """Indictates that this js_library target is one or more external npm packages in node_modules.
+        doc = """Internal use only. Indictates that this js_library target is one or more external npm packages in node_modules.
         This is used by the yarn_install & npm_install repository rules for npm dependencies installed by
         yarn & npm. When true, js_library will provide ExternalNpmPackageInfo.
         
@@ -73,6 +73,12 @@ _ATTRS = {
         See `examples/user_managed_deps` for a working example of user-managed npm dependencies.""",
         default = False,
     ),
+    "external_npm_package_path": attr.string(
+        doc = """Internal use only. The local workspace path that the linker should link these node_modules to.
+
+        Used only when external_npm_package is True. If empty, the linker will link these node_modules at the root.""",
+        default = "",
+    ),
     "is_windows": attr.bool(
         doc = "Internal use only. Automatically set by macro",
         mandatory = True,
@@ -230,6 +236,7 @@ def _impl(ctx):
             direct_sources = depset(transitive = direct_sources_depsets),
             sources = depset(transitive = npm_sources_depsets),
             workspace = workspace_name,
+            path = ctx.attr.external_npm_package_path,
         ))
 
     # Don't provide DeclarationInfo if there are no typings to provide.

internal/linker/index.js

@@ -34,7 +34,7 @@ function mkdirp(p) {
     return __awaiter(this, void 0, void 0, function* () {
         if (p && !(yield exists(p))) {
             yield mkdirp(path.dirname(p));
-            log_verbose(`mkdir( ${p} )`);
+            log_verbose(`creating directory ${p} in ${process.cwd()}`);
             try {
                 yield fs.promises.mkdir(p);
             }
@@ -98,7 +98,10 @@ function deleteDirectory(p) {
 }
 function symlink(target, p) {
     return __awaiter(this, void 0, void 0, function* () {
-        log_verbose(`symlink( ${p} -> ${target} )`);
+        if (!path.isAbsolute(target)) {
+            target = path.resolve(process.cwd(), target);
+        }
+        log_verbose(`creating symlink ${p} -> ${target}`);
         try {
             yield fs.promises.symlink(target, p, 'junction');
             return true;
@@ -117,33 +120,24 @@ function symlink(target, p) {
         }
     });
 }
-function resolveRoot(root, startCwd, isExecroot, runfiles) {
+function resolveExternalWorkspacePath(workspace, startCwd, isExecroot, execroot, runfiles) {
     return __awaiter(this, void 0, void 0, function* () {
         if (isExecroot) {
-            return root ? `${startCwd}/external/${root}` : `${startCwd}/node_modules`;
-        }
-        const match = startCwd.match(BAZEL_OUT_REGEX);
-        if (!match) {
-            if (!root) {
-                return `${startCwd}/node_modules`;
-            }
-            return path.resolve(`${startCwd}/../${root}`);
+            return `${execroot}/external/${workspace}`;
         }
-        const symlinkRoot = startCwd.slice(0, match.index);
-        process.chdir(symlinkRoot);
-        if (!root) {
-            return `${symlinkRoot}/node_modules`;
+        if (!execroot) {
+            return path.resolve(`${startCwd}/../${workspace}`);
         }
-        const fromManifest = runfiles.lookupDirectory(root);
+        const fromManifest = runfiles.lookupDirectory(workspace);
         if (fromManifest) {
             return fromManifest;
         }
         else {
-            const maybe = path.resolve(`${symlinkRoot}/external/${root}`);
-            if (fs.existsSync(maybe)) {
+            const maybe = path.resolve(`${execroot}/external/${workspace}`);
+            if (yield exists(maybe)) {
                 return maybe;
             }
-            return path.resolve(`${startCwd}/../${root}`);
+            return path.resolve(`${startCwd}/../${workspace}`);
         }
     });
 }
@@ -211,7 +205,7 @@ class Runfiles {
         if (result) {
             return result;
         }
-        const e = new Error(`could not resolve modulePath ${modulePath}`);
+        const e = new Error(`could not resolve module ${modulePath}`);
         e.code = 'MODULE_NOT_FOUND';
         throw e;
     }
@@ -272,6 +266,9 @@ function exists(p) {
     });
 }
 function existsSync(p) {
+    if (!p) {
+        return false;
+    }
     try {
         fs.lstatSync(p);
         return true;
@@ -328,19 +325,6 @@ function liftElement(element) {
     }
     return element;
 }
-function toParentLink(link) {
-    return [link[0], path.dirname(link[1])];
-}
-function allElementsAlign(name, elements) {
-    if (!elements[0].link) {
-        return false;
-    }
-    const parentLink = toParentLink(elements[0].link);
-    if (!elements.every(e => !!e.link && isDirectChildLink(parentLink, e.link))) {
-        return false;
-    }
-    return !!elements[0].link && allElementsAlignUnder(name, parentLink, elements);
-}
 function allElementsAlignUnder(parentName, parentLink, elements) {
     for (const { name, link, children } of elements) {
         if (!link || children) {
@@ -361,16 +345,10 @@ function allElementsAlignUnder(parentName, parentLink, elements) {
 function isDirectChildPath(parent, child) {
     return parent === path.dirname(child);
 }
-function isDirectChildLink([parentRel, parentPath], [childRel, childPath]) {
-    if (parentRel !== childRel) {
-        return false;
-    }
-    if (!isDirectChildPath(parentPath, childPath)) {
-        return false;
-    }
-    return true;
+function isDirectChildLink(parentLink, childLink) {
+    return parentLink === path.dirname(childLink);
 }
-function isNameLinkPathTopAligned(namePath, [, linkPath]) {
+function isNameLinkPathTopAligned(namePath, linkPath) {
     return path.basename(namePath) === path.basename(linkPath);
 }
 function visitDirectoryPreserveLinks(dirPath, visit) {
@@ -390,28 +368,96 @@ function visitDirectoryPreserveLinks(dirPath, visit) {
         }
     });
 }
+function findExecroot(startCwd) {
+    if (existsSync(`${startCwd}/bazel-out`)) {
+        return startCwd;
+    }
+    const bazelOutMatch = startCwd.match(BAZEL_OUT_REGEX);
+    return bazelOutMatch ? startCwd.slice(0, bazelOutMatch.index) : undefined;
+}
 function main(args, runfiles) {
     return __awaiter(this, void 0, void 0, function* () {
         if (!args || args.length < 1)
             throw new Error('requires one argument: modulesManifest path');
         const [modulesManifest] = args;
-        let { bin, root, modules, workspace } = JSON.parse(fs.readFileSync(modulesManifest));
+        log_verbose('manifest file:', modulesManifest);
+        let { workspace, bin, roots, modules } = JSON.parse(fs.readFileSync(modulesManifest));
         modules = modules || {};
-        log_verbose('manifest file', modulesManifest);
-        log_verbose('manifest contents', JSON.stringify({ workspace, bin, root, modules }, null, 2));
+        log_verbose('manifest contents:', JSON.stringify({ workspace, bin, roots, modules }, null, 2));
         const startCwd = process.cwd().replace(/\\/g, '/');
-        log_verbose('startCwd', startCwd);
-        const isExecroot = existsSync(`${startCwd}/bazel-out`);
-        log_verbose('isExecroot', isExecroot.toString());
-        const rootDir = yield resolveRoot(root, startCwd, isExecroot, runfiles);
-        log_verbose('resolved node_modules root', root, 'to', rootDir);
-        log_verbose('cwd', process.cwd());
-        if (!(yield exists(rootDir))) {
-            log_verbose('no third-party packages; mkdir node_modules at', root);
-            yield mkdirp(rootDir);
-        }
-        yield symlink(rootDir, 'node_modules');
-        process.chdir(rootDir);
+        log_verbose('startCwd:', startCwd);
+        const execroot = findExecroot(startCwd);
+        log_verbose('execroot:', execroot ? execroot : 'not found');
+        const isExecroot = startCwd == execroot;
+        log_verbose('isExecroot:', isExecroot.toString());
+        if (!isExecroot && execroot) {
+            process.chdir(execroot);
+            log_verbose('changed directory to execroot', execroot);
+        }
+        function symlinkWithUnlink(target, p, stats = null) {
+            return __awaiter(this, void 0, void 0, function* () {
+                if (!path.isAbsolute(target)) {
+                    target = path.resolve(process.cwd(), target);
+                }
+                if (stats === null) {
+                    stats = yield gracefulLstat(p);
+                }
+                if (runfiles.manifest && execroot && stats !== null && stats.isSymbolicLink()) {
+                    const symlinkPath = fs.readlinkSync(p).replace(/\\/g, '/');
+                    if (path.relative(symlinkPath, target) != '' &&
+                        !path.relative(execroot, symlinkPath).startsWith('..')) {
+                        log_verbose(`Out-of-date symlink for ${p} to ${symlinkPath} detected. Target should be ${target}. Unlinking.`);
+                        yield unlink(p);
+                    }
+                }
+                return symlink(target, p);
+            });
+        }
+        for (const packagePath of Object.keys(roots)) {
+            const workspace = roots[packagePath];
+            const workspacePath = yield resolveExternalWorkspacePath(workspace, startCwd, isExecroot, execroot, runfiles);
+            log_verbose(`resolved ${workspace} workspace path to ${workspacePath}`);
+            const workspaceNodeModules = `${workspacePath}/node_modules`;
+            if (packagePath) {
+                if (yield exists(workspaceNodeModules)) {
+                    let resolvedPackagePath;
+                    if (yield exists(packagePath)) {
+                        yield symlinkWithUnlink(workspaceNodeModules, `${packagePath}/node_modules`);
+                        resolvedPackagePath = packagePath;
+                    }
+                    if (!isExecroot) {
+                        const runfilesPackagePath = `${startCwd}/${packagePath}`;
+                        if (yield exists(runfilesPackagePath)) {
+                            if (resolvedPackagePath) {
+                                yield symlinkWithUnlink(`${resolvedPackagePath}/node_modules`, `${runfilesPackagePath}/node_modules`);
+                            }
+                            else {
+                                yield symlinkWithUnlink(workspaceNodeModules, `${runfilesPackagePath}/node_modules`);
+                            }
+                            resolvedPackagePath = runfilesPackagePath;
+                        }
+                    }
+                    const packagePathBin = `${bin}/${packagePath}`;
+                    if (resolvedPackagePath && (yield exists(packagePathBin))) {
+                        yield symlinkWithUnlink(`${resolvedPackagePath}/node_modules`, `${packagePathBin}/node_modules`);
+                    }
+                }
+            }
+            else {
+                if (yield exists(workspaceNodeModules)) {
+                    yield symlinkWithUnlink(workspaceNodeModules, `node_modules`);
+                }
+                else {
+                    log_verbose('no root npm workspace node_modules folder to link to; creating node_modules directory in', process.cwd());
+                    yield mkdirp('node_modules');
+                }
+            }
+        }
+        if (!roots || !roots['']) {
+            log_verbose('no root npm workspace; creating node_modules directory in ', process.cwd());
+            yield mkdirp('node_modules');
+        }
+        process.chdir('node_modules');
         function isLeftoverDirectoryFromLinker(stats, modulePath) {
             return __awaiter(this, void 0, void 0, function* () {
                 if (runfiles.manifest === undefined) {
@@ -451,44 +497,43 @@ function main(args, runfiles) {
             return __awaiter(this, void 0, void 0, function* () {
                 yield mkdirp(path.dirname(m.name));
                 if (m.link) {
-                    const [root, modulePath] = m.link;
+                    const modulePath = m.link;
                     let target;
-                    switch (root) {
-                        case 'execroot':
-                            if (isExecroot) {
-                                target = `${startCwd}/${modulePath}`;
-                                break;
-                            }
-                        case 'runfiles':
-                            let runfilesPath = modulePath;
-                            if (runfilesPath.startsWith(`${bin}/`)) {
-                                runfilesPath = runfilesPath.slice(bin.length + 1);
-                            }
-                            else if (runfilesPath === bin) {
-                                runfilesPath = '';
-                            }
-                            const externalPrefix = 'external/';
-                            if (runfilesPath.startsWith(externalPrefix)) {
-                                runfilesPath = runfilesPath.slice(externalPrefix.length);
-                            }
-                            else {
-                                runfilesPath = `${workspace}/${runfilesPath}`;
-                            }
-                            try {
-                                target = runfiles.resolve(runfilesPath);
-                                if (runfiles.manifest && root == 'execroot' && modulePath.startsWith(`${bin}/`)) {
-                                    if (!target.includes(`/${bin}/`)) {
-                                        const e = new Error(`could not resolve modulePath ${modulePath}`);
-                                        e.code = 'MODULE_NOT_FOUND';
-                                        throw e;
-                                    }
+                    if (isExecroot) {
+                        target = `${startCwd}/${modulePath}`;
+                    }
+                    if (!isExecroot || !existsSync(target)) {
+                        let runfilesPath = modulePath;
+                        if (runfilesPath.startsWith(`${bin}/`)) {
+                            runfilesPath = runfilesPath.slice(bin.length + 1);
+                        }
+                        else if (runfilesPath === bin) {
+                            runfilesPath = '';
+                        }
+                        const externalPrefix = 'external/';
+                        if (runfilesPath.startsWith(externalPrefix)) {
+                            runfilesPath = runfilesPath.slice(externalPrefix.length);
+                        }
+                        else {
+                            runfilesPath = `${workspace}/${runfilesPath}`;
+                        }
+                        try {
+                            target = runfiles.resolve(runfilesPath);
+                            if (runfiles.manifest && modulePath.startsWith(`${bin}/`)) {
+                                if (!target.match(BAZEL_OUT_REGEX)) {
+                                    const e = new Error(`could not resolve module ${runfilesPath} in output tree`);
+                                    e.code = 'MODULE_NOT_FOUND';
+                                    throw e;
                                 }
                             }
-                            catch (err) {
-                                target = undefined;
-                                log_verbose(`runfiles resolve failed for module '${m.name}': ${err.message}`);
-                            }
-                            break;
+                        }
+                        catch (err) {
+                            target = undefined;
+                            log_verbose(`runfiles resolve failed for module '${m.name}': ${err.message}`);
+                        }
+                    }
+                    if (target && !path.isAbsolute(target)) {
+                        target = path.resolve(process.cwd(), target);
                     }
                     const stats = yield gracefulLstat(m.name);
                     const isLeftOver = (stats !== null && (yield isLeftoverDirectoryFromLinker(stats, m.name)));
@@ -497,7 +542,7 @@ function main(args, runfiles) {
                             yield createSymlinkAndPreserveContents(stats, m.name, target);
                         }
                         else {
-                            yield symlink(target, m.name);
+                            yield symlinkWithUnlink(target, m.name, stats);
                         }
                     }
                     else {