Add node module resolution by default and use --path for custom package locations (#594)

Author: willemneal

.gitignore

@@ -3,4 +3,4 @@ docs/
 node_modules/
 out/
 raw/
-.history
+.history

cli/asc.js

@@ -21,6 +21,7 @@ const colorsUtil = require("./util/colors");
 const optionsUtil = require("./util/options");
 const mkdirp = require("./util/mkdirp");
 const EOL = process.platform === "win32" ? "\r\n" : "\n";
+const SEP = process.platform === "win32" ? "\\" : "/";
 
 // global.Binaryen = require("../lib/binaryen");
 
@@ -229,6 +230,10 @@ exports.main = function main(argv, options, callback) {
   // Begin parsing
   var parser = null;
 
+  // Maps package names to parent directory
+  var packages = new Map();
+  var importPathMap = new Map();
+
   // Include library files
   Object.keys(exports.libraryFiles).forEach(libPath => {
     if (libPath.indexOf("/") >= 0) return; // in sub-directory: imported on demand
@@ -254,13 +259,14 @@ exports.main = function main(argv, options, callback) {
         libFiles = [ path.basename(libDir) ];
         libDir = path.dirname(libDir);
       } else {
-        libFiles = listFiles(libDir);
+        libFiles = listFiles(libDir) || [];
       }
       for (let j = 0, l = libFiles.length; j < l; ++j) {
         let libPath = libFiles[j];
         let libText = readFile(libPath, libDir);
         if (libText === null) return callback(Error("Library file '" + libPath + "' not found."));
         stats.parseCount++;
+        exports.libraryFiles[libPath.replace(/\.ts$/, "")] = libText;
         stats.parseTime += measure(() => {
           parser = assemblyscript.parseFile(
             libText,
@@ -272,13 +278,32 @@ exports.main = function main(argv, options, callback) {
       }
     }
   }
+  args.path = args.path || [];
+  // Find all valid node_module paths starting at baseDir
+  function nodePaths(basePath, _path) {
+    return basePath.split(SEP)
+          .map((_, i, arr) => {
+            let dir = arr.slice(0, i + 1).join(SEP) || SEP;
+            let dirFrom = path.relative(baseDir, dir);
+            return path.join(dirFrom, _path);
+          })
+          .filter(dir => listFiles(dir, baseDir))
+          .reverse();
+  }
+  function getPaths(basePath) {
+    let paths = args.path.map(p => nodePaths(basePath, p));
+    return nodePaths(basePath, "node_modules").concat(...paths)
+  }
 
   // Parses the backlog of imported files after including entry files
   function parseBacklog() {
-    var sourcePath, sourceText;
+    var sourcePath, sourceText, sysPath;
+    // dependee is the path of the file that depends on sourcePath
     while ((sourcePath = parser.nextFile()) != null) {
+      dependee = importPathMap.get(assemblyscript.getDependee(parser, sourcePath)) || baseDir;
       sourceText = null;
-
+      sysPath = null;
+      
       // Load library file if explicitly requested
       if (sourcePath.startsWith(exports.libraryPrefix)) {
         const plainName = sourcePath.substring(exports.libraryPrefix.length);
@@ -294,11 +319,13 @@ exports.main = function main(argv, options, callback) {
             sourceText = readFile(plainName + ".ts", customLibDirs[i]);
             if (sourceText !== null) {
               sourcePath = exports.libraryPrefix + plainName + ".ts";
+              sysPath = path.join(customLibDirs[i], plainName + ".ts");
               break;
             } else {
               sourceText = readFile(indexName + ".ts", customLibDirs[i]);
               if (sourceText !== null) {
                 sourcePath = exports.libraryPrefix + indexName + ".ts";
+                sysPath = path.join(customLibDirs[i], indexName + ".ts");
                 break;
               }
             }
@@ -329,11 +356,13 @@ exports.main = function main(argv, options, callback) {
                 sourceText = readFile(plainName + ".ts", dir);
                 if (sourceText !== null) {
                   sourcePath = exports.libraryPrefix + plainName + ".ts";
+                  sysPath = path.join(dir, plainName + ".ts");
                   break;
                 } else {
                   sourceText = readFile(indexName + ".ts", dir);
                   if (sourceText !== null) {
                     sourcePath = exports.libraryPrefix + indexName + ".ts";
+                    sysPath = path.join(dir, indexName + ".ts");
                     break;
                   }
                 }
@@ -342,9 +371,77 @@ exports.main = function main(argv, options, callback) {
           }
         }
       }
+      /*
+      In this case the library wasn't found so we check paths
+      */
+      if (sourceText == null) {
+        if (args.traceResolution) {
+            stderr.write("Looking for " + sourcePath + " imported by " + dependee + " " + EOL);
+        }
+        paths = getPaths(path.join(baseDir, dependee));
+        let _package = sourcePath.replace(/\~lib\/([^\/]*).*/, "$1");
+        for (let _path of paths) {
+          let ascMain = (() => {
+            if (packages.has(_package)) {
+              return packages.get(_package);
+            }
+            let p = path.join(_path, _package, "package.json");
+            let res = readFile(p, baseDir);
+            if (res) {
+              let package_json;
+              try {
+                package_json = JSON.parse(res);
+              } catch (e) {
+                return callback(Error("Parsing " + p + " failed"));
+              }
+              let mainFile = package_json.ascMain;
+              if (mainFile && (typeof mainFile === 'string')) {
+                let newPackage = mainFile.replace(/(.*)\/index\.ts/, '$1');
+                packages.set(_package, newPackage);
+                return newPackage;
+              }
+            }
+            return "assembly";
+          })()
+          let realPath = (_p) => {
+            if (_p.startsWith(exports.libraryPrefix)){
+              _p = _p.substring(exports.libraryPrefix.length);
+            }
+            let first = _p.substring(0, _p.indexOf("/"));
+            let second = _p.substring(_p.indexOf("/") + 1);
+            return path.join(_path, first, ascMain, second);
+          }
+          if (args.traceResolution) {
+            stderr.write("    in " + realPath(sourcePath));
+          }
+          const plainName = sourcePath;
+          const indexName = sourcePath + "/index";
+          sourceText = readFile(realPath(plainName) + ".ts", baseDir);
+          if (sourceText !== null) {
+            sourcePath = plainName + ".ts";
+          } else {
+            sourceText = readFile(realPath(indexName) + ".ts", baseDir);
+            if (sourceText !== null) {
+              sourcePath = indexName + ".ts";
+            }
+          }
+          if (sourceText !== null) {
+            if (args.traceResolution) {
+              stderr.write("\nFound at " + realPath(sourcePath) + EOL);
+            }
+            let newPath = path.join(_path, _package);
+            sysPath = newPath;
+            break;
+          }
+          if (args.traceResolution) {
+            stderr.write(EOL);
+          }
+        }
+      }
       if (sourceText == null) {
         return callback(Error("Import file '" + sourcePath + ".ts' not found."));
       }
+      importPathMap.set(sourcePath.replace(/\.ts$/, ""), sysPath);
       stats.parseCount++;
       stats.parseTime += measure(() => {
         assemblyscript.parseFile(sourceText, sourcePath, false, parser);
@@ -418,6 +515,12 @@ exports.main = function main(argv, options, callback) {
   // Finish parsing
   const program = assemblyscript.finishParsing(parser);
 
+  // Print files and exit if listFiles
+  if (args.listFiles) {
+    stderr.write(program.sources.map(s => s.normalizedPath).sort().join(EOL) + EOL);
+    return callback(null);
+  }
+
   // Set up optimization levels
   var optimizeLevel = 0;
   var shrinkLevel = 0;
@@ -718,11 +821,12 @@ exports.main = function main(argv, options, callback) {
   return callback(null);
 
   function readFileNode(filename, baseDir) {
+    let name = path.resolve(baseDir, filename);
     try {
       let text;
       stats.readCount++;
       stats.readTime += measure(() => {
-        text = fs.readFileSync(path.join(baseDir, filename), { encoding: "utf8" });
+        text = fs.readFileSync(name, { encoding: "utf8" });
       });
       return text;
     } catch (e) {
@@ -755,7 +859,7 @@ exports.main = function main(argv, options, callback) {
       });
       return files;
     } catch (e) {
-      return [];
+      return null;
     }
   }
 

cli/asc.json

@@ -216,5 +216,22 @@
   "-O0z": { "value": { "optimizeLevel": 0, "shrinkLevel": 2 } },
   "-O1z": { "value": { "optimizeLevel": 1, "shrinkLevel": 2 } },
   "-O2z": { "value": { "optimizeLevel": 2, "shrinkLevel": 2 } },
-  "-O3z": { "value": { "optimizeLevel": 3, "shrinkLevel": 2 } }
+  "-O3z": { "value": { "optimizeLevel": 3, "shrinkLevel": 2 } },
+  "path": {
+    "description": [
+      "Comma separated paths to look for dependencies.",
+      "Looks for folders with package.json that includes a 'ascMain' field",
+      "or defaults to having an '/assembly' folder."],
+    "type": "S"
+  },
+  "traceResolution": {
+    "description": "Enable tracing of package resolution.",
+    "type": "b",
+    "default": false
+  },
+  "listFiles": {
+    "description": "List files to be compiled and exit.",
+    "type": "b",
+    "default": false
+  }
 }

package.json

@@ -47,9 +47,10 @@
     "check": "npm run check:config && npm run check:compiler",
     "check:config": "tsc --noEmit -p src --diagnostics --listFiles",
     "check:compiler": "tslint -c tslint.json --project src --formatters-dir lib/lint/formatters --format as",
-    "test": "npm run test:parser && npm run test:compiler",
+    "test": "npm run test:parser && npm run test:compiler && npm run test:packages",
     "test:parser": "node tests/parser",
     "test:compiler": "node tests/compiler",
+    "test:packages": "cd tests/packages && npm run test",
     "make": "npm run clean && npm test && npm run build && npm test",
     "all": "npm run check && npm run make",
     "docs": "typedoc --tsconfig tsconfig-docs.json --mode modules --name \"AssemblyScript Compiler API\" --out ./docs/api --ignoreCompilerErrors --excludeNotExported --excludePrivate --excludeExternals --exclude **/std/** --includeDeclarations --readme src/README.md",

src/index.ts

@@ -26,6 +26,11 @@ export function nextFile(parser: Parser): string | null {
   return parser.nextFile();
 }
 
+/** Obtains the path of the dependee of a given imported file. */
+export function getDependee(parser: Parser, file: string): string | null {
+  return parser.getDependee(file);
+}
+
 /** Obtains the next diagnostic message. Returns `null` once complete. */
 export function nextDiagnostic(parser: Parser): DiagnosticMessage | null {
   var program = parser.program;

src/parser.ts

@@ -102,6 +102,10 @@ export class Parser extends DiagnosticEmitter {
   donelog: Set<string> = new Set();
   /** Optional handler to intercept comments while tokenizing. */
   onComment: CommentHandler | null = null;
+  /** Current file being parsed. */
+  currentSource: Source;
+  /** Dependency map **/
+  dependees: Map<string, Source> = new Map();
 
   /** Constructs a new parser. */
   constructor() {
@@ -117,7 +121,6 @@ export class Parser extends DiagnosticEmitter {
   ): void {
     var normalizedPath = normalizePath(path);
     var internalPath = mangleInternalPath(normalizedPath);
-
     // check if already processed
     if (this.donelog.has(internalPath)) return;
     this.donelog.add(internalPath); // do not parse again
@@ -135,6 +138,7 @@ export class Parser extends DiagnosticEmitter {
     );
     var program = this.program;
     program.sources.push(source);
+    this.currentSource = source;
 
     // tokenize and parse
     var tn = new Tokenizer(source, program.diagnostics);
@@ -373,12 +377,22 @@ export class Parser extends DiagnosticEmitter {
     return backlog.length ? backlog.shift() : null;
   }
 
+  /** Obtains the dependee for a given import */
+  getDependee(dependent: string): string | null {
+    var source = this.dependees.get(dependent);
+    if (source) {
+      return source.internalPath;
+    }
+    return null;
+  }
+
   /** Finishes parsing and returns the program. */
   finish(): Program {
     if (this.backlog.length) throw new Error("backlog is not empty");
     this.backlog = [];
     this.seenlog.clear();
     this.donelog.clear();
+    this.dependees.clear();
     return this.program;
   }
 
@@ -2291,6 +2305,7 @@ export class Parser extends DiagnosticEmitter {
       let ret = Node.createExportStatement(members, path, isDeclare, tn.range(startPos, tn.pos));
       let internalPath = ret.internalPath;
       if (internalPath !== null && !this.seenlog.has(internalPath)) {
+        this.dependees.set(internalPath, this.currentSource);
         this.backlog.push(internalPath);
         this.seenlog.add(internalPath);
       }
@@ -2306,8 +2321,8 @@ export class Parser extends DiagnosticEmitter {
           if (!source.exportPaths) source.exportPaths = new Set();
           source.exportPaths.add(internalPath);
           if (!this.seenlog.has(internalPath)) {
+            this.dependees.set(internalPath, this.currentSource);
             this.backlog.push(internalPath);
-            this.seenlog.add(internalPath);
           }
           tn.skip(Token.SEMICOLON);
           return ret;
@@ -2472,8 +2487,8 @@ export class Parser extends DiagnosticEmitter {
         }
         let internalPath = ret.internalPath;
         if (!this.seenlog.has(internalPath)) {
+          this.dependees.set(internalPath, this.currentSource);
           this.backlog.push(internalPath);
-          this.seenlog.add(internalPath);
         }
         tn.skip(Token.SEMICOLON);
         return ret;

tests/packages/.gitignore

@@ -0,0 +1,4 @@
+packages/*/package-lock.json
+package-lock.json
+!node_modules/
+!packages/**/*/node_modules/

tests/packages/package.json

@@ -0,0 +1,15 @@
+{
+  "scripts": {
+    "test": "npm run a && npm run b && npm run c && npm run d && npm run e && npm run f && npm run g && npm run as",
+    "a": "cd ./packages/a && node ../../../../bin/asc assembly/index.ts --noEmit --runtime stub --validate",
+    "as": "cd ./packages/as && node ../../../../bin/asc as/index.ts --noEmit --runtime stub --validate",
+    "b": "cd ./packages/b && node ../../../../bin/asc assembly/index.ts --noEmit --runtime stub && node ../../../../bin/asc assembly/index.ts --noEmit --runtime stub --listFiles",
+    "c": "cd ./packages/c && node ../../../../bin/asc assembly/index.ts --noEmit --runtime stub --validate",
+    "d": "cd ./packages/d && node ../../../../bin/asc assembly/index.ts --path packages --noEmit --runtime stub --validate --traceResolution",
+    "e": "cd ./packages/d/packages/e && node ../../../../../../bin/asc assembly/index.ts --noEmit --runtime stub --validate",
+    "f": "cd ./packages/d/packages/e/packages/f && node ../../../../../../../../bin/asc assembly/index.ts --noEmit --runtime stub --validate",
+    "g": "cd ./packages/g && node test.js"
+  },
+  "author": "Willem Wyndham",
+  "license": "Apache-2.0"
+}

tests/packages/packages/a/assembly/a.ts

@@ -0,0 +1,3 @@
+export function A(): string {
+  return "A";
+}

tests/packages/packages/a/assembly/index.ts

@@ -0,0 +1 @@
+export * from "./a";

tests/packages/packages/as/as/as.ts

@@ -0,0 +1,3 @@
+export function AS(str: string): string {
+  return str + "S";
+}

tests/packages/packages/as/as/index.ts

@@ -0,0 +1 @@
+export * from "./as";

tests/packages/packages/as/package.json

@@ -0,0 +1,3 @@
+{
+  "ascMain": "as/index.ts"
+}

tests/packages/packages/b/assembly/b.ts

@@ -0,0 +1,9 @@
+import { A } from "a";
+
+export function B(): string {
+  return "B";
+}
+
+export function AB(): string {
+  return A() + B();
+}

tests/packages/packages/b/assembly/index.ts

@@ -0,0 +1 @@
+export * from "./b";