From 4dfb26ecca238ea2a57861627aa44ebd18ccbdaf Mon Sep 17 00:00:00 2001
From: dcode <dcode@dcode.io>
Date: Sat, 28 Sep 2019 00:01:09 +0200
Subject: [PATCH 1/4] Add option to run compiler tests in parallel

---
 tests/compiler.js | 148 +++++++++++++++++++++++++++++++++++-----------
 1 file changed, 112 insertions(+), 36 deletions(-)

diff --git a/tests/compiler.js b/tests/compiler.js
index d2d9cf66dd..1570d62590 100644
--- a/tests/compiler.js
+++ b/tests/compiler.js
@@ -8,6 +8,8 @@ const optionsUtil = require("../cli/util/options");
 const diff = require("./util/diff");
 const asc = require("../cli/asc.js");
 const rtrace = require("../lib/rtrace");
+const cluster = require("cluster");
+const numCPUs = require('os').cpus().length;
 
 const config = {
   "create": {
@@ -34,13 +36,18 @@ const config = {
       "Enables verbose rtrace output."
     ]
   },
+  "parallel": {
+    "description": [
+      "Runs tests in parallel."
+    ]
+  },
   "help": {
     "description": "Prints this message and exits.",
     "type": "b",
     "alias": "h"
   }
 };
-const opts = optionsUtil.parse(process.argv.slice(2),config);
+const opts = optionsUtil.parse(process.argv.slice(2), config);
 const args = opts.options;
 const argv = opts.arguments;
 
@@ -66,7 +73,7 @@ var skippedMessages = new Map();
 const basedir = path.join(__dirname, "compiler");
 
 // Get a list of all tests
-var tests = glob.sync("**/!(_*).ts", { cwd: basedir });
+var tests = glob.sync("**/!(_*).ts", { cwd: basedir }).map(name => name.replace(/\.ts$/, ""));
 
 // Run specific tests only if arguments are provided
 if (argv.length) {
@@ -77,11 +84,10 @@ if (argv.length) {
   }
 }
 
-// TODO: asc's callback is synchronous here. This might change.
-tests.forEach(filename => {
-  console.log(colorsUtil.white("Testing compiler/" + filename) + "\n");
+// Runs a single test
+function runTest(basename) {
+  console.log(colorsUtil.white("Testing compiler/" + basename) + "\n");
 
-  const basename = filename.replace(/\.ts$/, "");
   const configPath = path.join(basedir, basename + ".json");
   const config = fs.existsSync(configPath)
     ? require(configPath)
@@ -118,6 +124,7 @@ tests.forEach(filename => {
       console.log("- " + colorsUtil.yellow("feature SKIPPED") + " (" + missing_features.join(", ") + ")\n");
       skippedTests.add(basename);
       skippedMessages.set(basename, "feature not enabled");
+      if (cluster.isWorker) process.send({ cmd: "skipped", message: skippedMessages.get(basename) });
       return;
     }
   }
@@ -129,11 +136,9 @@ tests.forEach(filename => {
 
   var failed = false;
 
-  // TODO: also save stdout/stderr and diff it (-> expected failures)
-
   // Build unoptimized
   var cmd = [
-    filename,
+    basename + ".ts",
     "--baseDir", basedir,
     "--validate",
     "--measure",
@@ -142,10 +147,7 @@ tests.forEach(filename => {
   ];
   if (asc_flags)
     Array.prototype.push.apply(cmd, asc_flags);
-  if (args.createBinary)
-    cmd.push("--binaryFile", basename + ".untouched.wasm");
-  else
-    cmd.push("--binaryFile", "temp.wasm");
+  cmd.push("--binaryFile", basename + ".untouched.wasm");
   asc.main(cmd, {
     stdout: stdout,
     stderr: stderr
@@ -214,7 +216,7 @@ tests.forEach(filename => {
 
     // Build optimized
     var cmd = [
-      filename,
+      basename + ".ts",
       "--baseDir", basedir,
       "--validate",
       "--measure",
@@ -237,7 +239,7 @@ tests.forEach(filename => {
         failedTests.add(basename);
         return 1;
       }
-      let untouchedBuffer = fs.readFileSync(path.join(basedir, "temp.wasm"));
+      let untouchedBuffer = fs.readFileSync(path.join(basedir, basename + ".untouched.wasm"));
       let optimizedBuffer = stdout.toBuffer();
       const gluePath = path.join(basedir, basename + ".js");
       var glue = {};
@@ -258,29 +260,11 @@ tests.forEach(filename => {
     if (failed) return 1;
   });
   if (v8_no_flags) v8.setFlagsFromString(v8_no_flags);
-});
-
-if (skippedTests.size) {
-  console.log(colorsUtil.yellow("WARNING: ") + colorsUtil.white(skippedTests.size + " compiler tests have been skipped:\n"));
-  skippedTests.forEach(name => {
-    var message = skippedMessages.has(name) ? colorsUtil.gray("[" + skippedMessages.get(name) + "]") : "";
-    console.log("  " + name + " " + message);
-  });
-  console.log();
-}
-if (failedTests.size) {
-  process.exitCode = 1;
-  console.log(colorsUtil.red("ERROR: ") + colorsUtil.white(failedTests.size + " compiler tests had failures:\n"));
-  failedTests.forEach(name => {
-    var message = failedMessages.has(name) ? colorsUtil.gray("[" + failedMessages.get(name) + "]") : "";
-    console.log("  " + name + " " + message);
-  });
-  console.log();
-}
-if (!process.exitCode) {
-  console.log("[ " + colorsUtil.white("OK") + " ]");
+  if (!args.createBinary) fs.unlink(path.join(basedir, basename + ".untouched.wasm"), err => {});
+  if (cluster.isWorker) process.send({ cmd: "done", failed: failed, message: failedMessages.get(basename) });
 }
 
+// Tests if instantiation of a module succeeds
 function testInstantiate(basename, binaryBuffer, name, glue) {
   var failed = false;
   try {
@@ -370,3 +354,95 @@ function testInstantiate(basename, binaryBuffer, name, glue) {
   }
   return false;
 }
+
+// Evaluates the overall test result
+function evaluate() {
+  if (skippedTests.size) {
+    console.log(colorsUtil.yellow("WARNING: ") + colorsUtil.white(skippedTests.size + " compiler tests have been skipped:\n"));
+    skippedTests.forEach(name => {
+      var message = skippedMessages.has(name) ? colorsUtil.gray("[" + skippedMessages.get(name) + "]") : "";
+      console.log("  " + name + " " + message);
+    });
+    console.log();
+  }
+  if (failedTests.size) {
+    process.exitCode = 1;
+    console.log(colorsUtil.red("ERROR: ") + colorsUtil.white(failedTests.size + " compiler tests had failures:\n"));
+    failedTests.forEach(name => {
+      var message = failedMessages.has(name) ? colorsUtil.gray("[" + failedMessages.get(name) + "]") : "";
+      console.log("  " + name + " " + message);
+    });
+    console.log();
+  }
+  if (!process.exitCode) {
+    console.log("[ " + colorsUtil.white("OK") + " ]");
+  }
+}
+
+// Run tests in parallel if requested
+if (args.parallel && tests.length > 1) {
+  if (cluster.isWorker) {
+    colorsUtil.supported = true;
+    process.on("message", msg => {
+      if (msg.cmd == "run") {
+        try {
+          runTest(msg.test);
+        } catch (e) {
+          process.send({ cmd: "done", failed: true, message: e.message });
+        }
+        process.send({ cmd: "ready" });
+      } else {
+        throw Error("invalid command: " + msg.cmd);
+      }
+    });
+    process.send({ cmd: "ready" });
+  } else {
+    let workers = [];
+    let current = [];
+    let outputs = [];
+    let index = 0;
+    let numWorkers = Math.min(numCPUs, tests.length);
+    console.log("Spawning " + numWorkers + " workers ...");
+    cluster.settings.silent = true;
+    for (let i = 0; i < numWorkers; ++i) {
+      let worker = cluster.fork();
+      workers[i] = worker;
+      current[i] = null;
+      outputs[i] = [];
+      worker.process.stdout.on("data", buf => outputs[i].push(buf));
+      worker.process.stderr.on("data", buf => outputs[i].push(buf));
+      worker.on("message", msg => {
+        if (msg.cmd == "ready") {
+          if (index >= tests.length) {
+            workers[i] = null;
+            worker.kill();
+            return;
+          }
+          let testName = tests[index++];
+          current[i] = testName;
+          outputs[i] = [];
+          worker.send({ cmd: "run", test: testName });
+        } else if (msg.cmd == "done") {
+          process.stdout.write(Buffer.concat(outputs[i]).toString());
+          if (msg.failed) failedTests.add(current[i]);
+          if (msg.message) failedMessages.set(current[i], msg.message);
+        } else if (msg.cmd == "skipped") {
+          process.stdout.write(Buffer.concat(outputs[i]).toString());
+          skippedTests.add(current[i]);
+          if (msg.message) skippedMessages.set(current[i], msg.message);
+        } else {
+          throw Error("invalid command: " + msg.cmd);
+        }
+      });
+      worker.on("disconnect", () => {
+        if (workers[i]) throw Error("worker#" + i + " died unexpectedly");
+        if (workers.every(w => w == null)) evaluate();
+      });
+    }
+  }
+
+// Otherwise run tests sequentially
+} else {
+  tests.forEach(runTest);
+  evaluate();
+}

From ecfc957ea8fa75d3bb68d40450b7e324f31ba652 Mon Sep 17 00:00:00 2001
From: dcode <dcode@dcode.io>
Date: Sat, 28 Sep 2019 01:06:33 +0200
Subject: [PATCH 2/4] tweaks

---
 package-lock.json |  6 +++++
 package.json      |  1 +
 tests/compiler.js | 64 +++++++++++++++++++++++------------------------
 3 files changed, 38 insertions(+), 33 deletions(-)

diff --git a/package-lock.json b/package-lock.json
index 062e94a6ba..fccff050c9 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -3274,6 +3274,12 @@
         "sha.js": "^2.4.8"
       }
     },
+    "physical-cpu-count": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/physical-cpu-count/-/physical-cpu-count-2.0.0.tgz",
+      "integrity": "sha1-GN4vl+S/epVRrXURlCtUlverpmA=",
+      "dev": true
+    },
     "picomatch": {
       "version": "2.0.7",
       "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.0.7.tgz",
diff --git a/package.json b/package.json
index f213fcc56a..d008faa551 100644
--- a/package.json
+++ b/package.json
@@ -22,6 +22,7 @@
     "@types/node": "^12.7.5",
     "browser-process-hrtime": "^1.0.0",
     "diff": "^4.0.1",
+    "physical-cpu-count": "^2.0.0",
     "ts-loader": "^6.1.1",
     "ts-node": "^6.2.0",
     "tslint": "^5.20.0",
diff --git a/tests/compiler.js b/tests/compiler.js
index 1570d62590..132005d905 100644
--- a/tests/compiler.js
+++ b/tests/compiler.js
@@ -9,7 +9,7 @@ const diff = require("./util/diff");
 const asc = require("../cli/asc.js");
 const rtrace = require("../lib/rtrace");
 const cluster = require("cluster");
-const numCPUs = require('os').cpus().length;
+const coreCount = require("physical-cpu-count");
 
 const config = {
   "create": {
@@ -72,16 +72,17 @@ var skippedMessages = new Map();
 
 const basedir = path.join(__dirname, "compiler");
 
-// Get a list of all tests
-var tests = glob.sync("**/!(_*).ts", { cwd: basedir }).map(name => name.replace(/\.ts$/, ""));
-
-// Run specific tests only if arguments are provided
-if (argv.length) {
-  tests = tests.filter(filename => argv.indexOf(filename.replace(/\.ts$/, "")) >= 0);
-  if (!tests.length) {
-    console.error("No matching tests: " + argv.join(" "));
-    process.exit(1);
+// Gets a list of all relevant tests
+function getTests() {
+  var tests =  glob.sync("**/!(_*).ts", { cwd: basedir }).map(name => name.replace(/\.ts$/, ""));
+  if (argv.length) { // run matching tests only
+    tests = tests.filter(filename => argv.indexOf(filename.replace(/\.ts$/, "")) >= 0);
+    if (!tests.length) {
+      console.error("No matching tests: " + argv.join(" "));
+      process.exit(1);
+    }
   }
+  return tests;
 }
 
 // Runs a single test
@@ -356,7 +357,7 @@ function testInstantiate(basename, binaryBuffer, name, glue) {
 }
 
 // Evaluates the overall test result
-function evaluate() {
+function evaluateResult() {
   if (skippedTests.size) {
     console.log(colorsUtil.yellow("WARNING: ") + colorsUtil.white(skippedTests.size + " compiler tests have been skipped:\n"));
     skippedTests.forEach(name => {
@@ -380,30 +381,28 @@ function evaluate() {
 }
 
 // Run tests in parallel if requested
-if (args.parallel && tests.length > 1) {
+if (args.parallel && coreCount > 1) {
   if (cluster.isWorker) {
     colorsUtil.supported = true;
     process.on("message", msg => {
-      if (msg.cmd == "run") {
-        try {
-          runTest(msg.test);
-        } catch (e) {
-          process.send({ cmd: "done", failed: true, message: e.message });
-        }
-        process.send({ cmd: "ready" });
-      } else {
-        throw Error("invalid command: " + msg.cmd);
+      if (msg.cmd != "run") throw Error("invalid command: " + msg.cmd);
+      try {
+        runTest(msg.test);
+      } catch (e) {
+        process.send({ cmd: "done", failed: true, message: e.message });
       }
+      process.send({ cmd: "ready" });
     });
     process.send({ cmd: "ready" });
   } else {
-    let workers = [];
-    let current = [];
-    let outputs = [];
-    let index = 0;
-    let numWorkers = Math.min(numCPUs, tests.length);
-    console.log("Spawning " + numWorkers + " workers ...");
+    const tests = getTests();
+    const workers = [];
+    const current = [];
+    const outputs = [];
+    let numWorkers = coreCount - 1;
+    console.log("Spawning " + numWorkers + " workers (detected " + coreCount + " physical CPUs ) ...");
     cluster.settings.silent = true;
+    let index = 0;
     for (let i = 0; i < numWorkers; ++i) {
       let worker = cluster.fork();
       workers[i] = worker;
@@ -418,10 +417,9 @@ if (args.parallel && tests.length > 1) {
             worker.kill();
             return;
           }
-          let testName = tests[index++];
-          current[i] = testName;
+          current[i] = tests[index++];
           outputs[i] = [];
-          worker.send({ cmd: "run", test: testName });
+          worker.send({ cmd: "run", test: current[i] });
         } else if (msg.cmd == "done") {
           process.stdout.write(Buffer.concat(outputs[i]).toString());
           if (msg.failed) failedTests.add(current[i]);
@@ -436,13 +434,13 @@ if (args.parallel && tests.length > 1) {
       });
       worker.on("disconnect", () => {
         if (workers[i]) throw Error("worker#" + i + " died unexpectedly");
-        if (workers.every(w => w == null)) evaluate();
+        if (!--numWorkers) evaluateResult();
       });
     }
   }
 
 // Otherwise run tests sequentially
 } else {
-  tests.forEach(runTest);
-  evaluate();
+  getTests().forEach(runTest);
+  evaluateResult();
 }

From afee7184b4493908bd74349794f52db15fcdd8c9 Mon Sep 17 00:00:00 2001
From: dcode <dcode@dcode.io>
Date: Sat, 28 Sep 2019 02:07:18 +0200
Subject: [PATCH 3/4] more tweaks

---
 tests/compiler.js | 21 +++++++++++++++++++--
 1 file changed, 19 insertions(+), 2 deletions(-)

diff --git a/tests/compiler.js b/tests/compiler.js
index 132005d905..b99dcb9bec 100644
--- a/tests/compiler.js
+++ b/tests/compiler.js
@@ -1,3 +1,4 @@
+const startTime = Date.now();
 const fs  = require("fs");
 const path = require("path");
 const os = require("os");
@@ -375,6 +376,7 @@ function evaluateResult() {
     });
     console.log();
   }
+  console.log("Time: " + (Date.now() - startTime) + " ms\n");
   if (!process.exitCode) {
     console.log("[ " + colorsUtil.white("OK") + " ]");
   }
@@ -396,11 +398,26 @@ if (args.parallel && coreCount > 1) {
     process.send({ cmd: "ready" });
   } else {
     const tests = getTests();
+    for (let name of [ // Run tests known to take exceptionally long first
+      "std/libm",
+      "std/set",
+      "std/map",
+      "std/string",
+      "std/array",
+      "std/typedarray",
+      "std/math",
+    ]) {
+      let p = tests.indexOf(name);
+      if (p >= 0) {
+        tests.splice(p, 1);
+        tests.unshift(name);
+      }
+    }
     const workers = [];
     const current = [];
     const outputs = [];
-    let numWorkers = coreCount - 1;
-    console.log("Spawning " + numWorkers + " workers (detected " + coreCount + " physical CPUs ) ...");
+    let numWorkers = Math.min(coreCount - 1, tests.length);
+    console.log("Spawning " + numWorkers + " workers ...");
     cluster.settings.silent = true;
     let index = 0;
     for (let i = 0; i < numWorkers; ++i) {

From 91493a563c358fb5585574714788753e9182598a Mon Sep 17 00:00:00 2001
From: dcode <dcode@dcode.io>
Date: Sat, 28 Sep 2019 03:02:04 +0200
Subject: [PATCH 4/4] less is more

---
 tests/compiler.js | 40 +++++++++++++---------------------------
 1 file changed, 13 insertions(+), 27 deletions(-)

diff --git a/tests/compiler.js b/tests/compiler.js
index b99dcb9bec..82c0c3b772 100644
--- a/tests/compiler.js
+++ b/tests/compiler.js
@@ -393,26 +393,13 @@ if (args.parallel && coreCount > 1) {
       } catch (e) {
         process.send({ cmd: "done", failed: true, message: e.message });
       }
-      process.send({ cmd: "ready" });
     });
     process.send({ cmd: "ready" });
   } else {
     const tests = getTests();
-    for (let name of [ // Run tests known to take exceptionally long first
-      "std/libm",
-      "std/set",
-      "std/map",
-      "std/string",
-      "std/array",
-      "std/typedarray",
-      "std/math",
-    ]) {
-      let p = tests.indexOf(name);
-      if (p >= 0) {
-        tests.splice(p, 1);
-        tests.unshift(name);
-      }
-    }
+    // const sizes = new Map();
+    // tests.forEach(name => sizes.set(name, fs.statSync(path.join(basedir, name + ".ts")).size));
+    // tests.sort((a, b) => sizes.get(b) - sizes.get(a));
     const workers = [];
     const current = [];
     const outputs = [];
@@ -428,16 +415,7 @@ if (args.parallel && coreCount > 1) {
       worker.process.stdout.on("data", buf => outputs[i].push(buf));
       worker.process.stderr.on("data", buf => outputs[i].push(buf));
       worker.on("message", msg => {
-        if (msg.cmd == "ready") {
-          if (index >= tests.length) {
-            workers[i] = null;
-            worker.kill();
-            return;
-          }
-          current[i] = tests[index++];
-          outputs[i] = [];
-          worker.send({ cmd: "run", test: current[i] });
-        } else if (msg.cmd == "done") {
+        if (msg.cmd == "done") {
           process.stdout.write(Buffer.concat(outputs[i]).toString());
           if (msg.failed) failedTests.add(current[i]);
           if (msg.message) failedMessages.set(current[i], msg.message);
@@ -445,9 +423,17 @@ if (args.parallel && coreCount > 1) {
           process.stdout.write(Buffer.concat(outputs[i]).toString());
           skippedTests.add(current[i]);
           if (msg.message) skippedMessages.set(current[i], msg.message);
-        } else {
+        } else if (msg.cmd != "ready") {
           throw Error("invalid command: " + msg.cmd);
         }
+        if (index >= tests.length) {
+          workers[i] = null;
+          worker.kill();
+          return;
+        }
+        current[i] = tests[index++];
+        outputs[i] = [];
+        worker.send({ cmd: "run", test: current[i] });
       });
       worker.on("disconnect", () => {
         if (workers[i]) throw Error("worker#" + i + " died unexpectedly");