Update failed test tracking to support Mocha 6+

Author: rbuckton

.dockerignore

@@ -42,6 +42,7 @@ yarn.lock
 yarn-error.log
 .parallelperf.*
 .failed-tests
+.failed-tests.json
 TEST-results.xml
 package-lock.json
 tests

.gitignore

@@ -83,6 +83,7 @@ tests/cases/user/*/**/*.d.ts
 !tests/cases/user/discord.js/
 tests/baselines/reference/dt
 .failed-tests
+.failed-tests.json
 TEST-results.xml
 package-lock.json
 tests/cases/user/TypeScript-React-Starter/TypeScript-React-Starter

.npmignore

@@ -14,6 +14,7 @@ Jakefile.js
 .eslintignore
 .editorconfig
 .failed-tests
+.failed-tests.json
 .git
 .git/
 .gitattributes

Gulpfile.js

@@ -454,11 +454,11 @@ task("runtests", series(preBuild, preTest, runTests, postTest));
 task("runtests").description = "Runs the tests using the built run.js file.";
 task("runtests").flags = {
     "-t --tests=<regex>": "Pattern for tests to run.",
-    "   --failed": "Runs tests listed in '.failed-tests'.",
+    "   --failed": "Runs tests listed in '.failed-tests.json'.",
     "-r --reporter=<reporter>": "The mocha reporter to use.",
     "-d --debug": "Runs tests in debug mode (NodeJS 6 and earlier)",
     "-i --inspect": "Runs tests in inspector mode (NodeJS 8 and later)",
-    "   --keepFailed": "Keep tests in .failed-tests even if they pass",
+    "   --keepFailed": "Keep tests in '.failed-tests.json even if they pass",
     "   --light": "Run tests in light mode (fewer verifications, but tests run faster)",
     "   --dirty": "Run tests without first cleaning test output directories",
     "   --stackTraceLimit=<limit>": "Sets the maximum number of stack frames to display. Use 'full' to show all frames.",
@@ -476,7 +476,7 @@ task("runtests-parallel").description = "Runs all the tests in parallel using th
 task("runtests-parallel").flags = {
     "   --no-lint": "disables lint.",
     "   --light": "Run tests in light mode (fewer verifications, but tests run faster).",
-    "   --keepFailed": "Keep tests in .failed-tests even if they pass.",
+    "   --keepFailed": "Keep tests in '.failed-tests.json' even if they pass.",
     "   --dirty": "Run tests without first cleaning test output directories.",
     "   --stackTraceLimit=<limit>": "Sets the maximum number of stack frames to display. Use 'full' to show all frames.",
     "   --workers=<number>": "The number of parallel workers to use.",
@@ -634,9 +634,9 @@ task("watch", series(preBuild, preTest, parallel(watchLib, watchDiagnostics, wat
 task("watch").description = "Watches for changes and rebuilds and runs tests in parallel.";
 task("watch").flags = {
     "-t --tests=<regex>": "Pattern for tests to run. Forces tests to be run in a single worker.",
-    "   --failed": "Runs tests listed in '.failed-tests'. Forces tests to be run in a single worker.",
+    "   --failed": "Runs tests listed in '.failed-tests.json'. Forces tests to be run in a single worker.",
     "-r --reporter=<reporter>": "The mocha reporter to use.",
-    "   --keepFailed": "Keep tests in .failed-tests even if they pass",
+    "   --keepFailed": "Keep tests in '.failed-tests.json' even if they pass",
     "   --light": "Run tests in light mode (fewer verifications, but tests run faster)",
     "   --dirty": "Run tests without first cleaning test output directories",
     "   --stackTraceLimit=<limit>": "Sets the maximum number of stack frames to display. Use 'full' to show all frames.",

scripts/build/tests.js

@@ -77,12 +77,15 @@ async function runConsoleTests(runJs, defaultReporter, runInParallel, watchMode,
     // timeout normally isn"t necessary but Travis-CI has been timing out on compiler baselines occasionally
     // default timeout is 2sec which really should be enough, but maybe we just need a small amount longer
     if (!runInParallel) {
-        args.push(failed ? "scripts/run-failed-tests.js" : mochaJs);
+        args.push(mochaJs);
         args.push("-R", "scripts/failed-tests");
         args.push("-O", '"reporter=' + reporter + (keepFailed ? ",keepFailed=true" : "") + '"');
         if (tests) {
             args.push("-g", `"${tests}"`);
         }
+        if (failed) {
+            args.push("--config", ".failed-tests.json");
+        }
         if (colors) {
             args.push("--colors");
         }

scripts/failed-tests.js

@@ -7,7 +7,7 @@ const os = require("os");
 const failingHookRegExp = /^(.*) "(before|after) (all|each)" hook$/;
 
 /**
- * .failed-tests reporter
+ * .failed-tests.json reporter
  *
  * @typedef {Object} ReporterOptions
  * @property {string} [file]
@@ -25,7 +25,7 @@ class FailedTestsReporter extends Mocha.reporters.Base {
         if (!runner) return;
 
         const reporterOptions = this.reporterOptions = options.reporterOptions || {};
-        if (reporterOptions.file === undefined) reporterOptions.file = ".failed-tests";
+        if (reporterOptions.file === undefined) reporterOptions.file = ".failed-tests.json";
         if (reporterOptions.keepFailed === undefined) reporterOptions.keepFailed = false;
         if (reporterOptions.reporter) {
             /** @type {Mocha.ReporterConstructor} */
@@ -70,7 +70,7 @@ class FailedTestsReporter extends Mocha.reporters.Base {
      * @param {(err?: NodeJS.ErrnoException) => void} done
      */
     static writeFailures(file, passes, failures, keepFailed, done) {
-        const failingTests = new Set(fs.existsSync(file) ? readTests() : undefined);
+        const failingTests = new Set(readTests());
         const possiblyPassingSuites = /**@type {Set<string>}*/(new Set());
 
         // Remove tests that are now passing and track suites that are now
@@ -104,9 +104,8 @@ class FailedTestsReporter extends Mocha.reporters.Base {
         if (failingTests.size > 0) {
             const failed = Array
                 .from(failingTests)
-                .sort()
-                .join(os.EOL);
-            fs.writeFile(file, failed, "utf8", done);
+                .sort();
+            fs.writeFile(file, JSON.stringify({ grep: failed }, undefined, "  "), "utf8", done);
         }
         else if (!keepFailed && fs.existsSync(file)) {
             fs.unlink(file, done);
@@ -116,10 +115,12 @@ class FailedTestsReporter extends Mocha.reporters.Base {
         }
 
         function readTests() {
-            return fs.readFileSync(file, "utf8")
-                .split(/\r?\n/g)
-                .map(line => line.trim())
-                .filter(line => line.length > 0);
+            try {
+                return JSON.parse(fs.readFileSync(file, "utf8")).grep;
+            }
+            catch {
+                return undefined;
+            }
         }
     }
 

scripts/run-failed-tests.js

@@ -548,7 +548,7 @@ namespace Harness.Parallel.Host {
                 else {
                     failedTestReporter = new FailedTestReporter(replayRunner, {
                         reporterOptions: {
-                            file: path.resolve(".failed-tests"),
+                            file: path.resolve(".failed-tests.json"),
                             keepFailed: Harness.keepFailed // eslint-disable-line @typescript-eslint/no-unnecessary-qualifier
                         }
                     });