Fix unittest parallel reporting (#18583)

weswigham · web-flow · commit 5f49357bf658 · 2017-09-19T16:52:56.000-07:00
* Some tests depended on late execution * Emulate mocha execution order * Polyfill a synchronous done to handle that one unittest * Accpept updates tsconfig baselines fixed by #18534
diff --git a/src/harness/parallel/worker.ts b/src/harness/parallel/worker.ts
@@ -1,60 +1,132 @@
 namespace Harness.Parallel.Worker {
     let errors: ErrorInfo[] = [];
     let passing = 0;
+    let reportedUnitTests = false;
+
+    type Executor = {name: string, callback: Function, kind: "suite" | "test"} | never;
+
     function resetShimHarnessAndExecute(runner: RunnerBase) {
-        errors = [];
-        passing = 0;
+        if (reportedUnitTests) {
+            errors = [];
+            passing = 0;
+            testList.length = 0;
+        }
+        reportedUnitTests = true;
         runner.initializeTests();
+        testList.forEach(({ name, callback, kind }) => executeCallback(name, callback, kind));
         return { errors, passing };
     }
 
+
+    let beforeEachFunc: Function;
+    const namestack: string[] = [];
+    let testList: Executor[] = [];
     function shimMochaHarness() {
         (global as any).before = undefined;
         (global as any).after = undefined;
         (global as any).beforeEach = undefined;
-        let beforeEachFunc: Function;
-        describe = ((_name, callback) => {
-            const fakeContext: Mocha.ISuiteCallbackContext = {
-                retries() { return this; },
-                slow() { return this; },
-                timeout() { return this; },
-            };
-            (before as any) = (cb: Function) => cb();
-            let afterFunc: Function;
-            (after as any) = (cb: Function) => afterFunc = cb;
-            const savedBeforeEach = beforeEachFunc;
-            (beforeEach as any) = (cb: Function) => beforeEachFunc = cb;
-            callback.call(fakeContext);
-            afterFunc && afterFunc();
-            afterFunc = undefined;
-            beforeEachFunc = savedBeforeEach;
+        describe = ((name, callback) => {
+            testList.push({ name, callback, kind: "suite" });
         }) as Mocha.IContextDefinition;
         it = ((name, callback) => {
-            const fakeContext: Mocha.ITestCallbackContext = {
-                skip() { return this; },
-                timeout() { return this; },
-                retries() { return this; },
-                slow() { return this; },
-            };
-            // TODO: If we ever start using async test completions, polyfill the `done` parameter/promise return handling
-            if (beforeEachFunc) {
-                try {
-                    beforeEachFunc();
-                }
-                catch (error) {
-                    errors.push({ error: error.message, stack: error.stack, name });
-                    return;
-                }
+            if (!testList) {
+                throw new Error("Tests must occur within a describe block");
             }
+            testList.push({ name, callback, kind: "test" });
+        }) as Mocha.ITestDefinition;
+    }
+
+    function executeSuiteCallback(name: string, callback: Function) {
+        const fakeContext: Mocha.ISuiteCallbackContext = {
+            retries() { return this; },
+            slow() { return this; },
+            timeout() { return this; },
+        };
+        namestack.push(name);
+        let beforeFunc: Function;
+        (before as any) = (cb: Function) => beforeFunc = cb;
+        let afterFunc: Function;
+        (after as any) = (cb: Function) => afterFunc = cb;
+        const savedBeforeEach = beforeEachFunc;
+        (beforeEach as any) = (cb: Function) => beforeEachFunc = cb;
+        const savedTestList = testList;
+
+        testList = [];
+        callback.call(fakeContext);
+        beforeFunc && beforeFunc();
+        beforeFunc = undefined;
+        testList.forEach(({ name, callback, kind }) => executeCallback(name, callback, kind));
+        testList.length = 0;
+        testList = savedTestList;
+
+        afterFunc && afterFunc();
+        afterFunc = undefined;
+        beforeEachFunc = savedBeforeEach;
+        namestack.pop();
+    }
+
+    function executeCallback(name: string, callback: Function, kind: "suite" | "test") {
+        if (kind === "suite") {
+            executeSuiteCallback(name, callback);
+        }
+        else {
+            executeTestCallback(name, callback);
+        }
+    }
+
+    function executeTestCallback(name: string, callback: Function) {
+        const fakeContext: Mocha.ITestCallbackContext = {
+            skip() { return this; },
+            timeout() { return this; },
+            retries() { return this; },
+            slow() { return this; },
+        };
+        name = [...namestack, name].join(" ");
+        if (beforeEachFunc) {
             try {
+                beforeEachFunc();
+            }
+            catch (error) {
+                errors.push({ error: error.message, stack: error.stack, name });
+                return;
+            }
+        }
+        if (callback.length === 0) {
+            try {
+                // TODO: If we ever start using async test completions, polyfill promise return handling
                 callback.call(fakeContext);
             }
             catch (error) {
                 errors.push({ error: error.message, stack: error.stack, name });
                 return;
             }
             passing++;
-        }) as Mocha.ITestDefinition;
+        }
+        else {
+            // Uses `done` callback
+            let completed = false;
+            try {
+                callback.call(fakeContext, (err: any) => {
+                    if (completed) {
+                        throw new Error(`done() callback called multiple times; ensure it is only called once.`);
+                    }
+                    if (err) {
+                        errors.push({ error: err.toString(), stack: "", name });
+                    }
+                    else {
+                        passing++;
+                    }
+                    completed = true;
+                });
+            }
+            catch (error) {
+                errors.push({ error: error.message, stack: error.stack, name });
+                return;
+            }
+            if (!completed) {
+                errors.push({ error: "Test completes asynchronously, which is unsupported by the parallel harness", stack: "", name });
+            }
+        }
     }
 
     export function start() {
diff --git a/tests/baselines/reference/tsConfig/Default initialized TSConfig/tsconfig.json b/tests/baselines/reference/tsConfig/Default initialized TSConfig/tsconfig.json
@@ -2,7 +2,7 @@
   "compilerOptions": {
     /* Basic Options */
     "target": "es5",                          /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', or 'ESNEXT'. */
-    "module": "commonjs",                     /* Specify module code generation: 'none', commonjs', 'amd', 'system', 'umd', 'es2015', or 'ESNext'. */
+    "module": "commonjs",                     /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', or 'ESNext'. */
     // "lib": [],                             /* Specify library files to be included in the compilation:  */
     // "allowJs": true,                       /* Allow javascript files to be compiled. */
     // "checkJs": true,                       /* Report errors in .js files. */
diff --git a/tests/baselines/reference/tsConfig/Initialized TSConfig with boolean value compiler options/tsconfig.json b/tests/baselines/reference/tsConfig/Initialized TSConfig with boolean value compiler options/tsconfig.json
@@ -2,7 +2,7 @@
   "compilerOptions": {
     /* Basic Options */
     "target": "es5",                          /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', or 'ESNEXT'. */
-    "module": "commonjs",                     /* Specify module code generation: 'none', commonjs', 'amd', 'system', 'umd', 'es2015', or 'ESNext'. */
+    "module": "commonjs",                     /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', or 'ESNext'. */
     // "lib": [],                             /* Specify library files to be included in the compilation:  */
     // "allowJs": true,                       /* Allow javascript files to be compiled. */
     // "checkJs": true,                       /* Report errors in .js files. */
diff --git a/tests/baselines/reference/tsConfig/Initialized TSConfig with enum value compiler options/tsconfig.json b/tests/baselines/reference/tsConfig/Initialized TSConfig with enum value compiler options/tsconfig.json
@@ -2,7 +2,7 @@
   "compilerOptions": {
     /* Basic Options */
     "target": "es5",                          /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', or 'ESNEXT'. */
-    "module": "commonjs",                     /* Specify module code generation: 'none', commonjs', 'amd', 'system', 'umd', 'es2015', or 'ESNext'. */
+    "module": "commonjs",                     /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', or 'ESNext'. */
     // "lib": [],                             /* Specify library files to be included in the compilation:  */
     // "allowJs": true,                       /* Allow javascript files to be compiled. */
     // "checkJs": true,                       /* Report errors in .js files. */
diff --git a/tests/baselines/reference/tsConfig/Initialized TSConfig with files options/tsconfig.json b/tests/baselines/reference/tsConfig/Initialized TSConfig with files options/tsconfig.json
@@ -2,7 +2,7 @@
   "compilerOptions": {
     /* Basic Options */
     "target": "es5",                          /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', or 'ESNEXT'. */
-    "module": "commonjs",                     /* Specify module code generation: 'none', commonjs', 'amd', 'system', 'umd', 'es2015', or 'ESNext'. */
+    "module": "commonjs",                     /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', or 'ESNext'. */
     // "lib": [],                             /* Specify library files to be included in the compilation:  */
     // "allowJs": true,                       /* Allow javascript files to be compiled. */
     // "checkJs": true,                       /* Report errors in .js files. */
diff --git a/tests/baselines/reference/tsConfig/Initialized TSConfig with incorrect compiler option value/tsconfig.json b/tests/baselines/reference/tsConfig/Initialized TSConfig with incorrect compiler option value/tsconfig.json
@@ -2,7 +2,7 @@
   "compilerOptions": {
     /* Basic Options */
     "target": "es5",                          /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', or 'ESNEXT'. */
-    "module": "commonjs",                     /* Specify module code generation: 'none', commonjs', 'amd', 'system', 'umd', 'es2015', or 'ESNext'. */
+    "module": "commonjs",                     /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', or 'ESNext'. */
     "lib": ["es5","es2015.promise"],          /* Specify library files to be included in the compilation:  */
     // "allowJs": true,                       /* Allow javascript files to be compiled. */
     // "checkJs": true,                       /* Report errors in .js files. */
diff --git a/tests/baselines/reference/tsConfig/Initialized TSConfig with incorrect compiler option/tsconfig.json b/tests/baselines/reference/tsConfig/Initialized TSConfig with incorrect compiler option/tsconfig.json
@@ -2,7 +2,7 @@
   "compilerOptions": {
     /* Basic Options */
     "target": "es5",                          /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', or 'ESNEXT'. */
-    "module": "commonjs",                     /* Specify module code generation: 'none', commonjs', 'amd', 'system', 'umd', 'es2015', or 'ESNext'. */
+    "module": "commonjs",                     /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', or 'ESNext'. */
     // "lib": [],                             /* Specify library files to be included in the compilation:  */
     // "allowJs": true,                       /* Allow javascript files to be compiled. */
     // "checkJs": true,                       /* Report errors in .js files. */
diff --git a/tests/baselines/reference/tsConfig/Initialized TSConfig with list compiler options with enum value/tsconfig.json b/tests/baselines/reference/tsConfig/Initialized TSConfig with list compiler options with enum value/tsconfig.json
@@ -2,7 +2,7 @@
   "compilerOptions": {
     /* Basic Options */
     "target": "es5",                          /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', or 'ESNEXT'. */
-    "module": "commonjs",                     /* Specify module code generation: 'none', commonjs', 'amd', 'system', 'umd', 'es2015', or 'ESNext'. */
+    "module": "commonjs",                     /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', or 'ESNext'. */
     "lib": ["es5","es2015.core"],             /* Specify library files to be included in the compilation:  */
     // "allowJs": true,                       /* Allow javascript files to be compiled. */
     // "checkJs": true,                       /* Report errors in .js files. */
diff --git a/tests/baselines/reference/tsConfig/Initialized TSConfig with list compiler options/tsconfig.json b/tests/baselines/reference/tsConfig/Initialized TSConfig with list compiler options/tsconfig.json
@@ -2,7 +2,7 @@
   "compilerOptions": {
     /* Basic Options */
     "target": "es5",                          /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', or 'ESNEXT'. */
-    "module": "commonjs",                     /* Specify module code generation: 'none', commonjs', 'amd', 'system', 'umd', 'es2015', or 'ESNext'. */
+    "module": "commonjs",                     /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', or 'ESNext'. */
     // "lib": [],                             /* Specify library files to be included in the compilation:  */
     // "allowJs": true,                       /* Allow javascript files to be compiled. */
     // "checkJs": true,                       /* Report errors in .js files. */
