Skip to content

Balance unit tests across multiple threads #18956

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Oct 11, 2017
Merged

Balance unit tests across multiple threads #18956

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
1a4598d
Functioning parallel unittests
weswigham Oct 4, 2017
dc60b39
Merge branch 'master' into loadbalance-unit-tests
weswigham Oct 4, 2017
85396a6
Merge branch 'master' into loadbalance-unit-tests
weswigham Oct 9, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 36 additions & 12 deletions src/harness/parallel/host.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,20 +38,45 @@ namespace Harness.Parallel.Host {
return undefined;
}

function hashName(runner: TestRunnerKind, test: string) {
function hashName(runner: TestRunnerKind | "unittest", test: string) {
return `tsrunner-${runner}://${test}`;
}

let tasks: { runner: TestRunnerKind | "unittest", file: string, size: number }[] = [];
const newTasks: { runner: TestRunnerKind | "unittest", file: string, size: number }[] = [];
let unknownValue: string | undefined;
export function start() {
const perfData = readSavedPerfData(configOption);
let totalCost = 0;
if (runUnitTests) {
(global as any).describe = (suiteName: string) => {
// Note, sub-suites are not indexed (we assume such granularity is not required)
let size = 0;
if (perfData) {
size = perfData[hashName("unittest", suiteName)];
if (size === undefined) {
newTasks.push({ runner: "unittest", file: suiteName, size: 0 });
unknownValue = suiteName;
return;
}
}
tasks.push({ runner: "unittest", file: suiteName, size });
totalCost += size;
};
}
else {
(global as any).describe = ts.noop;
}

setTimeout(() => startDelayed(perfData, totalCost), 0); // Do real startup on next tick, so all unit tests have been collected
}

function startDelayed(perfData: {[testHash: string]: number}, totalCost: number) {
initializeProgressBarsDependencies();
console.log("Discovering tests...");
console.log(`Discovered ${tasks.length} unittest suites` + (newTasks.length ? ` and ${newTasks.length} new suites.` : "."));
console.log("Discovering runner-based tests...");
const discoverStart = +(new Date());
const { statSync }: { statSync(path: string): { size: number }; } = require("fs");
let tasks: { runner: TestRunnerKind, file: string, size: number }[] = [];
const newTasks: { runner: TestRunnerKind, file: string, size: number }[] = [];
const perfData = readSavedPerfData(configOption);
let totalCost = 0;
let unknownValue: string | undefined;
for (const runner of runners) {
const files = runner.enumerateTestFiles();
for (const file of files) {
Expand Down Expand Up @@ -87,8 +112,7 @@ namespace Harness.Parallel.Host {
}
tasks.sort((a, b) => a.size - b.size);
tasks = tasks.concat(newTasks);
// 1 fewer batches than threads to account for unittests running on the final thread
const batchCount = runners.length === 1 ? workerCount : workerCount - 1;
const batchCount = workerCount;
const packfraction = 0.9;
const chunkSize = 1000; // ~1KB or 1s for sending batches near the end of a test
const batchSize = (totalCost / workerCount) * packfraction; // Keep spare tests for unittest thread in reserve
Expand All @@ -113,7 +137,7 @@ namespace Harness.Parallel.Host {
let closedWorkers = 0;
for (let i = 0; i < workerCount; i++) {
// TODO: Just send the config over the IPC channel or in the command line arguments
const config: TestConfig = { light: Harness.lightMode, listenForWork: true, runUnitTests: runners.length === 1 ? false : i === workerCount - 1 };
const config: TestConfig = { light: Harness.lightMode, listenForWork: true, runUnitTests: runners.length !== 1 };
const configPath = ts.combinePaths(taskConfigsFolder, `task-config${i}.json`);
Harness.IO.writeFile(configPath, JSON.stringify(config));
const child = fork(__filename, [`--config="${configPath}"`]);
Expand Down Expand Up @@ -187,7 +211,7 @@ namespace Harness.Parallel.Host {
// It's only really worth doing an initial batching if there are a ton of files to go through
if (totalFiles > 1000) {
console.log("Batching initial test lists...");
const batches: { runner: TestRunnerKind, file: string, size: number }[][] = new Array(batchCount);
const batches: { runner: TestRunnerKind | "unittest", file: string, size: number }[][] = new Array(batchCount);
const doneBatching = new Array(batchCount);
let scheduledTotal = 0;
batcher: while (true) {
Expand Down Expand Up @@ -230,7 +254,7 @@ namespace Harness.Parallel.Host {
if (payload) {
worker.send({ type: "batch", payload });
}
else { // Unittest thread - send off just one test
else { // Out of batches, send off just one test
const payload = tasks.pop();
ts.Debug.assert(!!payload); // The reserve kept above should ensure there is always an initial task available, even in suboptimal scenarios
worker.send({ type: "test", payload });
Expand Down
4 changes: 2 additions & 2 deletions src/harness/parallel/shared.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,14 +1,14 @@
/// <reference path="./host.ts" />
/// <reference path="./worker.ts" />
namespace Harness.Parallel {
export type ParallelTestMessage = { type: "test", payload: { runner: TestRunnerKind, file: string } } | never;
export type ParallelTestMessage = { type: "test", payload: { runner: TestRunnerKind | "unittest", file: string } } | never;
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why not add "unittest" to TestRunnerKind?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We use TestRunnerKind outside the parallel runner to mean "anything for which we have a mapping to a runner class". Unfortunately, the logic doesn't generalize well to normal unit tests (as they don't conform to that API).

export type ParallelBatchMessage = { type: "batch", payload: ParallelTestMessage["payload"][] } | never;
export type ParallelCloseMessage = { type: "close" } | never;
export type ParallelHostMessage = ParallelTestMessage | ParallelCloseMessage | ParallelBatchMessage;

export type ParallelErrorMessage = { type: "error", payload: { error: string, stack: string, name?: string[] } } | never;
export type ErrorInfo = ParallelErrorMessage["payload"] & { name: string[] };
export type ParallelResultMessage = { type: "result", payload: { passing: number, errors: ErrorInfo[], duration: number, runner: TestRunnerKind, file: string } } | never;
export type ParallelResultMessage = { type: "result", payload: { passing: number, errors: ErrorInfo[], duration: number, runner: TestRunnerKind | "unittest", file: string } } | never;
export type ParallelBatchProgressMessage = { type: "progress", payload: ParallelResultMessage["payload"] } | never;
export type ParallelClientMessage = ParallelErrorMessage | ParallelResultMessage | ParallelBatchProgressMessage;
}
60 changes: 42 additions & 18 deletions src/harness/parallel/worker.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,22 +1,13 @@
namespace Harness.Parallel.Worker {
let errors: ErrorInfo[] = [];
let passing = 0;
let reportedUnitTests = false;

type Executor = {name: string, callback: Function, kind: "suite" | "test"} | never;

function resetShimHarnessAndExecute(runner: RunnerBase) {
if (reportedUnitTests) {
errors = [];
passing = 0;
testList.length = 0;
}
reportedUnitTests = true;
if (testList.length) {
// Execute unit tests
testList.forEach(({ name, callback, kind }) => executeCallback(name, callback, kind));
testList.length = 0;
}
errors = [];
passing = 0;
testList.length = 0;
const start = +(new Date());
runner.initializeTests();
testList.forEach(({ name, callback, kind }) => executeCallback(name, callback, kind));
Expand Down Expand Up @@ -226,13 +217,46 @@ namespace Harness.Parallel.Worker {
shimMochaHarness();
}

function handleTest(runner: TestRunnerKind, file: string) {
if (!runners.has(runner)) {
runners.set(runner, createRunner(runner));
function handleTest(runner: TestRunnerKind | "unittest", file: string) {
collectUnitTestsIfNeeded();
if (runner === unittest) {
return executeUnitTest(file);
}
else {
if (!runners.has(runner)) {
runners.set(runner, createRunner(runner));
}
const instance = runners.get(runner);
instance.tests = [file];
return { ...resetShimHarnessAndExecute(instance), runner, file };
}
const instance = runners.get(runner);
instance.tests = [file];
return { ...resetShimHarnessAndExecute(instance), runner, file };
}
}

const unittest: "unittest" = "unittest";
let unitTests: {[name: string]: Function};
function collectUnitTestsIfNeeded() {
if (!unitTests && testList.length) {
unitTests = {};
for (const test of testList) {
unitTests[test.name] = test.callback;
}
testList.length = 0;
}
}

function executeUnitTest(name: string) {
if (!unitTests) {
throw new Error(`Asked to run unit test ${name}, but no unit tests were discovered!`);
}
if (unitTests[name]) {
errors = [];
passing = 0;
const start = +(new Date());
executeSuiteCallback(name, unitTests[name]);
delete unitTests[name];
return { file: name, runner: unittest, errors, passing, duration: +(new Date()) - start };
}
throw new Error(`Unit test with name "${name}" was asked to be run, but such a test does not exist!`);
}
}