Allow customized sorting of test files prior to execution

This allows you to better control the distribution of test files across parallel runs. You can also control execution order for regular runs, including on your local machine.

Co-authored-by: Mark Wubben <[email protected]>

Author: erezrokah

docs/06-configuration.md

@@ -58,6 +58,7 @@ Arguments passed to the CLI will always take precedence over the CLI options con
 - `require`: extra modules to require before tests are run. Modules are required in the [worker processes](./01-writing-tests.md#process-isolation)
 - `timeout`: Timeouts in AVA behave differently than in other test frameworks. AVA resets a timer after each test, forcing tests to quit if no new test results were received within the specified timeout. This can be used to handle stalled tests. See our [timeout documentation](./07-test-timeouts.md) for more options.
 - `nodeArguments`: Configure Node.js arguments used to launch worker processes.
+- `sortTestFiles`: A comparator function to sort test files with. Available only when using a `ava.config.*` file. See an example use case [here](recipes/splitting-tests-ci.md).
 
 Note that providing files on the CLI overrides the `files` option.
 

docs/recipes/splitting-tests-ci.md

@@ -0,0 +1,22 @@
+# Splitting tests in CI
+
+AVA automatically detects whether your CI environment supports parallel builds using [ci-parallel-vars](https://www.npmjs.com/package/ci-parallel-vars). When parallel builds support is detected, AVA sorts the all detected test files by name, and splits them into chunks. Each CI machine is assigned a chunk (subset) of the tests, and then each chunk is run in parallel.
+
+To better distribute the tests across the machines, you can configure a custom comparator function:
+
+**`ava.config.js`:**
+
+```js
+import fs from 'node:fs';
+
+// Assuming 'test-data.json' structure is:
+// {
+// 	'tests/test1.js': { order: 1 },
+// 	'tests/test2.js': { order: 0 }
+// }
+const testData = JSON.parse(fs.readFileSync('test-data.json', 'utf8'));
+
+export default {
+	sortTestFiles: (file1, file2) => testData[file1].order - testData[file2].order,
+};
+```

lib/api.js

@@ -177,13 +177,19 @@ export default class Api extends Emittery {
 				const fileCount = selectedFiles.length;
 
 				// The files must be in the same order across all runs, so sort them.
-				selectedFiles = selectedFiles.sort((a, b) => a.localeCompare(b, [], {numeric: true}));
+				const defaultComparator = (a, b) => a.localeCompare(b, [], {numeric: true});
+				selectedFiles = selectedFiles.sort(this.options.sortTestFiles || defaultComparator);
 				selectedFiles = chunkd(selectedFiles, currentIndex, totalRuns);
 
 				const currentFileCount = selectedFiles.length;
 
 				runStatus = new RunStatus(fileCount, {currentFileCount, currentIndex, totalRuns}, selectionInsights);
 			} else {
+				// If a custom sorter was configured, use it.
+				if (this.options.sortTestFiles) {
+					selectedFiles = selectedFiles.sort(this.options.sortTestFiles);
+				}
+
 				runStatus = new RunStatus(selectedFiles.length, null, selectionInsights);
 			}
 
@@ -261,8 +267,8 @@ export default class Api extends Emittery {
 				}
 
 				const lineNumbers = getApplicableLineNumbers(globs.normalizeFileForMatching(apiOptions.projectDir, file), filter);
-				// Removing `providers` field because they cannot be transfered to the worker threads.
-				const {providers, ...forkOptions} = apiOptions;
+				// Removing `providers` and `sortTestFiles` fields because they cannot be transferred to the worker threads.
+				const {providers, sortTestFiles, ...forkOptions} = apiOptions;
 				const options = {
 					...forkOptions,
 					providerStates,

lib/cli.js

@@ -321,6 +321,10 @@ export default async function loadCli() { // eslint-disable-line complexity
 		exit('’sources’ has been removed. Use ’ignoredByWatcher’ to provide glob patterns of files that the watcher should ignore.');
 	}
 
+	if (Reflect.has(conf, 'sortTestFiles') && typeof conf.sortTestFiles !== 'function') {
+		exit('’sortTestFiles’ must be a comparator function.');
+	}
+
 	let projectPackageObject;
 	try {
 		projectPackageObject = JSON.parse(fs.readFileSync(path.resolve(projectDir, 'package.json')));
@@ -413,6 +417,7 @@ export default async function loadCli() { // eslint-disable-line complexity
 		moduleTypes,
 		nodeArguments,
 		parallelRuns,
+		sortTestFiles: conf.sortTestFiles,
 		projectDir,
 		providers,
 		ranFromCli: true,

readme.md

@@ -139,15 +139,16 @@ We have a growing list of [common pitfalls](docs/08-common-pitfalls.md) you may
 
 ### Recipes
 
-- [Shared workers](docs/recipes/shared-workers.md)
 - [Test setup](docs/recipes/test-setup.md)
-- [Code coverage](docs/recipes/code-coverage.md)
+- [TypeScript](docs/recipes/typescript.md)
+- [Shared workers](docs/recipes/shared-workers.md)
 - [Watch mode](docs/recipes/watch-mode.md)
-- [Endpoint testing](docs/recipes/endpoint-testing.md)
 - [When to use `t.plan()`](docs/recipes/when-to-use-plan.md)
-- [Browser testing](docs/recipes/browser-testing.md)
-- [TypeScript](docs/recipes/typescript.md)
 - [Passing arguments to your test files](docs/recipes/passing-arguments-to-your-test-files.md)
+- [Splitting tests in CI](docs/recipes/splitting-tests-ci.md)
+- [Code coverage](docs/recipes/code-coverage.md)
+- [Endpoint testing](docs/recipes/endpoint-testing.md)
+- [Browser testing](docs/recipes/browser-testing.md)
 - [Testing Vue.js components](docs/recipes/vue.md)
 - [Debugging tests with Chrome DevTools](docs/recipes/debugging-with-chrome-devtools.md)
 - [Debugging tests with VSCode](docs/recipes/debugging-with-vscode.md)

test-tap/fixture/parallel-runs/custom-comparator/0-1.cjs

@@ -0,0 +1,5 @@
+const test = require('../../../../entrypoints/main.cjs');
+
+test('at expected index', t => {
+	t.is(process.env.CI_NODE_INDEX, '2');
+});

test-tap/fixture/parallel-runs/custom-comparator/0-2.cjs

@@ -0,0 +1,5 @@
+const test = require('../../../../entrypoints/main.cjs');
+
+test('at expected index', t => {
+	t.is(process.env.CI_NODE_INDEX, '2');
+});

test-tap/fixture/parallel-runs/custom-comparator/0-3.cjs

@@ -0,0 +1,5 @@
+const test = require('../../../../entrypoints/main.cjs');
+
+test('at expected index', t => {
+	t.is(process.env.CI_NODE_INDEX, '2');
+});

test-tap/fixture/parallel-runs/custom-comparator/1-1.cjs

@@ -0,0 +1,5 @@
+const test = require('../../../../entrypoints/main.cjs');
+
+test('at expected index', t => {
+	t.is(process.env.CI_NODE_INDEX, '1');
+});

test-tap/fixture/parallel-runs/custom-comparator/1-2.cjs

@@ -0,0 +1,5 @@
+const test = require('../../../../entrypoints/main.cjs');
+
+test('at expected index', t => {
+	t.is(process.env.CI_NODE_INDEX, '1');
+});

test-tap/fixture/parallel-runs/custom-comparator/1-3.cjs

@@ -0,0 +1,5 @@
+const test = require('../../../../entrypoints/main.cjs');
+
+test('at expected index', t => {
+	t.is(process.env.CI_NODE_INDEX, '1');
+});

test-tap/fixture/parallel-runs/custom-comparator/2-1.cjs

@@ -0,0 +1,5 @@
+const test = require('../../../../entrypoints/main.cjs');
+
+test('at expected index', t => {
+	t.is(process.env.CI_NODE_INDEX, '0');
+});

test-tap/fixture/parallel-runs/custom-comparator/2-2.cjs

@@ -0,0 +1,5 @@
+const test = require('../../../../entrypoints/main.cjs');
+
+test('at expected index', t => {
+	t.is(process.env.CI_NODE_INDEX, '0');
+});

test-tap/fixture/parallel-runs/custom-comparator/2-3.cjs

@@ -0,0 +1,5 @@
+const test = require('../../../../entrypoints/main.cjs');
+
+test('at expected index', t => {
+	t.is(process.env.CI_NODE_INDEX, '0');
+});

test-tap/fixture/parallel-runs/custom-comparator/ava.config.js

@@ -0,0 +1,5 @@
+export default {
+	files: ['*.cjs'],
+	// Descending order
+	sortTestFiles: (a, b) => b.localeCompare(a, [], {numeric: true}),
+};

test-tap/fixture/parallel-runs/custom-comparator/package.json

@@ -0,0 +1,3 @@
+{
+	"type": "module"
+}

test-tap/fixture/sort-tests/0.cjs

@@ -0,0 +1,5 @@
+const test = require('../../../entrypoints/main.cjs');
+
+test('should run third', t => {
+	t.pass();
+});

test-tap/fixture/sort-tests/1.cjs

@@ -0,0 +1,5 @@
+const test = require('../../../entrypoints/main.cjs');
+
+test('should run second', t => {
+	t.pass();
+});

test-tap/fixture/sort-tests/2.cjs

@@ -0,0 +1,5 @@
+const test = require('../../../entrypoints/main.cjs');
+
+test('should run first', t => {
+	t.pass();
+});

test-tap/fixture/sort-tests/ava.config.js

@@ -0,0 +1,7 @@
+export default {
+	files: ['*.cjs'],
+	// Descending order
+	sortTestFiles: (a, b) => b.localeCompare(a, [], {numeric: true}),
+	concurrency: 1,
+	verbose: true,
+};

test-tap/fixture/sort-tests/package.json

@@ -0,0 +1,3 @@
+{
+	"type": "module"
+}

test-tap/integration/assorted.js

@@ -152,3 +152,11 @@ test('load .js test files as ESM modules', t => {
 		t.end();
 	});
 });
+
+test('uses sortTestFiles to sort test files', t => {
+	execCli([], {dirname: 'fixture/sort-tests'}, (error, stdout) => {
+		t.error(error);
+		t.match(stdout, /should run first[\s\S]+?should run second[\s\S]+?should run third/);
+		t.end();
+	});
+});

test-tap/integration/parallel-runs.js

@@ -43,3 +43,17 @@ test('fail when there are no files', t => {
 		}, error => t.ok(error));
 	}
 });
+
+test('correctly applies custom comparator', t => {
+	t.plan(3);
+	for (let i = 0; i < 3; i++) {
+		execCli([], {
+			dirname: 'fixture/parallel-runs/custom-comparator',
+			env: {
+				AVA_FORCE_CI: 'ci',
+				CI_NODE_INDEX: String(i),
+				CI_NODE_TOTAL: '3',
+			},
+		}, error => t.error(error));
+	}
+});