test-run-command: learn to run (parts of) the testsuite

Instead of relying on the presence of `make`, or `prove`, we might just
as well use our own facilities to run the test suite.

This helps e.g. when trying to verify a Git for Windows installation
without requiring to download a full Git for Windows SDK (which would use
up 600+ megabytes of bandwidth, and over a gigabyte of disk space).

Of course, it still requires the test helpers to be build *somewhere*,
and the Git version should at least roughly match the version from which
the test suite comes.

At the same time, this new way to run the test suite allows to validate
that a BusyBox-backed MinGit works as expected (verifying that BusyBox'
functionality is enough to at least pass the test suite).

Signed-off-by: Johannes Schindelin <[email protected]>

Author: dscho

t/helper/test-run-command.c

@@ -10,9 +10,14 @@
 
 #include "test-tool.h"
 #include "git-compat-util.h"
+#include "cache.h"
 #include "run-command.h"
 #include "argv-array.h"
 #include "strbuf.h"
+#include "parse-options.h"
+#include "string-list.h"
+#include "thread-utils.h"
+#include "wildmatch.h"
 #include <string.h>
 #include <errno.h>
 
@@ -50,6 +55,141 @@ static int task_finished(int result,
 	return 1;
 }
 
+struct testsuite {
+	struct string_list tests, failed;
+	int next;
+	int quiet, immediate, verbose, trace;
+};
+
+static int next_test(struct child_process *cp, struct strbuf *err, void *cb,
+		     void **task_cb)
+{
+	struct testsuite *suite = cb;
+	const char *test;
+	if (suite->next >= suite->tests.nr)
+		return 0;
+
+	test = suite->tests.items[suite->next++].string;
+	argv_array_pushl(&cp->args, "sh", test, NULL);
+	if (suite->quiet)
+		argv_array_push(&cp->args, "--quiet");
+	if (suite->immediate)
+		argv_array_push(&cp->args, "-i");
+	if (suite->verbose)
+		argv_array_push(&cp->args, "-v");
+	if (suite->trace)
+		argv_array_push(&cp->args, "-x");
+
+	strbuf_addf(err, "Output of '%s':\n", test);
+	*task_cb = (void *)test;
+
+	return 1;
+}
+
+static int test_finished(int result, struct strbuf *err, void *cb,
+			 void *task_cb)
+{
+	struct testsuite *suite = cb;
+	const char *name = (const char *)task_cb;
+
+	if (result)
+		string_list_append(&suite->failed, name);
+
+	strbuf_addf(err, "%s: '%s'\n", result ? "FAIL" : "SUCCESS", name);
+
+	return 0;
+}
+
+static int test_failed(struct strbuf *out, void *cb, void *task_cb)
+{
+	struct testsuite *suite = cb;
+	const char *name = (const char *)task_cb;
+
+	string_list_append(&suite->failed, name);
+	strbuf_addf(out, "FAILED TO START: '%s'\n", name);
+
+	return 0;
+}
+
+static const char * const testsuite_usage[] = {
+	"test-run-command testsuite [<options>] [<pattern>...]",
+	NULL
+};
+
+static int testsuite(int argc, const char **argv)
+{
+	struct testsuite suite;
+	int max_jobs = 1, i, ret;
+	DIR *dir;
+	struct dirent *d;
+	struct option options[] = {
+		OPT_BOOL('i', "immediate", &suite.immediate,
+			 "stop at first failed test case(s)"),
+		OPT_INTEGER('j', "jobs", &max_jobs, "run <N> jobs in parallel"),
+		OPT_BOOL('q', "quiet", &suite.quiet, "be terse"),
+		OPT_BOOL('v', "verbose", &suite.verbose, "be verbose"),
+		OPT_BOOL('x', "trace", &suite.trace, "trace shell commands"),
+		OPT_END()
+	};
+
+	memset(&suite, 0, sizeof(suite));
+	suite.tests.strdup_strings = suite.failed.strdup_strings = 1;
+
+	argc = parse_options(argc, argv, NULL, options,
+			testsuite_usage, PARSE_OPT_STOP_AT_NON_OPTION);
+
+	if (max_jobs <= 0)
+		max_jobs = online_cpus();
+
+	dir = opendir(".");
+	if (!dir)
+		die("Could not open the current directory");
+	while ((d = readdir(dir))) {
+		const char *p = d->d_name;
+
+		if (*p != 't' || !isdigit(p[1]) || !isdigit(p[2]) ||
+		    !isdigit(p[3]) || !isdigit(p[4]) || p[5] != '-' ||
+		    !ends_with(p, ".sh"))
+			continue;
+
+		/* No pattern: match all */
+		if (!argc) {
+			string_list_append(&suite.tests, p);
+			continue;
+		}
+
+		for (i = 0; i < argc; i++)
+			if (!wildmatch(argv[i], p, 0)) {
+				string_list_append(&suite.tests, p);
+				break;
+			}
+	}
+	closedir(dir);
+
+	if (!suite.tests.nr)
+		die("No tests match!");
+	if (max_jobs > suite.tests.nr)
+		max_jobs = suite.tests.nr;
+
+	fprintf(stderr, "Running %d tests (%d at a time)\n",
+		suite.tests.nr, max_jobs);
+
+	ret = run_processes_parallel(max_jobs, next_test, test_failed,
+				     test_finished, &suite);
+
+	if (suite.failed.nr > 0) {
+		ret = 1;
+		fprintf(stderr, "%d tests failed:\n\n", suite.failed.nr);
+		for (i = 0; i < suite.failed.nr; i++)
+			fprintf(stderr, "\t%s\n", suite.failed.items[i].string);
+	}
+
+	string_list_clear(&suite.tests, 0);
+	string_list_clear(&suite.failed, 0);
+
+	return !!ret;
+}
+
 static int inherit_handle(const char *argv0)
 {
 	struct child_process cp = CHILD_PROCESS_INIT;
@@ -95,6 +235,9 @@ int cmd__run_command(int argc, const char **argv)
 	struct child_process proc = CHILD_PROCESS_INIT;
 	int jobs;
 
+	if (argc > 1 && !strcmp(argv[1], "testsuite"))
+		exit(testsuite(argc - 1, argv + 1));
+
 	if (argc < 2)
 		return 1;
 	if (!strcmp(argv[1], "inherited-handle"))