From ef0522a5afb6ded76ac7932cbfd9450f3a6ffacb Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 29 May 2024 15:23:31 +0200 Subject: [PATCH 1/4] gh-119727: Add --sequentially option to regrtest --- Lib/test/libregrtest/cmdline.py | 5 +++++ Lib/test/libregrtest/main.py | 20 ++++++++++++++----- Lib/test/test_regrtest.py | 7 +++++++ ...-05-29-15-28-08.gh-issue-119727.dVkaZM.rst | 2 ++ 4 files changed, 29 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Tests/2024-05-29-15-28-08.gh-issue-119727.dVkaZM.rst diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index d4dac77b250ad6..47c2a363e6951f 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -174,6 +174,7 @@ def __init__(self, **kwargs) -> None: self.tempdir = None self._add_python_opts = True self.xmlpath = None + self.sequentially = False super().__init__(**kwargs) @@ -307,6 +308,10 @@ def _create_parser(): group.add_argument('-j', '--multiprocess', metavar='PROCESSES', dest='use_mp', type=int, help='run PROCESSES processes at once') + group.add_argument('--sequentially', action='store_true', + help='always run all tests sequentially, ' + 'ignore -jN option, ' + 'and failed tests are also rerun sequentially') group.add_argument('-T', '--coverage', action='store_true', dest='trace', help='turn on code coverage tracing using the trace ' diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 9e7a7d60880091..b0b8c519722c12 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -89,7 +89,10 @@ def __init__(self, ns: Namespace, _add_python_opts: bool = False): self.cmdline_args: TestList = ns.args # Workers - if ns.use_mp is None: + self.sequentially: bool = ns.sequentially + if self.sequentially: + num_workers = 0 # run sequentially + elif ns.use_mp is None: num_workers = 0 # run sequentially elif ns.use_mp <= 0: num_workers = -1 # use the number of CPUs @@ -236,7 +239,7 @@ def list_tests(tests: TestTuple): def _rerun_failed_tests(self, runtests: RunTests): # Configure the runner to re-run tests - if self.num_workers == 0: + if self.num_workers == 0 and not self.sequentially: # Always run tests in fresh processes to have more deterministic # initial state. Don't re-run tests in parallel but limit to a # single worker process to have side effects (on the system load @@ -246,7 +249,6 @@ def _rerun_failed_tests(self, runtests: RunTests): tests, match_tests_dict = self.results.prepare_rerun() # Re-run failed tests - self.log(f"Re-running {len(tests)} failed tests in verbose mode in subprocesses") runtests = runtests.copy( tests=tests, rerun=True, @@ -256,7 +258,15 @@ def _rerun_failed_tests(self, runtests: RunTests): match_tests_dict=match_tests_dict, output_on_failure=False) self.logger.set_tests(runtests) - self._run_tests_mp(runtests, self.num_workers) + + msg = f"Re-running {len(tests)} failed tests in verbose mode" + if not self.sequentially: + msg = f"{msg} in subprocesses" + self.log(msg) + self._run_tests_mp(runtests, self.num_workers) + else: + self.log(msg) + self.run_tests_sequentially(runtests) return runtests def rerun_failed_tests(self, runtests: RunTests): @@ -599,7 +609,7 @@ def _add_cross_compile_opts(self, regrtest_opts): keep_environ = True if cross_compile and hostrunner: - if self.num_workers == 0: + if self.num_workers == 0 and not self.sequentially: # For now use only two cores for cross-compiled builds; # hostrunner can be expensive. regrtest_opts.extend(['-j', '2']) diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 17eff617a56aa4..74f69b6fc99955 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -473,6 +473,13 @@ def test_verbose3_huntrleaks(self): self.assertEqual(regrtest.hunt_refleak.runs, 10) self.assertFalse(regrtest.output_on_failure) + def test_sequentially(self): + args = ['-j2', '--sequentially'] + with support.captured_stderr(): + regrtest = self.create_regrtest(args) + self.assertEqual(regrtest.num_workers, 0) + self.assertTrue(regrtest.sequentially) + @dataclasses.dataclass(slots=True) class Rerun: diff --git a/Misc/NEWS.d/next/Tests/2024-05-29-15-28-08.gh-issue-119727.dVkaZM.rst b/Misc/NEWS.d/next/Tests/2024-05-29-15-28-08.gh-issue-119727.dVkaZM.rst new file mode 100644 index 00000000000000..58ef0b0ff6f4e8 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2024-05-29-15-28-08.gh-issue-119727.dVkaZM.rst @@ -0,0 +1,2 @@ +Add ``--sequentially`` command line option to Python test runner (regrtest). +Patch by Victor Stinner. From 46a1e77f75d9da5a35b145cb0c4a39d6ba0163b1 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 29 May 2024 15:33:21 +0200 Subject: [PATCH 2/4] Change NEED_TTY formatting --- Lib/test/libregrtest/worker.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/test/libregrtest/worker.py b/Lib/test/libregrtest/worker.py index 15d32b5baa04d0..86cc30835fdbda 100644 --- a/Lib/test/libregrtest/worker.py +++ b/Lib/test/libregrtest/worker.py @@ -14,9 +14,9 @@ USE_PROCESS_GROUP = (hasattr(os, "setsid") and hasattr(os, "killpg")) -NEED_TTY = set(''' - test_ioctl -'''.split()) +NEED_TTY = { + 'test_ioctl', +} def create_worker_process(runtests: WorkerRunTests, output_fd: int, From 4d50b872f298d202ce80096fd980227aa6d7cefb Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 3 Jun 2024 17:58:22 +0200 Subject: [PATCH 3/4] Rename the option to: --single-process --- Lib/test/libregrtest/cmdline.py | 16 +++++++++++----- Lib/test/libregrtest/main.py | 20 +++++++++----------- Lib/test/test_regrtest.py | 12 +++++++++--- 3 files changed, 29 insertions(+), 19 deletions(-) diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index 47c2a363e6951f..2ff4715e82a41b 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -174,7 +174,7 @@ def __init__(self, **kwargs) -> None: self.tempdir = None self._add_python_opts = True self.xmlpath = None - self.sequentially = False + self.single_process = False super().__init__(**kwargs) @@ -308,10 +308,12 @@ def _create_parser(): group.add_argument('-j', '--multiprocess', metavar='PROCESSES', dest='use_mp', type=int, help='run PROCESSES processes at once') - group.add_argument('--sequentially', action='store_true', - help='always run all tests sequentially, ' - 'ignore -jN option, ' - 'and failed tests are also rerun sequentially') + group.add_argument('--single-process', action='store_true', + dest='single_process', + help='always run all tests sequentially in ' + 'a single process, ignore -jN option, ' + 'and failed tests are also rerun sequentially ' + 'in the same process') group.add_argument('-T', '--coverage', action='store_true', dest='trace', help='turn on code coverage tracing using the trace ' @@ -440,6 +442,10 @@ def _parse_args(args, **kwargs): else: ns._add_python_opts = False + # --singleprocess overrides -jN option + if ns.single_process: + ns.use_mp = None + # When both --slow-ci and --fast-ci options are present, # --slow-ci has the priority if ns.slow_ci: diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index b0b8c519722c12..5148d3070513e8 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -89,15 +89,13 @@ def __init__(self, ns: Namespace, _add_python_opts: bool = False): self.cmdline_args: TestList = ns.args # Workers - self.sequentially: bool = ns.sequentially - if self.sequentially: - num_workers = 0 # run sequentially - elif ns.use_mp is None: - num_workers = 0 # run sequentially + self.single_process: bool = ns.single_process + if self.single_process or ns.use_mp is None: + num_workers = 0 # run sequentially in a single process elif ns.use_mp <= 0: - num_workers = -1 # use the number of CPUs + num_workers = -1 # run in parallel, use the number of CPUs else: - num_workers = ns.use_mp + num_workers = ns.use_mp # run in parallel self.num_workers: int = num_workers self.worker_json: StrJSON | None = ns.worker_json @@ -239,7 +237,7 @@ def list_tests(tests: TestTuple): def _rerun_failed_tests(self, runtests: RunTests): # Configure the runner to re-run tests - if self.num_workers == 0 and not self.sequentially: + if self.num_workers == 0 and not self.single_process: # Always run tests in fresh processes to have more deterministic # initial state. Don't re-run tests in parallel but limit to a # single worker process to have side effects (on the system load @@ -260,7 +258,7 @@ def _rerun_failed_tests(self, runtests: RunTests): self.logger.set_tests(runtests) msg = f"Re-running {len(tests)} failed tests in verbose mode" - if not self.sequentially: + if not self.single_process: msg = f"{msg} in subprocesses" self.log(msg) self._run_tests_mp(runtests, self.num_workers) @@ -381,7 +379,7 @@ def run_tests_sequentially(self, runtests) -> None: tests = count(jobs, 'test') else: tests = 'tests' - msg = f"Run {tests} sequentially" + msg = f"Run {tests} sequentially in a single process" if runtests.timeout: msg += " (timeout: %s)" % format_duration(runtests.timeout) self.log(msg) @@ -609,7 +607,7 @@ def _add_cross_compile_opts(self, regrtest_opts): keep_environ = True if cross_compile and hostrunner: - if self.num_workers == 0 and not self.sequentially: + if self.num_workers == 0 and not self.single_process: # For now use only two cores for cross-compiled builds; # hostrunner can be expensive. regrtest_opts.extend(['-j', '2']) diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 74f69b6fc99955..97ce797f0f6acb 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -473,12 +473,18 @@ def test_verbose3_huntrleaks(self): self.assertEqual(regrtest.hunt_refleak.runs, 10) self.assertFalse(regrtest.output_on_failure) - def test_sequentially(self): - args = ['-j2', '--sequentially'] + def test_single_process(self): + args = ['-j2', '--single-process'] with support.captured_stderr(): regrtest = self.create_regrtest(args) self.assertEqual(regrtest.num_workers, 0) - self.assertTrue(regrtest.sequentially) + self.assertTrue(regrtest.single_process) + + args = ['--fast-ci', '--single-process'] + with support.captured_stderr(): + regrtest = self.create_regrtest(args) + self.assertEqual(regrtest.num_workers, 0) + self.assertTrue(regrtest.single_process) @dataclasses.dataclass(slots=True) From 90fb9fe7f557233913369b030c5347a89640fcda Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 3 Jun 2024 18:10:31 +0200 Subject: [PATCH 4/4] Update Misc/NEWS.d/next/Tests/2024-05-29-15-28-08.gh-issue-119727.dVkaZM.rst Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- .../next/Tests/2024-05-29-15-28-08.gh-issue-119727.dVkaZM.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Tests/2024-05-29-15-28-08.gh-issue-119727.dVkaZM.rst b/Misc/NEWS.d/next/Tests/2024-05-29-15-28-08.gh-issue-119727.dVkaZM.rst index 58ef0b0ff6f4e8..bf28d8bb77b8a2 100644 --- a/Misc/NEWS.d/next/Tests/2024-05-29-15-28-08.gh-issue-119727.dVkaZM.rst +++ b/Misc/NEWS.d/next/Tests/2024-05-29-15-28-08.gh-issue-119727.dVkaZM.rst @@ -1,2 +1,2 @@ -Add ``--sequentially`` command line option to Python test runner (regrtest). +Add ``--single-process`` command line option to Python test runner (regrtest). Patch by Victor Stinner.