[3.12] gh-108834: Sync libregrtest with the main branch (#108966)

* gh-108834: regrtest reruns failed tests in subprocesses (#108839)

When using --rerun option, regrtest now re-runs failed tests
in verbose mode in fresh worker processes to have more
deterministic behavior. So it can write its final report even
if a test killed a worker progress.

Add --fail-rerun option to regrtest: exit with non-zero exit code
if a test failed pass passed when re-run in verbose mode (in a
fresh process). That's now more useful since tests can pass
when re-run in a fresh worker progress, whereas they failed
when run after other tests when tests are run sequentially.

Rename --verbose2 option (-w) to --rerun. Keep --verbose2 as a
deprecated alias.

Changes:

* Fix and enhance statistics in regrtest summary. Add "(filtered)"
  when --match and/or --ignore options are used.
* Add RunTests class.
* Add TestResult.get_rerun_match_tests() method
* Rewrite code to serialize/deserialize worker arguments as JSON
  using a new WorkerJob class.
* Fix stats when a test is run with --forever --rerun.
* If failed test names cannot be parsed, log a warning and don't
  filter tests.
* test_regrtest.test_rerun_success() now uses a marker file, since
  the test is re-run in a separated process.
* Add tests on normalize_test_name() function.
* Add test_success() and test_skip() tests to test_regrtest.

(cherry picked from commit 31c2945)

* gh-108834: regrtest --fail-rerun exits with code 5 (#108896)

When the --fail-rerun option is used and a test fails and then pass,
regrtest now uses exit code 5 ("rerun) instead of 2 ("bad test").

(cherry picked from commit 1170d5a)

* gh-108416: Mark slow but not CPU bound test methods with requires_resource('walltime') (GH-108480)

(cherry picked from commit 1e0d627)

* Manually sync Lib/test/libregrtest/ from main

---------

Co-authored-by: Serhiy Storchaka <[email protected]>

Author: vstinner

Lib/test/_test_multiprocessing.py

@@ -675,6 +675,7 @@ def test_close(self):
 
         close_queue(q)
 
+    @support.requires_resource('walltime')
     def test_many_processes(self):
         if self.TYPE == 'threads':
             self.skipTest('test not appropriate for {}'.format(self.TYPE))
@@ -4953,6 +4954,7 @@ def test_wait_slow(self):
     def test_wait_socket_slow(self):
         self.test_wait_socket(True)
 
+    @support.requires_resource('walltime')
     def test_wait_timeout(self):
         from multiprocessing.connection import wait
 
@@ -4981,6 +4983,7 @@ def signal_and_sleep(cls, sem, period):
         sem.release()
         time.sleep(period)
 
+    @support.requires_resource('walltime')
     def test_wait_integer(self):
         from multiprocessing.connection import wait
 

Lib/test/bisect_cmd.py

@@ -109,9 +109,10 @@ def parse_args():
 
 def main():
     args = parse_args()
-    if '-w' in args.test_args or '--verbose2' in args.test_args:
-        print("WARNING: -w/--verbose2 option should not be used to bisect!")
-        print()
+    for opt in ('-w', '--rerun', '--verbose2'):
+        if opt in args.test_args:
+            print(f"WARNING: {opt} option should not be used to bisect!")
+            print()
 
     if args.input:
         with open(args.input) as fp:

Lib/test/libregrtest/cmdline.py

@@ -107,6 +107,8 @@
 
     cpu -       Used for certain CPU-heavy tests.
 
+    walltime -  Long running but not CPU-bound tests.
+
     subprocess  Run all tests for the subprocess module.
 
     urlfetch -  It is okay to download files required on testing.
@@ -129,7 +131,7 @@
 
 
 ALL_RESOURCES = ('audio', 'curses', 'largefile', 'network',
-                 'decimal', 'cpu', 'subprocess', 'urlfetch', 'gui')
+                 'decimal', 'cpu', 'subprocess', 'urlfetch', 'gui', 'walltime')
 
 # Other resources excluded from --use=all:
 #
@@ -156,7 +158,7 @@ def __init__(self, **kwargs) -> None:
         self.coverdir = 'coverage'
         self.runleaks = False
         self.huntrleaks = False
-        self.verbose2 = False
+        self.rerun = False
         self.verbose3 = False
         self.print_slow = False
         self.random_seed = None
@@ -213,8 +215,10 @@ def _create_parser():
     group = parser.add_argument_group('Verbosity')
     group.add_argument('-v', '--verbose', action='count',
                        help='run tests in verbose mode with output to stdout')
-    group.add_argument('-w', '--verbose2', action='store_true',
+    group.add_argument('-w', '--rerun', action='store_true',
                        help='re-run failed tests in verbose mode')
+    group.add_argument('--verbose2', action='store_true', dest='rerun',
+                       help='deprecated alias to --rerun')
     group.add_argument('-W', '--verbose3', action='store_true',
                        help='display test output on failure')
     group.add_argument('-q', '--quiet', action='store_true',
@@ -309,6 +313,9 @@ def _create_parser():
     group.add_argument('--fail-env-changed', action='store_true',
                        help='if a test file alters the environment, mark '
                             'the test as failed')
+    group.add_argument('--fail-rerun', action='store_true',
+                       help='if a test failed and then passed when re-run, '
+                            'mark the tests as failed')
 
     group.add_argument('--junit-xml', dest='xmlpath', metavar='FILENAME',
                        help='writes JUnit-style XML results to the specified '
@@ -380,7 +387,7 @@ def _parse_args(args, **kwargs):
         ns.python = shlex.split(ns.python)
     if ns.failfast and not (ns.verbose or ns.verbose3):
         parser.error("-G/--failfast needs either -v or -W")
-    if ns.pgo and (ns.verbose or ns.verbose2 or ns.verbose3):
+    if ns.pgo and (ns.verbose or ns.rerun or ns.verbose3):
         parser.error("--pgo/-v don't go together!")
     if ns.pgo_extended:
         ns.pgo = True  # pgo_extended implies pgo