gh-109566: regrtest reexecutes the process

regrtest now replaces the process with a new process to add options
to the Python command line. Add options: "-u -W default -bb". When
--fast-ci or --slow-ci option is used, the "-E" option is also added.

The following methods to run the Python test suite don't replace the
process:

* "import test.autotest"
* "from test.regrtest import main; main()"

Changes:

* PCbuild/rt.bat and Tools/scripts/run_tests.py no longer need to add
  "-u -W default -bb -E" options to Python: it's now done by
  regrtest.
* Fix Tools/scripts/run_tests.py: flush stdout before replacing the
  process. Previously, buffered messages were lost.

Author: vstinner

Lib/test/autotest.py

@@ -2,4 +2,4 @@
 # It can be especially handy if you're in an interactive shell, e.g.,
 # from test import autotest.
 from test.libregrtest.main import main
-main()
+main(reexec=False)

Lib/test/libregrtest/cmdline.py

@@ -184,6 +184,7 @@ def __init__(self, **kwargs) -> None:
         self.threshold = None
         self.fail_rerun = False
         self.tempdir = None
+        self.no_reexec = False
 
         super().__init__(**kwargs)
 
@@ -343,6 +344,8 @@ def _create_parser():
                        help='override the working directory for the test run')
     group.add_argument('--cleanup', action='store_true',
                        help='remove old test_python_* directories')
+    group.add_argument('--no-reexec', action='store_true',
+                       help="internal option, don't use it")
     return parser
 
 

Lib/test/libregrtest/main.py

@@ -1,6 +1,7 @@
 import os
 import random
 import re
+import shlex
 import sys
 import time
 
@@ -20,7 +21,7 @@
     StrPath, StrJSON, TestName, TestList, TestTuple, FilterTuple,
     strip_py_suffix, count, format_duration,
     printlist, get_temp_dir, get_work_dir, exit_timeout,
-    display_header, cleanup_temp_dir,
+    display_header, cleanup_temp_dir, print_warning,
     MS_WINDOWS)
 
 
@@ -47,7 +48,7 @@ class Regrtest:
     directly to set the values that would normally be set by flags
     on the command line.
     """
-    def __init__(self, ns: Namespace):
+    def __init__(self, ns: Namespace, reexec: bool = True):
         # Log verbosity
         self.verbose: int = int(ns.verbose)
         self.quiet: bool = ns.quiet
@@ -69,6 +70,7 @@ def __init__(self, ns: Namespace):
         self.want_cleanup: bool = ns.cleanup
         self.want_rerun: bool = ns.rerun
         self.want_run_leaks: bool = ns.runleaks
+        self.want_reexec: bool = (reexec and not ns.no_reexec)
 
         # Select tests
         if ns.match_tests:
@@ -95,6 +97,7 @@ def __init__(self, ns: Namespace):
         self.worker_json: StrJSON | None = ns.worker_json
 
         # Options to run tests
+        self.ci_mode: bool = (ns.fast_ci or ns.slow_ci)
         self.fail_fast: bool = ns.failfast
         self.fail_env_changed: bool = ns.fail_env_changed
         self.fail_rerun: bool = ns.fail_rerun
@@ -483,7 +486,38 @@ def run_tests(self, selected: TestTuple, tests: TestList | None) -> int:
                 # processes.
                 return self._run_tests(selected, tests)
 
+    def _reexecute_python(self):
+        if self.python_cmd:
+            # Do nothing if --python=cmd option is used
+            return
+
+        python_opts = [
+            '-u',                 # Unbuffered stdout and stderr
+            '-W', 'default',      # Add warnings filter 'default'
+            '-bb',                # Error on bytes/str comparison
+        ]
+        if self.ci_mode:
+            python_opts.append('-E')  # Ignore PYTHON* environment variables
+
+        cmd = [*sys.orig_argv, "--no-reexec"]
+        cmd[1:1] = python_opts
+
+        # Make sure that messages before execv() are logged
+        sys.stdout.flush()
+        sys.stderr.flush()
+
+        try:
+            os.execv(cmd[0], cmd)
+            # execv() do no return and so we don't get to this line on success
+        except OSError as exc:
+            cmd_text = shlex.join(cmd)
+            print_warning(f"Failed to reexecute Python: {exc!r}\n"
+                          f"Command: {cmd_text}")
+
     def main(self, tests: TestList | None = None):
+        if self.want_reexec:
+            self._reexecute_python()
+
         if self.junit_filename and not os.path.isabs(self.junit_filename):
             self.junit_filename = os.path.abspath(self.junit_filename)
 
@@ -515,7 +549,7 @@ def main(self, tests: TestList | None = None):
         sys.exit(exitcode)
 
 
-def main(tests=None, **kwargs):
+def main(tests=None, reexec=True, **kwargs):
     """Run the Python suite."""
     ns = _parse_args(sys.argv[1:], **kwargs)
-    Regrtest(ns).main(tests=tests)
+    Regrtest(ns, reexec=reexec).main(tests=tests)

Lib/test/regrtest.py

@@ -40,7 +40,7 @@ def _main():
     # sanity check
     assert __file__ == os.path.abspath(sys.argv[0])
 
-    main()
+    main(reexec=False)
 
 
 if __name__ == '__main__':

Lib/test/test_regrtest.py

@@ -382,7 +382,8 @@ def check_ci_mode(self, args, use_resources):
         # Check Regrtest attributes which are more reliable than Namespace
         # which has an unclear API
         regrtest = main.Regrtest(ns)
-        self.assertNotEqual(regrtest.num_workers, 0)
+        self.assertTrue(regrtest.ci_mode)
+        self.assertGreaterEqual(regrtest.num_workers, 1)
         self.assertTrue(regrtest.want_rerun)
         self.assertTrue(regrtest.randomize)
         self.assertIsNone(regrtest.random_seed)

Misc/NEWS.d/next/Tests/2023-09-26-18-12-01.gh-issue-109566.CP0Vhf.rst

@@ -0,0 +1,4 @@
+regrtest now replaces the process with a new process to add options to the
+Python command line. Add options: ``-u -W default -bb``. When ``--fast-ci``
+or ``--slow-ci`` option is used, the ``-E`` option is also added. Patch by
+Victor Stinner.

PCbuild/rt.bat

@@ -48,7 +48,7 @@ if NOT "%1"=="" (set regrtestargs=%regrtestargs% %1) & shift & goto CheckOpts
 
 if not defined prefix set prefix=%pcbuild%amd64
 set exe=%prefix%\python%suffix%.exe
-set cmd="%exe%" %dashO% -u -Wd -E -bb -m test %regrtestargs%
+set cmd="%exe%" %dashO% -m test %regrtestargs%
 if defined qmode goto Qmode
 
 echo Deleting .pyc files ...

Tools/scripts/run_tests.py

@@ -23,11 +23,7 @@ def is_python_flag(arg):
 
 
 def main(regrtest_args):
-    args = [sys.executable,
-            '-u',                 # Unbuffered stdout and stderr
-            '-W', 'default',      # Warnings set to 'default'
-            '-bb',                # Warnings about bytes/bytearray
-            ]
+    args = [sys.executable]
 
     cross_compile = '_PYTHON_HOST_PLATFORM' in os.environ
     if (hostrunner := os.environ.get("_PYTHON_HOSTRUNNER")) is None:
@@ -47,7 +43,6 @@ def main(regrtest_args):
         }
     else:
         environ = os.environ.copy()
-        args.append("-E")
 
     # Allow user-specified interpreter options to override our defaults.
     args.extend(test.support.args_from_interpreter_flags())
@@ -70,7 +65,8 @@ def main(regrtest_args):
 
     args.extend(regrtest_args)
 
-    print(shlex.join(args))
+    print(shlex.join(args), flush=True)
+
     if sys.platform == 'win32':
         from subprocess import call
         sys.exit(call(args))