pythongh-109276, pythongh-109508: Fix libregrtest stdout (python#109903)

Remove replace_stdout(): call sys.stdout.reconfigure() instead of set
the error handler to backslashreplace.

display_header() logs an empty line and flush stdout.

Remove encoding workaround in display_header() since stdout error
handler is now set to backslashreplace earlier.

Author: vstinner

Doc/using/configure.rst

@@ -965,7 +965,7 @@ Main Makefile targets
   this the default target of the ``make`` command (``make all`` or just
   ``make``).
 
-* ``make test``: Build Python and run the Python test suite with ``--slow-ci``
+* ``make test``: Build Python and run the Python test suite with ``--fast-ci``
   option. Variables:
 
   * ``TESTOPTS``: additional regrtest command line options.

Lib/test/libregrtest/main.py

@@ -513,9 +513,11 @@ def _reexecute_python(self):
             print_warning(f"Failed to reexecute Python: {exc!r}\n"
                           f"Command: {cmd_text}")
 
-    def main(self, tests: TestList | None = None):
-        if self.want_reexec and self.ci_mode:
-            self._reexecute_python()
+    def _init(self):
+        # Set sys.stdout encoder error handler to backslashreplace,
+        # similar to sys.stderr error handler, to avoid UnicodeEncodeError
+        # when printing a traceback or any other non-encodable character.
+        sys.stdout.reconfigure(errors="backslashreplace")
 
         if self.junit_filename and not os.path.isabs(self.junit_filename):
             self.junit_filename = os.path.abspath(self.junit_filename)
@@ -524,6 +526,12 @@ def main(self, tests: TestList | None = None):
 
         self.tmp_dir = get_temp_dir(self.tmp_dir)
 
+    def main(self, tests: TestList | None = None):
+        if self.want_reexec and self.ci_mode:
+            self._reexecute_python()
+
+        self._init()
+
         if self.want_cleanup:
             cleanup_temp_dir(self.tmp_dir)
             sys.exit(0)

Lib/test/libregrtest/setup.py

@@ -11,7 +11,7 @@
 from .runtests import RunTests
 from .utils import (
     setup_unraisable_hook, setup_threading_excepthook, fix_umask,
-    replace_stdout, adjust_rlimit_nofile)
+    adjust_rlimit_nofile)
 
 
 UNICODE_GUARD_ENV = "PYTHONREGRTEST_UNICODE_GUARD"
@@ -49,7 +49,7 @@ def setup_process():
             faulthandler.register(signum, chain=True, file=stderr_fd)
 
     adjust_rlimit_nofile()
-    replace_stdout()
+
     support.record_original_stdout(sys.stdout)
 
     # Some times __path__ and __file__ are not absolute (e.g. while running from

Lib/test/libregrtest/utils.py

@@ -1,4 +1,3 @@
-import atexit
 import contextlib
 import faulthandler
 import locale
@@ -495,32 +494,6 @@ def normalize_test_name(test_full_name, *, is_error=False):
     return short_name
 
 
-def replace_stdout():
-    """Set stdout encoder error handler to backslashreplace (as stderr error
-    handler) to avoid UnicodeEncodeError when printing a traceback"""
-    stdout = sys.stdout
-    try:
-        fd = stdout.fileno()
-    except ValueError:
-        # On IDLE, sys.stdout has no file descriptor and is not a TextIOWrapper
-        # object. Leaving sys.stdout unchanged.
-        #
-        # Catch ValueError to catch io.UnsupportedOperation on TextIOBase
-        # and ValueError on a closed stream.
-        return
-
-    sys.stdout = open(fd, 'w',
-        encoding=stdout.encoding,
-        errors="backslashreplace",
-        closefd=False,
-        newline='\n')
-
-    def restore_stdout():
-        sys.stdout.close()
-        sys.stdout = stdout
-    atexit.register(restore_stdout)
-
-
 def adjust_rlimit_nofile():
     """
     On macOS the default fd limit (RLIMIT_NOFILE) is sometimes too low (256)
@@ -548,20 +521,12 @@ def adjust_rlimit_nofile():
 
 
 def display_header(use_resources: tuple[str, ...]):
-    encoding = sys.stdout.encoding
-
     # Print basic platform information
     print("==", platform.python_implementation(), *sys.version.split())
     print("==", platform.platform(aliased=True),
                   "%s-endian" % sys.byteorder)
     print("== Python build:", ' '.join(get_build_info()))
-
-    cwd = os.getcwd()
-    # gh-109508: support.os_helper.FS_NONASCII, used by get_work_dir(), cannot
-    # be encoded to the filesystem encoding on purpose, escape non-encodable
-    # characters with backslashreplace error handler.
-    formatted_cwd = cwd.encode(encoding, "backslashreplace").decode(encoding)
-    print("== cwd:", formatted_cwd)
+    print("== cwd:", os.getcwd())
 
     cpu_count = os.cpu_count()
     if cpu_count:
@@ -588,18 +553,18 @@ def display_header(use_resources: tuple[str, ...]):
         sanitizers.append("memory")
     if ubsan:
         sanitizers.append("undefined behavior")
-    if not sanitizers:
-        return
-
-    print(f"== sanitizers: {', '.join(sanitizers)}")
-    for sanitizer, env_var in (
-        (asan, "ASAN_OPTIONS"),
-        (msan, "MSAN_OPTIONS"),
-        (ubsan, "UBSAN_OPTIONS"),
-    ):
-        options= os.environ.get(env_var)
-        if sanitizer and options is not None:
-            print(f"== {env_var}={options!r}")
+    if sanitizers:
+        print(f"== sanitizers: {', '.join(sanitizers)}")
+        for sanitizer, env_var in (
+            (asan, "ASAN_OPTIONS"),
+            (msan, "MSAN_OPTIONS"),
+            (ubsan, "UBSAN_OPTIONS"),
+        ):
+            options= os.environ.get(env_var)
+            if sanitizer and options is not None:
+                print(f"== {env_var}={options!r}")
+
+    print(flush=True)
 
 
 def cleanup_temp_dir(tmp_dir: StrPath):