gh-101634: regrtest reports decoding error as failed test

When running the Python test suite with -jN option, if a worker stdout
cannot be decoded from the locale encoding report a failed testn so the
exitcode is non-zero. Patch by Victor Stinner.

Author: vstinner

Lib/test/libregrtest/runtest_mp.py

@@ -277,6 +277,7 @@ def _runtest(self, test_name: str) -> MultiprocessResult:
             encoding = locale.getencoding()
         else:
             encoding = sys.stdout.encoding
+
         # gh-94026: Write stdout+stderr to a tempfile as workaround for
         # non-blocking pipes on Emscripten with NodeJS.
         with tempfile.TemporaryFile('w+', encoding=encoding) as stdout_fh:
@@ -481,6 +482,8 @@ def _process_result(self, item: QueueOutput) -> bool:
             # Thread got an exception
             format_exc = item[1]
             print_warning(f"regrtest worker thread failed: {format_exc}")
+            result = ChildError("<regrtest worker>")
+            self.regrtest.accumulate_result(result)
             return True
 
         self.test_index += 1

Lib/test/test_regrtest.py

@@ -422,7 +422,7 @@ def check_executed_tests(self, output, tests, skipped=(), failed=(),
                              env_changed=(), omitted=(),
                              rerun={}, no_test_ran=(),
                              randomize=False, interrupted=False,
-                             fail_env_changed=False):
+                             fail_env_changed=False, executed=None, good=None):
         if isinstance(tests, str):
             tests = [tests]
         if isinstance(skipped, str):
@@ -435,12 +435,14 @@ def check_executed_tests(self, output, tests, skipped=(), failed=(),
             omitted = [omitted]
         if isinstance(no_test_ran, str):
             no_test_ran = [no_test_ran]
+        if executed is None:
+            executed = tests
 
-        executed = self.parse_executed_tests(output)
+        get_executed = self.parse_executed_tests(output)
         if randomize:
-            self.assertEqual(set(executed), set(tests), output)
+            self.assertEqual(set(get_executed), set(executed), output)
         else:
-            self.assertEqual(executed, tests, output)
+            self.assertEqual(get_executed, executed, output)
 
         def plural(count):
             return 's' if count != 1 else ''
@@ -482,8 +484,9 @@ def list_regex(line_format, tests):
             regex = list_regex('%s test%s run no tests', no_test_ran)
             self.check_line(output, regex)
 
-        good = (len(tests) - len(skipped) - len(failed)
-                - len(omitted) - len(env_changed) - len(no_test_ran))
+        if good is None:
+            good = (len(tests) - len(skipped) - len(failed)
+                    - len(omitted) - len(env_changed) - len(no_test_ran))
         if good:
             regex = r'%s test%s OK\.$' % (good, plural(good))
             if not skipped and not failed and good > 1:
@@ -1551,6 +1554,40 @@ def test_leak_tmp_file(self):
                           f"files (1): mytmpfile",
                           output)
 
+    def test_mp_decode_error(self):
+        # gh-101634: If a worker stdout cannot be decoded, report a failed test
+        # and a non-zero exit code.
+        if sys.platform == 'win32':
+            encoding = locale.getencoding()
+        else:
+            encoding = sys.stdout.encoding
+
+        nonascii = b"byte:\xa0\xa9\xff\n"
+        try:
+            nonascii.decode(encoding)
+        except UnicodeDecodeError:
+            pass
+        else:
+            self.skipTest(f"{encoding} can decode non-ASCII bytes {nonascii!a}")
+
+        code = textwrap.dedent(fr"""
+            import sys
+            # bytes which cannot be decoded from UTF-8
+            nonascii = {nonascii!a}
+            sys.stdout.buffer.write(nonascii)
+            sys.stdout.buffer.flush()
+        """)
+        testname = self.create_test(code=code)
+
+        output = self.run_tests("--fail-env-changed", "-v", "-j1", testname,
+                                exitcode=EXITCODE_BAD_TEST)
+        self.check_executed_tests(output, [testname],
+                                  executed=[],
+                                  good=0,
+                                  failed="<regrtest worker>",
+                                  omitted=[testname],
+                                  randomize=True)
+
 
 class TestUtils(unittest.TestCase):
     def test_format_duration(self):

Misc/NEWS.d/next/Tests/2023-06-28-02-51-08.gh-issue-101634.Rayczr.rst

@@ -0,0 +1,3 @@
+When running the Python test suite with ``-jN`` option, if a worker stdout
+cannot be decoded from the locale encoding report a failed testn so the
+exitcode is non-zero. Patch by Victor Stinner.