Skip to content

Commit 5e88a04

Browse files
committed
gh-109276: libregrtest: WASM use filename for JSON
On Emscripten and WASI platforms, or if --python command line option is used, libregrtest now uses a filename for the JSON file. Emscripten and WASI buildbot workers run the main test process with a different Python (Linux) which spawns Emscripten/WASI processes using the command specified in --python command line option. Passing a file descriptor from the parent process to the child process doesn't work in this case. * Add JsonFile and JsonFileType classes * Add RunTests.json_file_use_filename() method. * Add a test in test_regrtest on the --python command line option. * test_regrtest: add parallel=False parameter.
1 parent 90cf345 commit 5e88a04

File tree

6 files changed

+160
-57
lines changed

6 files changed

+160
-57
lines changed

Lib/test/libregrtest/main.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -406,7 +406,7 @@ def create_run_tests(self, tests: TestTuple):
406406
python_cmd=self.python_cmd,
407407
randomize=self.randomize,
408408
random_seed=self.random_seed,
409-
json_fd=None,
409+
json_file=None,
410410
)
411411

412412
def _run_tests(self, selected: TestTuple, tests: TestList | None) -> int:

Lib/test/libregrtest/run_workers.py

Lines changed: 39 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import contextlib
12
import dataclasses
23
import faulthandler
34
import os.path
@@ -17,10 +18,10 @@
1718
from .logger import Logger
1819
from .result import TestResult, State
1920
from .results import TestResults
20-
from .runtests import RunTests
21+
from .runtests import RunTests, JsonFile, JsonFileType
2122
from .single import PROGRESS_MIN_TIME
2223
from .utils import (
23-
StrPath, StrJSON, TestName, MS_WINDOWS,
24+
StrPath, StrJSON, TestName, MS_WINDOWS, TMP_PREFIX,
2425
format_duration, print_warning, count, plural)
2526
from .worker import create_worker_process, USE_PROCESS_GROUP
2627

@@ -155,11 +156,10 @@ def mp_result_error(
155156
) -> MultiprocessResult:
156157
return MultiprocessResult(test_result, stdout, err_msg)
157158

158-
def _run_process(self, runtests: RunTests, output_fd: int, json_fd: int,
159+
def _run_process(self, runtests: RunTests, output_fd: int,
159160
tmp_dir: StrPath | None = None) -> int:
160161
try:
161-
popen = create_worker_process(runtests, output_fd, json_fd,
162-
tmp_dir)
162+
popen = create_worker_process(runtests, output_fd, tmp_dir)
163163

164164
self._killed = False
165165
self._popen = popen
@@ -226,21 +226,40 @@ def _runtest(self, test_name: TestName) -> MultiprocessResult:
226226
match_tests = None
227227
err_msg = None
228228

229+
stdout_file = tempfile.TemporaryFile('w+', encoding=encoding)
230+
231+
json_file_use_filename = self.runtests.json_file_use_filename()
232+
if json_file_use_filename:
233+
prefix = TMP_PREFIX + 'json_'
234+
# create an empty file to make the creation atomic
235+
# (to prevent races with other worker threads)
236+
json_fd, json_filename = tempfile.mkstemp(prefix=prefix)
237+
os.close(json_fd)
238+
json_tmpfile = contextlib.nullcontext()
239+
else:
240+
json_tmpfile = tempfile.TemporaryFile('w+', encoding='utf8')
241+
229242
# gh-94026: Write stdout+stderr to a tempfile as workaround for
230243
# non-blocking pipes on Emscripten with NodeJS.
231-
with (tempfile.TemporaryFile('w+', encoding=encoding) as stdout_file,
232-
tempfile.TemporaryFile('w+', encoding='utf8') as json_file):
244+
with (stdout_file, json_tmpfile):
233245
stdout_fd = stdout_file.fileno()
234-
json_fd = json_file.fileno()
235-
if MS_WINDOWS:
236-
json_fd = msvcrt.get_osfhandle(json_fd)
246+
if json_file_use_filename:
247+
json_file = JsonFile(json_filename, JsonFileType.FILENAME)
248+
else:
249+
json_fd = json_tmpfile.fileno()
250+
if MS_WINDOWS:
251+
json_handle = msvcrt.get_osfhandle(json_fd)
252+
json_file = JsonFile(json_handle, JsonFileType.WINDOWS_HANDLE)
253+
else:
254+
json_file = JsonFile(json_fd, JsonFileType.UNIX_FD)
255+
237256

238257
kwargs = {}
239258
if match_tests:
240259
kwargs['match_tests'] = match_tests
241260
worker_runtests = self.runtests.copy(
242261
tests=tests,
243-
json_fd=json_fd,
262+
json_file=json_file,
244263
**kwargs)
245264

246265
# gh-93353: Check for leaked temporary files in the parent process,
@@ -254,13 +273,12 @@ def _runtest(self, test_name: TestName) -> MultiprocessResult:
254273
tmp_dir = os.path.abspath(tmp_dir)
255274
try:
256275
retcode = self._run_process(worker_runtests,
257-
stdout_fd, json_fd, tmp_dir)
276+
stdout_fd, tmp_dir)
258277
finally:
259278
tmp_files = os.listdir(tmp_dir)
260279
os_helper.rmtree(tmp_dir)
261280
else:
262-
retcode = self._run_process(worker_runtests,
263-
stdout_fd, json_fd)
281+
retcode = self._run_process(worker_runtests, stdout_fd)
264282
tmp_files = ()
265283
stdout_file.seek(0)
266284

@@ -274,9 +292,14 @@ def _runtest(self, test_name: TestName) -> MultiprocessResult:
274292
return self.mp_result_error(result, err_msg=err_msg)
275293

276294
try:
295+
if json_file_use_filename:
296+
with open(json_filename, encoding='utf8') as json_fp:
297+
worker_json: StrJSON = json_fp.read()
298+
else:
299+
json_tmpfile.seek(0)
300+
worker_json: StrJSON = json_tmpfile.read()
301+
277302
# deserialize run_tests_worker() output
278-
json_file.seek(0)
279-
worker_json: StrJSON = json_file.read()
280303
if worker_json:
281304
result = TestResult.from_json(worker_json)
282305
else:

Lib/test/libregrtest/runtests.py

Lines changed: 65 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,62 @@
1+
import contextlib
12
import dataclasses
23
import json
4+
import os
5+
import subprocess
36
from typing import Any
47

8+
from test import support
9+
510
from .utils import (
611
StrPath, StrJSON, TestTuple, FilterTuple, FilterDict)
712

813

14+
class JsonFileType:
15+
UNIX_FD = "UNIX_FD"
16+
WINDOWS_HANDLE = "WINDOWS_HANDLE"
17+
FILENAME = "FILENAME"
18+
19+
20+
@dataclasses.dataclass(slots=True, frozen=True)
21+
class JsonFile:
22+
# See RunTests.json_file_use_filename()
23+
file: int | StrPath
24+
file_type: str
25+
26+
def configure_subprocess(self, popen_kwargs: dict) -> None:
27+
match self.file_type:
28+
case JsonFileType.UNIX_FD:
29+
# Unix file descriptor
30+
popen_kwargs['pass_fds'] = [self.file]
31+
case JsonFileType.WINDOWS_HANDLE:
32+
# Windows handle
33+
startupinfo = subprocess.STARTUPINFO()
34+
startupinfo.lpAttributeList = {"handle_list": [self.file]}
35+
popen_kwargs['startupinfo'] = startupinfo
36+
case JsonFileType.FILENAME:
37+
# Filename: nothing to do to
38+
pass
39+
40+
@contextlib.contextmanager
41+
def inherit_subprocess(self):
42+
if self.file_type == JsonFileType.WINDOWS_HANDLE:
43+
os.set_handle_inheritable(self.file, True)
44+
try:
45+
yield
46+
finally:
47+
os.set_handle_inheritable(self.file, False)
48+
else:
49+
yield
50+
51+
def open(self, mode, *, encoding):
52+
file = self.file
53+
if self.file_type == JsonFileType.WINDOWS_HANDLE:
54+
import msvcrt
55+
# Create a file descriptor from the handle
56+
file = msvcrt.open_osfhandle(file, os.O_WRONLY)
57+
return open(file, mode, encoding=encoding)
58+
59+
960
@dataclasses.dataclass(slots=True, frozen=True)
1061
class HuntRefleak:
1162
warmups: int
@@ -38,9 +89,7 @@ class RunTests:
3889
python_cmd: tuple[str] | None
3990
randomize: bool
4091
random_seed: int | None
41-
# On Unix, it's a file descriptor.
42-
# On Windows, it's a handle.
43-
json_fd: int | None
92+
json_file: JsonFile | None
4493

4594
def copy(self, **override):
4695
state = dataclasses.asdict(self)
@@ -74,6 +123,17 @@ def as_json(self) -> StrJSON:
74123
def from_json(worker_json: StrJSON) -> 'RunTests':
75124
return json.loads(worker_json, object_hook=_decode_runtests)
76125

126+
def json_file_use_filename(self) -> bool:
127+
# json_file type depends on the platform:
128+
# - Unix: file descriptor (int)
129+
# - Windows: handle (int)
130+
# - Emscripten/WASI or if --python is used: filename (str)
131+
return (
132+
self.python_cmd
133+
or support.is_emscripten
134+
or support.is_wasi
135+
)
136+
77137

78138
class _EncodeRunTests(json.JSONEncoder):
79139
def default(self, o: Any) -> dict[str, Any]:
@@ -90,6 +150,8 @@ def _decode_runtests(data: dict[str, Any]) -> RunTests | dict[str, Any]:
90150
data.pop('__runtests__')
91151
if data['hunt_refleak']:
92152
data['hunt_refleak'] = HuntRefleak(**data['hunt_refleak'])
153+
if data['json_file']:
154+
data['json_file'] = JsonFile(**data['json_file'])
93155
return RunTests(**data)
94156
else:
95157
return data

Lib/test/libregrtest/utils.py

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,12 @@
1717

1818

1919
MS_WINDOWS = (sys.platform == 'win32')
20-
WORK_DIR_PREFIX = 'test_python_'
21-
WORKER_WORK_DIR_PREFIX = f'{WORK_DIR_PREFIX}worker_'
20+
21+
# All temporary files and temporary directories created by libregrtest should
22+
# use TMP_PREFIX so cleanup_temp_dir() can remove them all.
23+
TMP_PREFIX = 'test_python_'
24+
WORK_DIR_PREFIX = TMP_PREFIX
25+
WORKER_WORK_DIR_PREFIX = WORK_DIR_PREFIX + 'worker_'
2226

2327
# bpo-38203: Maximum delay in seconds to exit Python (call Py_Finalize()).
2428
# Used to protect against threading._shutdown() hang.
@@ -387,7 +391,7 @@ def get_work_dir(parent_dir: StrPath, worker: bool = False) -> StrPath:
387391
# testing (see the -j option).
388392
# Emscripten and WASI have stubbed getpid(), Emscripten has only
389393
# milisecond clock resolution. Use randint() instead.
390-
if sys.platform in {"emscripten", "wasi"}:
394+
if support.is_emscripten or support.is_wasi:
391395
nounce = random.randint(0, 1_000_000)
392396
else:
393397
nounce = os.getpid()
@@ -580,7 +584,7 @@ def display_header():
580584
def cleanup_temp_dir(tmp_dir: StrPath):
581585
import glob
582586

583-
path = os.path.join(glob.escape(tmp_dir), WORK_DIR_PREFIX + '*')
587+
path = os.path.join(glob.escape(tmp_dir), TMP_PREFIX + '*')
584588
print("Cleanup %s directory" % tmp_dir)
585589
for name in glob.glob(path):
586590
if os.path.isdir(name):

Lib/test/libregrtest/worker.py

Lines changed: 15 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
from test.support import os_helper
88

99
from .setup import setup_process, setup_test_dir
10-
from .runtests import RunTests
10+
from .runtests import RunTests, JsonFile
1111
from .single import run_single_test
1212
from .utils import (
1313
StrPath, StrJSON, FilterTuple, MS_WINDOWS,
@@ -17,8 +17,7 @@
1717
USE_PROCESS_GROUP = (hasattr(os, "setsid") and hasattr(os, "killpg"))
1818

1919

20-
def create_worker_process(runtests: RunTests,
21-
output_fd: int, json_fd: int,
20+
def create_worker_process(runtests: RunTests, output_fd: int,
2221
tmp_dir: StrPath | None = None) -> subprocess.Popen:
2322
python_cmd = runtests.python_cmd
2423
worker_json = runtests.as_json()
@@ -55,34 +54,24 @@ def create_worker_process(runtests: RunTests,
5554
close_fds=True,
5655
cwd=work_dir,
5756
)
58-
if not MS_WINDOWS:
59-
kwargs['pass_fds'] = [json_fd]
60-
else:
61-
startupinfo = subprocess.STARTUPINFO()
62-
startupinfo.lpAttributeList = {"handle_list": [json_fd]}
63-
kwargs['startupinfo'] = startupinfo
64-
if USE_PROCESS_GROUP:
65-
kwargs['start_new_session'] = True
66-
67-
if MS_WINDOWS:
68-
os.set_handle_inheritable(json_fd, True)
69-
try:
57+
58+
# Pass json_file to the worker process
59+
json_file = runtests.json_file
60+
json_file.configure_subprocess(kwargs)
61+
62+
with json_file.inherit_subprocess():
7063
return subprocess.Popen(cmd, **kwargs)
71-
finally:
72-
if MS_WINDOWS:
73-
os.set_handle_inheritable(json_fd, False)
7464

7565

7666
def worker_process(worker_json: StrJSON) -> NoReturn:
7767
runtests = RunTests.from_json(worker_json)
7868
test_name = runtests.tests[0]
7969
match_tests: FilterTuple | None = runtests.match_tests
80-
json_fd: int = runtests.json_fd
81-
82-
if MS_WINDOWS:
83-
import msvcrt
84-
json_fd = msvcrt.open_osfhandle(json_fd, os.O_WRONLY)
85-
70+
# json_file type depends on the platform:
71+
# - Unix: file descriptor (int)
72+
# - Windows: handle (int)
73+
# - Emscripten/WASI or if --python is used: filename (str)
74+
json_file: JsonFile = runtests.json_file
8675

8776
setup_test_dir(runtests.test_dir)
8877
setup_process()
@@ -96,8 +85,8 @@ def worker_process(worker_json: StrJSON) -> NoReturn:
9685

9786
result = run_single_test(test_name, runtests)
9887

99-
with open(json_fd, 'w', encoding='utf-8') as json_file:
100-
result.write_json_into(json_file)
88+
with json_file.open('w', encoding='utf-8') as json_fp:
89+
result.write_json_into(json_fp)
10190

10291
sys.exit(0)
10392

0 commit comments

Comments
 (0)