Skip to content

Commit 996e3e8

Browse files
ambvgvanrossum
authored andcommitted
Speed up testpythoneval (#2635)
Splits eval-test into simple buckets by first letter of test name, enabling parallel execution. This speeds up execution of the test suite by around 25% on my laptop. The split enables more consistent loading of all CPU cores during the entire run of ./runtests.py. To achieve this, I had to modify testpythoneval.py to not write all testcase inputs to the same temporary path. Before: SUMMARY all 204 tasks and 1811 tests passed *** OK *** total time in run: 554.571954 total time in check: 214.105742 total time in lint: 130.914682 total time in pytest: 92.031659 ./runtests.py -j4 -v 744.76s user 74.10s system 235% cpu 5:48.34 total After: SUMMARY all 225 tasks and 3823 tests passed *** OK *** total time in run: 640.698327 total time in check: 178.758370 total time in lint: 149.604402 total time in pytest: 78.356671 ./runtests.py -j4 -v 850.81s user 81.09s system 353% cpu 4:23.69 total Total wall clock time fell from 5:48 to 4:23. Note: the test sum is now over-reported. Looks like the driver counts also the filtered out tests in eval-test. I don't have cycles now to hunt this down.
1 parent 560f2ec commit 996e3e8

File tree

2 files changed

+88
-35
lines changed

2 files changed

+88
-35
lines changed

mypy/test/testpythoneval.py

Lines changed: 64 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -10,16 +10,20 @@
1010
this suite would slow down the main suite too much.
1111
"""
1212

13+
from contextlib import contextmanager
14+
import errno
1315
import os
1416
import os.path
17+
import re
1518
import subprocess
1619
import sys
1720

1821
import typing
22+
from typing import Dict, List, Tuple
1923

2024
from mypy.myunit import Suite, SkipTestCaseException
2125
from mypy.test.config import test_data_prefix, test_temp_dir
22-
from mypy.test.data import parse_test_cases
26+
from mypy.test.data import DataDrivenTestCase, parse_test_cases
2327
from mypy.test.helpers import assert_string_arrays_equal
2428
from mypy.util import try_find_python2_interpreter
2529

@@ -33,6 +37,7 @@
3337

3438
# Path to Python 3 interpreter
3539
python3_path = sys.executable
40+
program_re = re.compile(r'\b_program.py\b')
3641

3742

3843
class PythonEvaluationSuite(Suite):
@@ -48,56 +53,83 @@ def cases(self):
4853
return c
4954

5055

51-
def test_python_evaluation(testcase):
52-
python2_interpreter = try_find_python2_interpreter()
53-
# Use Python 2 interpreter if running a Python 2 test case.
54-
if testcase.name.lower().endswith('python2'):
55-
if not python2_interpreter:
56+
def test_python_evaluation(testcase: DataDrivenTestCase) -> None:
57+
"""Runs Mypy in a subprocess.
58+
59+
If this passes without errors, executes the script again with a given Python
60+
version.
61+
"""
62+
mypy_cmdline = [
63+
python3_path,
64+
os.path.join(testcase.old_cwd, 'scripts', 'mypy'),
65+
'--show-traceback',
66+
]
67+
py2 = testcase.name.lower().endswith('python2')
68+
if py2:
69+
mypy_cmdline.append('--py2')
70+
interpreter = try_find_python2_interpreter()
71+
if not interpreter:
5672
# Skip, can't find a Python 2 interpreter.
5773
raise SkipTestCaseException()
58-
interpreter = python2_interpreter
59-
args = ['--py2']
60-
py2 = True
6174
else:
6275
interpreter = python3_path
63-
args = []
64-
py2 = False
65-
args.append('--show-traceback')
76+
6677
# Write the program to a file.
67-
program = '_program.py'
78+
program = '_' + testcase.name + '.py'
79+
mypy_cmdline.append(program)
6880
program_path = os.path.join(test_temp_dir, program)
6981
with open(program_path, 'w') as file:
7082
for s in testcase.input:
7183
file.write('{}\n'.format(s))
7284
# Type check the program.
7385
# This uses the same PYTHONPATH as the current process.
74-
process = subprocess.Popen([python3_path,
75-
os.path.join(testcase.old_cwd, 'scripts', 'mypy')]
76-
+ args + [program],
77-
stdout=subprocess.PIPE,
78-
stderr=subprocess.STDOUT,
79-
cwd=test_temp_dir)
80-
outb = process.stdout.read()
81-
# Split output into lines.
82-
out = [s.rstrip('\n\r') for s in str(outb, 'utf8').splitlines()]
83-
if not process.wait():
86+
returncode, out = run(mypy_cmdline)
87+
if returncode == 0:
8488
# Set up module path for the execution.
8589
# This needs the typing module but *not* the mypy module.
8690
vers_dir = '2.7' if py2 else '3.2'
8791
typing_path = os.path.join(testcase.old_cwd, 'lib-typing', vers_dir)
8892
assert os.path.isdir(typing_path)
8993
env = os.environ.copy()
9094
env['PYTHONPATH'] = typing_path
91-
process = subprocess.Popen([interpreter, program],
92-
stdout=subprocess.PIPE,
93-
stderr=subprocess.STDOUT,
94-
cwd=test_temp_dir,
95-
env=env)
96-
outb = process.stdout.read()
97-
# Split output into lines.
98-
out += [s.rstrip('\n\r') for s in str(outb, 'utf8').splitlines()]
95+
returncode, interp_out = run([interpreter, program], env=env)
96+
out += interp_out
9997
# Remove temp file.
10098
os.remove(program_path)
101-
assert_string_arrays_equal(testcase.output, out,
99+
assert_string_arrays_equal(adapt_output(testcase), out,
102100
'Invalid output ({}, line {})'.format(
103101
testcase.file, testcase.line))
102+
103+
104+
def split_lines(*streams: bytes) -> List[str]:
105+
"""Returns a single list of string lines from the byte streams in args."""
106+
return [
107+
s.rstrip('\n\r')
108+
for stream in streams
109+
for s in str(stream, 'utf8').splitlines()
110+
]
111+
112+
113+
def adapt_output(testcase: DataDrivenTestCase) -> List[str]:
114+
"""Translates the generic _program.py into the actual filename."""
115+
program = '_' + testcase.name + '.py'
116+
return [program_re.sub(program, line) for line in testcase.output]
117+
118+
119+
def run(
120+
cmdline: List[str], *, env: Dict[str, str] = None, timeout: int = 30
121+
) -> Tuple[int, List[str]]:
122+
"""A poor man's subprocess.run() for 3.3 and 3.4 compatibility."""
123+
process = subprocess.Popen(
124+
cmdline,
125+
env=env,
126+
stdout=subprocess.PIPE,
127+
stderr=subprocess.PIPE,
128+
cwd=test_temp_dir,
129+
)
130+
try:
131+
out, err = process.communicate(timeout=timeout)
132+
except subprocess.TimeoutExpired:
133+
out = err = b''
134+
process.kill()
135+
return process.returncode, split_lines(out, err)

runtests.py

Lines changed: 24 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -28,9 +28,12 @@ def get_versions(): # type: () -> typing.List[str]
2828

2929
from mypy.waiter import Waiter, LazySubprocess
3030
from mypy import util
31+
from mypy.test.config import test_data_prefix
32+
from mypy.test.testpythoneval import python_eval_files, python_34_eval_files
3133

3234
import itertools
3335
import os
36+
import re
3437

3538

3639
# Ideally, all tests would be `discover`able so that they can be driven
@@ -233,9 +236,27 @@ def add_myunit(driver: Driver) -> None:
233236

234237

235238
def add_pythoneval(driver: Driver) -> None:
236-
driver.add_python_mod('eval-test', 'mypy.myunit',
237-
'-m', 'mypy.test.testpythoneval', *driver.arglist,
238-
coverage=True)
239+
cases = set()
240+
case_re = re.compile(r'^\[case ([^\]]+)\]$')
241+
for file in python_eval_files + python_34_eval_files:
242+
with open(os.path.join(test_data_prefix, file), 'r') as f:
243+
for line in f:
244+
m = case_re.match(line)
245+
if m:
246+
case_name = m.group(1)
247+
assert case_name[:4] == 'test'
248+
cases.add(case_name[4:5])
249+
250+
for prefix in sorted(cases):
251+
driver.add_python_mod(
252+
'eval-test-' + prefix,
253+
'mypy.myunit',
254+
'-m',
255+
'mypy.test.testpythoneval',
256+
'test_testpythoneval_PythonEvaluationSuite.test' + prefix + '*',
257+
*driver.arglist,
258+
coverage=True
259+
)
239260

240261

241262
def add_cmdline(driver: Driver) -> None:

0 commit comments

Comments
 (0)