Skip to content

Commit 84d9d14

Browse files
authored
bpo-29512: Add test.bisect, bisect failing tests (#2452)
Add a new "python3 -m test.bisect" tool to bisect failing tests. It can be used to find which test method(s) leak references, leak files, etc.
1 parent 23e7944 commit 84d9d14

File tree

1 file changed

+167
-0
lines changed

1 file changed

+167
-0
lines changed

Lib/test/bisect.py

Lines changed: 167 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,167 @@
1+
#!/usr/bin/env python3
2+
"""
3+
Command line tool to bisect failing CPython tests.
4+
5+
Find the test_os test method which alters the environment:
6+
7+
./python -m test.bisect --fail-env-changed test_os
8+
9+
Find a reference leak in "test_os", write the list of failing tests into the
10+
"bisect" file:
11+
12+
./python -m test.bisect -o bisect -R 3:3 test_os
13+
14+
Load an existing list of tests from a file using -i option:
15+
16+
./python -m test --list-cases -m FileTests test_os > tests
17+
./python -m test.bisect -i tests test_os
18+
"""
19+
20+
import argparse
21+
import datetime
22+
import os.path
23+
import math
24+
import random
25+
import subprocess
26+
import sys
27+
import tempfile
28+
import time
29+
30+
31+
def write_tests(filename, tests):
32+
with open(filename, "w") as fp:
33+
for name in tests:
34+
print(name, file=fp)
35+
fp.flush()
36+
37+
38+
def write_output(filename, tests):
39+
if not filename:
40+
return
41+
print("Write %s tests into %s" % (len(tests), filename))
42+
write_tests(filename, tests)
43+
return filename
44+
45+
46+
def format_shell_args(args):
47+
return ' '.join(args)
48+
49+
50+
def list_cases(args):
51+
cmd = [sys.executable, '-m', 'test', '--list-cases']
52+
cmd.extend(args.test_args)
53+
proc = subprocess.run(cmd,
54+
stdout=subprocess.PIPE,
55+
universal_newlines=True)
56+
exitcode = proc.returncode
57+
if exitcode:
58+
cmd = format_shell_args(cmd)
59+
print("Failed to list tests: %s failed with exit code %s"
60+
% (cmd, exitcode))
61+
sys.exit(exitcode)
62+
tests = proc.stdout.splitlines()
63+
return tests
64+
65+
66+
def run_tests(args, tests, huntrleaks=None):
67+
tmp = tempfile.mktemp()
68+
try:
69+
write_tests(tmp, tests)
70+
71+
cmd = [sys.executable, '-m', 'test', '--matchfile', tmp]
72+
cmd.extend(args.test_args)
73+
print("+ %s" % format_shell_args(cmd))
74+
proc = subprocess.run(cmd)
75+
return proc.returncode
76+
finally:
77+
if os.path.exists(tmp):
78+
os.unlink(tmp)
79+
80+
81+
def parse_args():
82+
parser = argparse.ArgumentParser()
83+
parser.add_argument('-i', '--input',
84+
help='Test names produced by --list-tests written '
85+
'into a file. If not set, run --list-tests')
86+
parser.add_argument('-o', '--output',
87+
help='Result of the bisection')
88+
parser.add_argument('-n', '--max-tests', type=int, default=1,
89+
help='Maximum number of tests to stop the bisection '
90+
'(default: 1)')
91+
parser.add_argument('-N', '--max-iter', type=int, default=100,
92+
help='Maximum number of bisection iterations '
93+
'(default: 100)')
94+
# FIXME: document that following arguments are test arguments
95+
96+
args, test_args = parser.parse_known_args()
97+
args.test_args = test_args
98+
return args
99+
100+
101+
def main():
102+
args = parse_args()
103+
104+
if args.input:
105+
with open(args.input) as fp:
106+
tests = [line.strip() for line in fp]
107+
else:
108+
tests = list_cases(args)
109+
110+
print("Start bisection with %s tests" % len(tests))
111+
print("Test arguments: %s" % format_shell_args(args.test_args))
112+
print("Bisection will stop when getting %s or less tests "
113+
"(-n/--max-tests option), or after %s iterations "
114+
"(-N/--max-iter option)"
115+
% (args.max_tests, args.max_iter))
116+
output = write_output(args.output, tests)
117+
print()
118+
119+
start_time = time.monotonic()
120+
iteration = 1
121+
try:
122+
while len(tests) > args.max_tests and iteration <= args.max_iter:
123+
ntest = len(tests)
124+
ntest = max(ntest // 2, 1)
125+
subtests = random.sample(tests, ntest)
126+
127+
print("[+] Iteration %s: run %s tests/%s"
128+
% (iteration, len(subtests), len(tests)))
129+
print()
130+
131+
exitcode = run_tests(args, subtests)
132+
133+
print("ran %s tests/%s" % (ntest, len(tests)))
134+
print("exit", exitcode)
135+
if exitcode:
136+
print("Tests failed: use this new subtest")
137+
tests = subtests
138+
output = write_output(args.output, tests)
139+
else:
140+
print("Tests succeeded: skip this subtest, try a new subbset")
141+
print()
142+
iteration += 1
143+
except KeyboardInterrupt:
144+
print()
145+
print("Bisection interrupted!")
146+
print()
147+
148+
print("Tests (%s):" % len(tests))
149+
for test in tests:
150+
print("* %s" % test)
151+
print()
152+
153+
if output:
154+
print("Output written into %s" % output)
155+
156+
dt = math.ceil(time.monotonic() - start_time)
157+
if len(tests) <= args.max_tests:
158+
print("Bisection completed in %s iterations and %s"
159+
% (iteration, datetime.timedelta(seconds=dt)))
160+
sys.exit(1)
161+
else:
162+
print("Bisection failed after %s iterations and %s"
163+
% (iteration, datetime.timedelta(seconds=dt)))
164+
165+
166+
if __name__ == "__main__":
167+
main()

0 commit comments

Comments
 (0)