Skip to content

Commit 86f0354

Browse files
authored
[3.7] bpo-36560: regrtest: don't collect the GC twice (GH-12747) (GH-12749)
* bpo-36560: Fix reference leak hunting in regrtest (GH-12744) Fix reference leak hunting in regrtest: compute also deltas (of reference count, allocated memory blocks, file descriptor count) during warmup, to ensure that everything is initialized before starting to hunt reference leaks. Other changes: * Replace gc.collect() with support.gc_collect() * Move calls to read memory statistics from dash_R_cleanup() to dash_R() * Pass regrtest 'ns' to dash_R() * dash_R() is now more quiet with --quiet option (don't display progress). * Precompute the full range for "for it in range(repcount):" to ensure that the iteration doesn't allocate anything new. * dash_R() now is responsible to call warm_caches(). (cherry picked from commit 5aaac94) * bpo-36560: regrtest: don't collect the GC twice (GH-12747) dash_R() function of libregrtest doesn't call support.gc_collect() directly anymore: it's already called by dash_R_cleanup(). Call dash_R_cleanup() before starting the loop. (cherry picked from commit bb44478)
1 parent 0a16bb1 commit 86f0354

File tree

4 files changed

+48
-33
lines changed

4 files changed

+48
-33
lines changed

Lib/test/libregrtest/refleak.py

Lines changed: 43 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ def _get_dump(cls):
1818
cls._abc_negative_cache, cls._abc_negative_cache_version)
1919

2020

21-
def dash_R(the_module, test, indirect_test, huntrleaks):
21+
def dash_R(ns, the_module, test_name, test_func):
2222
"""Run a test multiple times, looking for reference leaks.
2323
2424
Returns:
@@ -32,6 +32,10 @@ def dash_R(the_module, test, indirect_test, huntrleaks):
3232
raise Exception("Tracking reference leaks requires a debug build "
3333
"of Python")
3434

35+
# Avoid false positives due to various caches
36+
# filling slowly with random data:
37+
warm_caches()
38+
3539
# Save current values for dash_R_cleanup() to restore.
3640
fs = warnings.filters[:]
3741
ps = copyreg.dispatch_table.copy()
@@ -57,31 +61,52 @@ def dash_R(the_module, test, indirect_test, huntrleaks):
5761
def get_pooled_int(value):
5862
return int_pool.setdefault(value, value)
5963

60-
nwarmup, ntracked, fname = huntrleaks
64+
nwarmup, ntracked, fname = ns.huntrleaks
6165
fname = os.path.join(support.SAVEDCWD, fname)
6266
repcount = nwarmup + ntracked
67+
68+
# Pre-allocate to ensure that the loop doesn't allocate anything new
69+
rep_range = list(range(repcount))
6370
rc_deltas = [0] * repcount
6471
alloc_deltas = [0] * repcount
6572
fd_deltas = [0] * repcount
73+
getallocatedblocks = sys.getallocatedblocks
74+
gettotalrefcount = sys.gettotalrefcount
75+
fd_count = support.fd_count
6676

67-
print("beginning", repcount, "repetitions", file=sys.stderr)
68-
print(("1234567890"*(repcount//10 + 1))[:repcount], file=sys.stderr,
69-
flush=True)
7077
# initialize variables to make pyflakes quiet
7178
rc_before = alloc_before = fd_before = 0
72-
for i in range(repcount):
73-
indirect_test()
74-
alloc_after, rc_after, fd_after = dash_R_cleanup(fs, ps, pic, zdc,
75-
abcs)
76-
print('.', end='', file=sys.stderr, flush=True)
77-
if i >= nwarmup:
78-
rc_deltas[i] = get_pooled_int(rc_after - rc_before)
79-
alloc_deltas[i] = get_pooled_int(alloc_after - alloc_before)
80-
fd_deltas[i] = get_pooled_int(fd_after - fd_before)
79+
80+
if not ns.quiet:
81+
print("beginning", repcount, "repetitions", file=sys.stderr)
82+
print(("1234567890"*(repcount//10 + 1))[:repcount], file=sys.stderr,
83+
flush=True)
84+
85+
dash_R_cleanup(fs, ps, pic, zdc, abcs)
86+
87+
for i in rep_range:
88+
test_func()
89+
dash_R_cleanup(fs, ps, pic, zdc, abcs)
90+
91+
# dash_R_cleanup() ends with collecting cyclic trash:
92+
# read memory statistics immediately after.
93+
alloc_after = getallocatedblocks()
94+
rc_after = gettotalrefcount()
95+
fd_after = fd_count()
96+
97+
if not ns.quiet:
98+
print('.', end='', file=sys.stderr, flush=True)
99+
100+
rc_deltas[i] = get_pooled_int(rc_after - rc_before)
101+
alloc_deltas[i] = get_pooled_int(alloc_after - alloc_before)
102+
fd_deltas[i] = get_pooled_int(fd_after - fd_before)
103+
81104
alloc_before = alloc_after
82105
rc_before = rc_after
83106
fd_before = fd_after
84-
print(file=sys.stderr)
107+
108+
if not ns.quiet:
109+
print(file=sys.stderr)
85110

86111
# These checkers return False on success, True on failure
87112
def check_rc_deltas(deltas):
@@ -112,7 +137,7 @@ def check_fd_deltas(deltas):
112137
deltas = deltas[nwarmup:]
113138
if checker(deltas):
114139
msg = '%s leaked %s %s, sum=%s' % (
115-
test, deltas, item_name, sum(deltas))
140+
test_name, deltas, item_name, sum(deltas))
116141
print(msg, file=sys.stderr, flush=True)
117142
with open(fname, "a") as refrep:
118143
print(msg, file=refrep)
@@ -122,7 +147,7 @@ def check_fd_deltas(deltas):
122147

123148

124149
def dash_R_cleanup(fs, ps, pic, zdc, abcs):
125-
import gc, copyreg
150+
import copyreg
126151
import collections.abc
127152

128153
# Restore some original values.
@@ -154,16 +179,8 @@ def dash_R_cleanup(fs, ps, pic, zdc, abcs):
154179

155180
clear_caches()
156181

157-
# Collect cyclic trash and read memory statistics immediately after.
158-
func1 = sys.getallocatedblocks
159-
func2 = sys.gettotalrefcount
160-
gc.collect()
161-
return func1(), func2(), support.fd_count()
162-
163182

164183
def clear_caches():
165-
import gc
166-
167184
# Clear the warnings registry, so they can be displayed again
168185
for mod in sys.modules.values():
169186
if hasattr(mod, '__warningregistry__'):
@@ -256,7 +273,7 @@ def clear_caches():
256273
for f in typing._cleanups:
257274
f()
258275

259-
gc.collect()
276+
support.gc_collect()
260277

261278

262279
def warm_caches():

Lib/test/libregrtest/runtest.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -177,7 +177,7 @@ def test_runner():
177177
raise Exception("errors while loading tests")
178178
support.run_unittest(tests)
179179
if ns.huntrleaks:
180-
refleak = dash_R(the_module, test, test_runner, ns.huntrleaks)
180+
refleak = dash_R(ns, the_module, test, test_runner)
181181
else:
182182
test_runner()
183183
test_time = time.perf_counter() - start_time

Lib/test/libregrtest/setup.py

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,6 @@
1010
except ImportError:
1111
gc = None
1212

13-
from test.libregrtest.refleak import warm_caches
14-
1513

1614
def setup_tests(ns):
1715
try:
@@ -79,10 +77,6 @@ def setup_tests(ns):
7977
if ns.huntrleaks:
8078
unittest.BaseTestSuite._cleanup = False
8179

82-
# Avoid false positives due to various caches
83-
# filling slowly with random data:
84-
warm_caches()
85-
8680
if ns.memlimit is not None:
8781
support.set_memlimit(ns.memlimit)
8882

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
Fix reference leak hunting in regrtest: compute also deltas (of reference
2+
count, allocated memory blocks, file descriptor count) during warmup, to
3+
ensure that everything is initialized before starting to hunt reference
4+
leaks.

0 commit comments

Comments
 (0)