From e5116eb9534b287660f36e07c80dd75d9801a912 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Fri, 24 Mar 2017 13:52:11 +0100 Subject: [PATCH 1/2] bpo-29861: release references to multiprocessing Pool tasks (#743) * bpo-29861: release references to multiprocessing Pool tasks Release references to tasks, their arguments and their results as soon as they are finished, instead of keeping them alive until another task arrives. * Comments in test (cherry picked from commit 8988945cdc27ffa86ba8c624e095b51c459f5154) --- Lib/multiprocessing/pool.py | 7 +++- Lib/test/_test_multiprocessing.py | 28 ++++++++++++++ Misc/NEWS | 62 +------------------------------ 3 files changed, 36 insertions(+), 61 deletions(-) diff --git a/Lib/multiprocessing/pool.py b/Lib/multiprocessing/pool.py index 6d25469e1620c0..7d5ab67b4704cd 100644 --- a/Lib/multiprocessing/pool.py +++ b/Lib/multiprocessing/pool.py @@ -128,6 +128,8 @@ def worker(inqueue, outqueue, initializer=None, initargs=(), maxtasks=None, util.debug("Possible encoding error while sending result: %s" % ( wrapped)) put((job, i, (False, wrapped))) + + task = job = result = func = args = kwds = None completed += 1 util.debug('worker exiting after %d tasks' % completed) @@ -402,10 +404,11 @@ def _handle_tasks(taskqueue, put, outqueue, pool, cache): if set_length: util.debug('doing set_length()') set_length(i+1) + finally: + task = taskseq = job = None else: util.debug('task handler got sentinel') - try: # tell result handler to finish when cache is empty util.debug('task handler sending sentinel to result handler') @@ -445,6 +448,7 @@ def _handle_results(outqueue, get, cache): cache[job]._set(i, obj) except KeyError: pass + task = job = obj = None while cache and thread._state != TERMINATE: try: @@ -461,6 +465,7 @@ def _handle_results(outqueue, get, cache): cache[job]._set(i, obj) except KeyError: pass + task = job = obj = None if hasattr(outqueue, '_reader'): util.debug('ensuring that outqueue is not full') diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 9b789c2a3385b2..b053342f1c26c1 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -18,6 +18,7 @@ import logging import struct import operator +import weakref import test.support import test.support.script_helper @@ -1668,6 +1669,19 @@ def sqr(x, wait=0.0): def mul(x, y): return x*y +def identity(x): + return x + +class CountedObject(object): + n_instances = 0 + + def __new__(cls): + cls.n_instances += 1 + return object.__new__(cls) + + def __del__(self): + type(self).n_instances -= 1 + class SayWhenError(ValueError): pass def exception_throwing_generator(total, when): @@ -1676,6 +1690,7 @@ def exception_throwing_generator(total, when): raise SayWhenError("Somebody said when") yield i + class _TestPool(BaseTestCase): @classmethod @@ -1910,6 +1925,19 @@ def test_wrapped_exception(self): with self.assertRaises(RuntimeError): p.apply(self._test_wrapped_exception) + def test_release_task_refs(self): + # Issue #29861: task arguments and results should not be kept + # alive after we are done with them. + objs = [CountedObject() for i in range(10)] + refs = [weakref.ref(o) for o in objs] + self.pool.map(identity, objs) + + del objs + self.assertEqual(set(wr() for wr in refs), {None}) + # With a process pool, copies of the objects are returned, check + # they were released too. + self.assertEqual(CountedObject.n_instances, 0) + def raising(): raise KeyError("key") diff --git a/Misc/NEWS b/Misc/NEWS index e063455b986413..1afc00908c83df 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,24 +10,6 @@ Release date: XXXX-XX-XX Core and Builtins ----------------- -- bpo-28876: ``bool(range)`` works even if ``len(range)`` - raises :exc:`OverflowError`. - -- bpo-29600: Fix wrapping coroutine return values in StopIteration. - -- Issue #29537: Restore runtime compatibility with bytecode files generated by - CPython 3.5.0 to 3.5.2, and adjust the eval loop to avoid the problems that - could be caused by the malformed variant of the BUILD_MAP_UNPACK_WITH_CALL - opcode that they may contain. Patch by Petr Viktorin, Serhiy Storchaka, - and Nick Coghlan. - -- Issue #28598: Support __rmod__ for subclasses of str being called before - str.__mod__. Patch by Martijn Pieters. - -- bpo-29602: Fix incorrect handling of signed zeros in complex constructor for - complex subclasses and for inputs having a __complex__ method. Patch - by Serhiy Storchaka. - - bpo-29347: Fixed possibly dereferencing undefined pointers when creating weakref objects. @@ -46,43 +28,8 @@ Extension Modules Library ------- -- bpo-29884: faulthandler: Restore the old sigaltstack during teardown. - Patch by Christophe Zeitouny. - -- bpo-25455: Fixed crashes in repr of recursive buffered file-like objects. - -- bpo-29800: Fix crashes in partial.__repr__ if the keys of partial.keywords - are not strings. Patch by Michael Seifert. - -- bpo-29742: get_extra_info() raises exception if get called on closed ssl transport. - Patch by Nikolay Kim. - -- bpo-8256: Fixed possible failing or crashing input() if attributes "encoding" - or "errors" of sys.stdin or sys.stdout are not set or are not strings. - -- bpo-28298: Fix a bug that prevented array 'Q', 'L' and 'I' from accepting big - intables (objects that have __int__) as elements. Patch by Oren Milman. - -- bpo-29615: SimpleXMLRPCDispatcher no longer chains KeyError (or any other - exception) to exception(s) raised in the dispatched methods. - Patch by Petr Motejlek. - -- bpo-29704: asyncio.subprocess.SubprocessStreamProtocol no longer closes before - all pipes are closed. - -- bpo-29703: Fix asyncio to support instantiation of new event loops - in child processes. - -- bpo-29376: Fix assertion error in threading._DummyThread.is_alive(). - -- bpo-29110: Fix file object leak in aifc.open() when file is given as a - filesystem path and is not in valid AIFF format. Patch by Anthony Zhang. - -- Issue #28961: Fix unittest.mock._Call helper: don't ignore the name parameter - anymore. Patch written by Jiajun Huang. - -- bpo-29532: Altering a kwarg dictionary passed to functools.partial() - no longer affects a partial object after creation. +- bpo-29861: Release references to tasks, their arguments and their results + as soon as they are finished in multiprocessing.Pool. - Issue #28556: Various updates to typing module: typing.Counter, typing.ChainMap, improved ABC caching, etc. Original PRs by Jelle Zijlstra, Ivan Levkivskyi, @@ -141,11 +88,6 @@ C API Documentation ------------- -- bpo-28929: Link the documentation to its source file on GitHub. - -- bpo-25008: Document smtpd.py as effectively deprecated and add a pointer to - aiosmtpd, a third-party asyncio-based replacement. - - Issue #26355: Add canonical header link on each page to corresponding major version of the documentation. Patch by Matthias Bussonnier. From 97d8e7b6d60a64b165f6c63fa79c45c9e1582738 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Fri, 24 Mar 2017 14:35:04 +0100 Subject: [PATCH 2/2] Fix Misc/NEWS?? --- Misc/NEWS | 61 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/Misc/NEWS b/Misc/NEWS index 1afc00908c83df..cc219d4bed0d49 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,24 @@ Release date: XXXX-XX-XX Core and Builtins ----------------- +- bpo-28876: ``bool(range)`` works even if ``len(range)`` + raises :exc:`OverflowError`. + +- bpo-29600: Fix wrapping coroutine return values in StopIteration. + +- Issue #29537: Restore runtime compatibility with bytecode files generated by + CPython 3.5.0 to 3.5.2, and adjust the eval loop to avoid the problems that + could be caused by the malformed variant of the BUILD_MAP_UNPACK_WITH_CALL + opcode that they may contain. Patch by Petr Viktorin, Serhiy Storchaka, + and Nick Coghlan. + +- Issue #28598: Support __rmod__ for subclasses of str being called before + str.__mod__. Patch by Martijn Pieters. + +- bpo-29602: Fix incorrect handling of signed zeros in complex constructor for + complex subclasses and for inputs having a __complex__ method. Patch + by Serhiy Storchaka. + - bpo-29347: Fixed possibly dereferencing undefined pointers when creating weakref objects. @@ -31,6 +49,44 @@ Library - bpo-29861: Release references to tasks, their arguments and their results as soon as they are finished in multiprocessing.Pool. +- bpo-29884: faulthandler: Restore the old sigaltstack during teardown. + Patch by Christophe Zeitouny. + +- bpo-25455: Fixed crashes in repr of recursive buffered file-like objects. + +- bpo-29800: Fix crashes in partial.__repr__ if the keys of partial.keywords + are not strings. Patch by Michael Seifert. + +- bpo-29742: get_extra_info() raises exception if get called on closed ssl transport. + Patch by Nikolay Kim. + +- bpo-8256: Fixed possible failing or crashing input() if attributes "encoding" + or "errors" of sys.stdin or sys.stdout are not set or are not strings. + +- bpo-28298: Fix a bug that prevented array 'Q', 'L' and 'I' from accepting big + intables (objects that have __int__) as elements. Patch by Oren Milman. + +- bpo-29615: SimpleXMLRPCDispatcher no longer chains KeyError (or any other + exception) to exception(s) raised in the dispatched methods. + Patch by Petr Motejlek. + +- bpo-29704: asyncio.subprocess.SubprocessStreamProtocol no longer closes before + all pipes are closed. + +- bpo-29703: Fix asyncio to support instantiation of new event loops + in child processes. + +- bpo-29376: Fix assertion error in threading._DummyThread.is_alive(). + +- bpo-29110: Fix file object leak in aifc.open() when file is given as a + filesystem path and is not in valid AIFF format. Patch by Anthony Zhang. + +- Issue #28961: Fix unittest.mock._Call helper: don't ignore the name parameter + anymore. Patch written by Jiajun Huang. + +- bpo-29532: Altering a kwarg dictionary passed to functools.partial() + no longer affects a partial object after creation. + - Issue #28556: Various updates to typing module: typing.Counter, typing.ChainMap, improved ABC caching, etc. Original PRs by Jelle Zijlstra, Ivan Levkivskyi, Manuel Krebber, and Łukasz Langa. @@ -88,6 +144,11 @@ C API Documentation ------------- +- bpo-28929: Link the documentation to its source file on GitHub. + +- bpo-25008: Document smtpd.py as effectively deprecated and add a pointer to + aiosmtpd, a third-party asyncio-based replacement. + - Issue #26355: Add canonical header link on each page to corresponding major version of the documentation. Patch by Matthias Bussonnier.