From 6fb4cba8c6fed534a06476f2c0c047149f42a75f Mon Sep 17 00:00:00 2001 From: Roy Hyunjin Han Date: Thu, 30 Mar 2023 14:02:41 -0400 Subject: [PATCH 01/13] Add support for manager.list().clear() ``` from multiprocessing import Manager with Manager() as manager: xs = manager.list() xs.clear() ``` For now, we can use the workaround `del xs[:]` --- Lib/multiprocessing/managers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/multiprocessing/managers.py b/Lib/multiprocessing/managers.py index b6534939b4d98b..348df9a0d7a0a6 100644 --- a/Lib/multiprocessing/managers.py +++ b/Lib/multiprocessing/managers.py @@ -1148,7 +1148,7 @@ def set(self, value): BaseListProxy = MakeProxyType('BaseListProxy', ( '__add__', '__contains__', '__delitem__', '__getitem__', '__len__', '__mul__', '__reversed__', '__rmul__', '__setitem__', - 'append', 'count', 'extend', 'index', 'insert', 'pop', 'remove', + 'append', 'clear', 'count', 'extend', 'index', 'insert', 'pop', 'remove', 'reverse', 'sort', '__imul__' )) class ListProxy(BaseListProxy): From b3c8810b4cc4de4269a743a07fb516d92816b46f Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Thu, 30 Mar 2023 18:19:54 +0000 Subject: [PATCH 02/13] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20b?= =?UTF-8?q?lurb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../next/Library/2023-03-30-18-19-53.gh-issue-103134.bHrn91.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Library/2023-03-30-18-19-53.gh-issue-103134.bHrn91.rst diff --git a/Misc/NEWS.d/next/Library/2023-03-30-18-19-53.gh-issue-103134.bHrn91.rst b/Misc/NEWS.d/next/Library/2023-03-30-18-19-53.gh-issue-103134.bHrn91.rst new file mode 100644 index 00000000000000..4f830a672a17e7 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-03-30-18-19-53.gh-issue-103134.bHrn91.rst @@ -0,0 +1 @@ +Add support for `multiprocessing.Manager().list().clear()` From fc803caf566b15af5721f26e265edbb7d2e34668 Mon Sep 17 00:00:00 2001 From: Roy Hyunjin Han Date: Thu, 30 Mar 2023 14:24:09 -0400 Subject: [PATCH 03/13] Update 2023-03-30-18-19-53.gh-issue-103134.bHrn91.rst --- .../next/Library/2023-03-30-18-19-53.gh-issue-103134.bHrn91.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Library/2023-03-30-18-19-53.gh-issue-103134.bHrn91.rst b/Misc/NEWS.d/next/Library/2023-03-30-18-19-53.gh-issue-103134.bHrn91.rst index 4f830a672a17e7..03af43d6faef27 100644 --- a/Misc/NEWS.d/next/Library/2023-03-30-18-19-53.gh-issue-103134.bHrn91.rst +++ b/Misc/NEWS.d/next/Library/2023-03-30-18-19-53.gh-issue-103134.bHrn91.rst @@ -1 +1 @@ -Add support for `multiprocessing.Manager().list().clear()` +Add support for ``multiprocessing.Manager().list().clear()`` From a748d41fc1b80367bda28eb97260cd5ea059a7e0 Mon Sep 17 00:00:00 2001 From: Roy Hyunjin Han Date: Fri, 31 Mar 2023 01:15:44 -0400 Subject: [PATCH 04/13] Update ListProxy and DictProxy ``` from multiprocessing import Manager with Manager() as manager: xs = manager.list() xs.clear xs.copy d = manager.dict() d | {} # __or__ {} | d # __ror__ reversed(d) # __reversed__ d.fromkeys ``` suggested by @terryjreedy tested manually in Python 3.10.8 https://github.com/python/cpython/issues/103134 --- Lib/multiprocessing/managers.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Lib/multiprocessing/managers.py b/Lib/multiprocessing/managers.py index 348df9a0d7a0a6..4597e01d800c4c 100644 --- a/Lib/multiprocessing/managers.py +++ b/Lib/multiprocessing/managers.py @@ -1148,8 +1148,8 @@ def set(self, value): BaseListProxy = MakeProxyType('BaseListProxy', ( '__add__', '__contains__', '__delitem__', '__getitem__', '__len__', '__mul__', '__reversed__', '__rmul__', '__setitem__', - 'append', 'clear', 'count', 'extend', 'index', 'insert', 'pop', 'remove', - 'reverse', 'sort', '__imul__' + 'append', 'clear', 'copy', 'count', 'extend', 'index', 'insert', 'pop', + 'remove', 'reverse', 'sort', '__imul__' )) class ListProxy(BaseListProxy): def __iadd__(self, value): @@ -1162,8 +1162,9 @@ def __imul__(self, value): DictProxy = MakeProxyType('DictProxy', ( '__contains__', '__delitem__', '__getitem__', '__iter__', '__len__', - '__setitem__', 'clear', 'copy', 'get', 'items', - 'keys', 'pop', 'popitem', 'setdefault', 'update', 'values' + '__or__', '__reversed__', '__ror__', '__setitem__', 'clear', 'copy', + 'fromkeys', 'get', 'items', 'keys', 'pop', 'popitem', 'setdefault', + 'update', 'values' )) DictProxy._method_to_typeid_ = { '__iter__': 'Iterator', From 22de5a171c93f19e342c870272ac99eb19f0c28a Mon Sep 17 00:00:00 2001 From: Roy Hyunjin Han Date: Fri, 31 Mar 2023 01:23:18 -0400 Subject: [PATCH 05/13] Update Misc/NEWS.d/next/Library/2023-03-30-18-19-53.gh-issue-103134.bHrn91.rst Co-authored-by: Terry Jan Reedy --- .../next/Library/2023-03-30-18-19-53.gh-issue-103134.bHrn91.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Library/2023-03-30-18-19-53.gh-issue-103134.bHrn91.rst b/Misc/NEWS.d/next/Library/2023-03-30-18-19-53.gh-issue-103134.bHrn91.rst index 03af43d6faef27..e38d17fe231586 100644 --- a/Misc/NEWS.d/next/Library/2023-03-30-18-19-53.gh-issue-103134.bHrn91.rst +++ b/Misc/NEWS.d/next/Library/2023-03-30-18-19-53.gh-issue-103134.bHrn91.rst @@ -1 +1 @@ -Add support for ``multiprocessing.Manager().list().clear()`` +Add support for ``multiprocessing.Manager().list().clear()`` and ``.copy()``. From 8602f0a99295ee48a6dc213b4389a26e4f0c9ca2 Mon Sep 17 00:00:00 2001 From: Roy Hyunjin Han Date: Fri, 31 Mar 2023 01:29:12 -0400 Subject: [PATCH 06/13] Update 2023-03-30-18-19-53.gh-issue-103134.bHrn91.rst --- .../next/Library/2023-03-30-18-19-53.gh-issue-103134.bHrn91.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Library/2023-03-30-18-19-53.gh-issue-103134.bHrn91.rst b/Misc/NEWS.d/next/Library/2023-03-30-18-19-53.gh-issue-103134.bHrn91.rst index e38d17fe231586..864a044e84f215 100644 --- a/Misc/NEWS.d/next/Library/2023-03-30-18-19-53.gh-issue-103134.bHrn91.rst +++ b/Misc/NEWS.d/next/Library/2023-03-30-18-19-53.gh-issue-103134.bHrn91.rst @@ -1 +1 @@ -Add support for ``multiprocessing.Manager().list().clear()`` and ``.copy()``. +Add support for ``multiprocessing.Manager().list().clear()`` and ``.copy()``. Add support for ``multiprocessing.Manager().dict().fromkeys`` and ``reversed(d)``, ``d | {}``, ``{} | d``, where ``d = multiprocessing.Manager().dict()``. From 6fe264a48bfb91ee97705b8889e6f8644834abc3 Mon Sep 17 00:00:00 2001 From: Roy Hyunjin Han Date: Fri, 31 Mar 2023 02:10:38 -0400 Subject: [PATCH 07/13] Add DictProxy.__ior__ for d |= {'b': 2} --- Lib/multiprocessing/managers.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Lib/multiprocessing/managers.py b/Lib/multiprocessing/managers.py index 4597e01d800c4c..68b3a17c942d99 100644 --- a/Lib/multiprocessing/managers.py +++ b/Lib/multiprocessing/managers.py @@ -1161,10 +1161,10 @@ def __imul__(self, value): DictProxy = MakeProxyType('DictProxy', ( - '__contains__', '__delitem__', '__getitem__', '__iter__', '__len__', - '__or__', '__reversed__', '__ror__', '__setitem__', 'clear', 'copy', - 'fromkeys', 'get', 'items', 'keys', 'pop', 'popitem', 'setdefault', - 'update', 'values' + '__contains__', '__delitem__', '__getitem__', '__ior__', '__iter__', + '__len__', '__or__', '__reversed__', '__ror__', '__setitem__', 'clear', + 'copy', 'fromkeys', 'get', 'items', 'keys', 'pop', 'popitem', + 'setdefault', 'update', 'values' )) DictProxy._method_to_typeid_ = { '__iter__': 'Iterator', From 8c2afdb660caab24d926956b7fc6d4fa759126fa Mon Sep 17 00:00:00 2001 From: Roy Hyunjin Han Date: Fri, 31 Mar 2023 02:18:53 -0400 Subject: [PATCH 08/13] Update 2023-03-30-18-19-53.gh-issue-103134.bHrn91.rst --- .../next/Library/2023-03-30-18-19-53.gh-issue-103134.bHrn91.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Library/2023-03-30-18-19-53.gh-issue-103134.bHrn91.rst b/Misc/NEWS.d/next/Library/2023-03-30-18-19-53.gh-issue-103134.bHrn91.rst index 864a044e84f215..bc033805706727 100644 --- a/Misc/NEWS.d/next/Library/2023-03-30-18-19-53.gh-issue-103134.bHrn91.rst +++ b/Misc/NEWS.d/next/Library/2023-03-30-18-19-53.gh-issue-103134.bHrn91.rst @@ -1 +1 @@ -Add support for ``multiprocessing.Manager().list().clear()`` and ``.copy()``. Add support for ``multiprocessing.Manager().dict().fromkeys`` and ``reversed(d)``, ``d | {}``, ``{} | d``, where ``d = multiprocessing.Manager().dict()``. +Add support for ``multiprocessing.Manager().list().clear()`` and ``.copy()``. Add support for ``multiprocessing.Manager().dict().fromkeys`` and ``reversed(d)``, ``d | {}``, ``{} | d``, ``d |= {'b': 2}``, where ``d = multiprocessing.Manager().dict()``. From 1cf0985deb3ea174d3d6feb809d51d6f25775fce Mon Sep 17 00:00:00 2001 From: Roy Hyunjin Han Date: Sat, 29 Apr 2023 20:55:57 -0400 Subject: [PATCH 09/13] Fix __ior__ and add tests thanks to https://github.com/python/cpython/issues/103134#issuecomment-1493429209 @sunmy2019 --- Lib/multiprocessing/managers.py | 18 +++++++++++------- Lib/test/_test_multiprocessing.py | 20 +++++++++++++++++--- 2 files changed, 28 insertions(+), 10 deletions(-) diff --git a/Lib/multiprocessing/managers.py b/Lib/multiprocessing/managers.py index 68b3a17c942d99..b958c17c28e611 100644 --- a/Lib/multiprocessing/managers.py +++ b/Lib/multiprocessing/managers.py @@ -1146,10 +1146,10 @@ def set(self, value): BaseListProxy = MakeProxyType('BaseListProxy', ( - '__add__', '__contains__', '__delitem__', '__getitem__', '__len__', - '__mul__', '__reversed__', '__rmul__', '__setitem__', + '__add__', '__contains__', '__delitem__', '__getitem__', '__imul__', + '__len__', '__mul__', '__reversed__', '__rmul__', '__setitem__', 'append', 'clear', 'copy', 'count', 'extend', 'index', 'insert', 'pop', - 'remove', 'reverse', 'sort', '__imul__' + 'remove', 'reverse', 'sort', )) class ListProxy(BaseListProxy): def __iadd__(self, value): @@ -1160,15 +1160,19 @@ def __imul__(self, value): return self -DictProxy = MakeProxyType('DictProxy', ( +BaseDictProxy = MakeProxyType('DictProxy', ( '__contains__', '__delitem__', '__getitem__', '__ior__', '__iter__', '__len__', '__or__', '__reversed__', '__ror__', '__setitem__', 'clear', 'copy', 'fromkeys', 'get', 'items', 'keys', 'pop', 'popitem', - 'setdefault', 'update', 'values' + 'setdefault', 'update', 'values', )) -DictProxy._method_to_typeid_ = { - '__iter__': 'Iterator', +class DictProxy(BaseDictProxy): + _method_to_typeid_ = { + '__iter__': 'Iterator', } + def __ior__(self, value): + self._callmethod('__ior__', (value,)) + return self ArrayProxy = MakeProxyType('ArrayProxy', ( diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 9a2db24b4bd597..b5d098bbaaed40 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -5706,12 +5706,16 @@ def _test_list(cls, obj): case.assertEqual(obj[0], 5) case.assertEqual(obj.count(5), 1) case.assertEqual(obj.index(5), 0) + obj += [7] + obj.extend(obj * 2 + obj.copy()) obj.sort() obj.reverse() for x in obj: pass - case.assertEqual(len(obj), 1) - case.assertEqual(obj.pop(0), 5) + case.assertEqual(len(obj), 8) + case.assertEqual(obj.pop(0), 7) + obj.clear() + case.assertEqual(len(obj), 0) def test_list(self): o = self.manager.list() @@ -5730,7 +5734,17 @@ def _test_dict(cls, obj): case.assertListEqual(list(obj.keys()), ['foo']) case.assertListEqual(list(obj.values()), [5]) case.assertDictEqual(obj.copy(), {'foo': 5}) - case.assertTupleEqual(obj.popitem(), ('foo', 5)) + obj |= {'bar': 6} + case.assertListEqual(list(reversed(obj)), ['bar', 'foo']) + case.assertDictEqual({'bar': 7} | obj, {'foo': 5, 'bar': 6}) + case.assertDictEqual(obj | {'bar': 7}, {'foo': 5, 'bar': 7}) + case.assertDictEqual(obj.fromkeys(['bar'], 6), {'bar': 6}) + case.assertTupleEqual(obj.popitem(), ('bar', 6)) + obj.setdefault('bar', 0) + obj.update({'bar': 7}) + case.assertEqual(obj.pop('bar'), 7) + obj.clear() + case.assertEqual(len(obj), 0) def test_dict(self): o = self.manager.dict() From 83ecdb31a6fc0ea0f7177c9533477c03b5d110f3 Mon Sep 17 00:00:00 2001 From: Roy Hyunjin Han Date: Sat, 27 Apr 2024 15:48:39 -0400 Subject: [PATCH 10/13] Test the type and the value of the results as requested by @serhiy-storchaka --- Lib/test/_test_multiprocessing.py | 32 ++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 886b5d678863dd..0d95ec9fa30b63 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -6065,12 +6065,26 @@ def _test_list(cls, obj): case.assertEqual(obj.count(5), 1) case.assertEqual(obj.index(5), 0) obj += [7] - obj.extend(obj * 2 + obj.copy()) + case.assertIsInstance(obj, multiprocessing.managers.ListProxy) + case.assertListEqual(list(obj), [5, 7]) + obj *= 2 + case.assertIsInstance(obj, multiprocessing.managers.ListProxy) + case.assertListEqual(list(obj), [5, 7, 5, 7]) + double_obj = obj * 2 + case.assertIsInstance(double_obj, list) + case.assertListEqual(list(double_obj), [5, 7, 5, 7, 5, 7, 5, 7]) + double_obj = 2 * obj + case.assertIsInstance(double_obj, list) + case.assertListEqual(list(double_obj), [5, 7, 5, 7, 5, 7, 5, 7]) + copied_obj = obj.copy() + case.assertIsInstance(copied_obj, list) + case.assertListEqual(list(copied_obj), [5, 7, 5, 7]) + obj.extend(double_obj + copied_obj) obj.sort() obj.reverse() for x in obj: pass - case.assertEqual(len(obj), 8) + case.assertEqual(len(obj), 16) case.assertEqual(obj.pop(0), 7) obj.clear() case.assertEqual(len(obj), 0) @@ -6093,9 +6107,17 @@ def _test_dict(cls, obj): case.assertListEqual(list(obj.values()), [5]) case.assertDictEqual(obj.copy(), {'foo': 5}) obj |= {'bar': 6} - case.assertListEqual(list(reversed(obj)), ['bar', 'foo']) - case.assertDictEqual({'bar': 7} | obj, {'foo': 5, 'bar': 6}) - case.assertDictEqual(obj | {'bar': 7}, {'foo': 5, 'bar': 7}) + case.assertIsInstance(obj, multiprocessing.managers.DictProxy) + case.assertDictEqual(dict(obj), {'foo': 5, 'bar': 6}) + x = reversed(obj) + case.assertIsInstance(x, type(iter([]))) + case.assertListEqual(list(x), ['bar', 'foo']) + x = {'bar': 7, 'baz': 7} | obj + case.assertIsInstance(x, dict) + case.assertDictEqual(dict(x), {'foo': 5, 'bar': 6, 'baz': 7}) + x = obj | {'bar': 7, 'baz': 7} + case.assertIsInstance(x, dict) + case.assertDictEqual(dict(x), {'foo': 5, 'bar': 7, 'baz': 7}) case.assertDictEqual(obj.fromkeys(['bar'], 6), {'bar': 6}) case.assertTupleEqual(obj.popitem(), ('bar', 6)) obj.setdefault('bar', 0) From 6f2ea5b888c697efef04e5b71d072dd970756842 Mon Sep 17 00:00:00 2001 From: Roy Hyunjin Han Date: Sat, 27 Apr 2024 15:54:28 -0400 Subject: [PATCH 11/13] Remove trailing whitespace --- Lib/multiprocessing/managers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/multiprocessing/managers.py b/Lib/multiprocessing/managers.py index b3b071d9ea95f0..0f5f9f64c2de9e 100644 --- a/Lib/multiprocessing/managers.py +++ b/Lib/multiprocessing/managers.py @@ -1181,7 +1181,7 @@ class DictProxy(_BaseDictProxy): def __ior__(self, value): self._callmethod('__ior__', (value,)) return self - + __class_getitem__ = classmethod(types.GenericAlias) ArrayProxy = MakeProxyType('ArrayProxy', ( From 577e5a30682adc644734a7c15d909b1e8619fabb Mon Sep 17 00:00:00 2001 From: Roy Hyunjin Han Date: Mon, 20 May 2024 09:41:56 -0400 Subject: [PATCH 12/13] Update _test_multiprocessing.py Check that the dictionary returned by dict_proxy.from_keys is a dict as defined in dict.from_keys as requested by @encukou --- Lib/test/_test_multiprocessing.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 43ab218a296fec..301541a666e140 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -6118,8 +6118,12 @@ def _test_dict(cls, obj): x = obj | {'bar': 7, 'baz': 7} case.assertIsInstance(x, dict) case.assertDictEqual(dict(x), {'foo': 5, 'bar': 7, 'baz': 7}) - case.assertDictEqual(obj.fromkeys(['bar'], 6), {'bar': 6}) - case.assertTupleEqual(obj.popitem(), ('bar', 6)) + x = obj.fromkeys(['bar'], 6) + case.assertIsInstance(x, dict) + case.assertDictEqual(x, {'bar': 6}) + x = obj.popitem() + case.assertIsInstance(x, tuple) + case.assertTupleEqual(x, ('bar', 6)) obj.setdefault('bar', 0) obj.update({'bar': 7}) case.assertEqual(obj.pop('bar'), 7) From 72d1055cd6e43fdd012218fe2abdb06fdcb87fd4 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Mon, 20 May 2024 10:06:00 -0400 Subject: [PATCH 13/13] Add links to the NEWS blurb --- .../Library/2023-03-30-18-19-53.gh-issue-103134.bHrn91.rst | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Library/2023-03-30-18-19-53.gh-issue-103134.bHrn91.rst b/Misc/NEWS.d/next/Library/2023-03-30-18-19-53.gh-issue-103134.bHrn91.rst index bc033805706727..11559dce0ae2b4 100644 --- a/Misc/NEWS.d/next/Library/2023-03-30-18-19-53.gh-issue-103134.bHrn91.rst +++ b/Misc/NEWS.d/next/Library/2023-03-30-18-19-53.gh-issue-103134.bHrn91.rst @@ -1 +1,6 @@ -Add support for ``multiprocessing.Manager().list().clear()`` and ``.copy()``. Add support for ``multiprocessing.Manager().dict().fromkeys`` and ``reversed(d)``, ``d | {}``, ``{} | d``, ``d |= {'b': 2}``, where ``d = multiprocessing.Manager().dict()``. +Add additional methods to :ref:`proxy objects ` +in the :mod:`!multiprocessing` module: + +* :meth:`!clear` and :meth:`!copy` for proxies of :class:`list` +* :meth:`~dict.fromkeys`, ``reversed(d)``, ``d | {}``, ``{} | d``, + ``d |= {'b': 2}`` for proxies of :class:`dict`