From 1b1ae0773cb8d10fb3295efa367e09af383a005f Mon Sep 17 00:00:00 2001 From: sobolevn Date: Thu, 16 Sep 2021 17:23:55 +0300 Subject: [PATCH 1/7] bpo-45128: fixes `test_multiprocessing_fork` mysterious crash --- Lib/test/test_logging.py | 9 +++++++++ .../next/Tests/2021-09-16-17-22-35.bpo-45128.Jz6fl2.rst | 2 ++ 2 files changed, 11 insertions(+) create mode 100644 Misc/NEWS.d/next/Tests/2021-09-16-17-22-35.bpo-45128.Jz6fl2.rst diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index 9998f1a02e4c9c..f1f3471e295587 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -4432,12 +4432,17 @@ def test_dict_arg(self): def _extract_logrecord_process_name(key, logMultiprocessing, conn=None): prev_logMultiprocessing = logging.logMultiprocessing logging.logMultiprocessing = logMultiprocessing + old_mp = None try: import multiprocessing as mp name = mp.current_process().name r1 = logging.makeLogRecord({'msg': f'msg1_{key}'}) + + old_mp = sys.modules['multiprocessing'] del sys.modules['multiprocessing'] + assert old_mp + r2 = logging.makeLogRecord({'msg': f'msg2_{key}'}) results = {'processName' : name, @@ -4446,6 +4451,10 @@ def _extract_logrecord_process_name(key, logMultiprocessing, conn=None): } finally: logging.logMultiprocessing = prev_logMultiprocessing + if old_mp is not None: + # We need to restore the previous `multiprocessing` module: + # https://bugs.python.org/issue45128 + sys.modules['multiprocessing'] = old_mp if conn: conn.send(results) else: diff --git a/Misc/NEWS.d/next/Tests/2021-09-16-17-22-35.bpo-45128.Jz6fl2.rst b/Misc/NEWS.d/next/Tests/2021-09-16-17-22-35.bpo-45128.Jz6fl2.rst new file mode 100644 index 00000000000000..e54120e3449244 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2021-09-16-17-22-35.bpo-45128.Jz6fl2.rst @@ -0,0 +1,2 @@ +Fix crash ``test_multiprocessing_fork`` due to ``LogRecordTest`` and +``sys.modules`` manipulation. From c9028b27ab112e75a2c4d9a4c181be336dfb9732 Mon Sep 17 00:00:00 2001 From: sobolevn Date: Thu, 16 Sep 2021 17:29:48 +0300 Subject: [PATCH 2/7] Extra test --- Lib/test/test_logging.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index f1f3471e295587..f5de4e24788f34 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -4496,6 +4496,17 @@ def test_multiprocessing(self): if multiprocessing_imported: import multiprocessing + def test_multiprocessing_again(self): + # https://bugs.python.org/issue45128 + import sys + import multiprocessing.queues + + self.test_multiprocessing() + + import multiprocessing + import multiprocessing.connection + from multiprocessing.connection import wait + connection = multiprocessing.connection # It was AttributeError here def test_optional(self): r = logging.makeLogRecord({}) From a70dd41c29762956265d24c5bbc4c678f9348818 Mon Sep 17 00:00:00 2001 From: sobolevn Date: Thu, 16 Sep 2021 17:30:13 +0300 Subject: [PATCH 3/7] Extra test --- Lib/test/test_logging.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index f5de4e24788f34..0385f099c425f6 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -4506,7 +4506,7 @@ def test_multiprocessing_again(self): import multiprocessing import multiprocessing.connection from multiprocessing.connection import wait - connection = multiprocessing.connection # It was AttributeError here + multiprocessing.connection # It was AttributeError here def test_optional(self): r = logging.makeLogRecord({}) From 4fd13810fdf769254d1a7bc73157caeb310ec54c Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Thu, 16 Sep 2021 18:33:37 +0300 Subject: [PATCH 4/7] Update Misc/NEWS.d/next/Tests/2021-09-16-17-22-35.bpo-45128.Jz6fl2.rst Co-authored-by: Victor Stinner --- Misc/NEWS.d/next/Tests/2021-09-16-17-22-35.bpo-45128.Jz6fl2.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Tests/2021-09-16-17-22-35.bpo-45128.Jz6fl2.rst b/Misc/NEWS.d/next/Tests/2021-09-16-17-22-35.bpo-45128.Jz6fl2.rst index e54120e3449244..f9662a42a745ba 100644 --- a/Misc/NEWS.d/next/Tests/2021-09-16-17-22-35.bpo-45128.Jz6fl2.rst +++ b/Misc/NEWS.d/next/Tests/2021-09-16-17-22-35.bpo-45128.Jz6fl2.rst @@ -1,2 +1,2 @@ -Fix crash ``test_multiprocessing_fork`` due to ``LogRecordTest`` and +Fix crash ``test_multiprocessing_fork`` due to ``test_logging`` and ``sys.modules`` manipulation. From 6caffde643787060ee70783aad75505091e2407b Mon Sep 17 00:00:00 2001 From: sobolevn Date: Thu, 16 Sep 2021 18:41:59 +0300 Subject: [PATCH 5/7] Isolates test_multiprocessing --- Lib/test/test_logging.py | 32 ++++++++++++-------------------- 1 file changed, 12 insertions(+), 20 deletions(-) diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index 0385f099c425f6..fb8e0876771e95 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -4432,18 +4432,16 @@ def test_dict_arg(self): def _extract_logrecord_process_name(key, logMultiprocessing, conn=None): prev_logMultiprocessing = logging.logMultiprocessing logging.logMultiprocessing = logMultiprocessing - old_mp = None try: import multiprocessing as mp name = mp.current_process().name r1 = logging.makeLogRecord({'msg': f'msg1_{key}'}) - old_mp = sys.modules['multiprocessing'] - del sys.modules['multiprocessing'] - assert old_mp - - r2 = logging.makeLogRecord({'msg': f'msg2_{key}'}) + with support.swap_item(sys.modules, 'multiprocessing', None): + # We need to restore the previous `multiprocessing` module: + # https://bugs.python.org/issue45128 + r2 = logging.makeLogRecord({'msg': f'msg2_{key}'}) results = {'processName' : name, 'r1.processName': r1.processName, @@ -4451,16 +4449,12 @@ def _extract_logrecord_process_name(key, logMultiprocessing, conn=None): } finally: logging.logMultiprocessing = prev_logMultiprocessing - if old_mp is not None: - # We need to restore the previous `multiprocessing` module: - # https://bugs.python.org/issue45128 - sys.modules['multiprocessing'] = old_mp if conn: conn.send(results) else: return results - def test_multiprocessing(self): + def test_multiprocessing(self, first_pass=True): multiprocessing_imported = 'multiprocessing' in sys.modules try: # logMultiprocessing is True by default @@ -4496,17 +4490,15 @@ def test_multiprocessing(self): if multiprocessing_imported: import multiprocessing - def test_multiprocessing_again(self): - # https://bugs.python.org/issue45128 - import sys - import multiprocessing.queues + if first_pass: # https://bugs.python.org/issue45128 + import multiprocessing.queues - self.test_multiprocessing() + self.test_multiprocessing(first_pass=False) - import multiprocessing - import multiprocessing.connection - from multiprocessing.connection import wait - multiprocessing.connection # It was AttributeError here + import multiprocessing + import multiprocessing.connection + from multiprocessing.connection import wait + multiprocessing.connection # It was AttributeError here def test_optional(self): r = logging.makeLogRecord({}) From a48afcb0facc4dac58c242f78ba0a646d55531c4 Mon Sep 17 00:00:00 2001 From: sobolevn Date: Thu, 16 Sep 2021 19:08:38 +0300 Subject: [PATCH 6/7] Removes regression test --- Lib/test/test_logging.py | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index fb8e0876771e95..e8ab5adea65fee 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -4438,9 +4438,8 @@ def _extract_logrecord_process_name(key, logMultiprocessing, conn=None): r1 = logging.makeLogRecord({'msg': f'msg1_{key}'}) + # https://bugs.python.org/issue45128 with support.swap_item(sys.modules, 'multiprocessing', None): - # We need to restore the previous `multiprocessing` module: - # https://bugs.python.org/issue45128 r2 = logging.makeLogRecord({'msg': f'msg2_{key}'}) results = {'processName' : name, @@ -4454,7 +4453,7 @@ def _extract_logrecord_process_name(key, logMultiprocessing, conn=None): else: return results - def test_multiprocessing(self, first_pass=True): + def test_multiprocessing(self): multiprocessing_imported = 'multiprocessing' in sys.modules try: # logMultiprocessing is True by default @@ -4490,16 +4489,6 @@ def test_multiprocessing(self, first_pass=True): if multiprocessing_imported: import multiprocessing - if first_pass: # https://bugs.python.org/issue45128 - import multiprocessing.queues - - self.test_multiprocessing(first_pass=False) - - import multiprocessing - import multiprocessing.connection - from multiprocessing.connection import wait - multiprocessing.connection # It was AttributeError here - def test_optional(self): r = logging.makeLogRecord({}) NOT_NONE = self.assertIsNotNone From f11bc503d60e87f72f85f75122637f82a158a664 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Thu, 16 Sep 2021 21:46:49 +0300 Subject: [PATCH 7/7] Update 2021-09-16-17-22-35.bpo-45128.Jz6fl2.rst --- Misc/NEWS.d/next/Tests/2021-09-16-17-22-35.bpo-45128.Jz6fl2.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Tests/2021-09-16-17-22-35.bpo-45128.Jz6fl2.rst b/Misc/NEWS.d/next/Tests/2021-09-16-17-22-35.bpo-45128.Jz6fl2.rst index f9662a42a745ba..b50eb32b3faa8e 100644 --- a/Misc/NEWS.d/next/Tests/2021-09-16-17-22-35.bpo-45128.Jz6fl2.rst +++ b/Misc/NEWS.d/next/Tests/2021-09-16-17-22-35.bpo-45128.Jz6fl2.rst @@ -1,2 +1,2 @@ -Fix crash ``test_multiprocessing_fork`` due to ``test_logging`` and +Fix ``test_multiprocessing_fork`` failure due to ``test_logging`` and ``sys.modules`` manipulation.