From 666ca33e14e7cef8b4c80d5d9763aaa87b531e7a Mon Sep 17 00:00:00 2001 From: Eddie Elizondo Date: Sun, 31 Dec 2023 10:23:51 -0800 Subject: [PATCH 01/16] Reenable non-debug interned string cleanup through _PyUnicode_ClearInterned --- Objects/unicodeobject.c | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 4b03cc3f4da5fa..bbdb63187c160c 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -14961,17 +14961,6 @@ _PyUnicode_ClearInterned(PyInterpreterState *interp) } assert(PyDict_CheckExact(interned)); - /* TODO: - * Currently, the runtime is not able to guarantee that it can exit without - * allocations that carry over to a future initialization of Python within - * the same process. i.e: - * ./python -X showrefcount -c 'import itertools' - * [237 refs, 237 blocks] - * - * Therefore, this should remain disabled for until there is a strict guarantee - * that no memory will be left after `Py_Finalize`. - */ -#ifdef Py_DEBUG /* For all non-singleton interned strings, restore the two valid references to that instance from within the intern string dictionary and let the normal reference counting process clean up these instances. */ @@ -15028,7 +15017,6 @@ _PyUnicode_ClearInterned(PyInterpreterState *interp) for (Py_ssize_t i=0; i < ids->size; i++) { Py_XINCREF(ids->array[i]); } -#endif /* Py_DEBUG */ clear_interned_dict(interp); } From 23d1aff6f124b60be294c6f0e843f1fe8d053b56 Mon Sep 17 00:00:00 2001 From: Eddie Elizondo Date: Sun, 31 Dec 2023 10:50:08 -0800 Subject: [PATCH 02/16] Revert test changes due to previously leftover leaks --- Lib/test/_test_embed_structseq.py | 40 ++++++++++++++----------------- 1 file changed, 18 insertions(+), 22 deletions(-) diff --git a/Lib/test/_test_embed_structseq.py b/Lib/test/_test_embed_structseq.py index 834daa4df55fec..868f9f83e8be77 100644 --- a/Lib/test/_test_embed_structseq.py +++ b/Lib/test/_test_embed_structseq.py @@ -1,31 +1,27 @@ import sys import types +import unittest -# Note: This test file can't import `unittest` since the runtime can't -# currently guarantee that it will not leak memory. Doing so will mark -# the test as passing but with reference leaks. This can safely import -# the `unittest` library once there's a strict guarantee of no leaks -# during runtime shutdown. # bpo-46417: Test that structseq types used by the sys module are still # valid when Py_Finalize()/Py_Initialize() are called multiple times. -class TestStructSeq: +class TestStructSeq(unittest.TestCase): # test PyTypeObject members - def _check_structseq(self, obj_type): + def check_structseq(self, obj_type): # ob_refcnt - assert sys.getrefcount(obj_type) > 1 + self.assertGreaterEqual(sys.getrefcount(obj_type), 1) # tp_base - assert issubclass(obj_type, tuple) + self.assertTrue(issubclass(obj_type, tuple)) # tp_bases - assert obj_type.__bases__ == (tuple,) + self.assertEqual(obj_type.__bases__, (tuple,)) # tp_dict - assert isinstance(obj_type.__dict__, types.MappingProxyType) + self.assertIsInstance(obj_type.__dict__, types.MappingProxyType) # tp_mro - assert obj_type.__mro__ == (obj_type, tuple, object) + self.assertEqual(obj_type.__mro__, (obj_type, tuple, object)) # tp_name - assert isinstance(type.__name__, str) + self.assertIsInstance(type.__name__, str) # tp_subclasses - assert obj_type.__subclasses__() == [] + self.assertEqual(obj_type.__subclasses__(), []) def test_sys_attrs(self): for attr_name in ( @@ -36,23 +32,23 @@ def test_sys_attrs(self): 'thread_info', # ThreadInfoType 'version_info', # VersionInfoType ): - attr = getattr(sys, attr_name) - self._check_structseq(type(attr)) + with self.subTest(attr=attr_name): + attr = getattr(sys, attr_name) + self.check_structseq(type(attr)) def test_sys_funcs(self): func_names = ['get_asyncgen_hooks'] # AsyncGenHooksType if hasattr(sys, 'getwindowsversion'): func_names.append('getwindowsversion') # WindowsVersionType for func_name in func_names: - func = getattr(sys, func_name) - obj = func() - self._check_structseq(type(obj)) + with self.subTest(func=func_name): + func = getattr(sys, func_name) + obj = func() + self.check_structseq(type(obj)) try: - tests = TestStructSeq() - tests.test_sys_attrs() - tests.test_sys_funcs() + unittest.main() except SystemExit as exc: if exc.args[0] != 0: raise From ed09466ad0084ead701b334422da6558537b80ff Mon Sep 17 00:00:00 2001 From: Eddie Elizondo Date: Mon, 15 Jan 2024 18:31:56 +0000 Subject: [PATCH 03/16] Add documentation --- Doc/c-api/init.rst | 17 ++++++++++------ Doc/whatsnew/3.13.rst | 11 ++++++++++ ...-01-15-18-11-48.gh-issue-113190.OwQX64.rst | 20 +++++++++++++++++++ 3 files changed, 42 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-01-15-18-11-48.gh-issue-113190.OwQX64.rst diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index f8fd48e781d6da..3724528cbdcda8 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -386,11 +386,15 @@ Initializing and finalizing the interpreter Undo all initializations made by :c:func:`Py_Initialize` and subsequent use of Python/C API functions, and destroy all sub-interpreters (see :c:func:`Py_NewInterpreter` below) that were created and not yet destroyed since - the last call to :c:func:`Py_Initialize`. Ideally, this frees all memory - allocated by the Python interpreter. This is a no-op when called for a second + the last call to :c:func:`Py_Initialize`. This is a no-op when called for a second time (without calling :c:func:`Py_Initialize` again first). Normally the return value is ``0``. If there were errors during finalization (flushing buffered data), ``-1`` is returned. + + Note that Python will do a best effort at freeing all memory allocated by the Python + interpreter. Therefore, any C-Extension should make sure to correctly clean up all + of the preveiously allocated PyObjects before using them in subsequent calls to + `Py_Initialize`. Otherwise it introduces vulnerabilities and incorrect behavior. This function is provided for a number of reasons. An embedding application might want to restart Python without having to restart the application itself. @@ -406,10 +410,11 @@ Initializing and finalizing the interpreter loaded extension modules loaded by Python are not unloaded. Small amounts of memory allocated by the Python interpreter may not be freed (if you find a leak, please report it). Memory tied up in circular references between objects is not - freed. Some memory allocated by extension modules may not be freed. Some - extensions may not work properly if their initialization routine is called more - than once; this can happen if an application calls :c:func:`Py_Initialize` and - :c:func:`Py_FinalizeEx` more than once. + freed. Interned strings will all be deallocated regarldess of their reference count. + Some memory allocated by extension modules may not be freed. Some extensions may not + work properly if their initialization routine is called more than once; this can + happen if an application calls :c:func:`Py_Initialize` and :c:func:`Py_FinalizeEx` + more than once. .. audit-event:: cpython._PySys_ClearAuditHooks "" c.Py_FinalizeEx diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 05b9b87a63252f..648c1a72b3d6dc 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -1363,6 +1363,17 @@ New Features * Add :c:func:`Py_HashPointer` function to hash a pointer. (Contributed by Victor Stinner in :gh:`111545`.) +* Modified :c:func:`_PyUnicode_ClearInterned` function to always delete all + interned strings during a call to :c:func:`Py_Finalize`. This makes all + is backwards incompatible to any C-Extension that holds onto an interned + string after a call to c:func:`Py_Finalize` and is then reused after a + call to c:func:`Py_Initialize`. Any issues arising from this behavior will + normally result in crashes during the exectuion of the subsequent call to + c:func:`Py_Initatilize` from accessing uninitialized memory. To fix, use + an address sanitizer (i.e. ASAN) to identify any use-after-free coming from + an interned string and deallocate it during module shutdown. + (Contribued by Eddie Elizondo in :gh:`113601`.) + Porting to Python 3.13 ---------------------- diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-01-15-18-11-48.gh-issue-113190.OwQX64.rst b/Misc/NEWS.d/next/Core and Builtins/2024-01-15-18-11-48.gh-issue-113190.OwQX64.rst new file mode 100644 index 00000000000000..f549057d6863d5 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-01-15-18-11-48.gh-issue-113190.OwQX64.rst @@ -0,0 +1,20 @@ +This updates the interned string deallocation function +`_PyUnicode_ClearInterned` to delete all interned strings during runtime +finalization when calling `Py_Finalize`, regardless of their reference +count. + +Worth noting that if an extension accidentally holds onto a interned string, +event after calling Py_Finalize, it will result in use-after-free error, +leaving the user to a potential vulnerabilities. That said, the history of +how these interned strings are handled have been changing during throughout +different versions. For example: + +* In Python 3.9 and older, interned strings were never deleted. Only special +build for Valgrind and Purity cleared them at exit +* In Python 3.10 and 3.11, interned strings are always deleted at exit +* In Python 3.12, interned strings are deleted only if Python is built in +debug mode: they are not deleted in release mode +* In Python 3.13, interned strings will now be akll deleted + +Given how we've changed guarantees throughout different versions, we do not +expect that users actively rely on this behavior for their extensions. From 2608119bf24f5f0f34606d57dbd9d43b52067387 Mon Sep 17 00:00:00 2001 From: Eddie Elizondo Date: Mon, 15 Jan 2024 18:35:07 +0000 Subject: [PATCH 04/16] Lint docs --- Doc/whatsnew/3.13.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 648c1a72b3d6dc..03c6fd78b3eda4 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -1366,7 +1366,7 @@ New Features * Modified :c:func:`_PyUnicode_ClearInterned` function to always delete all interned strings during a call to :c:func:`Py_Finalize`. This makes all is backwards incompatible to any C-Extension that holds onto an interned - string after a call to c:func:`Py_Finalize` and is then reused after a + string after a call to c:func:`Py_Finalize` and is then reused after a call to c:func:`Py_Initialize`. Any issues arising from this behavior will normally result in crashes during the exectuion of the subsequent call to c:func:`Py_Initatilize` from accessing uninitialized memory. To fix, use From 17e88173bd0c11a28e020c635e3e9b162ff71dd7 Mon Sep 17 00:00:00 2001 From: Eddie Elizondo Date: Mon, 15 Jan 2024 18:38:45 +0000 Subject: [PATCH 05/16] Lint docs v2 --- Doc/c-api/init.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index 3724528cbdcda8..bddbbb7abb274d 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -390,7 +390,7 @@ Initializing and finalizing the interpreter time (without calling :c:func:`Py_Initialize` again first). Normally the return value is ``0``. If there were errors during finalization (flushing buffered data), ``-1`` is returned. - + Note that Python will do a best effort at freeing all memory allocated by the Python interpreter. Therefore, any C-Extension should make sure to correctly clean up all of the preveiously allocated PyObjects before using them in subsequent calls to @@ -410,7 +410,7 @@ Initializing and finalizing the interpreter loaded extension modules loaded by Python are not unloaded. Small amounts of memory allocated by the Python interpreter may not be freed (if you find a leak, please report it). Memory tied up in circular references between objects is not - freed. Interned strings will all be deallocated regarldess of their reference count. + freed. Interned strings will all be deallocated regarldess of their reference count. Some memory allocated by extension modules may not be freed. Some extensions may not work properly if their initialization routine is called more than once; this can happen if an application calls :c:func:`Py_Initialize` and :c:func:`Py_FinalizeEx` From dd1e8b4c72d5a42ddfef783879b867443458832e Mon Sep 17 00:00:00 2001 From: Eddie Elizondo Date: Mon, 15 Jan 2024 18:46:34 +0000 Subject: [PATCH 06/16] Lint docs v3 --- Doc/c-api/init.rst | 5 +++-- Doc/whatsnew/3.13.rst | 6 +++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index bddbbb7abb274d..23681c32df7491 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -390,11 +390,12 @@ Initializing and finalizing the interpreter time (without calling :c:func:`Py_Initialize` again first). Normally the return value is ``0``. If there were errors during finalization (flushing buffered data), ``-1`` is returned. - + Note that Python will do a best effort at freeing all memory allocated by the Python interpreter. Therefore, any C-Extension should make sure to correctly clean up all of the preveiously allocated PyObjects before using them in subsequent calls to - `Py_Initialize`. Otherwise it introduces vulnerabilities and incorrect behavior. + :c:func:`Py_Initialize`. Otherwise it could introduce vulnerabilities and incorrect + behavior. This function is provided for a number of reasons. An embedding application might want to restart Python without having to restart the application itself. diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 03c6fd78b3eda4..26602294bb596a 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -1366,10 +1366,10 @@ New Features * Modified :c:func:`_PyUnicode_ClearInterned` function to always delete all interned strings during a call to :c:func:`Py_Finalize`. This makes all is backwards incompatible to any C-Extension that holds onto an interned - string after a call to c:func:`Py_Finalize` and is then reused after a - call to c:func:`Py_Initialize`. Any issues arising from this behavior will + string after a call to :c:func:`Py_Finalize` and is then reused after a + call to :c:func:`Py_Initialize`. Any issues arising from this behavior will normally result in crashes during the exectuion of the subsequent call to - c:func:`Py_Initatilize` from accessing uninitialized memory. To fix, use + :c:func:`Py_Initatilize` from accessing uninitialized memory. To fix, use an address sanitizer (i.e. ASAN) to identify any use-after-free coming from an interned string and deallocate it during module shutdown. (Contribued by Eddie Elizondo in :gh:`113601`.) From ce5536bee653d2854f7179630b6033c667dcb809 Mon Sep 17 00:00:00 2001 From: Eddie Elizondo Date: Mon, 15 Jan 2024 18:46:44 +0000 Subject: [PATCH 07/16] Lint docs v4 --- .../2024-01-15-18-11-48.gh-issue-113190.OwQX64.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-01-15-18-11-48.gh-issue-113190.OwQX64.rst b/Misc/NEWS.d/next/Core and Builtins/2024-01-15-18-11-48.gh-issue-113190.OwQX64.rst index f549057d6863d5..906a3966dabc3c 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2024-01-15-18-11-48.gh-issue-113190.OwQX64.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2024-01-15-18-11-48.gh-issue-113190.OwQX64.rst @@ -1,6 +1,6 @@ This updates the interned string deallocation function -`_PyUnicode_ClearInterned` to delete all interned strings during runtime -finalization when calling `Py_Finalize`, regardless of their reference +``_PyUnicode_ClearInterned`` to delete all interned strings during runtime +finalization when calling ``Py_Finalize``, regardless of their reference count. Worth noting that if an extension accidentally holds onto a interned string, From 071438f5749fc2d073632266f78b5f64bf2df326 Mon Sep 17 00:00:00 2001 From: Eddie Elizondo Date: Mon, 15 Jan 2024 18:56:10 +0000 Subject: [PATCH 08/16] Lint docs v5 --- Doc/whatsnew/3.13.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 26602294bb596a..6dbb66912833c0 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -1363,13 +1363,13 @@ New Features * Add :c:func:`Py_HashPointer` function to hash a pointer. (Contributed by Victor Stinner in :gh:`111545`.) -* Modified :c:func:`_PyUnicode_ClearInterned` function to always delete all +* Modified the ``_PyUnicode_ClearInterned`` function to always delete all interned strings during a call to :c:func:`Py_Finalize`. This makes all is backwards incompatible to any C-Extension that holds onto an interned string after a call to :c:func:`Py_Finalize` and is then reused after a call to :c:func:`Py_Initialize`. Any issues arising from this behavior will normally result in crashes during the exectuion of the subsequent call to - :c:func:`Py_Initatilize` from accessing uninitialized memory. To fix, use + :c:func:`Py_Initialize` from accessing uninitialized memory. To fix, use an address sanitizer (i.e. ASAN) to identify any use-after-free coming from an interned string and deallocate it during module shutdown. (Contribued by Eddie Elizondo in :gh:`113601`.) From afc7c1d3410a2e7ff396299ce186068e160b25a0 Mon Sep 17 00:00:00 2001 From: Eddie Elizondo Date: Mon, 15 Jan 2024 19:13:46 +0000 Subject: [PATCH 09/16] Lint docs v6 --- .../2024-01-15-18-11-48.gh-issue-113190.OwQX64.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-01-15-18-11-48.gh-issue-113190.OwQX64.rst b/Misc/NEWS.d/next/Core and Builtins/2024-01-15-18-11-48.gh-issue-113190.OwQX64.rst index 906a3966dabc3c..0e5caadbb8320b 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2024-01-15-18-11-48.gh-issue-113190.OwQX64.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2024-01-15-18-11-48.gh-issue-113190.OwQX64.rst @@ -9,12 +9,12 @@ leaving the user to a potential vulnerabilities. That said, the history of how these interned strings are handled have been changing during throughout different versions. For example: -* In Python 3.9 and older, interned strings were never deleted. Only special -build for Valgrind and Purity cleared them at exit -* In Python 3.10 and 3.11, interned strings are always deleted at exit -* In Python 3.12, interned strings are deleted only if Python is built in -debug mode: they are not deleted in release mode -* In Python 3.13, interned strings will now be akll deleted + * In Python 3.9 and older, interned strings were never deleted. Only special + build for Valgrind and Purity cleared them at exit + * In Python 3.10 and 3.11, interned strings are always deleted at exit + * In Python 3.12, interned strings are deleted only if Python is built in + debug mode: they are not deleted in release mode + * In Python 3.13, interned strings will now be akll deleted Given how we've changed guarantees throughout different versions, we do not expect that users actively rely on this behavior for their extensions. From 2ec1326497e286c6d5ca8f701caa9908737baaa7 Mon Sep 17 00:00:00 2001 From: Eddie Elizondo Date: Mon, 15 Jan 2024 19:24:02 +0000 Subject: [PATCH 10/16] Lint docs v7 --- .../2024-01-15-18-11-48.gh-issue-113190.OwQX64.rst | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-01-15-18-11-48.gh-issue-113190.OwQX64.rst b/Misc/NEWS.d/next/Core and Builtins/2024-01-15-18-11-48.gh-issue-113190.OwQX64.rst index 0e5caadbb8320b..3784e36aaa81d4 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2024-01-15-18-11-48.gh-issue-113190.OwQX64.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2024-01-15-18-11-48.gh-issue-113190.OwQX64.rst @@ -9,12 +9,14 @@ leaving the user to a potential vulnerabilities. That said, the history of how these interned strings are handled have been changing during throughout different versions. For example: - * In Python 3.9 and older, interned strings were never deleted. Only special - build for Valgrind and Purity cleared them at exit - * In Python 3.10 and 3.11, interned strings are always deleted at exit - * In Python 3.12, interned strings are deleted only if Python is built in - debug mode: they are not deleted in release mode - * In Python 3.13, interned strings will now be akll deleted + +* In Python 3.9 and older, interned strings were never deleted. Only special + build for Valgrind and Purity cleared them at exit. +* In Python 3.10 and 3.11, interned strings are always deleted at exit. +* In Python 3.12, interned strings are deleted only if Python is built in + debug mode: they are not deleted in release mode. +* In Python 3.13, interned strings will now be akll deleted. + Given how we've changed guarantees throughout different versions, we do not expect that users actively rely on this behavior for their extensions. From e8b9b45b22d51f611ceaaca60799764d40e889cf Mon Sep 17 00:00:00 2001 From: Eddie Elizondo Date: Mon, 15 Jan 2024 19:28:29 +0000 Subject: [PATCH 11/16] Lint docs v8 --- .../2024-01-15-18-11-48.gh-issue-113190.OwQX64.rst | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-01-15-18-11-48.gh-issue-113190.OwQX64.rst b/Misc/NEWS.d/next/Core and Builtins/2024-01-15-18-11-48.gh-issue-113190.OwQX64.rst index 3784e36aaa81d4..180a5c95f243fe 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2024-01-15-18-11-48.gh-issue-113190.OwQX64.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2024-01-15-18-11-48.gh-issue-113190.OwQX64.rst @@ -9,14 +9,12 @@ leaving the user to a potential vulnerabilities. That said, the history of how these interned strings are handled have been changing during throughout different versions. For example: - * In Python 3.9 and older, interned strings were never deleted. Only special - build for Valgrind and Purity cleared them at exit. +build for Valgrind and Purity cleared them at exit. * In Python 3.10 and 3.11, interned strings are always deleted at exit. * In Python 3.12, interned strings are deleted only if Python is built in - debug mode: they are not deleted in release mode. +debug mode: they are not deleted in release mode. * In Python 3.13, interned strings will now be akll deleted. - Given how we've changed guarantees throughout different versions, we do not expect that users actively rely on this behavior for their extensions. From 175bc406911166aba6a496f45c297557a0883a94 Mon Sep 17 00:00:00 2001 From: Eddie Elizondo Date: Mon, 15 Jan 2024 19:37:37 +0000 Subject: [PATCH 12/16] Lint docs v9 --- ...4-01-15-18-11-48.gh-issue-113190.OwQX64.rst | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-01-15-18-11-48.gh-issue-113190.OwQX64.rst b/Misc/NEWS.d/next/Core and Builtins/2024-01-15-18-11-48.gh-issue-113190.OwQX64.rst index 180a5c95f243fe..f43cc4fcb4d359 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2024-01-15-18-11-48.gh-issue-113190.OwQX64.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2024-01-15-18-11-48.gh-issue-113190.OwQX64.rst @@ -7,14 +7,14 @@ Worth noting that if an extension accidentally holds onto a interned string, event after calling Py_Finalize, it will result in use-after-free error, leaving the user to a potential vulnerabilities. That said, the history of how these interned strings are handled have been changing during throughout -different versions. For example: +different versions. -* In Python 3.9 and older, interned strings were never deleted. Only special -build for Valgrind and Purity cleared them at exit. -* In Python 3.10 and 3.11, interned strings are always deleted at exit. -* In Python 3.12, interned strings are deleted only if Python is built in -debug mode: they are not deleted in release mode. -* In Python 3.13, interned strings will now be akll deleted. +For example, in Python 3.9 and older, interned strings were never deleted. +Only special builds for Valgrind and Purity cleared them at exit. In Python +3.10 and 3.11, interned strings are always deleted at exit. In Python 3.12, +interned strings are deleted only if Python is built in debug mode: they +are not deleted in release mode. In Python 3.13, interned strings will +now be all deleted. -Given how we've changed guarantees throughout different versions, we do not -expect that users actively rely on this behavior for their extensions. +Given how the guarantees changed throughout different versions, it is not +expected that users actively rely on this behavior for their extensions. From ffe9d0323c4c95a6b583cf5b5eafc59dff0b620a Mon Sep 17 00:00:00 2001 From: Eddie Elizondo Date: Tue, 16 Jan 2024 02:24:01 +0000 Subject: [PATCH 13/16] Fix NEWS message --- .../2024-01-15-18-11-48.gh-issue-113190.OwQX64.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-01-15-18-11-48.gh-issue-113190.OwQX64.rst b/Misc/NEWS.d/next/Core and Builtins/2024-01-15-18-11-48.gh-issue-113190.OwQX64.rst index f43cc4fcb4d359..114fd14d148f2b 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2024-01-15-18-11-48.gh-issue-113190.OwQX64.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2024-01-15-18-11-48.gh-issue-113190.OwQX64.rst @@ -4,7 +4,7 @@ finalization when calling ``Py_Finalize``, regardless of their reference count. Worth noting that if an extension accidentally holds onto a interned string, -event after calling Py_Finalize, it will result in use-after-free error, +even after calling Py_Finalize, it will result in use-after-free error, leaving the user to a potential vulnerabilities. That said, the history of how these interned strings are handled have been changing during throughout different versions. From 080a8536394586120d661cf6c650c44a5db7b596 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 19 Jul 2024 13:45:50 +0200 Subject: [PATCH 14/16] Remove outdated comment --- Objects/unicodeobject.c | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 55502fefec9128..af7c50389632e7 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -15632,18 +15632,7 @@ _PyUnicode_ClearInterned(PyInterpreterState *interp) int shared = 0; switch (PyUnicode_CHECK_INTERNED(s)) { case SSTATE_INTERNED_IMMORTAL: - /* Make immortal interned strings mortal again. - * - * Currently, the runtime is not able to guarantee that it can exit - * without allocations that carry over to a future initialization - * of Python within the same process. i.e: - * ./python -X showrefcount -c 'import itertools' - * [237 refs, 237 blocks] - * - * This should remain disabled (`Py_DEBUG` only) until there is a - * strict guarantee that no memory will be left after - * `Py_Finalize`. - */ + /* Make immortal interned strings mortal again. */ // Skip the Immortal Instance check and restore // the two references (key and value) ignored // by PyUnicode_InternInPlace(). From 6aee7f5b1deb0f907057b031cc1805ca29196d29 Mon Sep 17 00:00:00 2001 From: Eddie Elizondo Date: Sun, 11 Aug 2024 11:37:37 -0400 Subject: [PATCH 15/16] Addressed Comments --- Doc/whatsnew/3.14.rst | 3 +-- ...-01-15-18-11-48.gh-issue-113190.OwQX64.rst | 21 +------------------ 2 files changed, 2 insertions(+), 22 deletions(-) diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 18c0830c46a986..ab2c6350108c5e 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -417,8 +417,7 @@ New Features which has an ambiguous return value. (Contributed by Irit Katriel and Erlend Aasland in :gh:`105201`.) -* Modified the ``_PyUnicode_ClearInterned`` function to always delete all - interned strings during a call to :c:func:`Py_Finalize`. This +* :c:func:`Py_Finalize` now deletes all interned strings. This is backwards incompatible to any C-Extension that holds onto an interned string after a call to :c:func:`Py_Finalize` and is then reused after a call to :c:func:`Py_Initialize`. Any issues arising from this behavior will diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-01-15-18-11-48.gh-issue-113190.OwQX64.rst b/Misc/NEWS.d/next/Core and Builtins/2024-01-15-18-11-48.gh-issue-113190.OwQX64.rst index 114fd14d148f2b..4c12870c3df548 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2024-01-15-18-11-48.gh-issue-113190.OwQX64.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2024-01-15-18-11-48.gh-issue-113190.OwQX64.rst @@ -1,20 +1 @@ -This updates the interned string deallocation function -``_PyUnicode_ClearInterned`` to delete all interned strings during runtime -finalization when calling ``Py_Finalize``, regardless of their reference -count. - -Worth noting that if an extension accidentally holds onto a interned string, -even after calling Py_Finalize, it will result in use-after-free error, -leaving the user to a potential vulnerabilities. That said, the history of -how these interned strings are handled have been changing during throughout -different versions. - -For example, in Python 3.9 and older, interned strings were never deleted. -Only special builds for Valgrind and Purity cleared them at exit. In Python -3.10 and 3.11, interned strings are always deleted at exit. In Python 3.12, -interned strings are deleted only if Python is built in debug mode: they -are not deleted in release mode. In Python 3.13, interned strings will -now be all deleted. - -Given how the guarantees changed throughout different versions, it is not -expected that users actively rely on this behavior for their extensions. +:c:func:`Py_Finalize` now deletes all interned strings. From dc3b54fe1269fb6ee6a33882def3a7f53b35c3a2 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 15 Aug 2024 13:28:20 +0200 Subject: [PATCH 16/16] Update whatsnew --- Doc/whatsnew/3.14.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index ab2c6350108c5e..c1a968994c06f2 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -423,7 +423,7 @@ New Features call to :c:func:`Py_Initialize`. Any issues arising from this behavior will normally result in crashes during the exectuion of the subsequent call to :c:func:`Py_Initialize` from accessing uninitialized memory. To fix, use - an address sanitizer (i.e. ASAN) to identify any use-after-free coming from + an address sanitizer to identify any use-after-free coming from an interned string and deallocate it during module shutdown. (Contribued by Eddie Elizondo in :gh:`113601`.)