Merge branch 'main' into gh-121735/zip-module-resources

Author: jaraco

Doc/c-api/init.rst

@@ -394,8 +394,7 @@ Initializing and finalizing the interpreter
    Undo all initializations made by :c:func:`Py_Initialize` and subsequent use of
    Python/C API functions, and destroy all sub-interpreters (see
    :c:func:`Py_NewInterpreter` below) that were created and not yet destroyed since
-   the last call to :c:func:`Py_Initialize`.  Ideally, this frees all memory
-   allocated by the Python interpreter.  This is a no-op when called for a second
+   the last call to :c:func:`Py_Initialize`.  This is a no-op when called for a second
    time (without calling :c:func:`Py_Initialize` again first).
 
    Since this is the reverse of :c:func:`Py_Initialize`, it should be called
@@ -407,6 +406,12 @@ Initializing and finalizing the interpreter
    If there were errors during finalization (flushing buffered data),
    ``-1`` is returned.
 
+   Note that Python will do a best effort at freeing all memory allocated by the Python
+   interpreter.  Therefore, any C-Extension should make sure to correctly clean up all
+   of the preveiously allocated PyObjects before using them in subsequent calls to
+   :c:func:`Py_Initialize`.  Otherwise it could introduce vulnerabilities and incorrect
+   behavior.
+
    This function is provided for a number of reasons.  An embedding application
    might want to restart Python without having to restart the application itself.
    An application that has loaded the Python interpreter from a dynamically
@@ -421,10 +426,11 @@ Initializing and finalizing the interpreter
    loaded extension modules loaded by Python are not unloaded.  Small amounts of
    memory allocated by the Python interpreter may not be freed (if you find a leak,
    please report it).  Memory tied up in circular references between objects is not
-   freed.  Some memory allocated by extension modules may not be freed.  Some
-   extensions may not work properly if their initialization routine is called more
-   than once; this can happen if an application calls :c:func:`Py_Initialize` and
-   :c:func:`Py_FinalizeEx` more than once.
+   freed.  Interned strings will all be deallocated regarldess of their reference count.
+   Some memory allocated by extension modules may not be freed.  Some extensions may not
+   work properly if their initialization routine is called more than once; this can
+   happen if an application calls :c:func:`Py_Initialize` and :c:func:`Py_FinalizeEx`
+   more than once.
 
    .. audit-event:: cpython._PySys_ClearAuditHooks "" c.Py_FinalizeEx
 

Doc/conf.py

@@ -546,6 +546,8 @@
     r'https://msdn.microsoft.com/.*': 'https://learn.microsoft.com/.*',
     r'https://docs.microsoft.com/.*': 'https://learn.microsoft.com/.*',
     r'https://go.microsoft.com/fwlink/\?LinkID=\d+': 'https://learn.microsoft.com/.*',
+    # Debian's man page redirects to its current stable version
+    r'https://manpages.debian.org/\w+\(\d(\w+)?\)': r'https://manpages.debian.org/\w+/[\w/\-\.]*\.\d(\w+)?\.en\.html',
     # Language redirects
     r'https://toml.io': 'https://toml.io/en/',
     r'https://www.redhat.com': 'https://www.redhat.com/en',

Doc/library/enum.rst

@@ -570,6 +570,8 @@ Data Types
          >>> len(white)
          3
 
+      .. versionadded:: 3.11
+
    .. method:: __bool__(self):
 
       Returns *True* if any members in flag, *False* otherwise::

Doc/tutorial/appendix.rst

@@ -14,8 +14,8 @@ There are two variants of the interactive :term:`REPL`.  The classic
 basic interpreter is supported on all platforms with minimal line
 control capabilities.
 
-On Unix-like systems (e.g. Linux or macOS) with :mod:`curses` and
-:mod:`readline` support, a new interactive shell is used by default.
+On Windows, or Unix-like systems with :mod:`curses` support,
+a new interactive shell is used by default.
 This one supports color, multiline editing, history browsing, and
 paste mode.  To disable color, see :ref:`using-on-controlling-color` for
 details.  Function keys provide some additional functionality.

Doc/whatsnew/3.13.rst

@@ -419,6 +419,16 @@ New Features
   which has an ambiguous return value.
   (Contributed by Irit Katriel and Erlend Aasland in :gh:`105201`.)
 
+* :c:func:`Py_Finalize` now deletes all interned strings. This
+  is backwards incompatible to any C-Extension that holds onto an interned
+  string after a call to :c:func:`Py_Finalize` and is then reused after a
+  call to :c:func:`Py_Initialize`.  Any issues arising from this behavior will
+  normally result in crashes during the exectuion of the subsequent call to
+  :c:func:`Py_Initialize` from accessing uninitialized memory. To fix, use
+  an address sanitizer to identify any use-after-free coming from
+  an interned string and deallocate it during module shutdown.
+  (Contribued by Eddie Elizondo in :gh:`113601`.)
+
 Porting to Python 3.14
 ----------------------
 

Doc/whatsnew/3.14.rst

@@ -128,6 +128,13 @@ static inline void _PyFrame_Copy(_PyInterpreterFrame *src, _PyInterpreterFrame *
     // Don't leave a dangling pointer to the old frame when creating generators
     // and coroutines:
     dest->previous = NULL;
+
+#ifdef Py_GIL_DISABLED
+    PyCodeObject *co = (PyCodeObject *)dest->f_executable;
+    for (int i = stacktop; i < co->co_nlocalsplus + co->co_stacksize; i++) {
+        dest->localsplus[i] = PyStackRef_NULL;
+    }
+#endif
 }
 
 /* Consumes reference to func and locals.
@@ -153,6 +160,16 @@ _PyFrame_Initialize(
     for (int i = null_locals_from; i < code->co_nlocalsplus; i++) {
         frame->localsplus[i] = PyStackRef_NULL;
     }
+
+#ifdef Py_GIL_DISABLED
+    // On GIL disabled, we walk the entire stack in GC. Since stacktop
+    // is not always in sync with the real stack pointer, we have
+    // no choice but to traverse the entire stack.
+    // This just makes sure we don't pass the GC invalid stack values.
+    for (int i = code->co_nlocalsplus; i < code->co_nlocalsplus + code->co_stacksize; i++) {
+        frame->localsplus[i] = PyStackRef_NULL;
+    }
+#endif
 }
 
 /* Gets the pointer to the locals array
@@ -314,6 +331,13 @@ _PyFrame_PushTrampolineUnchecked(PyThreadState *tstate, PyCodeObject *code, int
     frame->instr_ptr = _PyCode_CODE(code);
     frame->owner = FRAME_OWNED_BY_THREAD;
     frame->return_offset = 0;
+
+#ifdef Py_GIL_DISABLED
+    assert(code->co_nlocalsplus == 0);
+    for (int i = 0; i < code->co_stacksize; i++) {
+        frame->localsplus[i] = PyStackRef_NULL;
+    }
+#endif
     return frame;
 }
 

Include/internal/pycore_frame.h

@@ -381,6 +381,8 @@ extern void _PyGC_ClearAllFreeLists(PyInterpreterState *interp);
 extern void _Py_ScheduleGC(PyThreadState *tstate);
 extern void _Py_RunGC(PyThreadState *tstate);
 
+// GC visit callback for tracked interpreter frames
+extern int _PyGC_VisitFrameStack(struct _PyInterpreterFrame *frame, visitproc visit, void *arg);
 
 #ifdef __cplusplus
 }

Include/internal/pycore_gc.h

@@ -36,9 +36,18 @@ typedef enum {
 #  define MI_TSAN 1
 #endif
 
+#ifdef __cplusplus
+extern "C++" {
+#endif
+
 #include "mimalloc/mimalloc.h"
 #include "mimalloc/mimalloc/types.h"
 #include "mimalloc/mimalloc/internal.h"
+
+#ifdef __cplusplus
+}
+#endif
+
 #endif
 
 #ifdef Py_GIL_DISABLED

Include/internal/pycore_mimalloc.h

@@ -44,6 +44,15 @@ struct _gilstate_runtime_state {
 
 /* Runtime audit hook state */
 
+#define _Py_Debug_Cookie "xdebugpy"
+
+#ifdef Py_GIL_DISABLED
+# define _Py_Debug_gilruntimestate_enabled offsetof(struct _gil_runtime_state, enabled)
+# define _Py_Debug_Free_Threaded 1
+#else
+# define _Py_Debug_gilruntimestate_enabled 0
+# define _Py_Debug_Free_Threaded 0
+#endif
 typedef struct _Py_AuditHookEntry {
     struct _Py_AuditHookEntry *next;
     Py_AuditHookFunction hookCFunction;
@@ -53,6 +62,7 @@ typedef struct _Py_AuditHookEntry {
 typedef struct _Py_DebugOffsets {
     char cookie[8];
     uint64_t version;
+    uint64_t free_threaded;
     // Runtime state offset;
     struct _runtime_state {
         uint64_t size;
@@ -71,6 +81,8 @@ typedef struct _Py_DebugOffsets {
         uint64_t sysdict;
         uint64_t builtins;
         uint64_t ceval_gil;
+        uint64_t gil_runtime_state;
+        uint64_t gil_runtime_state_enabled;
         uint64_t gil_runtime_state_locked;
         uint64_t gil_runtime_state_holder;
     } interpreter_state;
@@ -122,20 +134,57 @@ typedef struct _Py_DebugOffsets {
     struct _type_object {
         uint64_t size;
         uint64_t tp_name;
+        uint64_t tp_repr;
+        uint64_t tp_flags;
     } type_object;
 
     // PyTuple object offset;
     struct _tuple_object {
         uint64_t size;
         uint64_t ob_item;
+        uint64_t ob_size;
     } tuple_object;
 
+    // PyList object offset;
+    struct _list_object {
+        uint64_t size;
+        uint64_t ob_item;
+        uint64_t ob_size;
+    } list_object;
+
+    // PyDict object offset;
+    struct _dict_object {
+        uint64_t size;
+        uint64_t ma_keys;
+        uint64_t ma_values;
+    } dict_object;
+
+    // PyFloat object offset;
+    struct _float_object {
+        uint64_t size;
+        uint64_t ob_fval;
+    } float_object;
+
+    // PyLong object offset;
+    struct _long_object {
+        uint64_t size;
+        uint64_t lv_tag;
+        uint64_t ob_digit;
+    } long_object;
+
+    // PyBytes object offset;
+    struct _bytes_object {
+        uint64_t size;
+        uint64_t ob_size;
+        uint64_t ob_sval;
+    } bytes_object;
+
     // Unicode object offset;
     struct _unicode_object {
         uint64_t size;
         uint64_t state;
         uint64_t length;
-        size_t asciiobject_size;
+        uint64_t asciiobject_size;
     } unicode_object;
 
     // GC runtime state offset;

Include/internal/pycore_runtime.h

@@ -29,11 +29,12 @@ extern PyTypeObject _PyExc_MemoryError;
 /* The static initializers defined here should only be used
    in the runtime init code (in pystate.c and pylifecycle.c). */
 
-#define _PyRuntimeState_INIT(runtime) \
+#define _PyRuntimeState_INIT(runtime, debug_cookie) \
     { \
         .debug_offsets = { \
-            .cookie = "xdebugpy", \
+            .cookie = debug_cookie, \
             .version = PY_VERSION_HEX, \
+            .free_threaded = _Py_Debug_Free_Threaded, \
             .runtime_state = { \
                 .size = sizeof(_PyRuntimeState), \
                 .finalizing = offsetof(_PyRuntimeState, _finalizing), \
@@ -49,6 +50,8 @@ extern PyTypeObject _PyExc_MemoryError;
                 .sysdict = offsetof(PyInterpreterState, sysdict), \
                 .builtins = offsetof(PyInterpreterState, builtins), \
                 .ceval_gil = offsetof(PyInterpreterState, ceval.gil), \
+                .gil_runtime_state = offsetof(PyInterpreterState, _gil), \
+                .gil_runtime_state_enabled = _Py_Debug_gilruntimestate_enabled, \
                 .gil_runtime_state_locked = offsetof(PyInterpreterState, _gil.locked), \
                 .gil_runtime_state_holder = offsetof(PyInterpreterState, _gil.last_holder), \
             }, \
@@ -90,10 +93,37 @@ extern PyTypeObject _PyExc_MemoryError;
             .type_object = { \
                 .size = sizeof(PyTypeObject), \
                 .tp_name = offsetof(PyTypeObject, tp_name), \
+                .tp_repr = offsetof(PyTypeObject, tp_repr), \
+                .tp_flags = offsetof(PyTypeObject, tp_flags), \
             }, \
             .tuple_object = { \
                 .size = sizeof(PyTupleObject), \
                 .ob_item = offsetof(PyTupleObject, ob_item), \
+                .ob_size = offsetof(PyTupleObject, ob_base.ob_size), \
+            }, \
+            .list_object = { \
+                .size = sizeof(PyListObject), \
+                .ob_item = offsetof(PyListObject, ob_item), \
+                .ob_size = offsetof(PyListObject, ob_base.ob_size), \
+            }, \
+            .dict_object = { \
+                .size = sizeof(PyDictObject), \
+                .ma_keys = offsetof(PyDictObject, ma_keys), \
+                .ma_values = offsetof(PyDictObject, ma_values), \
+            }, \
+            .float_object = { \
+                .size = sizeof(PyFloatObject), \
+                .ob_fval = offsetof(PyFloatObject, ob_fval), \
+            }, \
+            .long_object = { \
+                .size = sizeof(PyLongObject), \
+                .lv_tag = offsetof(_PyLongValue, lv_tag), \
+                .ob_digit = offsetof(_PyLongValue, ob_digit), \
+            }, \
+            .bytes_object = { \
+                .size = sizeof(PyBytesObject), \
+                .ob_size = offsetof(PyBytesObject, ob_base.ob_size), \
+                .ob_sval = offsetof(PyBytesObject, ob_sval), \
             }, \
             .unicode_object = { \
                 .size = sizeof(PyUnicodeObject), \

Include/internal/pycore_runtime_init.h

@@ -150,8 +150,7 @@ PyStackRef_FromPyObjectNew(PyObject *obj)
     // Make sure we don't take an already tagged value.
     assert(((uintptr_t)obj & Py_TAG_BITS) == 0);
     assert(obj != NULL);
-    // TODO (gh-117139): Add deferred objects later.
-    if (_Py_IsImmortal(obj)) {
+    if (_Py_IsImmortal(obj) || _PyObject_HasDeferredRefcount(obj)) {
         return (_PyStackRef){ .bits = (uintptr_t)obj | Py_TAG_DEFERRED };
     }
     else {
@@ -220,7 +219,8 @@ PyStackRef_DUP(_PyStackRef stackref)
 {
     if (PyStackRef_IsDeferred(stackref)) {
         assert(PyStackRef_IsNull(stackref) ||
-            _Py_IsImmortal(PyStackRef_AsPyObjectBorrow(stackref)));
+            _Py_IsImmortal(PyStackRef_AsPyObjectBorrow(stackref)) ||
+            _PyObject_HasDeferredRefcount(PyStackRef_AsPyObjectBorrow(stackref)));
         return stackref;
     }
     Py_INCREF(PyStackRef_AsPyObjectBorrow(stackref));