gh-111968: Introduce _Py_freelist_state and _PyFreeListState_GET API

Author: corona10

Include/internal/pycore_gc.h

@@ -209,7 +209,7 @@ extern Py_ssize_t _PyGC_CollectNoFail(PyThreadState *tstate);
 // Functions to clear types free lists
 extern void _PyTuple_ClearFreeList(PyInterpreterState *interp);
 extern void _PyFloat_ClearFreeList(PyInterpreterState *interp);
-extern void _PyList_ClearFreeList(PyInterpreterState *interp);
+extern void _PyList_ClearFreeList(PyFreeListState *state);
 extern void _PyDict_ClearFreeList(PyInterpreterState *interp);
 extern void _PyAsyncGen_ClearFreeLists(PyInterpreterState *interp);
 extern void _PyContext_ClearFreeList(PyInterpreterState *interp);

Include/internal/pycore_interp.h

@@ -174,6 +174,9 @@ struct _is {
     // One bit is set for each non-NULL entry in code_watchers
     uint8_t active_code_watchers;
 
+#if !defined(Py_GIL_DISABLED)
+    struct _Py_freelist_state freelist_state;
+#endif
     struct _py_object_state object_state;
     struct _Py_unicode_state unicode;
     struct _Py_float_state float_state;
@@ -185,7 +188,6 @@ struct _is {
     PySliceObject *slice_cache;
 
     struct _Py_tuple_state tuple;
-    struct _Py_list_state list;
     struct _Py_dict_state dict_state;
     struct _Py_async_gen_state async_gen;
     struct _Py_context_state context;

Include/internal/pycore_pystate.h

@@ -9,6 +9,7 @@ extern "C" {
 #endif
 
 #include "pycore_runtime.h"       // _PyRuntime
+#include "pycore_tstate.h"        // _PyThreadStateImpl
 
 
 // Values for PyThreadState.state. A thread must be in the "attached" state
@@ -239,6 +240,19 @@ PyAPI_FUNC(const PyConfig*) _Py_GetConfig(void);
 // See also PyInterpreterState_Get() and _PyInterpreterState_GET().
 extern PyInterpreterState* _PyGILState_GetInterpreterStateUnsafe(void);
 
+static inline PyFreeListState* _PyFreeListState_GET(void) {
+    PyThreadState *tstate = _PyThreadState_GET();
+#ifdef Py_DEBUG
+    _Py_EnsureTstateNotNULL(tstate);
+#endif
+
+#ifdef Py_GIL_DISABLED
+    return &((_PyThreadStateImpl*)tstate)->freelist_state;
+#else
+    return &tstate->interp->freelist_state;
+#endif
+}
+
 #ifdef __cplusplus
 }
 #endif

Include/internal/pycore_tstate.h

@@ -11,6 +11,10 @@ extern "C" {
 #include "pycore_mimalloc.h"      // struct _mimalloc_thread_state
 
 
+typedef struct _Py_freelist_state {
+    struct _Py_list_state list;
+} _Py_freelist_state;
+
 // Every PyThreadState is actually allocated as a _PyThreadStateImpl. The
 // PyThreadState fields are exposed as part of the C API, although most fields
 // are intended to be private. The _PyThreadStateImpl fields not exposed.
@@ -20,6 +24,7 @@ typedef struct _PyThreadStateImpl {
 
 #ifdef Py_GIL_DISABLED
     struct _mimalloc_thread_state mimalloc;
+    struct _Py_freelist_state freelist_state;
 #endif
 
 } _PyThreadStateImpl;

Include/pytypedefs.h

@@ -23,6 +23,7 @@ typedef struct _frame PyFrameObject;
 
 typedef struct _ts PyThreadState;
 typedef struct _is PyInterpreterState;
+typedef struct _Py_freelist_state PyFreeListState;
 
 #ifdef __cplusplus
 }

Modules/gcmodule.c

@@ -1070,14 +1070,33 @@ delete_garbage(PyThreadState *tstate, GCState *gcstate,
 static void
 clear_freelists(PyInterpreterState *interp)
 {
+    // TODO: Unify with clear_all_freelists
     _PyTuple_ClearFreeList(interp);
     _PyFloat_ClearFreeList(interp);
-    _PyList_ClearFreeList(interp);
     _PyDict_ClearFreeList(interp);
     _PyAsyncGen_ClearFreeLists(interp);
     _PyContext_ClearFreeList(interp);
 }
 
+static void
+clear_all_freelists(PyInterpreterState *interp)
+{
+    clear_freelists(interp);
+#if defined(Py_GIL_DISABLED)
+    HEAD_LOCK(&_PyRuntime);
+    _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)interp->threads.head;
+    while (tstate != NULL) {
+        _PyList_ClearFreeList(&tstate->freelist_state);
+        tstate = tstate->base.next;
+    }
+    HEAD_UNLOCK(&_PyRuntime);
+#else
+    // Only free-lists per interpreter are existed.
+    PyFreeListState* state = _PyFreeListState_GET();
+    _PyList_ClearFreeList(state);
+#endif
+}
+
 // Show stats for objects in each generations
 static void
 show_stats_each_generations(GCState *gcstate)
@@ -1486,6 +1505,7 @@ gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason)
      * generation */
     if (generation == NUM_GENERATIONS-1) {
         clear_freelists(tstate->interp);
+        clear_all_freelists(tstate->interp);
     }
 
     if (_PyErr_Occurred(tstate)) {

Objects/listobject.c

@@ -24,8 +24,9 @@ _Py_DECLARE_STR(list_err, "list index out of range");
 static struct _Py_list_state *
 get_list_state(void)
 {
-    PyInterpreterState *interp = _PyInterpreterState_GET();
-    return &interp->list;
+    PyFreeListState *state = _PyFreeListState_GET();
+    assert(state != NULL);
+    return &state->list;
 }
 #endif
 
@@ -120,12 +121,12 @@ list_preallocate_exact(PyListObject *self, Py_ssize_t size)
 }
 
 void
-_PyList_ClearFreeList(PyInterpreterState *interp)
+_PyList_ClearFreeList(PyFreeListState *state)
 {
 #if PyList_MAXFREELIST > 0
-    struct _Py_list_state *state = &interp->list;
-    while (state->numfree) {
-        PyListObject *op = state->free_list[--state->numfree];
+    struct _Py_list_state *list_state = &state->list;
+    while (list_state->numfree) {
+        PyListObject *op = list_state->free_list[--list_state->numfree];
         assert(PyList_CheckExact(op));
         PyObject_GC_Del(op);
     }
@@ -135,10 +136,10 @@ _PyList_ClearFreeList(PyInterpreterState *interp)
 void
 _PyList_Fini(PyInterpreterState *interp)
 {
-    _PyList_ClearFreeList(interp);
+    PyFreeListState *state = _PyFreeListState_GET();
+    _PyList_ClearFreeList(state);
 #if defined(Py_DEBUG) && PyList_MAXFREELIST > 0
-    struct _Py_list_state *state = &interp->list;
-    state->numfree = -1;
+    state->list.numfree = -1;
 #endif
 }