bpo-44032: Move data stack to thread from FrameObject. (GH-26076)

* Remove 'zombie' frames. We won't need them once we are allocating fixed-size frames.

* Add co_nlocalplus field to code object to avoid recomputing size of locals + frees + cells.

* Move locals, cells and freevars out of frame object into separate memory buffer.

* Use per-threadstate allocated memory chunks for local variables.

* Move globals and builtins from frame object to per-thread stack.

* Move (slow) locals frame object to per-thread stack.

* Move internal frame functions to internal header.

Author: markshannon

Include/cpython/code.h

@@ -40,8 +40,8 @@ struct PyCodeObject {
     PyObject *co_name;          /* unicode (name, for reference) */
     PyObject *co_linetable;     /* string (encoding addr<->lineno mapping) See
                                    Objects/lnotab_notes.txt for details. */
+    int co_nlocalsplus;         /* Number of locals + free + cell variables */
     PyObject *co_exceptiontable; /* Byte string encoding exception handling table */
-    void *co_zombieframe;       /* for optimization only (see frameobject.c) */
     PyObject *co_weakreflist;   /* to support weakrefs to code objects */
     /* Scratch space for extra data relating to the code object.
        Type is a void* to keep the format private in codeobject.c to force

Include/cpython/frameobject.h

@@ -20,12 +20,9 @@ enum _framestate {
 typedef signed char PyFrameState;
 
 struct _frame {
-    PyObject_VAR_HEAD
+    PyObject_HEAD
     struct _frame *f_back;      /* previous frame, or NULL */
     PyCodeObject *f_code;       /* code segment */
-    PyObject *f_builtins;       /* builtin symbol table (PyDictObject) */
-    PyObject *f_globals;        /* global symbol table (PyDictObject) */
-    PyObject *f_locals;         /* local symbol table (any mapping) */
     PyObject **f_valuestack;    /* points after the last local */
     PyObject *f_trace;          /* Trace function */
     /* Borrowed reference to a generator, or NULL */
@@ -36,7 +33,8 @@ struct _frame {
     PyFrameState f_state;       /* What state the frame is in */
     char f_trace_lines;         /* Emit per-line trace events? */
     char f_trace_opcodes;       /* Emit per-opcode trace events? */
-    PyObject *f_localsplus[1];  /* locals+stack, dynamically sized */
+    char f_own_locals_memory;   /* This frame owns the memory for the locals */
+    PyObject **f_localsptr;     /* Pointer to locals, cells, free */
 };
 
 static inline int _PyFrame_IsRunnable(struct _frame *f) {
@@ -62,7 +60,7 @@ PyAPI_FUNC(PyFrameObject *) PyFrame_New(PyThreadState *, PyCodeObject *,
 
 /* only internal use */
 PyFrameObject*
-_PyFrame_New_NoTrack(PyThreadState *, PyFrameConstructor *, PyObject *);
+_PyFrame_New_NoTrack(PyThreadState *, PyFrameConstructor *, PyObject *, PyObject **);
 
 
 /* The rest of the interface is specific for frame objects */

Include/cpython/pystate.h

@@ -57,6 +57,12 @@ typedef struct _err_stackitem {
 
 } _PyErr_StackItem;
 
+typedef struct _stack_chunk {
+    struct _stack_chunk *previous;
+    size_t size;
+    size_t top;
+    PyObject * data[1]; /* Variable sized */
+} _PyStackChunk;
 
 // The PyThreadState typedef is in Include/pystate.h.
 struct _ts {
@@ -149,6 +155,9 @@ struct _ts {
 
     CFrame root_cframe;
 
+    _PyStackChunk *datastack_chunk;
+    PyObject **datastack_top;
+    PyObject **datastack_limit;
     /* XXX signal handlers should also be here */
 
 };

Include/genobject.h

@@ -18,7 +18,7 @@ extern "C" {
     /* Note: gi_frame can be NULL if the generator is "finished" */         \
     PyFrameObject *prefix##_frame;                                          \
     /* The code object backing the generator */                             \
-    PyObject *prefix##_code;                                                \
+    PyCodeObject *prefix##_code;                                                \
     /* List of weak reference. */                                           \
     PyObject *prefix##_weakreflist;                                         \
     /* Name of the generator. */                                            \

Include/internal/pycore_frame.h

@@ -0,0 +1,38 @@
+#ifndef Py_INTERNAL_FRAME_H
+#define Py_INTERNAL_FRAME_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+enum {
+    FRAME_SPECIALS_GLOBALS_OFFSET = 0,
+    FRAME_SPECIALS_BUILTINS_OFFSET = 1,
+    FRAME_SPECIALS_LOCALS_OFFSET = 2,
+    FRAME_SPECIALS_SIZE = 3
+};
+
+static inline PyObject **
+_PyFrame_Specials(PyFrameObject *f) {
+    return &f->f_valuestack[-FRAME_SPECIALS_SIZE];
+}
+
+/* Returns a *borrowed* reference. */
+static inline PyObject *
+_PyFrame_GetGlobals(PyFrameObject *f)
+{
+    return _PyFrame_Specials(f)[FRAME_SPECIALS_GLOBALS_OFFSET];
+}
+
+/* Returns a *borrowed* reference. */
+static inline PyObject *
+_PyFrame_GetBuiltins(PyFrameObject *f)
+{
+    return _PyFrame_Specials(f)[FRAME_SPECIALS_BUILTINS_OFFSET];
+}
+
+int _PyFrame_TakeLocals(PyFrameObject *f);
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_INTERNAL_FRAME_H */

Include/internal/pycore_pymem.h

@@ -94,6 +94,11 @@ struct _PyTraceMalloc_Config {
 
 PyAPI_DATA(struct _PyTraceMalloc_Config) _Py_tracemalloc_config;
 
+/* Allocate memory directly from the O/S virtual memory system,
+ * where supported. Otherwise fallback on malloc */
+void *_PyObject_VirtualAlloc(size_t size);
+void _PyObject_VirtualFree(void *, size_t size);
+
 
 #ifdef __cplusplus
 }

Include/internal/pycore_pystate.h

@@ -147,6 +147,9 @@ PyAPI_FUNC(int) _PyState_AddModule(
 
 PyAPI_FUNC(int) _PyOS_InterruptOccurred(PyThreadState *tstate);
 
+PyObject **_PyThreadState_PushLocals(PyThreadState *, int size);
+void _PyThreadState_PopLocals(PyThreadState *, PyObject **);
+
 #ifdef __cplusplus
 }
 #endif

Lib/test/test_gdb.py

@@ -666,15 +666,16 @@ def test_builtin_method(self):
 
     def test_frames(self):
         gdb_output = self.get_stack_trace('''
+import sys
 def foo(a, b, c):
-    pass
+    return sys._getframe(0)
 
-foo(3, 4, 5)
-id(foo.__code__)''',
+f = foo(3, 4, 5)
+id(f)''',
                                           breakpoint='builtin_id',
-                                          cmds_after_breakpoint=['print (PyFrameObject*)(((PyCodeObject*)v)->co_zombieframe)']
+                                          cmds_after_breakpoint=['print (PyFrameObject*)v']
                                           )
-        self.assertTrue(re.match(r'.*\s+\$1 =\s+Frame 0x-?[0-9a-f]+, for file <string>, line 3, in foo \(\)\s+.*',
+        self.assertTrue(re.match(r'.*\s+\$1 =\s+Frame 0x-?[0-9a-f]+, for file <string>, line 4, in foo \(a=3.*',
                                  gdb_output,
                                  re.DOTALL),
                         'Unexpected gdb representation: %r\n%s' % (gdb_output, gdb_output))

Lib/test/test_sys.py

@@ -1274,11 +1274,7 @@ class C(object): pass
         # frame
         import inspect
         x = inspect.currentframe()
-        ncells = len(x.f_code.co_cellvars)
-        nfrees = len(x.f_code.co_freevars)
-        localsplus = x.f_code.co_stacksize + x.f_code.co_nlocals +\
-                  ncells + nfrees
-        check(x, vsize('8P3i3c' + localsplus*'P'))
+        check(x, size('5P3i4cP'))
         # function
         def func(): pass
         check(func, size('14P'))

Misc/NEWS.d/next/Core and Builtins/2021-05-14-20-03-32.bpo-44032.OzT1ob.rst

@@ -0,0 +1,2 @@
+Move 'fast' locals and other variables from the frame object to a per-thread
+datastack.

Objects/codeobject.c

@@ -280,6 +280,8 @@ PyCode_NewWithPosOnlyArgs(int argcount, int posonlyargcount, int kwonlyargcount,
     co->co_posonlyargcount = posonlyargcount;
     co->co_kwonlyargcount = kwonlyargcount;
     co->co_nlocals = nlocals;
+    co->co_nlocalsplus = nlocals +
+        (int)PyTuple_GET_SIZE(freevars) + (int)PyTuple_GET_SIZE(cellvars);
     co->co_stacksize = stacksize;
     co->co_flags = flags;
     Py_INCREF(code);
@@ -304,7 +306,6 @@ PyCode_NewWithPosOnlyArgs(int argcount, int posonlyargcount, int kwonlyargcount,
     co->co_linetable = linetable;
     Py_INCREF(exceptiontable);
     co->co_exceptiontable = exceptiontable;
-    co->co_zombieframe = NULL;
     co->co_weakreflist = NULL;
     co->co_extra = NULL;
 
@@ -968,8 +969,6 @@ code_dealloc(PyCodeObject *co)
     Py_XDECREF(co->co_exceptiontable);
     if (co->co_cell2arg != NULL)
         PyMem_Free(co->co_cell2arg);
-    if (co->co_zombieframe != NULL)
-        PyObject_GC_Del(co->co_zombieframe);
     if (co->co_weakreflist != NULL)
         PyObject_ClearWeakRefs((PyObject*)co);
     PyObject_Free(co);