bpo-44590: Lazily allocate frame objects (GH-27077)

* Convert "specials" array to InterpreterFrame struct, adding f_lasti, f_state and other non-debug FrameObject fields to it.

* Refactor, calls pushing the call to the interpreter upward toward _PyEval_Vector.

* Compute f_back when on thread stack, only filling in value when frame object outlives stack invocation.

* Move ownership of InterpreterFrame in generator from frame object to generator object.

* Do not create frame objects for Python calls.

* Do not create frame objects for generators.

Author: markshannon

Include/cpython/ceval.h

@@ -19,7 +19,7 @@ PyAPI_FUNC(PyObject *) _PyEval_GetBuiltinId(_Py_Identifier *);
    flag was set, else return 0. */
 PyAPI_FUNC(int) PyEval_MergeCompilerFlags(PyCompilerFlags *cf);
 
-PyAPI_FUNC(PyObject *) _PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int exc);
+PyAPI_FUNC(PyObject *) _PyEval_EvalFrameDefault(PyThreadState *tstate, struct _interpreter_frame *f, int exc);
 
 PyAPI_FUNC(void) _PyEval_SetSwitchInterval(unsigned long microseconds);
 PyAPI_FUNC(unsigned long) _PyEval_GetSwitchInterval(void);

Include/cpython/frameobject.h

@@ -4,50 +4,17 @@
 #  error "this header file must not be included directly"
 #endif
 
-/* These values are chosen so that the inline functions below all
- * compare f_state to zero.
- */
-enum _framestate {
-    FRAME_CREATED = -2,
-    FRAME_SUSPENDED = -1,
-    FRAME_EXECUTING = 0,
-    FRAME_RETURNED = 1,
-    FRAME_UNWINDING = 2,
-    FRAME_RAISED = 3,
-    FRAME_CLEARED = 4
-};
-
-typedef signed char PyFrameState;
-
 struct _frame {
     PyObject_HEAD
     struct _frame *f_back;      /* previous frame, or NULL */
-    PyObject **f_valuestack;    /* points after the last local */
+    struct _interpreter_frame *f_frame; /* points to the frame data */
     PyObject *f_trace;          /* Trace function */
-    /* Borrowed reference to a generator, or NULL */
-    PyObject *f_gen;
-    int f_stackdepth;           /* Depth of value stack */
-    int f_lasti;                /* Last instruction if called */
     int f_lineno;               /* Current line number. Only valid if non-zero */
-    PyFrameState f_state;       /* What state the frame is in */
     char f_trace_lines;         /* Emit per-line trace events? */
     char f_trace_opcodes;       /* Emit per-opcode trace events? */
     char f_own_locals_memory;   /* This frame owns the memory for the locals */
-    PyObject **f_localsptr;     /* Pointer to locals, cells, free */
 };
 
-static inline int _PyFrame_IsRunnable(struct _frame *f) {
-    return f->f_state < FRAME_EXECUTING;
-}
-
-static inline int _PyFrame_IsExecuting(struct _frame *f) {
-    return f->f_state == FRAME_EXECUTING;
-}
-
-static inline int _PyFrameHasCompleted(struct _frame *f) {
-    return f->f_state > FRAME_EXECUTING;
-}
-
 /* Standard object interface */
 
 PyAPI_DATA(PyTypeObject) PyFrame_Type;
@@ -59,7 +26,7 @@ PyAPI_FUNC(PyFrameObject *) PyFrame_New(PyThreadState *, PyCodeObject *,
 
 /* only internal use */
 PyFrameObject*
-_PyFrame_New_NoTrack(PyThreadState *, PyFrameConstructor *, PyObject *, PyObject **);
+_PyFrame_New_NoTrack(struct _interpreter_frame *, int);
 
 
 /* The rest of the interface is specific for frame objects */

Include/cpython/pystate.h

@@ -78,7 +78,7 @@ struct _ts {
     PyInterpreterState *interp;
 
     /* Borrowed reference to the current frame (it can be NULL) */
-    PyFrameObject *frame;
+    struct _interpreter_frame *frame;
     int recursion_depth;
     int recursion_headroom; /* Allow 50 more calls to handle any errors. */
     int stackcheck_counter;
@@ -223,7 +223,7 @@ PyAPI_FUNC(void) PyThreadState_DeleteCurrent(void);
 
 /* Frame evaluation API */
 
-typedef PyObject* (*_PyFrameEvalFunction)(PyThreadState *tstate, PyFrameObject *, int);
+typedef PyObject* (*_PyFrameEvalFunction)(PyThreadState *tstate, struct _interpreter_frame *, int);
 
 PyAPI_FUNC(_PyFrameEvalFunction) _PyInterpreterState_GetEvalFrameFunc(
     PyInterpreterState *interp);

Include/genobject.h

@@ -16,9 +16,9 @@ extern "C" {
 #define _PyGenObject_HEAD(prefix)                                           \
     PyObject_HEAD                                                           \
     /* Note: gi_frame can be NULL if the generator is "finished" */         \
-    PyFrameObject *prefix##_frame;                                          \
+    struct _interpreter_frame *prefix##_xframe;                             \
     /* The code object backing the generator */                             \
-    PyCodeObject *prefix##_code;                                                \
+    PyCodeObject *prefix##_code;                                            \
     /* List of weak reference. */                                           \
     PyObject *prefix##_weakreflist;                                         \
     /* Name of the generator. */                                            \

Include/internal/pycore_ceval.h

@@ -41,9 +41,9 @@ extern PyObject *_PyEval_BuiltinsFromGlobals(
 
 
 static inline PyObject*
-_PyEval_EvalFrame(PyThreadState *tstate, PyFrameObject *f, int throwflag)
+_PyEval_EvalFrame(PyThreadState *tstate, struct _interpreter_frame *frame, int throwflag)
 {
-    return tstate->interp->eval_frame(tstate, f, throwflag);
+    return tstate->interp->eval_frame(tstate, frame, throwflag);
 }
 
 extern PyObject *
@@ -107,6 +107,9 @@ static inline void _Py_LeaveRecursiveCall_inline(void)  {
 
 #define Py_LeaveRecursiveCall() _Py_LeaveRecursiveCall_inline()
 
+struct _interpreter_frame *_PyEval_GetFrame(void);
+
+PyObject *_Py_MakeCoro(PyFrameConstructor *, struct _interpreter_frame *);
 
 #ifdef __cplusplus
 }

Include/internal/pycore_frame.h

@@ -4,41 +4,123 @@
 extern "C" {
 #endif
 
-enum {
-    FRAME_SPECIALS_GLOBALS_OFFSET = 0,
-    FRAME_SPECIALS_BUILTINS_OFFSET = 1,
-    FRAME_SPECIALS_LOCALS_OFFSET = 2,
-    FRAME_SPECIALS_CODE_OFFSET = 3,
-    FRAME_SPECIALS_SIZE = 4
+/* These values are chosen so that the inline functions below all
+ * compare f_state to zero.
+ */
+enum _framestate {
+    FRAME_CREATED = -2,
+    FRAME_SUSPENDED = -1,
+    FRAME_EXECUTING = 0,
+    FRAME_RETURNED = 1,
+    FRAME_UNWINDING = 2,
+    FRAME_RAISED = 3,
+    FRAME_CLEARED = 4
 };
 
-static inline PyObject **
-_PyFrame_Specials(PyFrameObject *f) {
-    return &f->f_valuestack[-FRAME_SPECIALS_SIZE];
+typedef signed char PyFrameState;
+
+typedef struct _interpreter_frame {
+    PyObject *f_globals;
+    PyObject *f_builtins;
+    PyObject *f_locals;
+    PyCodeObject *f_code;
+    PyFrameObject *frame_obj;
+    /* Borrowed reference to a generator, or NULL */
+    PyObject *generator;
+    struct _interpreter_frame *previous;
+    int f_lasti;       /* Last instruction if called */
+    int stackdepth;  /* Depth of value stack */
+    int nlocalsplus;
+    PyFrameState f_state;       /* What state the frame is in */
+    PyObject *stack[1];
+} InterpreterFrame;
+
+static inline int _PyFrame_IsRunnable(InterpreterFrame *f) {
+    return f->f_state < FRAME_EXECUTING;
+}
+
+static inline int _PyFrame_IsExecuting(InterpreterFrame *f) {
+    return f->f_state == FRAME_EXECUTING;
 }
 
-/* Returns a *borrowed* reference. */
-static inline PyObject *
-_PyFrame_GetGlobals(PyFrameObject *f)
+static inline int _PyFrameHasCompleted(InterpreterFrame *f) {
+    return f->f_state > FRAME_EXECUTING;
+}
+
+#define FRAME_SPECIALS_SIZE ((sizeof(InterpreterFrame)-1)/sizeof(PyObject *))
+
+InterpreterFrame *
+_PyInterpreterFrame_HeapAlloc(PyFrameConstructor *con, PyObject *locals);
+
+static inline void
+_PyFrame_InitializeSpecials(
+    InterpreterFrame *frame, PyFrameConstructor *con,
+    PyObject *locals, int nlocalsplus)
 {
-    return _PyFrame_Specials(f)[FRAME_SPECIALS_GLOBALS_OFFSET];
+    frame->f_code = (PyCodeObject *)Py_NewRef(con->fc_code);
+    frame->f_builtins = Py_NewRef(con->fc_builtins);
+    frame->f_globals = Py_NewRef(con->fc_globals);
+    frame->f_locals = Py_XNewRef(locals);
+    frame->nlocalsplus = nlocalsplus;
+    frame->stackdepth = 0;
+    frame->frame_obj = NULL;
+    frame->generator = NULL;
+    frame->f_lasti = -1;
+    frame->f_state = FRAME_CREATED;
 }
 
-/* Returns a *borrowed* reference. */
-static inline PyObject *
-_PyFrame_GetBuiltins(PyFrameObject *f)
+/* Gets the pointer to the locals array
+ * that precedes this frame.
+ */
+static inline PyObject**
+_PyFrame_GetLocalsArray(InterpreterFrame *frame)
 {
-    return _PyFrame_Specials(f)[FRAME_SPECIALS_BUILTINS_OFFSET];
+    return ((PyObject **)frame) - frame->nlocalsplus;
 }
 
-/* Returns a *borrowed* reference. */
-static inline PyCodeObject *
-_PyFrame_GetCode(PyFrameObject *f)
+/* For use by _PyFrame_GetFrameObject
+  Do not call directly. */
+PyFrameObject *
+_PyFrame_MakeAndSetFrameObject(InterpreterFrame *frame);
+
+/* Gets the PyFrameObject for this frame, lazily
+ * creating it if necessary.
+ * Returns a borrowed referennce */
+static inline PyFrameObject *
+_PyFrame_GetFrameObject(InterpreterFrame *frame)
 {
-    return (PyCodeObject *)_PyFrame_Specials(f)[FRAME_SPECIALS_CODE_OFFSET];
+    PyFrameObject *res =  frame->frame_obj;
+    if (res != NULL) {
+        return res;
+    }
+    return _PyFrame_MakeAndSetFrameObject(frame);
 }
 
-int _PyFrame_TakeLocals(PyFrameObject *f);
+/* Clears all references in the frame.
+ * If take is non-zero, then the InterpreterFrame frame
+ * may be transfered to the frame object it references
+ * instead of being cleared. Either way
+ * the caller no longer owns the references
+ * in the frame.
+ * take should  be set to 1 for heap allocated
+ * frames like the ones in generators and coroutines.
+ */
+int
+_PyFrame_Clear(InterpreterFrame * frame, int take);
+
+int
+_PyFrame_Traverse(InterpreterFrame *frame, visitproc visit, void *arg);
+
+int
+_PyFrame_FastToLocalsWithError(InterpreterFrame *frame);
+
+void
+_PyFrame_LocalsToFast(InterpreterFrame *frame, int clear);
+
+InterpreterFrame *_PyThreadState_PushFrame(
+    PyThreadState *tstate, PyFrameConstructor *con, PyObject *locals);
+
+void _PyThreadState_PopFrame(PyThreadState *tstate, InterpreterFrame *frame);
 
 #ifdef __cplusplus
 }

Include/internal/pycore_pystate.h

@@ -147,9 +147,6 @@ PyAPI_FUNC(int) _PyState_AddModule(
 
 PyAPI_FUNC(int) _PyOS_InterruptOccurred(PyThreadState *tstate);
 
-PyObject **_PyThreadState_PushLocals(PyThreadState *, int size);
-void _PyThreadState_PopLocals(PyThreadState *, PyObject **);
-
 #ifdef __cplusplus
 }
 #endif

Lib/test/test_faulthandler.py

@@ -56,6 +56,7 @@ def temporary_filename():
         os_helper.unlink(filename)
 
 class FaultHandlerTests(unittest.TestCase):
+
     def get_output(self, code, filename=None, fd=None):
         """
         Run the specified code in Python (in a new child process) and read the

Lib/test/test_sys.py

@@ -1275,7 +1275,7 @@ class C(object): pass
         # frame
         import inspect
         x = inspect.currentframe()
-        check(x, size('4P3i4cP'))
+        check(x, size('3Pi3c'))
         # function
         def func(): pass
         check(func, size('14Pi'))

Makefile.pre.in

@@ -350,6 +350,7 @@ PYTHON_OBJS=	\
 		Python/context.o \
 		Python/dynamic_annotations.o \
 		Python/errors.o \
+		Python/frame.o \
 		Python/frozenmain.o \
 		Python/future.o \
 		Python/getargs.o \

Misc/NEWS.d/next/Core and Builtins/2021-07-09-12-08-17.bpo-44590.a2ntVX.rst

@@ -0,0 +1,5 @@
+All necessary data for executing a Python function (local variables, stack,
+etc) is now kept in a per-thread stack. Frame objects are lazily allocated
+on demand. This increases performance by about 7% on the standard benchmark
+suite. Introspection and debugging are unaffected as frame objects are
+always available when needed. Patch by Mark Shannon.

Modules/_tracemalloc.c

@@ -3,7 +3,7 @@
 #include "pycore_pymem.h"         // _Py_tracemalloc_config
 #include "pycore_traceback.h"
 #include "pycore_hashtable.h"
-#include "frameobject.h"          // PyFrame_GetBack()
+#include <pycore_frame.h>
 
 #include "clinic/_tracemalloc.c.h"
 /*[clinic input]
@@ -299,18 +299,16 @@ hashtable_compare_traceback(const void *key1, const void *key2)
 
 
 static void
-tracemalloc_get_frame(PyFrameObject *pyframe, frame_t *frame)
+tracemalloc_get_frame(InterpreterFrame *pyframe, frame_t *frame)
 {
     frame->filename = unknown_filename;
-    int lineno = PyFrame_GetLineNumber(pyframe);
+    int lineno = PyCode_Addr2Line(pyframe->f_code, pyframe->f_lasti*2);
     if (lineno < 0) {
         lineno = 0;
     }
     frame->lineno = (unsigned int)lineno;
 
-    PyCodeObject *code = PyFrame_GetCode(pyframe);
-    PyObject *filename = code->co_filename;
-    Py_DECREF(code);
+    PyObject *filename = pyframe->f_code->co_filename;
 
     if (filename == NULL) {
 #ifdef TRACE_DEBUG
@@ -395,7 +393,7 @@ traceback_get_frames(traceback_t *traceback)
         return;
     }
 
-    PyFrameObject *pyframe = PyThreadState_GetFrame(tstate);
+    InterpreterFrame *pyframe = tstate->frame;
     for (; pyframe != NULL;) {
         if (traceback->nframe < _Py_tracemalloc_config.max_nframe) {
             tracemalloc_get_frame(pyframe, &traceback->frames[traceback->nframe]);
@@ -406,8 +404,7 @@ traceback_get_frames(traceback_t *traceback)
             traceback->total_nframe++;
         }
 
-        PyFrameObject *back = PyFrame_GetBack(pyframe);
-        Py_DECREF(pyframe);
+        InterpreterFrame *back = pyframe->previous;
         pyframe = back;
     }
 }

Modules/_xxsubinterpretersmodule.c

@@ -4,6 +4,7 @@
 
 #include "Python.h"
 #include "frameobject.h"
+#include "pycore_frame.h"
 #include "interpreteridobject.h"
 
 
@@ -1834,13 +1835,12 @@ _is_running(PyInterpreterState *interp)
     }
 
     assert(!PyErr_Occurred());
-    PyFrameObject *frame = PyThreadState_GetFrame(tstate);
+    InterpreterFrame *frame = tstate->frame;
     if (frame == NULL) {
         return 0;
     }
 
     int executing = _PyFrame_IsExecuting(frame);
-    Py_DECREF(frame);
 
     return executing;
 }