From 5fce348085f7a1ea290d6700050910c4e5014f92 Mon Sep 17 00:00:00 2001 From: Matt Page Date: Fri, 11 Apr 2025 20:29:58 -0700 Subject: [PATCH] Only mark slow path functions used by the interpreter loop as noinline in free-threaded builds There is concern that this is slowing down the default build. --- Include/pyport.h | 8 ++++++++ Python/ceval.c | 2 +- Python/instrumentation.c | 14 +++++++------- Python/specialize.c | 28 ++++++++++++++-------------- 4 files changed, 30 insertions(+), 22 deletions(-) diff --git a/Include/pyport.h b/Include/pyport.h index 2a7192c2c55cdd..77f01f65970a39 100644 --- a/Include/pyport.h +++ b/Include/pyport.h @@ -374,6 +374,14 @@ extern "C" { # define Py_NO_INLINE #endif +// FT_NO_INLINE +// Disable inlining of a function only in the free-threaded build. +#if defined(Py_GIL_DISABLED) +# define FT_NO_INLINE Py_NO_INLINE +#else +# define FT_NO_INLINE +#endif + #include "exports.h" #ifdef Py_LIMITED_API diff --git a/Python/ceval.c b/Python/ceval.c index e4a63ad9287783..38840cfe9c3d7d 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1427,7 +1427,7 @@ skip_to_next_entry(unsigned char *p, unsigned char *end) { #define MAX_LINEAR_SEARCH 40 -static Py_NO_INLINE int +static FT_NO_INLINE int get_exception_handler(PyCodeObject *code, int index, int *level, int *handler, int *lasti) { unsigned char *start = (unsigned char *)PyBytes_AS_STRING(code->co_exceptiontable); diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 13bdd041becd69..6936f8678136a1 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -1200,7 +1200,7 @@ call_instrumentation_vector( return err; } -Py_NO_INLINE int +FT_NO_INLINE int _Py_call_instrumentation( PyThreadState *tstate, int event, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr) @@ -1209,7 +1209,7 @@ _Py_call_instrumentation( return call_instrumentation_vector(instr, tstate, event, frame, instr, 2, args); } -Py_NO_INLINE int +FT_NO_INLINE int _Py_call_instrumentation_arg( PyThreadState *tstate, int event, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg) @@ -1218,7 +1218,7 @@ _Py_call_instrumentation_arg( return call_instrumentation_vector(instr, tstate, event, frame, instr, 3, args); } -Py_NO_INLINE int +FT_NO_INLINE int _Py_call_instrumentation_2args( PyThreadState *tstate, int event, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg0, PyObject *arg1) @@ -1227,7 +1227,7 @@ _Py_call_instrumentation_2args( return call_instrumentation_vector(instr, tstate, event, frame, instr, 4, args); } -Py_NO_INLINE _Py_CODEUNIT * +FT_NO_INLINE _Py_CODEUNIT * _Py_call_instrumentation_jump( _Py_CODEUNIT *instr, PyThreadState *tstate, int event, _PyInterpreterFrame *frame, _Py_CODEUNIT *src, _Py_CODEUNIT *dest) @@ -1271,7 +1271,7 @@ call_instrumentation_vector_protected( assert(_PyErr_Occurred(tstate)); } -Py_NO_INLINE void +FT_NO_INLINE void _Py_call_instrumentation_exc2( PyThreadState *tstate, int event, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg0, PyObject *arg1) @@ -1294,7 +1294,7 @@ _Py_Instrumentation_GetLine(PyCodeObject *code, int index) return line; } -Py_NO_INLINE int +FT_NO_INLINE int _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, _Py_CODEUNIT *instr, _Py_CODEUNIT *prev) { PyCodeObject *code = _PyFrame_GetCode(frame); @@ -1396,7 +1396,7 @@ _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, return original_opcode; } -Py_NO_INLINE int +FT_NO_INLINE int _Py_call_instrumentation_instruction(PyThreadState *tstate, _PyInterpreterFrame* frame, _Py_CODEUNIT *instr) { PyCodeObject *code = _PyFrame_GetCode(frame); diff --git a/Python/specialize.c b/Python/specialize.c index ceb396c5b54815..21f97dfbce235c 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -818,7 +818,7 @@ specialize_module_load_attr( /* Attribute specialization */ -Py_NO_INLINE void +FT_NO_INLINE void _Py_Specialize_LoadSuperAttr(_PyStackRef global_super_st, _PyStackRef cls_st, _Py_CODEUNIT *instr, int load_method) { PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); PyObject *cls = PyStackRef_AsPyObjectBorrow(cls_st); @@ -1342,7 +1342,7 @@ specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* na return result; } -Py_NO_INLINE void +FT_NO_INLINE void _Py_Specialize_LoadAttr(_PyStackRef owner_st, _Py_CODEUNIT *instr, PyObject *name) { PyObject *owner = PyStackRef_AsPyObjectBorrow(owner_st); @@ -1373,7 +1373,7 @@ _Py_Specialize_LoadAttr(_PyStackRef owner_st, _Py_CODEUNIT *instr, PyObject *nam } } -Py_NO_INLINE void +FT_NO_INLINE void _Py_Specialize_StoreAttr(_PyStackRef owner_st, _Py_CODEUNIT *instr, PyObject *name) { PyObject *owner = PyStackRef_AsPyObjectBorrow(owner_st); @@ -1771,7 +1771,7 @@ specialize_load_global_lock_held( unspecialize(instr); } -Py_NO_INLINE void +FT_NO_INLINE void _Py_Specialize_LoadGlobal( PyObject *globals, PyObject *builtins, _Py_CODEUNIT *instr, PyObject *name) @@ -1891,7 +1891,7 @@ store_subscr_fail_kind(PyObject *container, PyObject *sub) } #endif -Py_NO_INLINE void +FT_NO_INLINE void _Py_Specialize_StoreSubscr(_PyStackRef container_st, _PyStackRef sub_st, _Py_CODEUNIT *instr) { PyObject *container = PyStackRef_AsPyObjectBorrow(container_st); @@ -2171,7 +2171,7 @@ specialize_c_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs) } } -Py_NO_INLINE void +FT_NO_INLINE void _Py_Specialize_Call(_PyStackRef callable_st, _Py_CODEUNIT *instr, int nargs) { PyObject *callable = PyStackRef_AsPyObjectBorrow(callable_st); @@ -2211,7 +2211,7 @@ _Py_Specialize_Call(_PyStackRef callable_st, _Py_CODEUNIT *instr, int nargs) } } -Py_NO_INLINE void +FT_NO_INLINE void _Py_Specialize_CallKw(_PyStackRef callable_st, _Py_CODEUNIT *instr, int nargs) { PyObject *callable = PyStackRef_AsPyObjectBorrow(callable_st); @@ -2566,7 +2566,7 @@ binary_op_extended_specialization(PyObject *lhs, PyObject *rhs, int oparg, return 0; } -Py_NO_INLINE void +FT_NO_INLINE void _Py_Specialize_BinaryOp(_PyStackRef lhs_st, _PyStackRef rhs_st, _Py_CODEUNIT *instr, int oparg, _PyStackRef *locals) { @@ -2724,7 +2724,7 @@ compare_op_fail_kind(PyObject *lhs, PyObject *rhs) } #endif // Py_STATS -Py_NO_INLINE void +FT_NO_INLINE void _Py_Specialize_CompareOp(_PyStackRef lhs_st, _PyStackRef rhs_st, _Py_CODEUNIT *instr, int oparg) { @@ -2787,7 +2787,7 @@ unpack_sequence_fail_kind(PyObject *seq) } #endif // Py_STATS -Py_NO_INLINE void +FT_NO_INLINE void _Py_Specialize_UnpackSequence(_PyStackRef seq_st, _Py_CODEUNIT *instr, int oparg) { PyObject *seq = PyStackRef_AsPyObjectBorrow(seq_st); @@ -2894,7 +2894,7 @@ int } #endif // Py_STATS -Py_NO_INLINE void +FT_NO_INLINE void _Py_Specialize_ForIter(_PyStackRef iter, _Py_CODEUNIT *instr, int oparg) { assert(ENABLE_SPECIALIZATION_FT); @@ -2949,7 +2949,7 @@ _Py_Specialize_ForIter(_PyStackRef iter, _Py_CODEUNIT *instr, int oparg) unspecialize(instr); } -Py_NO_INLINE void +FT_NO_INLINE void _Py_Specialize_Send(_PyStackRef receiver_st, _Py_CODEUNIT *instr) { PyObject *receiver = PyStackRef_AsPyObjectBorrow(receiver_st); @@ -3019,7 +3019,7 @@ check_type_always_true(PyTypeObject *ty) return 0; } -Py_NO_INLINE void +FT_NO_INLINE void _Py_Specialize_ToBool(_PyStackRef value_o, _Py_CODEUNIT *instr) { assert(ENABLE_SPECIALIZATION_FT); @@ -3093,7 +3093,7 @@ containsop_fail_kind(PyObject *value) { } #endif -Py_NO_INLINE void +FT_NO_INLINE void _Py_Specialize_ContainsOp(_PyStackRef value_st, _Py_CODEUNIT *instr) { PyObject *value = PyStackRef_AsPyObjectBorrow(value_st);