diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index f90af6a9ce7c26..992c365a1ce14f 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -1279,6 +1279,7 @@ with sub-interpreters: Hangs the current thread, rather than terminating it, if called while the interpreter is finalizing. + .. c:function:: void PyGILState_Release(PyGILState_STATE) Release any resources previously acquired. After this call, Python's state will @@ -1499,6 +1500,37 @@ All of the following functions must be called after :c:func:`Py_Initialize`. .. versionadded:: 3.8 +.. c:function:: int PyThreadState_Ensure(PyInterpreterState *interp, const char **errmsg) + + Similar to :c:func:`PyGILState_Ensure`, except that it returns with a status + code even in the case of failure, and takes an interpreter state. + Specifically, it returns a status code (``>= 0``) when the operation + succeeded, or sets *\*errmsg* (if *errmsg* is not NULL) and returns ``-1`` + on failure. + + On success, the thread state must be released by + :c:func:`PyThreadState_Release`. + + In the case of failure, it is *unsafe* to use the Python API following the + call. Releasing the obtained *state* via :c:func:`PyGILState_Release` must + only be done in the case of success. + + .. versionadded:: next + + +.. c:function:: void PyThreadState_Release(int state) + + Release any resources previously acquired. After this call, Python's state + will be the same as it was prior to the corresponding + :c:func:`PyThreadState_Ensure` call (but generally this state will be + unknown to the caller). + + Every call to :c:func:`PyThreadState_Ensure` must be matched by a call to + :c:func:`PyThreadState_Release` on the same thread. + + .. versionadded:: next + + .. c:function:: PyObject* PyUnstable_InterpreterState_GetMainModule(PyInterpreterState *interp) Return a :term:`strong reference` to the ``__main__`` :ref:`module object ` diff --git a/Doc/data/stable_abi.dat b/Doc/data/stable_abi.dat index 59e7a31bc2ef06..9f22fefc7c826d 100644 --- a/Doc/data/stable_abi.dat +++ b/Doc/data/stable_abi.dat @@ -636,12 +636,14 @@ func,PySys_WriteStdout,3.2,, type,PyThreadState,3.2,,opaque func,PyThreadState_Clear,3.2,, func,PyThreadState_Delete,3.2,, +func,PyThreadState_Ensure,3.14,, func,PyThreadState_Get,3.2,, func,PyThreadState_GetDict,3.2,, func,PyThreadState_GetFrame,3.10,, func,PyThreadState_GetID,3.10,, func,PyThreadState_GetInterpreter,3.10,, func,PyThreadState_New,3.2,, +func,PyThreadState_Release,3.14,, func,PyThreadState_SetAsyncExc,3.2,, func,PyThreadState_Swap,3.2,, func,PyThread_GetInfo,3.3,, diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 1e469e8738bfcb..e72a242a3be714 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -1372,6 +1372,11 @@ New features and get an attribute of the module. (Contributed by Victor Stinner in :gh:`128911`.) +* Add :c:func:`PyThreadState_Ensure` and :c:func:`PyThreadState_Release` + functions: similar to :c:func:`PyGILState_Ensure` and + :c:func:`PyGILState_Release`, but :c:func:`PyThreadState_Ensure` returns + ``-1`` on failure. Patch by Victor Stinner. + (Contributed by Victor Stinner in :gh:`124622`.) Limited C API changes --------------------- diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index cd6d9582496850..66c3b2860f0125 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -140,6 +140,8 @@ struct _ts { PyObject *dict; /* Stores per-thread state */ int gilstate_counter; + // PyThreadState_Ensure() call depth + int ensure_depth; PyObject *async_exc; /* Asynchronous exception to raise */ unsigned long thread_id; /* Thread id where this tstate was created */ diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index fea8665ae39ab5..f896e9ea3a9e9a 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -327,6 +327,12 @@ void _Py_unset_eval_breaker_bit_all(PyInterpreterState *interp, uintptr_t bit); PyAPI_FUNC(PyObject *) _PyFloat_FromDouble_ConsumeInputs(_PyStackRef left, _PyStackRef right, double value); +extern int _PyEval_AcquireLockOrFail( + PyThreadState *tstate, + const char **errmsg); +extern int _PyEval_RestoreThreadOrFail( + PyThreadState *tstate, + const char **errmsg); #ifdef __cplusplus } diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index 9ec59e60f609ab..a4e46c21d4e538 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -313,6 +313,10 @@ _Py_AssertHoldsTstateFunc(const char *func) #define _Py_AssertHoldsTstate() #endif +extern int _PyThreadState_AttachOrFail( + PyThreadState *tstate, + const char **errmsg); + #ifdef __cplusplus } #endif diff --git a/Include/pystate.h b/Include/pystate.h index 727b8fbfffe0e6..8a9d59924e05d8 100644 --- a/Include/pystate.h +++ b/Include/pystate.h @@ -111,6 +111,14 @@ PyAPI_FUNC(PyGILState_STATE) PyGILState_Ensure(void); */ PyAPI_FUNC(void) PyGILState_Release(PyGILState_STATE); +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030e0000 +/* New in 3.14 */ +PyAPI_FUNC(int) PyThreadState_Ensure( + PyInterpreterState *interp, + const char **errmsg); +PyAPI_FUNC(void) PyThreadState_Release(int state); +#endif + /* Helper/diagnostic function - get the current thread state for this thread. May return NULL if no GILState API has been used on the current thread. Note that the main thread always has such a diff --git a/Lib/test/test_stable_abi_ctypes.py b/Lib/test/test_stable_abi_ctypes.py index f3724ce6d4d15a..2c64a26eef5d0d 100644 --- a/Lib/test/test_stable_abi_ctypes.py +++ b/Lib/test/test_stable_abi_ctypes.py @@ -668,12 +668,14 @@ def test_windows_feature_macros(self): "PyThreadState_Clear", "PyThreadState_Delete", "PyThreadState_DeleteCurrent", + "PyThreadState_Ensure", "PyThreadState_Get", "PyThreadState_GetDict", "PyThreadState_GetFrame", "PyThreadState_GetID", "PyThreadState_GetInterpreter", "PyThreadState_New", + "PyThreadState_Release", "PyThreadState_SetAsyncExc", "PyThreadState_Swap", "PyThread_GetInfo", diff --git a/Misc/NEWS.d/next/C_API/2025-02-05-15-01-23.gh-issue-124622.jnROuN.rst b/Misc/NEWS.d/next/C_API/2025-02-05-15-01-23.gh-issue-124622.jnROuN.rst new file mode 100644 index 00000000000000..9d6db0182c4077 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2025-02-05-15-01-23.gh-issue-124622.jnROuN.rst @@ -0,0 +1,4 @@ +Add :c:func:`PyThreadState_Ensure` and :c:func:`PyThreadState_Release` +functions: similar to :c:func:`PyGILState_Ensure` and +:c:func:`PyGILState_Release`, but :c:func:`PyThreadState_Ensure` returns ``-1`` +on failure. Patch by Victor Stinner. diff --git a/Misc/stable_abi.toml b/Misc/stable_abi.toml index 9317be605f0065..e733ff821f0f09 100644 --- a/Misc/stable_abi.toml +++ b/Misc/stable_abi.toml @@ -2545,3 +2545,7 @@ added = '3.14' [function.Py_PACK_VERSION] added = '3.14' +[function.PyThreadState_Ensure] + added = '3.14' +[function.PyThreadState_Release] + added = '3.14' diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index c84646ccf03fa7..193fad706cee26 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -1263,19 +1263,24 @@ typedef struct { PyThread_type_lock start_event; PyThread_type_lock exit_event; PyObject *callback; + PyInterpreterState *interp; } test_c_thread_t; static void temporary_c_thread(void *data) { test_c_thread_t *test_c_thread = data; - PyGILState_STATE state; PyObject *res; PyThread_release_lock(test_c_thread->start_event); /* Allocate a Python thread state for this thread */ - state = PyGILState_Ensure(); + const char *errmsg; + int state = PyThreadState_Ensure(test_c_thread->interp, &errmsg); + if (state < 0) { + fprintf(stderr, "ERROR: PyThreadState_Ensure() failed: %s", errmsg); + abort(); + } res = PyObject_CallNoArgs(test_c_thread->callback); Py_CLEAR(test_c_thread->callback); @@ -1288,7 +1293,7 @@ temporary_c_thread(void *data) } /* Destroy the Python thread state for this thread */ - PyGILState_Release(state); + PyThreadState_Release(state); PyThread_release_lock(test_c_thread->exit_event); } @@ -1310,6 +1315,7 @@ call_in_temporary_c_thread(PyObject *self, PyObject *args) test_c_thread.start_event = PyThread_allocate_lock(); test_c_thread.exit_event = PyThread_allocate_lock(); test_c_thread.callback = NULL; + test_c_thread.interp = PyInterpreterState_Get(); if (!test_c_thread.start_event || !test_c_thread.exit_event) { PyErr_SetString(PyExc_RuntimeError, "could not allocate lock"); goto exit; @@ -1370,6 +1376,106 @@ join_temporary_c_thread(PyObject *self, PyObject *Py_UNUSED(ignored)) Py_RETURN_NONE; } +static PyObject* +_test_tstate_ensure(int subinterpreter, int clear_current_before) +{ + PyThreadState *oldts = PyThreadState_Get(); + + PyInterpreterState *interp; + PyThreadState *subinterp_tstate = NULL; + PyThreadState *expected_tstate = NULL; + if (subinterpreter) { + // Create a sub-interpreter + subinterp_tstate = Py_NewInterpreter(); + assert(PyThreadState_Get() == subinterp_tstate); + interp = PyThreadState_GetInterpreter(subinterp_tstate); + expected_tstate = subinterp_tstate; + } + else { + interp = PyThreadState_GetInterpreter(oldts); + expected_tstate = oldts; + } + + if (clear_current_before) { + PyThreadState_Swap(NULL); + assert(PyThreadState_GetUnchecked() == NULL); + } + + // First call + const char *errmsg; + int state1 = PyThreadState_Ensure(interp, &errmsg); + if (state1 < 0) { + fprintf(stderr, "ERROR: PyThreadState_Ensure() failed: %s", errmsg); + abort(); + } + PyThreadState *ensure1 = PyThreadState_GetUnchecked(); + if (clear_current_before) { + assert(ensure1 != expected_tstate); + } + else { + assert(ensure1 == expected_tstate); + } + + { + // Second call + int state2 = PyThreadState_Ensure(interp, &errmsg); + if (state2 < 0) { + fprintf(stderr, "ERROR: PyThreadState_Ensure() failed: %s", errmsg); + abort(); + } + PyThreadState *ensure2 = PyThreadState_GetUnchecked(); + assert(ensure2 == ensure1); + + PyThreadState_Release(state2); + } + PyThreadState_Release(state1); + + if (!clear_current_before) { + assert(PyThreadState_GetUnchecked() == expected_tstate); + } + else { + assert(PyThreadState_GetUnchecked() == NULL); + } + + if (subinterpreter) { + PyThreadState_Swap(subinterp_tstate); + Py_EndInterpreter(subinterp_tstate); + assert(PyThreadState_GetUnchecked() == NULL); + } + PyThreadState_Swap(oldts); + + Py_RETURN_NONE; +} + + +static PyObject * +test_tstate_ensure(PyObject *self, PyObject *Py_UNUSED(args)) +{ + return _test_tstate_ensure(0, 0); +} + + +static PyObject * +test_tstate_ensure_clear(PyObject *self, PyObject *Py_UNUSED(args)) +{ + return _test_tstate_ensure(0, 1); +} + + +static PyObject * +test_tstate_ensure_subinterp(PyObject *self, PyObject *Py_UNUSED(args)) +{ + return _test_tstate_ensure(1, 0); +} + + +static PyObject * +test_tstate_ensure_subinterp_clear(PyObject *self, PyObject *Py_UNUSED(args)) +{ + return _test_tstate_ensure(1, 1); +} + + /* marshal */ static PyObject* @@ -2537,6 +2643,10 @@ static PyMethodDef TestMethods[] = { {"call_in_temporary_c_thread", call_in_temporary_c_thread, METH_VARARGS, PyDoc_STR("set_error_class(error_class) -> None")}, {"join_temporary_c_thread", join_temporary_c_thread, METH_NOARGS}, + {"test_tstate_ensure", test_tstate_ensure, METH_NOARGS}, + {"test_tstate_ensure_clear", test_tstate_ensure_clear, METH_NOARGS}, + {"test_tstate_ensure_subinterp", test_tstate_ensure_subinterp, METH_NOARGS}, + {"test_tstate_ensure_subinterp_clear", test_tstate_ensure_subinterp_clear, METH_NOARGS}, {"pymarshal_write_long_to_file", pymarshal_write_long_to_file, METH_VARARGS}, {"pymarshal_write_object_to_file", diff --git a/PC/python3dll.c b/PC/python3dll.c index 84b3c735240b73..177760b054195b 100755 --- a/PC/python3dll.c +++ b/PC/python3dll.c @@ -631,12 +631,14 @@ EXPORT_FUNC(PyThread_tss_set) EXPORT_FUNC(PyThreadState_Clear) EXPORT_FUNC(PyThreadState_Delete) EXPORT_FUNC(PyThreadState_DeleteCurrent) +EXPORT_FUNC(PyThreadState_Ensure) EXPORT_FUNC(PyThreadState_Get) EXPORT_FUNC(PyThreadState_GetDict) EXPORT_FUNC(PyThreadState_GetFrame) EXPORT_FUNC(PyThreadState_GetID) EXPORT_FUNC(PyThreadState_GetInterpreter) EXPORT_FUNC(PyThreadState_New) +EXPORT_FUNC(PyThreadState_Release) EXPORT_FUNC(PyThreadState_SetAsyncExc) EXPORT_FUNC(PyThreadState_Swap) EXPORT_FUNC(PyTraceBack_Here) diff --git a/Python/ceval_gil.c b/Python/ceval_gil.c index 416eec01052224..7b0e3fbaad2cd6 100644 --- a/Python/ceval_gil.c +++ b/Python/ceval_gil.c @@ -277,12 +277,15 @@ drop_gil(PyInterpreterState *interp, PyThreadState *tstate, int final_release) /* Take the GIL. + Return 0 on success. + Return -1 if the thread must exit. + The function saves errno at entry and restores its value at exit. It may hang rather than return if the interpreter has been finalized. tstate must be non-NULL. */ -static void -take_gil(PyThreadState *tstate) +static int +take_gil_or_fail(PyThreadState *tstate, const char **errmsg) { int err = errno; @@ -304,7 +307,7 @@ take_gil(PyThreadState *tstate) C++. gh-87135: The best that can be done is to hang the thread as the public APIs calling this have no error reporting mechanism (!). */ - PyThread_hang_thread(); + goto tstate_must_exit; } assert(_PyThreadState_CheckConsistency(tstate)); @@ -312,7 +315,7 @@ take_gil(PyThreadState *tstate) struct _gil_runtime_state *gil = interp->ceval.gil; #ifdef Py_GIL_DISABLED if (!_Py_atomic_load_int_relaxed(&gil->enabled)) { - return; + goto done; } #endif @@ -348,9 +351,7 @@ take_gil(PyThreadState *tstate) if (drop_requested) { _Py_unset_eval_breaker_bit(holder_tstate, _PY_GIL_DROP_REQUEST_BIT); } - // gh-87135: hang the thread as *thread_exit() is not a safe - // API. It lacks stack unwind and local variable destruction. - PyThread_hang_thread(); + goto tstate_must_exit; } assert(_PyThreadState_CheckConsistency(tstate)); @@ -366,7 +367,7 @@ take_gil(PyThreadState *tstate) // return. COND_SIGNAL(gil->cond); MUTEX_UNLOCK(gil->mutex); - return; + goto done; } #endif @@ -401,7 +402,7 @@ take_gil(PyThreadState *tstate) /* tstate could be a dangling pointer, so don't pass it to drop_gil(). */ drop_gil(interp, NULL, 1); - PyThread_hang_thread(); + goto tstate_must_exit; } assert(_PyThreadState_CheckConsistency(tstate)); @@ -411,8 +412,28 @@ take_gil(PyThreadState *tstate) MUTEX_UNLOCK(gil->mutex); +#ifdef Py_GIL_DISABLED +done: +#endif errno = err; - return; + return 0; + +tstate_must_exit: + if (errmsg) { + *errmsg = "Python is being finalized"; + } + errno = err; + return -1; +} + +static void +take_gil(PyThreadState *tstate) +{ + if (take_gil_or_fail(tstate, NULL) < 0) { + // gh-87135: hang the thread as *thread_exit() is not a safe + // API. It lacks stack unwind and local variable destruction. + PyThread_hang_thread(); + } } void _PyEval_SetSwitchInterval(unsigned long microseconds) @@ -586,6 +607,13 @@ _PyEval_AcquireLock(PyThreadState *tstate) take_gil(tstate); } +int +_PyEval_AcquireLockOrFail(PyThreadState *tstate, const char **errmsg) +{ + _Py_EnsureTstateNotNULL(tstate); + return take_gil_or_fail(tstate, errmsg); +} + void _PyEval_ReleaseLock(PyInterpreterState *interp, PyThreadState *tstate, @@ -641,19 +669,32 @@ PyEval_SaveThread(void) return tstate; } -void -PyEval_RestoreThread(PyThreadState *tstate) + +int +_PyEval_RestoreThreadOrFail(PyThreadState *tstate, const char **errmsg) { #ifdef MS_WINDOWS int err = GetLastError(); #endif _Py_EnsureTstateNotNULL(tstate); - _PyThreadState_Attach(tstate); + if (_PyThreadState_AttachOrFail(tstate, errmsg) < 0) { + return -1; + } #ifdef MS_WINDOWS SetLastError(err); #endif + return 0; +} + + +void +PyEval_RestoreThread(PyThreadState *tstate) +{ + if (_PyEval_RestoreThreadOrFail(tstate, NULL) < 0) { + PyThread_hang_thread(); + } } diff --git a/Python/pystate.c b/Python/pystate.c index 89a652850e9363..598b99724629bb 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1499,6 +1499,7 @@ init_threadstate(_PyThreadStateImpl *_tstate, // PyGILState_Release must not try to delete this thread state. // This is cleared when PyGILState_Ensure() creates the thread state. tstate->gilstate_counter = 1; + tstate->ensure_depth = 1; tstate->current_frame = NULL; tstate->datastack_chunk = NULL; @@ -2070,8 +2071,8 @@ tstate_wait_attach(PyThreadState *tstate) } while (!tstate_try_attach(tstate)); } -void -_PyThreadState_Attach(PyThreadState *tstate) +int +_PyThreadState_AttachOrFail(PyThreadState *tstate, const char **errmsg) { #if defined(Py_DEBUG) // This is called from PyEval_RestoreThread(). Similar @@ -2086,7 +2087,9 @@ _PyThreadState_Attach(PyThreadState *tstate) while (1) { - _PyEval_AcquireLock(tstate); + if (_PyEval_AcquireLockOrFail(tstate, errmsg) < 0) { + return -1; + } // XXX assert(tstate_is_alive(tstate)); current_fast_set(&_PyRuntime, tstate); @@ -2121,6 +2124,15 @@ _PyThreadState_Attach(PyThreadState *tstate) #if defined(Py_DEBUG) errno = err; #endif + return 0; +} + +void +_PyThreadState_Attach(PyThreadState *tstate) +{ + if (_PyThreadState_AttachOrFail(tstate, NULL) < 0) { + PyThread_hang_thread(); + } } static void @@ -2740,6 +2752,7 @@ PyGILState_Check(void) return (tstate == tcur); } + PyGILState_STATE PyGILState_Ensure(void) { @@ -2793,6 +2806,7 @@ PyGILState_Ensure(void) return has_gil ? PyGILState_LOCKED : PyGILState_UNLOCKED; } + void PyGILState_Release(PyGILState_STATE oldstate) { @@ -2804,11 +2818,16 @@ PyGILState_Release(PyGILState_STATE oldstate) } /* We must hold the GIL and have our thread state current */ + /* XXX - remove the check - the assert should be fine, + but while this is very new (April 2003), the extra check + by release-only users can't hurt. + */ if (!holds_gil(tstate)) { _Py_FatalErrorFormat(__func__, "thread state %p must be current when releasing", tstate); } + assert(holds_gil(tstate)); --tstate->gilstate_counter; assert(tstate->gilstate_counter >= 0); /* illegal counter value */ @@ -2841,6 +2860,108 @@ PyGILState_Release(PyGILState_STATE oldstate) } +int +PyThreadState_Ensure(PyInterpreterState *interp, const char **errmsg) +{ + assert(_PyEval_ThreadsInitialized()); + + if (errmsg) { + *errmsg = NULL; + } + + PyThreadState *tcur = current_fast_get(); + + if (tcur != NULL && tcur->interp != interp) { + // The current thread state is from another interpreter + tcur = NULL; + } + + int has_gil; + if (tcur == NULL) { + /* Create a new Python thread state for this thread */ + // XXX Use PyInterpreterState_EnsureThreadState()? + tcur = new_threadstate(interp, _PyThreadState_WHENCE_GILSTATE); + if (tcur == NULL) { + Py_FatalError("Couldn't create thread-state for new thread"); + } + bind_tstate(tcur); + bind_gilstate_tstate(tcur); + + /* This is our thread state! We'll need to delete it in the + matching call to PyThreadState_Release(). */ + assert(tcur->ensure_depth == 1); + tcur->ensure_depth = 0; + + has_gil = 0; /* new thread state is never current */ + } + else { + has_gil = holds_gil(tcur); + } + + if (!has_gil) { + if (_PyEval_RestoreThreadOrFail(tcur, errmsg) < 0) { + return -1; + } + } + + /* Update our counter in the thread-state - no need for locks: + - tcur will remain valid as we hold the GIL. + - the counter is safe as we are the only thread "allowed" + to modify this value + */ + ++tcur->ensure_depth; + + return has_gil ? PyGILState_LOCKED : PyGILState_UNLOCKED; +} + + +void +PyThreadState_Release(int oldstate) +{ + PyThreadState *tstate = _PyThreadState_GET(); + if (tstate == NULL) { + Py_FatalError("auto-releasing thread-state, " + "but no thread-state for this thread"); + } + + /* We must hold the GIL and have our thread state current */ + if (!holds_gil(tstate)) { + _Py_FatalErrorFormat(__func__, + "thread state %p must be current when releasing", + tstate); + } + --tstate->ensure_depth; + assert(tstate->ensure_depth >= 0); /* illegal counter value */ + + /* If we're going to destroy this thread-state, we must + * clear it while the GIL is held, as destructors may run. + */ + if (tstate->ensure_depth == 0) { + /* can't have been locked when we created it */ + assert(oldstate == PyGILState_UNLOCKED); + // XXX Unbind tstate here. + // gh-119585: `PyThreadState_Clear()` may call destructors that + // themselves use PyGILState_Ensure and PyGILState_Release, so make + // sure that ensure_depth is not zero when calling it. + ++tstate->ensure_depth; + PyThreadState_Clear(tstate); + --tstate->ensure_depth; + /* Delete the thread-state. Note this releases the GIL too! + * It's vital that the GIL be held here, to avoid shutdown + * races; see bugs 225673 and 1061968 (that nasty bug has a + * habit of coming back). + */ + assert(tstate->ensure_depth == 0); + assert(current_fast_get() == tstate); + _PyThreadState_DeleteCurrent(tstate); + } + /* Release the lock if necessary */ + else if (oldstate == PyGILState_UNLOCKED) { + PyEval_SaveThread(); + } +} + + /*************/ /* Other API */ /*************/