bpo-35886: Make PyInterpreterState an opaque type in the public API. (GH-11731)

Move PyInterpreterState into the "internal" header files.

Author: ericsnowcurrently

Doc/whatsnew/3.8.rst

@@ -571,6 +571,16 @@ Changes in the Python API
 * ``PyGC_Head`` struct is changed completely.  All code touched the
   struct member should be rewritten.  (See :issue:`33597`)
 
+* The ``PyInterpreterState`` struct has been moved into the "internal"
+  header files (specifically Include/internal/pycore_pystate.h).  An
+  opaque ``PyInterpreterState`` is still available as part of the public
+  API (and stable ABI).  The docs indicate that none of the struct's
+  fields are public, so we hope no one has been using them.  However,
+  if you do rely on one or more of those private fields and have no
+  alternative then please open a BPO issue.  We'll work on helping
+  you adjust (possibly including adding accessor functions to the
+  public API).  (See :issue:`35886`.)
+
 * Asyncio tasks can now be named, either by passing the ``name`` keyword
   argument to :func:`asyncio.create_task` or
   the :meth:`~asyncio.loop.create_task` event loop method, or by

Include/cpython/pystate.h

@@ -6,8 +6,6 @@
 extern "C" {
 #endif
 
-typedef PyObject* (*_PyFrameEvalFunction)(struct _frame *, int);
-
 /* Placeholders while working on the new configuration API
  *
  * See PEP 432 for final anticipated contents
@@ -30,63 +28,9 @@ typedef struct {
     (_PyMainInterpreterConfig){.install_signal_handlers = -1}
 /* Note: _PyMainInterpreterConfig_INIT sets other fields to 0/NULL */
 
-typedef struct _is {
-
-    struct _is *next;
-    struct _ts *tstate_head;
-
-    int64_t id;
-    int64_t id_refcount;
-    PyThread_type_lock id_mutex;
-
-    PyObject *modules;
-    PyObject *modules_by_index;
-    PyObject *sysdict;
-    PyObject *builtins;
-    PyObject *importlib;
-
-    /* Used in Python/sysmodule.c. */
-    int check_interval;
-
-    /* Used in Modules/_threadmodule.c. */
-    long num_threads;
-    /* Support for runtime thread stack size tuning.
-       A value of 0 means using the platform's default stack size
-       or the size specified by the THREAD_STACK_SIZE macro. */
-    /* Used in Python/thread.c. */
-    size_t pythread_stacksize;
-
-    PyObject *codec_search_path;
-    PyObject *codec_search_cache;
-    PyObject *codec_error_registry;
-    int codecs_initialized;
-    int fscodec_initialized;
-
-    _PyCoreConfig core_config;
-    _PyMainInterpreterConfig config;
-#ifdef HAVE_DLOPEN
-    int dlopenflags;
-#endif
-
-    PyObject *builtins_copy;
-    PyObject *import_func;
-    /* Initialized to PyEval_EvalFrameDefault(). */
-    _PyFrameEvalFunction eval_frame;
-
-    Py_ssize_t co_extra_user_count;
-    freefunc co_extra_freefuncs[MAX_CO_EXTRA_USERS];
-
-#ifdef HAVE_FORK
-    PyObject *before_forkers;
-    PyObject *after_forkers_parent;
-    PyObject *after_forkers_child;
-#endif
-    /* AtExit module */
-    void (*pyexitfunc)(PyObject *);
-    PyObject *pyexitmodule;
+PyAPI_FUNC(_PyCoreConfig *) _PyInterpreterState_GetCoreConfig(PyInterpreterState *);
+PyAPI_FUNC(_PyMainInterpreterConfig *) _PyInterpreterState_GetMainConfig(PyInterpreterState *);
 
-    uint64_t tstate_next_unique_id;
-} PyInterpreterState;
 
 /* State unique per thread */
 
@@ -122,7 +66,8 @@ typedef struct _err_stackitem {
 } _PyErr_StackItem;
 
 
-typedef struct _ts {
+// The PyThreadState typedef is in Include/pystate.h.
+struct _ts {
     /* See Python/ceval.c for comments explaining most fields */
 
     struct _ts *prev;
@@ -214,7 +159,7 @@ typedef struct _ts {
 
     /* XXX signal handlers should also be here */
 
-} PyThreadState;
+};
 
 /* Get the current interpreter state.
 

Include/internal/pycore_pystate.h

@@ -17,37 +17,74 @@ extern "C" {
 #include "pycore_warnings.h"
 
 
-/* GIL state */
+/* interpreter state */
 
-struct _gilstate_runtime_state {
-    int check_enabled;
-    /* Assuming the current thread holds the GIL, this is the
-       PyThreadState for the current thread. */
-    _Py_atomic_address tstate_current;
-    PyThreadFrameGetter getframe;
-    /* The single PyInterpreterState used by this process'
-       GILState implementation
-    */
-    /* TODO: Given interp_main, it may be possible to kill this ref */
-    PyInterpreterState *autoInterpreterState;
-    Py_tss_t autoTSSkey;
-};
+typedef PyObject* (*_PyFrameEvalFunction)(struct _frame *, int);
+
+// The PyInterpreterState typedef is in Include/pystate.h.
+struct _is {
+
+    struct _is *next;
+    struct _ts *tstate_head;
+
+    int64_t id;
+    int64_t id_refcount;
+    PyThread_type_lock id_mutex;
+
+    PyObject *modules;
+    PyObject *modules_by_index;
+    PyObject *sysdict;
+    PyObject *builtins;
+    PyObject *importlib;
+
+    /* Used in Python/sysmodule.c. */
+    int check_interval;
+
+    /* Used in Modules/_threadmodule.c. */
+    long num_threads;
+    /* Support for runtime thread stack size tuning.
+       A value of 0 means using the platform's default stack size
+       or the size specified by the THREAD_STACK_SIZE macro. */
+    /* Used in Python/thread.c. */
+    size_t pythread_stacksize;
+
+    PyObject *codec_search_path;
+    PyObject *codec_search_cache;
+    PyObject *codec_error_registry;
+    int codecs_initialized;
+    int fscodec_initialized;
+
+    _PyCoreConfig core_config;
+    _PyMainInterpreterConfig config;
+#ifdef HAVE_DLOPEN
+    int dlopenflags;
+#endif
 
-/* hook for PyEval_GetFrame(), requested for Psyco */
-#define _PyThreadState_GetFrame _PyRuntime.gilstate.getframe
+    PyObject *builtins_copy;
+    PyObject *import_func;
+    /* Initialized to PyEval_EvalFrameDefault(). */
+    _PyFrameEvalFunction eval_frame;
 
-/* Issue #26558: Flag to disable PyGILState_Check().
-   If set to non-zero, PyGILState_Check() always return 1. */
-#define _PyGILState_check_enabled _PyRuntime.gilstate.check_enabled
+    Py_ssize_t co_extra_user_count;
+    freefunc co_extra_freefuncs[MAX_CO_EXTRA_USERS];
 
+#ifdef HAVE_FORK
+    PyObject *before_forkers;
+    PyObject *after_forkers_parent;
+    PyObject *after_forkers_child;
+#endif
+    /* AtExit module */
+    void (*pyexitfunc)(PyObject *);
+    PyObject *pyexitmodule;
 
-/* interpreter state */
+    uint64_t tstate_next_unique_id;
+};
 
-PyAPI_FUNC(PyInterpreterState *) _PyInterpreterState_LookUpID(PY_INT64_T);
+PyAPI_FUNC(struct _is*) _PyInterpreterState_LookUpID(PY_INT64_T);
 
-PyAPI_FUNC(int) _PyInterpreterState_IDInitref(PyInterpreterState *);
-PyAPI_FUNC(void) _PyInterpreterState_IDIncref(PyInterpreterState *);
-PyAPI_FUNC(void) _PyInterpreterState_IDDecref(PyInterpreterState *);
+PyAPI_FUNC(int) _PyInterpreterState_IDInitref(struct _is *);
+PyAPI_FUNC(void) _PyInterpreterState_IDIncref(struct _is *);
+PyAPI_FUNC(void) _PyInterpreterState_IDDecref(struct _is *);
 
 
 /* cross-interpreter data */
@@ -119,6 +156,30 @@ struct _xidregitem {
 };
 
 
+/* GIL state */
+
+struct _gilstate_runtime_state {
+    int check_enabled;
+    /* Assuming the current thread holds the GIL, this is the
+       PyThreadState for the current thread. */
+    _Py_atomic_address tstate_current;
+    PyThreadFrameGetter getframe;
+    /* The single PyInterpreterState used by this process'
+       GILState implementation
+    */
+    /* TODO: Given interp_main, it may be possible to kill this ref */
+    PyInterpreterState *autoInterpreterState;
+    Py_tss_t autoTSSkey;
+};
+
+/* hook for PyEval_GetFrame(), requested for Psyco */
+#define _PyThreadState_GetFrame _PyRuntime.gilstate.getframe
+
+/* Issue #26558: Flag to disable PyGILState_Check().
+   If set to non-zero, PyGILState_Check() always return 1. */
+#define _PyGILState_check_enabled _PyRuntime.gilstate.check_enabled
+
+
 /* Full Python runtime state */
 
 typedef struct pyruntimestate {

Include/pystate.h

@@ -20,22 +20,20 @@ struct _frame;
 struct _ts;
 struct _is;
 
-#ifdef Py_LIMITED_API
+/* struct _ts is defined in cpython/pystate.h */
 typedef struct _ts PyThreadState;
+/* struct _is is defined in internal/pycore_pystate.h */
 typedef struct _is PyInterpreterState;
-#else
-/* PyThreadState and PyInterpreterState are defined in cpython/pystate.h */
-#endif
 
 /* State unique per thread */
 
-PyAPI_FUNC(struct _is *) PyInterpreterState_New(void);
-PyAPI_FUNC(void) PyInterpreterState_Clear(struct _is *);
-PyAPI_FUNC(void) PyInterpreterState_Delete(struct _is *);
+PyAPI_FUNC(PyInterpreterState *) PyInterpreterState_New(void);
+PyAPI_FUNC(void) PyInterpreterState_Clear(PyInterpreterState *);
+PyAPI_FUNC(void) PyInterpreterState_Delete(PyInterpreterState *);
 
 #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03070000
 /* New in 3.7 */
-PyAPI_FUNC(int64_t) PyInterpreterState_GetID(struct _is *);
+PyAPI_FUNC(int64_t) PyInterpreterState_GetID(PyInterpreterState *);
 #endif
 #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03030000
 /* New in 3.3 */
@@ -44,9 +42,9 @@ PyAPI_FUNC(int) PyState_RemoveModule(struct PyModuleDef*);
 #endif
 PyAPI_FUNC(PyObject*) PyState_FindModule(struct PyModuleDef*);
 
-PyAPI_FUNC(struct _ts *) PyThreadState_New(struct _is *);
-PyAPI_FUNC(void) PyThreadState_Clear(struct _ts *);
-PyAPI_FUNC(void) PyThreadState_Delete(struct _ts *);
+PyAPI_FUNC(PyThreadState *) PyThreadState_New(PyInterpreterState *);
+PyAPI_FUNC(void) PyThreadState_Clear(PyThreadState *);
+PyAPI_FUNC(void) PyThreadState_Delete(PyThreadState *);
 PyAPI_FUNC(void) PyThreadState_DeleteCurrent(void);
 
 /* Get the current thread state.
@@ -57,7 +55,7 @@ PyAPI_FUNC(void) PyThreadState_DeleteCurrent(void);
    The caller must hold the GIL.
 
    See also PyThreadState_GET() and _PyThreadState_GET(). */
-PyAPI_FUNC(struct _ts *) PyThreadState_Get(void);
+PyAPI_FUNC(PyThreadState *) PyThreadState_Get(void);
 
 /* Get the current Python thread state.
 
@@ -70,7 +68,7 @@ PyAPI_FUNC(struct _ts *) PyThreadState_Get(void);
    See also PyThreadState_Get() and _PyThreadState_GET(). */
 #define PyThreadState_GET() PyThreadState_Get()
 
-PyAPI_FUNC(struct _ts *) PyThreadState_Swap(struct _ts *);
+PyAPI_FUNC(PyThreadState *) PyThreadState_Swap(PyThreadState *);
 PyAPI_FUNC(PyObject *) PyThreadState_GetDict(void);
 PyAPI_FUNC(int) PyThreadState_SetAsyncExc(unsigned long, PyObject *);
 
@@ -118,7 +116,7 @@ PyAPI_FUNC(void) PyGILState_Release(PyGILState_STATE);
    thread-state, even if no auto-thread-state call has been made
    on the main thread.
 */
-PyAPI_FUNC(struct _ts *) PyGILState_GetThisThreadState(void);
+PyAPI_FUNC(PyThreadState *) PyGILState_GetThisThreadState(void);
 
 
 #ifndef Py_LIMITED_API

Misc/NEWS.d/next/Core and Builtins/2019-02-01-18-12-14.bpo-35886.0Z-C0V.rst

@@ -0,0 +1,2 @@
+The implementation of PyInterpreterState has been moved into the internal
+header files (guarded by Py_BUILD_CORE).

Modules/_testcapimodule.c

@@ -4685,7 +4685,7 @@ static PyObject *
 get_core_config(PyObject *self, PyObject *Py_UNUSED(args))
 {
     PyInterpreterState *interp = _PyInterpreterState_Get();
-    const _PyCoreConfig *config = &interp->core_config;
+    const _PyCoreConfig *config = _PyInterpreterState_GetCoreConfig(interp);
     return _PyCoreConfig_AsDict(config);
 }
 
@@ -4694,7 +4694,7 @@ static PyObject *
 get_main_config(PyObject *self, PyObject *Py_UNUSED(args))
 {
     PyInterpreterState *interp = _PyInterpreterState_Get();
-    const _PyMainInterpreterConfig *config = &interp->config;
+    const _PyMainInterpreterConfig *config = _PyInterpreterState_GetMainConfig(interp);
     return _PyMainInterpreterConfig_AsDict(config);
 }
 

Programs/_testembed.c

@@ -315,7 +315,7 @@ dump_config_impl(void)
 
     /* core config */
     PyInterpreterState *interp = _PyInterpreterState_Get();
-    const _PyCoreConfig *core_config = &interp->core_config;
+    const _PyCoreConfig *core_config = _PyInterpreterState_GetCoreConfig(interp);
     dict = _PyCoreConfig_AsDict(core_config);
     if (dict == NULL) {
         goto error;
@@ -326,7 +326,7 @@ dump_config_impl(void)
     Py_CLEAR(dict);
 
     /* main config */
-    const _PyMainInterpreterConfig *main_config = &interp->config;
+    const _PyMainInterpreterConfig *main_config = _PyInterpreterState_GetMainConfig(interp);
     dict = _PyMainInterpreterConfig_AsDict(main_config);
     if (dict == NULL) {
         goto error;

Python/pystate.c

@@ -407,6 +407,17 @@ _PyInterpreterState_IDDecref(PyInterpreterState *interp)
     }
 }
 
+_PyCoreConfig *
+_PyInterpreterState_GetCoreConfig(PyInterpreterState *interp)
+{
+    return &interp->core_config;
+}
+
+_PyMainInterpreterConfig *
+_PyInterpreterState_GetMainConfig(PyInterpreterState *interp)
+{
+    return &interp->config;
+}
 
 /* Default implementation for _PyThreadState_GetFrame */
 static struct _frame *