Skip to content

Commit c6fe086

Browse files
gh-76785: Move the Cross-Interpreter Code to Its Own File (gh-111502)
This is partly to clear this stuff out of pystate.c, but also in preparation for moving some code out of _xxsubinterpretersmodule.c. This change also moves this stuff to the internal API (new: Include/internal/pycore_crossinterp.h). @vstinner did this previously and I undid it. Now I'm re-doing it. :/
1 parent 7b153d1 commit c6fe086

18 files changed

+849
-760
lines changed

Include/cpython/pystate.h

-77
Original file line numberDiff line numberDiff line change
@@ -258,80 +258,3 @@ PyAPI_FUNC(_PyFrameEvalFunction) _PyInterpreterState_GetEvalFrameFunc(
258258
PyAPI_FUNC(void) _PyInterpreterState_SetEvalFrameFunc(
259259
PyInterpreterState *interp,
260260
_PyFrameEvalFunction eval_frame);
261-
262-
263-
/* cross-interpreter data */
264-
265-
// _PyCrossInterpreterData is similar to Py_buffer as an effectively
266-
// opaque struct that holds data outside the object machinery. This
267-
// is necessary to pass safely between interpreters in the same process.
268-
typedef struct _xid _PyCrossInterpreterData;
269-
270-
typedef PyObject *(*xid_newobjectfunc)(_PyCrossInterpreterData *);
271-
typedef void (*xid_freefunc)(void *);
272-
273-
struct _xid {
274-
// data is the cross-interpreter-safe derivation of a Python object
275-
// (see _PyObject_GetCrossInterpreterData). It will be NULL if the
276-
// new_object func (below) encodes the data.
277-
void *data;
278-
// obj is the Python object from which the data was derived. This
279-
// is non-NULL only if the data remains bound to the object in some
280-
// way, such that the object must be "released" (via a decref) when
281-
// the data is released. In that case the code that sets the field,
282-
// likely a registered "crossinterpdatafunc", is responsible for
283-
// ensuring it owns the reference (i.e. incref).
284-
PyObject *obj;
285-
// interp is the ID of the owning interpreter of the original
286-
// object. It corresponds to the active interpreter when
287-
// _PyObject_GetCrossInterpreterData() was called. This should only
288-
// be set by the cross-interpreter machinery.
289-
//
290-
// We use the ID rather than the PyInterpreterState to avoid issues
291-
// with deleted interpreters. Note that IDs are never re-used, so
292-
// each one will always correspond to a specific interpreter
293-
// (whether still alive or not).
294-
int64_t interpid;
295-
// new_object is a function that returns a new object in the current
296-
// interpreter given the data. The resulting object (a new
297-
// reference) will be equivalent to the original object. This field
298-
// is required.
299-
xid_newobjectfunc new_object;
300-
// free is called when the data is released. If it is NULL then
301-
// nothing will be done to free the data. For some types this is
302-
// okay (e.g. bytes) and for those types this field should be set
303-
// to NULL. However, for most the data was allocated just for
304-
// cross-interpreter use, so it must be freed when
305-
// _PyCrossInterpreterData_Release is called or the memory will
306-
// leak. In that case, at the very least this field should be set
307-
// to PyMem_RawFree (the default if not explicitly set to NULL).
308-
// The call will happen with the original interpreter activated.
309-
xid_freefunc free;
310-
};
311-
312-
PyAPI_FUNC(void) _PyCrossInterpreterData_Init(
313-
_PyCrossInterpreterData *data,
314-
PyInterpreterState *interp, void *shared, PyObject *obj,
315-
xid_newobjectfunc new_object);
316-
PyAPI_FUNC(int) _PyCrossInterpreterData_InitWithSize(
317-
_PyCrossInterpreterData *,
318-
PyInterpreterState *interp, const size_t, PyObject *,
319-
xid_newobjectfunc);
320-
PyAPI_FUNC(void) _PyCrossInterpreterData_Clear(
321-
PyInterpreterState *, _PyCrossInterpreterData *);
322-
323-
PyAPI_FUNC(int) _PyObject_GetCrossInterpreterData(PyObject *, _PyCrossInterpreterData *);
324-
PyAPI_FUNC(PyObject *) _PyCrossInterpreterData_NewObject(_PyCrossInterpreterData *);
325-
PyAPI_FUNC(int) _PyCrossInterpreterData_Release(_PyCrossInterpreterData *);
326-
PyAPI_FUNC(int) _PyCrossInterpreterData_ReleaseAndRawFree(_PyCrossInterpreterData *);
327-
328-
PyAPI_FUNC(int) _PyObject_CheckCrossInterpreterData(PyObject *);
329-
330-
/* cross-interpreter data registry */
331-
332-
typedef int (*crossinterpdatafunc)(PyThreadState *tstate, PyObject *,
333-
_PyCrossInterpreterData *);
334-
335-
PyAPI_FUNC(int) _PyCrossInterpreterData_RegisterClass(PyTypeObject *, crossinterpdatafunc);
336-
PyAPI_FUNC(int) _PyCrossInterpreterData_UnregisterClass(PyTypeObject *);
337-
PyAPI_FUNC(crossinterpdatafunc) _PyCrossInterpreterData_Lookup(PyObject *);

Include/internal/pycore_ceval.h

-10
Original file line numberDiff line numberDiff line change
@@ -54,16 +54,6 @@ PyAPI_FUNC(int) _PyEval_AddPendingCall(
5454
void *arg,
5555
int flags);
5656

57-
typedef int (*_Py_simple_func)(void *);
58-
extern int _Py_CallInInterpreter(
59-
PyInterpreterState *interp,
60-
_Py_simple_func func,
61-
void *arg);
62-
extern int _Py_CallInInterpreterAndRawFree(
63-
PyInterpreterState *interp,
64-
_Py_simple_func func,
65-
void *arg);
66-
6757
extern void _PyEval_SignalAsyncExc(PyInterpreterState *interp);
6858
#ifdef HAVE_FORK
6959
extern PyStatus _PyEval_ReInitThreads(PyThreadState *tstate);

Include/internal/pycore_crossinterp.h

+139
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,139 @@
1+
#ifndef Py_INTERNAL_CROSSINTERP_H
2+
#define Py_INTERNAL_CROSSINTERP_H
3+
#ifdef __cplusplus
4+
extern "C" {
5+
#endif
6+
7+
#ifndef Py_BUILD_CORE
8+
# error "this header requires Py_BUILD_CORE define"
9+
#endif
10+
11+
12+
/***************************/
13+
/* cross-interpreter calls */
14+
/***************************/
15+
16+
typedef int (*_Py_simple_func)(void *);
17+
extern int _Py_CallInInterpreter(
18+
PyInterpreterState *interp,
19+
_Py_simple_func func,
20+
void *arg);
21+
extern int _Py_CallInInterpreterAndRawFree(
22+
PyInterpreterState *interp,
23+
_Py_simple_func func,
24+
void *arg);
25+
26+
27+
/**************************/
28+
/* cross-interpreter data */
29+
/**************************/
30+
31+
typedef struct _xid _PyCrossInterpreterData;
32+
typedef PyObject *(*xid_newobjectfunc)(_PyCrossInterpreterData *);
33+
typedef void (*xid_freefunc)(void *);
34+
35+
// _PyCrossInterpreterData is similar to Py_buffer as an effectively
36+
// opaque struct that holds data outside the object machinery. This
37+
// is necessary to pass safely between interpreters in the same process.
38+
struct _xid {
39+
// data is the cross-interpreter-safe derivation of a Python object
40+
// (see _PyObject_GetCrossInterpreterData). It will be NULL if the
41+
// new_object func (below) encodes the data.
42+
void *data;
43+
// obj is the Python object from which the data was derived. This
44+
// is non-NULL only if the data remains bound to the object in some
45+
// way, such that the object must be "released" (via a decref) when
46+
// the data is released. In that case the code that sets the field,
47+
// likely a registered "crossinterpdatafunc", is responsible for
48+
// ensuring it owns the reference (i.e. incref).
49+
PyObject *obj;
50+
// interp is the ID of the owning interpreter of the original
51+
// object. It corresponds to the active interpreter when
52+
// _PyObject_GetCrossInterpreterData() was called. This should only
53+
// be set by the cross-interpreter machinery.
54+
//
55+
// We use the ID rather than the PyInterpreterState to avoid issues
56+
// with deleted interpreters. Note that IDs are never re-used, so
57+
// each one will always correspond to a specific interpreter
58+
// (whether still alive or not).
59+
int64_t interpid;
60+
// new_object is a function that returns a new object in the current
61+
// interpreter given the data. The resulting object (a new
62+
// reference) will be equivalent to the original object. This field
63+
// is required.
64+
xid_newobjectfunc new_object;
65+
// free is called when the data is released. If it is NULL then
66+
// nothing will be done to free the data. For some types this is
67+
// okay (e.g. bytes) and for those types this field should be set
68+
// to NULL. However, for most the data was allocated just for
69+
// cross-interpreter use, so it must be freed when
70+
// _PyCrossInterpreterData_Release is called or the memory will
71+
// leak. In that case, at the very least this field should be set
72+
// to PyMem_RawFree (the default if not explicitly set to NULL).
73+
// The call will happen with the original interpreter activated.
74+
xid_freefunc free;
75+
};
76+
77+
PyAPI_FUNC(_PyCrossInterpreterData *) _PyCrossInterpreterData_New(void);
78+
PyAPI_FUNC(void) _PyCrossInterpreterData_Free(_PyCrossInterpreterData *data);
79+
80+
81+
/* defining cross-interpreter data */
82+
83+
PyAPI_FUNC(void) _PyCrossInterpreterData_Init(
84+
_PyCrossInterpreterData *data,
85+
PyInterpreterState *interp, void *shared, PyObject *obj,
86+
xid_newobjectfunc new_object);
87+
PyAPI_FUNC(int) _PyCrossInterpreterData_InitWithSize(
88+
_PyCrossInterpreterData *,
89+
PyInterpreterState *interp, const size_t, PyObject *,
90+
xid_newobjectfunc);
91+
PyAPI_FUNC(void) _PyCrossInterpreterData_Clear(
92+
PyInterpreterState *, _PyCrossInterpreterData *);
93+
94+
95+
/* using cross-interpreter data */
96+
97+
PyAPI_FUNC(int) _PyObject_CheckCrossInterpreterData(PyObject *);
98+
PyAPI_FUNC(int) _PyObject_GetCrossInterpreterData(PyObject *, _PyCrossInterpreterData *);
99+
PyAPI_FUNC(PyObject *) _PyCrossInterpreterData_NewObject(_PyCrossInterpreterData *);
100+
PyAPI_FUNC(int) _PyCrossInterpreterData_Release(_PyCrossInterpreterData *);
101+
PyAPI_FUNC(int) _PyCrossInterpreterData_ReleaseAndRawFree(_PyCrossInterpreterData *);
102+
103+
104+
/* cross-interpreter data registry */
105+
106+
// For now we use a global registry of shareable classes. An
107+
// alternative would be to add a tp_* slot for a class's
108+
// crossinterpdatafunc. It would be simpler and more efficient.
109+
110+
typedef int (*crossinterpdatafunc)(PyThreadState *tstate, PyObject *,
111+
_PyCrossInterpreterData *);
112+
113+
struct _xidregitem;
114+
115+
struct _xidregitem {
116+
struct _xidregitem *prev;
117+
struct _xidregitem *next;
118+
/* This can be a dangling pointer, but only if weakref is set. */
119+
PyTypeObject *cls;
120+
/* This is NULL for builtin types. */
121+
PyObject *weakref;
122+
size_t refcount;
123+
crossinterpdatafunc getdata;
124+
};
125+
126+
struct _xidregistry {
127+
PyThread_type_lock mutex;
128+
struct _xidregitem *head;
129+
};
130+
131+
PyAPI_FUNC(int) _PyCrossInterpreterData_RegisterClass(PyTypeObject *, crossinterpdatafunc);
132+
PyAPI_FUNC(int) _PyCrossInterpreterData_UnregisterClass(PyTypeObject *);
133+
PyAPI_FUNC(crossinterpdatafunc) _PyCrossInterpreterData_Lookup(PyObject *);
134+
135+
136+
#ifdef __cplusplus
137+
}
138+
#endif
139+
#endif /* !Py_INTERNAL_CROSSINTERP_H */

Include/internal/pycore_interp.h

+1-22
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ extern "C" {
1515
#include "pycore_ceval_state.h" // struct _ceval_state
1616
#include "pycore_code.h" // struct callable_cache
1717
#include "pycore_context.h" // struct _Py_context_state
18+
#include "pycore_crossinterp.h" // struct _xidregistry
1819
#include "pycore_dict_state.h" // struct _Py_dict_state
1920
#include "pycore_dtoa.h" // struct _dtoa_state
2021
#include "pycore_exceptions.h" // struct _Py_exc_state
@@ -41,28 +42,6 @@ struct _Py_long_state {
4142

4243
/* cross-interpreter data registry */
4344

44-
/* For now we use a global registry of shareable classes. An
45-
alternative would be to add a tp_* slot for a class's
46-
crossinterpdatafunc. It would be simpler and more efficient. */
47-
48-
struct _xidregitem;
49-
50-
struct _xidregitem {
51-
struct _xidregitem *prev;
52-
struct _xidregitem *next;
53-
/* This can be a dangling pointer, but only if weakref is set. */
54-
PyTypeObject *cls;
55-
/* This is NULL for builtin types. */
56-
PyObject *weakref;
57-
size_t refcount;
58-
crossinterpdatafunc getdata;
59-
};
60-
61-
struct _xidregistry {
62-
PyThread_type_lock mutex;
63-
struct _xidregitem *head;
64-
};
65-
6645

6746
/* interpreter state */
6847

Include/internal/pycore_runtime.h

+1
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ extern "C" {
1010

1111
#include "pycore_atexit.h" // struct _atexit_runtime_state
1212
#include "pycore_ceval_state.h" // struct _ceval_runtime_state
13+
#include "pycore_crossinterp.h" // struct _xidregistry
1314
#include "pycore_faulthandler.h" // struct _faulthandler_runtime_state
1415
#include "pycore_floatobject.h" // struct _Py_float_runtime_state
1516
#include "pycore_import.h" // struct _import_runtime_state

Lib/test/test__xxsubinterpreters.py

+6-6
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
import threading
88
import unittest
99

10-
import _testcapi
10+
import _testinternalcapi
1111
from test import support
1212
from test.support import import_helper
1313
from test.support import script_helper
@@ -146,17 +146,17 @@ class ShareableTypeTests(unittest.TestCase):
146146
def _assert_values(self, values):
147147
for obj in values:
148148
with self.subTest(obj):
149-
xid = _testcapi.get_crossinterp_data(obj)
150-
got = _testcapi.restore_crossinterp_data(xid)
149+
xid = _testinternalcapi.get_crossinterp_data(obj)
150+
got = _testinternalcapi.restore_crossinterp_data(xid)
151151

152152
self.assertEqual(got, obj)
153153
self.assertIs(type(got), type(obj))
154154

155155
def test_singletons(self):
156156
for obj in [None]:
157157
with self.subTest(obj):
158-
xid = _testcapi.get_crossinterp_data(obj)
159-
got = _testcapi.restore_crossinterp_data(xid)
158+
xid = _testinternalcapi.get_crossinterp_data(obj)
159+
got = _testinternalcapi.restore_crossinterp_data(xid)
160160

161161
# XXX What about between interpreters?
162162
self.assertIs(got, obj)
@@ -187,7 +187,7 @@ def test_non_shareable_int(self):
187187
for i in ints:
188188
with self.subTest(i):
189189
with self.assertRaises(OverflowError):
190-
_testcapi.get_crossinterp_data(i)
190+
_testinternalcapi.get_crossinterp_data(i)
191191

192192

193193
class ModuleTests(TestBase):

Makefile.pre.in

+2
Original file line numberDiff line numberDiff line change
@@ -409,6 +409,7 @@ PYTHON_OBJS= \
409409
Python/codecs.o \
410410
Python/compile.o \
411411
Python/context.o \
412+
Python/crossinterp.o \
412413
Python/dynamic_annotations.o \
413414
Python/errors.o \
414415
Python/executor.o \
@@ -1800,6 +1801,7 @@ PYTHON_HEADERS= \
18001801
$(srcdir)/Include/internal/pycore_complexobject.h \
18011802
$(srcdir)/Include/internal/pycore_condvar.h \
18021803
$(srcdir)/Include/internal/pycore_context.h \
1804+
$(srcdir)/Include/internal/pycore_crossinterp.h \
18031805
$(srcdir)/Include/internal/pycore_dict.h \
18041806
$(srcdir)/Include/internal/pycore_dict_state.h \
18051807
$(srcdir)/Include/internal/pycore_descrobject.h \

0 commit comments

Comments
 (0)