From be93a283167187478ba215748f1ebb0d31dcfd14 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 14 Dec 2016 06:52:38 -0700 Subject: [PATCH 01/78] Add the _interpreters module to the stdlib. --- Doc/library/_interpreters.rst | 37 +++++++++++++++++++++++++++++ Lib/test/test__interpreters.py | 11 +++++++++ Modules/_interpretersmodule.c | 43 ++++++++++++++++++++++++++++++++++ setup.py | 3 +++ 4 files changed, 94 insertions(+) create mode 100644 Doc/library/_interpreters.rst create mode 100644 Lib/test/test__interpreters.py create mode 100644 Modules/_interpretersmodule.c diff --git a/Doc/library/_interpreters.rst b/Doc/library/_interpreters.rst new file mode 100644 index 00000000000000..db0f2ef3942945 --- /dev/null +++ b/Doc/library/_interpreters.rst @@ -0,0 +1,37 @@ +:mod:`_interpreters` --- Low-level interpreters API +=================================================== + +.. module:: _interpreters + :synopsis: Low-level interpreters API. + +.. versionadded:: 3,7 + + :ref:`_sub-interpreter-support` + +threading + +-------------- + +This module provides low-level primitives for working with multiple +Python interpreters in the same process. + +.. XXX The :mod:`interpreters` module provides an easier to use and + higher-level API built on top of this module. + +This module is optional. It is provided by Python implementations which +support multiple interpreters. + +.. XXX For systems lacking the :mod:`_interpreters` module, the + :mod:`_dummy_interpreters` module is available. It duplicates this + module's interface and can be used as a drop-in replacement. + +It defines the following functions: + + +.. XXX TBD + + +**Caveats:** + +* ... + diff --git a/Lib/test/test__interpreters.py b/Lib/test/test__interpreters.py new file mode 100644 index 00000000000000..157ef203b74907 --- /dev/null +++ b/Lib/test/test__interpreters.py @@ -0,0 +1,11 @@ +import unittest +from test import support +interpreters = support.import_module('_interpreters') + + +class InterpretersTests(unittest.TestCase): + pass + + +if __name__ == "__main__": + unittest.main() diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c new file mode 100644 index 00000000000000..1103e1e37c48f1 --- /dev/null +++ b/Modules/_interpretersmodule.c @@ -0,0 +1,43 @@ + +/* interpreters module */ +/* low-level access to interpreter primitives */ + +#include "Python.h" + + +static PyMethodDef module_functions[] = { + {NULL, NULL} /* sentinel */ +}; + + +/* initialization function */ + +PyDoc_STRVAR(module_doc, +"This module provides primitive operations to manage Python interpreters.\n\ +The 'interpreters' module provides a more convenient interface."); + +static struct PyModuleDef interpretersmodule = { + PyModuleDef_HEAD_INIT, + "_interpreters", + module_doc, + -1, + module_functions, + NULL, + NULL, + NULL, + NULL +}; + + +PyMODINIT_FUNC +PyInit__interpreters(void) +{ + PyObject *module; + + module = PyModule_Create(&interpretersmodule); + if (module == NULL) + return NULL; + + + return module; +} diff --git a/setup.py b/setup.py index 258094e3ada29a..2c5cf51bc1ee65 100644 --- a/setup.py +++ b/setup.py @@ -745,6 +745,9 @@ def detect_modules(self): ['_xxtestfuzz/_xxtestfuzz.c', '_xxtestfuzz/fuzzer.c']) ) + # Python interface to subinterpreter C-API. + exts.append(Extension('_interpreters', ['_interpretersmodule.c'])) + # # Here ends the simple stuff. From here on, modules need certain # libraries, are platform-specific, or present other surprises. From 6f7a120fbe6bce55ad221a0b6df120b603fc6a22 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 29 Dec 2016 15:28:53 -0700 Subject: [PATCH 02/78] Add create() and destroy(). --- Doc/library/_interpreters.rst | 13 ++- Lib/test/test__interpreters.py | 157 ++++++++++++++++++++++++++++++- Modules/_interpretersmodule.c | 167 ++++++++++++++++++++++++++++++++- 3 files changed, 334 insertions(+), 3 deletions(-) diff --git a/Doc/library/_interpreters.rst b/Doc/library/_interpreters.rst index db0f2ef3942945..259b3ff67f13c3 100644 --- a/Doc/library/_interpreters.rst +++ b/Doc/library/_interpreters.rst @@ -28,7 +28,18 @@ support multiple interpreters. It defines the following functions: -.. XXX TBD +.. function:: create() + + Initialize a new Python interpreter and return its identifier. The + interpreter will be created in the current thread and will remain + idle until something is run in it. + + +.. function:: destroy(id) + + Finalize and destroy the identified interpreter. + +.. XXX must not be running? **Caveats:** diff --git a/Lib/test/test__interpreters.py b/Lib/test/test__interpreters.py index 157ef203b74907..dd84db647bf1d5 100644 --- a/Lib/test/test__interpreters.py +++ b/Lib/test/test__interpreters.py @@ -1,10 +1,165 @@ +import threading import unittest + from test import support interpreters = support.import_module('_interpreters') class InterpretersTests(unittest.TestCase): - pass + + def setUp(self): + self.ids = [] + self.lock = threading.Lock() + + def tearDown(self): + for id in self.ids: + try: + interpreters.destroy(id) + except RuntimeError: + pass # already destroyed + + def _create(self): + id = interpreters.create() + self.ids.append(id) + return id + + def test_create_in_main(self): + id = interpreters.create() + self.ids.append(id) + + self.assertIn(id, interpreters._enumerate()) + + def test_create_unique_id(self): + seen = set() + for _ in range(100): + id = self._create() + interpreters.destroy(id) + seen.add(id) + + self.assertEqual(len(seen), 100) + + def test_create_in_thread(self): + id = None + def f(): + nonlocal id + id = interpreters.create() + self.ids.append(id) + self.lock.acquire() + self.lock.release() + + t = threading.Thread(target=f) + with self.lock: + t.start() + t.join() + self.assertIn(id, interpreters._enumerate()) + + @unittest.skip('waiting for run_string()') + def test_create_in_subinterpreter(self): + raise NotImplementedError + + @unittest.skip('waiting for run_string()') + def test_create_in_threaded_subinterpreter(self): + raise NotImplementedError + + def test_create_after_destroy_all(self): + before = set(interpreters._enumerate()) + # Create 3 subinterpreters. + ids = [] + for _ in range(3): + id = interpreters.create() + ids.append(id) + # Now destroy them. + for id in ids: + interpreters.destroy(id) + # Finally, create another. + id = interpreters.create() + self.ids.append(id) + self.assertEqual(set(interpreters._enumerate()), before | {id}) + + def test_create_after_destroy_some(self): + before = set(interpreters._enumerate()) + # Create 3 subinterpreters. + id1 = interpreters.create() + id2 = interpreters.create() + self.ids.append(id2) + id3 = interpreters.create() + # Now destroy 2 of them. + interpreters.destroy(id1) + interpreters.destroy(id3) + # Finally, create another. + id = interpreters.create() + self.ids.append(id) + self.assertEqual(set(interpreters._enumerate()), before | {id, id2}) + + def test_destroy_one(self): + id1 = self._create() + id2 = self._create() + id3 = self._create() + self.assertIn(id2, interpreters._enumerate()) + interpreters.destroy(id2) + self.assertNotIn(id2, interpreters._enumerate()) + self.assertIn(id1, interpreters._enumerate()) + self.assertIn(id3, interpreters._enumerate()) + + def test_destroy_all(self): + before = set(interpreters._enumerate()) + ids = set() + for _ in range(3): + id = self._create() + ids.add(id) + self.assertEqual(set(interpreters._enumerate()), before | ids) + for id in ids: + interpreters.destroy(id) + self.assertEqual(set(interpreters._enumerate()), before) + + def test_destroy_main(self): + main, = interpreters._enumerate() + with self.assertRaises(RuntimeError): + interpreters.destroy(main) + + def f(): + with self.assertRaises(RuntimeError): + interpreters.destroy(main) + + t = threading.Thread(target=f) + t.start() + t.join() + + def test_destroy_already_destroyed(self): + id = interpreters.create() + interpreters.destroy(id) + with self.assertRaises(RuntimeError): + interpreters.destroy(id) + + def test_destroy_does_not_exist(self): + with self.assertRaises(RuntimeError): + interpreters.destroy(1_000_000) + + def test_destroy_bad_id(self): + with self.assertRaises(RuntimeError): + interpreters.destroy(-1) + + @unittest.skip('waiting for run_string()') + def test_destroy_from_current(self): + raise NotImplementedError + + @unittest.skip('waiting for run_string()') + def test_destroy_from_sibling(self): + raise NotImplementedError + + def test_destroy_from_other_thread(self): + id = interpreters.create() + self.ids.append(id) + def f(): + interpreters.destroy(id) + + t = threading.Thread(target=f) + t.start() + t.join() + + @unittest.skip('waiting for run_string()') + def test_destroy_still_running(self): + raise NotImplementedError if __name__ == "__main__": diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index 1103e1e37c48f1..97235034e722e7 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -5,8 +5,173 @@ #include "Python.h" +static PyObject * +_get_id(PyInterpreterState *interp) +{ + unsigned long id = PyInterpreterState_GetID(interp); + if (id == 0 && PyErr_Occurred() != NULL) + return NULL; + return PyLong_FromUnsignedLong(id); +} + +static PyInterpreterState * +_look_up(PyObject *requested_id) +{ + PyObject * id; + PyInterpreterState *interp; + + interp = PyInterpreterState_Head(); + while (interp != NULL) { + id = _get_id(interp); + if (id == NULL) + return NULL; + if (requested_id == id) + return interp; + interp = PyInterpreterState_Next(interp); + } + + PyErr_Format(PyExc_RuntimeError, + "unrecognized interpreter ID %R", requested_id); + return NULL; +} + +static PyObject * +_get_current(void) +{ + PyThreadState *tstate; + PyInterpreterState *interp; + + tstate = PyThreadState_Get(); + if (tstate == NULL) + return NULL; + interp = tstate->interp; + + // get ID + return _get_id(interp); +} + + +/* module level code ********************************************************/ + +// XXX track count? + +static PyObject * +interp_create(PyObject *self, PyObject *args) +{ + if (!PyArg_UnpackTuple(args, "create", 0, 0)) + return NULL; + + // Create and initialize the new interpreter. + PyThreadState *tstate, *save_tstate; + save_tstate = PyThreadState_Swap(NULL); + tstate = Py_NewInterpreter(); + PyThreadState_Swap(save_tstate); + if (tstate == NULL) { + /* Since no new thread state was created, there is no exception to + propagate; raise a fresh one after swapping in the old thread + state. */ + PyErr_SetString(PyExc_RuntimeError, "interpreter creation failed"); + return NULL; + } + return _get_id(tstate->interp); +} + +PyDoc_STRVAR(create_doc, +"create() -> ID\n\ +\n\ +Create a new interpreter and return a unique generated ID."); + + +static PyObject * +interp_destroy(PyObject *self, PyObject *args) +{ + PyObject *id; + if (!PyArg_UnpackTuple(args, "destroy", 1, 1, &id)) + return NULL; + if (!PyLong_Check(id)) { + PyErr_SetString(PyExc_TypeError, "ID must be an int"); + return NULL; + } + + // Ensure the ID is not the current interpreter. + PyObject *current = _get_current(); + if (current == NULL) + return NULL; + if (PyObject_RichCompareBool(id, current, Py_EQ) != 0) { + PyErr_SetString(PyExc_RuntimeError, + "cannot destroy the current interpreter"); + return NULL; + } + + // Look up the interpreter. + PyInterpreterState *interp = _look_up(id); + if (interp == NULL) + return NULL; + + // Destroy the interpreter. + //PyInterpreterState_Delete(interp); + PyThreadState *tstate, *save_tstate; + tstate = PyInterpreterState_ThreadHead(interp); // XXX Is this the right one? + save_tstate = PyThreadState_Swap(tstate); + // XXX Stop current execution? + Py_EndInterpreter(tstate); // XXX Handle possible errors? + PyThreadState_Swap(save_tstate); + + Py_RETURN_NONE; +} + +PyDoc_STRVAR(destroy_doc, +"destroy(ID)\n\ +\n\ +Destroy the identified interpreter.\n\ +\n\ +Attempting to destroy the current interpreter results in a RuntimeError.\n\ +So does an unrecognized ID."); + + +static PyObject * +interp_enumerate(PyObject *self) +{ + PyObject *ids, *id; + PyInterpreterState *interp; + + // XXX Handle multiple main interpreters. + + ids = PyList_New(0); + if (ids == NULL) + return NULL; + + interp = PyInterpreterState_Head(); + while (interp != NULL) { + id = _get_id(interp); + if (id == NULL) + return NULL; + // insert at front of list + if (PyList_Insert(ids, 0, id) < 0) + return NULL; + + interp = PyInterpreterState_Next(interp); + } + + return ids; +} + +PyDoc_STRVAR(enumerate_doc, +"enumerate() -> [ID]\n\ +\n\ +Return a list containing the ID of every existing interpreter."); + + static PyMethodDef module_functions[] = { - {NULL, NULL} /* sentinel */ + {"create", (PyCFunction)interp_create, + METH_VARARGS, create_doc}, + {"destroy", (PyCFunction)interp_destroy, + METH_VARARGS, destroy_doc}, + + {"_enumerate", (PyCFunction)interp_enumerate, + METH_NOARGS, enumerate_doc}, + + {NULL, NULL} /* sentinel */ }; From 941efdd693036e5d54513edb70a6c78e7386cd3d Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Sat, 31 Dec 2016 18:49:38 -0700 Subject: [PATCH 03/78] Finish nearly all the create/destroy tests. --- Lib/test/test__interpreters.py | 162 +++++++++++++++++++--------- Modules/_interpretersmodule.c | 186 +++++++++++++++++++++++++++------ 2 files changed, 265 insertions(+), 83 deletions(-) diff --git a/Lib/test/test__interpreters.py b/Lib/test/test__interpreters.py index dd84db647bf1d5..c46ca0236b96d2 100644 --- a/Lib/test/test__interpreters.py +++ b/Lib/test/test__interpreters.py @@ -1,67 +1,108 @@ +import contextlib +import os import threading import unittest from test import support + interpreters = support.import_module('_interpreters') -class InterpretersTests(unittest.TestCase): +@contextlib.contextmanager +def _blocked(): + r, w = os.pipe() + wait_script = """if True: + import select + # Wait for a "done" signal. + select.select([{}], [], []) + + #import time + #time.sleep(1_000_000) + """.format(r) + try: + yield wait_script + finally: + os.write(w, b'') # release! + os.close(r) + os.close(w) + - def setUp(self): - self.ids = [] - self.lock = threading.Lock() +class TestBase(unittest.TestCase): def tearDown(self): - for id in self.ids: + for id in interpreters._enumerate(): + if id == 0: # main + continue try: interpreters.destroy(id) except RuntimeError: pass # already destroyed - def _create(self): - id = interpreters.create() - self.ids.append(id) - return id - def test_create_in_main(self): +class CreateTests(TestBase): + + def test_in_main(self): id = interpreters.create() - self.ids.append(id) self.assertIn(id, interpreters._enumerate()) - def test_create_unique_id(self): + def test_unique_id(self): seen = set() for _ in range(100): - id = self._create() + id = interpreters.create() interpreters.destroy(id) seen.add(id) self.assertEqual(len(seen), 100) - def test_create_in_thread(self): + def test_in_thread(self): + lock = threading.Lock() id = None def f(): nonlocal id id = interpreters.create() - self.ids.append(id) - self.lock.acquire() - self.lock.release() + lock.acquire() + lock.release() t = threading.Thread(target=f) - with self.lock: + with lock: t.start() t.join() self.assertIn(id, interpreters._enumerate()) - @unittest.skip('waiting for run_string()') - def test_create_in_subinterpreter(self): - raise NotImplementedError + def test_in_subinterpreter(self): + main, = interpreters._enumerate() + id = interpreters.create() + interpreters._run_string(id, """if True: + import _interpreters + id = _interpreters.create() + #_interpreters.create() + """) + + ids = interpreters._enumerate() + self.assertIn(id, ids) + self.assertIn(main, ids) + self.assertEqual(len(ids), 3) + + def test_in_threaded_subinterpreter(self): + main, = interpreters._enumerate() + id = interpreters.create() + def f(): + interpreters._run_string(id, """if True: + import _interpreters + _interpreters.create() + """) + + t = threading.Thread(target=f) + t.start() + t.join() - @unittest.skip('waiting for run_string()') - def test_create_in_threaded_subinterpreter(self): - raise NotImplementedError + ids = interpreters._enumerate() + self.assertIn(id, ids) + self.assertIn(main, ids) + self.assertEqual(len(ids), 3) - def test_create_after_destroy_all(self): + def test_after_destroy_all(self): before = set(interpreters._enumerate()) # Create 3 subinterpreters. ids = [] @@ -73,46 +114,46 @@ def test_create_after_destroy_all(self): interpreters.destroy(id) # Finally, create another. id = interpreters.create() - self.ids.append(id) self.assertEqual(set(interpreters._enumerate()), before | {id}) - def test_create_after_destroy_some(self): + def test_after_destroy_some(self): before = set(interpreters._enumerate()) # Create 3 subinterpreters. id1 = interpreters.create() id2 = interpreters.create() - self.ids.append(id2) id3 = interpreters.create() # Now destroy 2 of them. interpreters.destroy(id1) interpreters.destroy(id3) # Finally, create another. id = interpreters.create() - self.ids.append(id) self.assertEqual(set(interpreters._enumerate()), before | {id, id2}) - def test_destroy_one(self): - id1 = self._create() - id2 = self._create() - id3 = self._create() + +class DestroyTests(TestBase): + + def test_one(self): + id1 = interpreters.create() + id2 = interpreters.create() + id3 = interpreters.create() self.assertIn(id2, interpreters._enumerate()) interpreters.destroy(id2) self.assertNotIn(id2, interpreters._enumerate()) self.assertIn(id1, interpreters._enumerate()) self.assertIn(id3, interpreters._enumerate()) - def test_destroy_all(self): + def test_all(self): before = set(interpreters._enumerate()) ids = set() for _ in range(3): - id = self._create() + id = interpreters.create() ids.add(id) self.assertEqual(set(interpreters._enumerate()), before | ids) for id in ids: interpreters.destroy(id) self.assertEqual(set(interpreters._enumerate()), before) - def test_destroy_main(self): + def test_main(self): main, = interpreters._enumerate() with self.assertRaises(RuntimeError): interpreters.destroy(main) @@ -125,31 +166,40 @@ def f(): t.start() t.join() - def test_destroy_already_destroyed(self): + def test_already_destroyed(self): id = interpreters.create() interpreters.destroy(id) with self.assertRaises(RuntimeError): interpreters.destroy(id) - def test_destroy_does_not_exist(self): + def test_does_not_exist(self): with self.assertRaises(RuntimeError): interpreters.destroy(1_000_000) - def test_destroy_bad_id(self): + def test_bad_id(self): with self.assertRaises(RuntimeError): interpreters.destroy(-1) - @unittest.skip('waiting for run_string()') - def test_destroy_from_current(self): - raise NotImplementedError + def test_from_current(self): + id = interpreters.create() + with self.assertRaises(RuntimeError): + interpreters._run_string(id, """if True: + import _interpreters + _interpreters.destroy({}) + """.format(id)) - @unittest.skip('waiting for run_string()') - def test_destroy_from_sibling(self): - raise NotImplementedError + def test_from_sibling(self): + main, = interpreters._enumerate() + id1 = interpreters.create() + id2 = interpreters.create() + interpreters._run_string(id1, """if True: + import _interpreters + _interpreters.destroy({}) + """.format(id2)) + self.assertEqual(set(interpreters._enumerate()), {main, id1}) - def test_destroy_from_other_thread(self): + def test_from_other_thread(self): id = interpreters.create() - self.ids.append(id) def f(): interpreters.destroy(id) @@ -157,9 +207,21 @@ def f(): t.start() t.join() - @unittest.skip('waiting for run_string()') - def test_destroy_still_running(self): - raise NotImplementedError + @unittest.skip('not working yet') + def test_still_running(self): + main, = interpreters._enumerate() + id = interpreters.create() + def f(): + interpreters._run_string(id, wait_script) + + t = threading.Thread(target=f) + with _blocked() as wait_script: + t.start() + with self.assertRaises(RuntimeError): + interpreters.destroy(id) + + t.join() + self.assertEqual(set(interpreters._enumerate()), {main, id}) if __name__ == "__main__": diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index 97235034e722e7..ce44df269bacd2 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -3,53 +3,131 @@ /* low-level access to interpreter primitives */ #include "Python.h" +#include "frameobject.h" -static PyObject * -_get_id(PyInterpreterState *interp) +static PyInterpreterState * +_get_current(void) { - unsigned long id = PyInterpreterState_GetID(interp); - if (id == 0 && PyErr_Occurred() != NULL) + PyThreadState *tstate; + + tstate = PyThreadState_Get(); + if (tstate == NULL) return NULL; - return PyLong_FromUnsignedLong(id); + return tstate->interp; } static PyInterpreterState * -_look_up(PyObject *requested_id) +_look_up_int64(PY_INT64_T requested_id) { - PyObject * id; - PyInterpreterState *interp; + if (requested_id < 0) + goto error; - interp = PyInterpreterState_Head(); + PyInterpreterState *interp = PyInterpreterState_Head(); while (interp != NULL) { - id = _get_id(interp); - if (id == NULL) + PY_INT64_T id = PyInterpreterState_GetID(interp); + if (id < 0) return NULL; if (requested_id == id) return interp; interp = PyInterpreterState_Next(interp); } +error: PyErr_Format(PyExc_RuntimeError, - "unrecognized interpreter ID %R", requested_id); + "unrecognized interpreter ID %lld", requested_id); return NULL; } -static PyObject * -_get_current(void) +static PyInterpreterState * +_look_up(PyObject *requested_id) { - PyThreadState *tstate; - PyInterpreterState *interp; + long long id = PyLong_AsLongLong(requested_id); + if (id == -1 && PyErr_Occurred() != NULL) + return NULL; + // XXX Fail if larger than INT64_MAX? + return _look_up_int64(id); +} - tstate = PyThreadState_Get(); - if (tstate == NULL) +static PyObject * +_get_id(PyInterpreterState *interp) +{ + PY_INT64_T id = PyInterpreterState_GetID(interp); + if (id < 0) return NULL; - interp = tstate->interp; + return PyLong_FromLongLong(id); +} - // get ID - return _get_id(interp); +static int +_is_running(PyInterpreterState *interp) +{ + PyThreadState *tstate = PyInterpreterState_ThreadHead(interp); + if (PyThreadState_Next(tstate) != NULL) { + PyErr_SetString(PyExc_RuntimeError, + "interpreter has more than one thread"); + return -1; + } + PyFrameObject *frame = _PyThreadState_GetFrame(tstate); + if (frame == NULL) { + if (PyErr_Occurred() != NULL) + return -1; + return 0; + } + return (int)(frame->f_executing); } +static int +_ensure_not_running(PyInterpreterState *interp) +{ + int is_running = _is_running(interp); + if (is_running < 0) + return -1; + if (is_running) { + PyErr_Format(PyExc_RuntimeError, "interpreter already running"); + return -1; + } + return 0; +} + +static int +_run_string(PyInterpreterState *interp, const char *codestr) +{ + PyObject *result = NULL; + PyObject *exc = NULL, *value = NULL, *tb = NULL; + + if (_ensure_not_running(interp) < 0) + return -1; + + // Switch to interpreter. + PyThreadState *tstate = PyInterpreterState_ThreadHead(interp); + PyThreadState *save_tstate = PyThreadState_Swap(tstate); + + // Run the string (see PyRun_SimpleStringFlags). + // XXX How to handle sys.exit()? + PyObject *m = PyImport_AddModule("__main__"); + if (m == NULL) { + PyErr_Fetch(&exc, &value, &tb); + goto done; + } + PyObject *d = PyModule_GetDict(m); + result = PyRun_StringFlags(codestr, Py_file_input, d, d, NULL); + if (result == NULL) { + // Get the exception from the subinterpreter. + PyErr_Fetch(&exc, &value, &tb); + goto done; + } + Py_DECREF(result); // We throw away the result. + +done: + // Switch back. + if (save_tstate != NULL) + PyThreadState_Swap(save_tstate); + + // Propagate any exception out to the caller. + PyErr_Restore(exc, value, tb); + + return (result == NULL) ? -1 : 0; +} /* module level code ********************************************************/ @@ -93,19 +171,25 @@ interp_destroy(PyObject *self, PyObject *args) return NULL; } - // Ensure the ID is not the current interpreter. - PyObject *current = _get_current(); + // Look up the interpreter. + PyInterpreterState *interp = _look_up(id); + if (interp == NULL) + return NULL; + + // Ensure we don't try to destroy the current interpreter. + PyInterpreterState *current = _get_current(); if (current == NULL) return NULL; - if (PyObject_RichCompareBool(id, current, Py_EQ) != 0) { + if (interp == current) { PyErr_SetString(PyExc_RuntimeError, "cannot destroy the current interpreter"); return NULL; } - // Look up the interpreter. - PyInterpreterState *interp = _look_up(id); - if (interp == NULL) + // Ensure the interpreter isn't running. + /* XXX We *could* support destroying a running interpreter but + aren't going to worry about it for now. */ + if (_ensure_not_running(interp) < 0) return NULL; // Destroy the interpreter. @@ -156,11 +240,44 @@ interp_enumerate(PyObject *self) return ids; } -PyDoc_STRVAR(enumerate_doc, -"enumerate() -> [ID]\n\ -\n\ -Return a list containing the ID of every existing interpreter."); +static PyObject * +interp_run_string(PyObject *self, PyObject *args) +{ + PyObject *id, *code; + if (!PyArg_UnpackTuple(args, "run_string", 2, 2, &id, &code)) + return NULL; + if (!PyLong_Check(id)) { + PyErr_SetString(PyExc_TypeError, "first arg (ID) must be an int"); + return NULL; + } + if (!PyUnicode_Check(code)) { + PyErr_SetString(PyExc_TypeError, + "second arg (code) must be a string"); + return NULL; + } + // Look up the interpreter. + PyInterpreterState *interp = _look_up(id); + if (interp == NULL) + return NULL; + + // Extract code. + Py_ssize_t size; + const char *codestr = PyUnicode_AsUTF8AndSize(code, &size); + if (codestr == NULL) + return NULL; + if (strlen(codestr) != (size_t)size) { + PyErr_SetString(PyExc_ValueError, + "source code string cannot contain null bytes"); + return NULL; + } + + // Run the code in the interpreter. + if (_run_string(interp, codestr) < 0) + return NULL; + else + Py_RETURN_NONE; +} static PyMethodDef module_functions[] = { {"create", (PyCFunction)interp_create, @@ -168,8 +285,11 @@ static PyMethodDef module_functions[] = { {"destroy", (PyCFunction)interp_destroy, METH_VARARGS, destroy_doc}, - {"_enumerate", (PyCFunction)interp_enumerate, - METH_NOARGS, enumerate_doc}, + {"_enumerate", (PyCFunction)interp_enumerate, + METH_NOARGS, NULL}, + + {"_run_string", (PyCFunction)interp_run_string, + METH_VARARGS, NULL}, {NULL, NULL} /* sentinel */ }; From 95e734287ac06bbc4ad23e723129ba2a44507597 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 29 Dec 2016 15:32:18 -0700 Subject: [PATCH 04/78] Add run_string(). --- Doc/library/_interpreters.rst | 18 +++- Lib/test/test__interpreters.py | 190 ++++++++++++++++++++++++++++++--- Modules/_interpretersmodule.c | 11 +- 3 files changed, 203 insertions(+), 16 deletions(-) diff --git a/Doc/library/_interpreters.rst b/Doc/library/_interpreters.rst index 259b3ff67f13c3..8d05622f5af106 100644 --- a/Doc/library/_interpreters.rst +++ b/Doc/library/_interpreters.rst @@ -27,7 +27,6 @@ support multiple interpreters. It defines the following functions: - .. function:: create() Initialize a new Python interpreter and return its identifier. The @@ -42,6 +41,23 @@ It defines the following functions: .. XXX must not be running? +.. function:: run_string(id, command) + + A wrapper around :c:func:`PyRun_SimpleString` which runs the provided + Python program using the identified interpreter. Providing an + invalid or unknown ID results in a RuntimeError, likewise if the main + interpreter or any other running interpreter is used. + + Any value returned from the code is thrown away, similar to what + threads do. If the code results in an exception then that exception + is raised in the thread in which run_string() was called, similar to + how :func:`exec` works. This aligns with how interpreters are not + inherently threaded. + +.. XXX must not be running already? +.. XXX sys.exit() (and SystemExit) is swallowed? + + **Caveats:** * ... diff --git a/Lib/test/test__interpreters.py b/Lib/test/test__interpreters.py index c46ca0236b96d2..f6392ae7abaeab 100644 --- a/Lib/test/test__interpreters.py +++ b/Lib/test/test__interpreters.py @@ -1,5 +1,9 @@ import contextlib import os +import os.path +import shutil +import tempfile +from textwrap import dedent import threading import unittest @@ -11,14 +15,14 @@ @contextlib.contextmanager def _blocked(): r, w = os.pipe() - wait_script = """if True: + wait_script = dedent(""" import select # Wait for a "done" signal. select.select([{}], [], []) #import time #time.sleep(1_000_000) - """.format(r) + """).format(r) try: yield wait_script finally: @@ -73,11 +77,10 @@ def f(): def test_in_subinterpreter(self): main, = interpreters._enumerate() id = interpreters.create() - interpreters._run_string(id, """if True: + interpreters.run_string(id, dedent(""" import _interpreters id = _interpreters.create() - #_interpreters.create() - """) + """)) ids = interpreters._enumerate() self.assertIn(id, ids) @@ -88,10 +91,10 @@ def test_in_threaded_subinterpreter(self): main, = interpreters._enumerate() id = interpreters.create() def f(): - interpreters._run_string(id, """if True: + interpreters.run_string(id, dedent(""" import _interpreters _interpreters.create() - """) + """)) t = threading.Thread(target=f) t.start() @@ -102,6 +105,7 @@ def f(): self.assertIn(main, ids) self.assertEqual(len(ids), 3) + def test_after_destroy_all(self): before = set(interpreters._enumerate()) # Create 3 subinterpreters. @@ -183,19 +187,19 @@ def test_bad_id(self): def test_from_current(self): id = interpreters.create() with self.assertRaises(RuntimeError): - interpreters._run_string(id, """if True: + interpreters.run_string(id, dedent(""" import _interpreters _interpreters.destroy({}) - """.format(id)) + """).format(id)) def test_from_sibling(self): main, = interpreters._enumerate() id1 = interpreters.create() id2 = interpreters.create() - interpreters._run_string(id1, """if True: + interpreters.run_string(id1, dedent(""" import _interpreters _interpreters.destroy({}) - """.format(id2)) + """).format(id2)) self.assertEqual(set(interpreters._enumerate()), {main, id1}) def test_from_other_thread(self): @@ -212,7 +216,7 @@ def test_still_running(self): main, = interpreters._enumerate() id = interpreters.create() def f(): - interpreters._run_string(id, wait_script) + interpreters.run_string(id, wait_script) t = threading.Thread(target=f) with _blocked() as wait_script: @@ -224,5 +228,165 @@ def f(): self.assertEqual(set(interpreters._enumerate()), {main, id}) -if __name__ == "__main__": +class RunStringTests(TestBase): + + SCRIPT = dedent(""" + with open('{}', 'w') as out: + out.write('{}') + """) + FILENAME = 'spam' + + def setUp(self): + self.id = interpreters.create() + self.dirname = None + self.filename = None + + def tearDown(self): + if self.dirname is not None: + shutil.rmtree(self.dirname) + super().tearDown() + + def _resolve_filename(self, name=None): + if name is None: + name = self.FILENAME + if self.dirname is None: + self.dirname = tempfile.mkdtemp() + return os.path.join(self.dirname, name) + + def _empty_file(self): + self.filename = self._resolve_filename() + support.create_empty_file(self.filename) + return self.filename + + def assert_file_contains(self, expected, filename=None): + if filename is None: + filename = self.filename + self.assertIsNot(filename, None) + with open(filename) as out: + content = out.read() + self.assertEqual(content, expected) + + def test_success(self): + filename = self._empty_file() + expected = 'spam spam spam spam spam' + script = self.SCRIPT.format(filename, expected) + interpreters.run_string(self.id, script) + + self.assert_file_contains(expected) + + def test_in_thread(self): + filename = self._empty_file() + expected = 'spam spam spam spam spam' + script = self.SCRIPT.format(filename, expected) + def f(): + interpreters.run_string(self.id, script) + + t = threading.Thread(target=f) + t.start() + t.join() + + self.assert_file_contains(expected) + + def test_create_thread(self): + filename = self._empty_file() + expected = 'spam spam spam spam spam' + script = dedent(""" + import threading + def f(): + with open('{}', 'w') as out: + out.write('{}') + + t = threading.Thread(target=f) + t.start() + t.join() + """).format(filename, expected) + interpreters.run_string(self.id, script) + + self.assert_file_contains(expected) + + @unittest.skip('not working yet') + @unittest.skipUnless(hasattr(os, 'fork'), "test needs os.fork()") + def test_fork(self): + filename = self._empty_file() + expected = 'spam spam spam spam spam' + script = dedent(""" + import os + import sys + pid = os.fork() + if pid == 0: + with open('{}', 'w') as out: + out.write('{}') + sys.exit(0) + """).format(filename, expected) + interpreters.run_string(self.id, script) + + self.assert_file_contains(expected) + + @unittest.skip('not working yet') + def test_already_running(self): + def f(): + interpreters.run_string(self.id, wait_script) + + t = threading.Thread(target=f) + with _blocked() as wait_script: + t.start() + with self.assertRaises(RuntimeError): + interpreters.run_string(self.id, 'print("spam")') + t.join() + + def test_does_not_exist(self): + id = 0 + while id in interpreters._enumerate(): + id += 1 + with self.assertRaises(RuntimeError): + interpreters.run_string(id, 'print("spam")') + + def test_error_id(self): + with self.assertRaises(RuntimeError): + interpreters.run_string(-1, 'print("spam")') + + def test_bad_id(self): + with self.assertRaises(TypeError): + interpreters.run_string('spam', 'print("spam")') + + def test_bad_code(self): + with self.assertRaises(TypeError): + interpreters.run_string(self.id, 10) + + def test_bytes_for_code(self): + with self.assertRaises(TypeError): + interpreters.run_string(self.id, b'print("spam")') + + def test_invalid_syntax(self): + with self.assertRaises(SyntaxError): + # missing close paren + interpreters.run_string(self.id, 'print("spam"') + + def test_failure(self): + with self.assertRaises(Exception) as caught: + interpreters.run_string(self.id, 'raise Exception("spam")') + self.assertEqual(str(caught.exception), 'spam') + + def test_sys_exit(self): + with self.assertRaises(SystemExit) as cm: + interpreters.run_string(self.id, dedent(""" + import sys + sys.exit() + """)) + self.assertIsNone(cm.exception.code) + + with self.assertRaises(SystemExit) as cm: + interpreters.run_string(self.id, dedent(""" + import sys + sys.exit(42) + """)) + self.assertEqual(cm.exception.code, 42) + + def test_SystemError(self): + with self.assertRaises(SystemExit) as cm: + interpreters.run_string(self.id, 'raise SystemExit(42)') + self.assertEqual(cm.exception.code, 42) + + +if __name__ == '__main__': unittest.main() diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index ce44df269bacd2..a46610e5a10899 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -279,6 +279,13 @@ interp_run_string(PyObject *self, PyObject *args) Py_RETURN_NONE; } +PyDoc_STRVAR(run_string_doc, +"run_string(ID, sourcetext) -> run_id\n\ +\n\ +Execute the provided string in the identified interpreter.\n\ +See PyRun_SimpleStrings."); + + static PyMethodDef module_functions[] = { {"create", (PyCFunction)interp_create, METH_VARARGS, create_doc}, @@ -288,8 +295,8 @@ static PyMethodDef module_functions[] = { {"_enumerate", (PyCFunction)interp_enumerate, METH_NOARGS, NULL}, - {"_run_string", (PyCFunction)interp_run_string, - METH_VARARGS, NULL}, + {"run_string", (PyCFunction)interp_run_string, + METH_VARARGS, run_string_doc}, {NULL, NULL} /* sentinel */ }; From 92aeaaca0e21b58ef7ab63ae746d2480f60cbd39 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 2 Jan 2017 15:42:08 -0700 Subject: [PATCH 05/78] Get tricky tests working. --- Lib/test/test__interpreters.py | 53 +++++++++++++++++++--------------- 1 file changed, 30 insertions(+), 23 deletions(-) diff --git a/Lib/test/test__interpreters.py b/Lib/test/test__interpreters.py index f6392ae7abaeab..a23246ea2a3f3a 100644 --- a/Lib/test/test__interpreters.py +++ b/Lib/test/test__interpreters.py @@ -13,22 +13,18 @@ @contextlib.contextmanager -def _blocked(): - r, w = os.pipe() +def _blocked(dirname): + filename = os.path.join(dirname, '.lock') wait_script = dedent(""" - import select - # Wait for a "done" signal. - select.select([{}], [], []) - - #import time - #time.sleep(1_000_000) - """).format(r) + import os.path + import time + while not os.path.exists('{}'): + time.sleep(0.1) + """).format(filename) try: yield wait_script finally: - os.write(w, b'') # release! - os.close(r) - os.close(w) + support.create_empty_file(filename) class TestBase(unittest.TestCase): @@ -211,15 +207,15 @@ def f(): t.start() t.join() - @unittest.skip('not working yet') def test_still_running(self): main, = interpreters._enumerate() id = interpreters.create() def f(): interpreters.run_string(id, wait_script) + dirname = tempfile.mkdtemp() t = threading.Thread(target=f) - with _blocked() as wait_script: + with _blocked(dirname) as wait_script: t.start() with self.assertRaises(RuntimeError): interpreters.destroy(id) @@ -243,7 +239,10 @@ def setUp(self): def tearDown(self): if self.dirname is not None: - shutil.rmtree(self.dirname) + try: + shutil.rmtree(self.dirname) + except FileNotFoundError: + pass # already deleted super().tearDown() def _resolve_filename(self, name=None): @@ -304,31 +303,39 @@ def f(): self.assert_file_contains(expected) - @unittest.skip('not working yet') @unittest.skipUnless(hasattr(os, 'fork'), "test needs os.fork()") def test_fork(self): filename = self._empty_file() expected = 'spam spam spam spam spam' script = dedent(""" import os - import sys + r, w = os.pipe() pid = os.fork() - if pid == 0: - with open('{}', 'w') as out: + if pid == 0: # child + import sys + filename = '{}' + with open(filename, 'w') as out: out.write('{}') - sys.exit(0) + os.write(w, b'done!') + else: + import select + try: + select.select([r], [], []) + finally: + os.close(r) + os.close(w) """).format(filename, expected) + # XXX Kill the child process in a unittest-friendly way. interpreters.run_string(self.id, script) - self.assert_file_contains(expected) - @unittest.skip('not working yet') def test_already_running(self): def f(): interpreters.run_string(self.id, wait_script) t = threading.Thread(target=f) - with _blocked() as wait_script: + dirname = tempfile.mkdtemp() + with _blocked(dirname) as wait_script: t.start() with self.assertRaises(RuntimeError): interpreters.run_string(self.id, 'print("spam")') From b6e2f3a5505a02d5d5c10c83ea3df60df84e3631 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 2 Jan 2017 15:46:07 -0700 Subject: [PATCH 06/78] Add a test for a still running interpreter when main exits. --- Lib/test/test__interpreters.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/Lib/test/test__interpreters.py b/Lib/test/test__interpreters.py index a23246ea2a3f3a..24724b662dd400 100644 --- a/Lib/test/test__interpreters.py +++ b/Lib/test/test__interpreters.py @@ -8,10 +8,22 @@ import unittest from test import support +from test.support import script_helper interpreters = support.import_module('_interpreters') +SCRIPT_THREADED_INTERP = """\ +import threading +import _interpreters +def f(): + _interpreters.run_string(id, {!r}) + +t = threading.Thread(target=f) +t.start() +""" + + @contextlib.contextmanager def _blocked(dirname): filename = os.path.join(dirname, '.lock') @@ -27,6 +39,27 @@ def _blocked(dirname): support.create_empty_file(filename) +class InterpreterTests(unittest.TestCase): + + def setUp(self): + self.dirname = tempfile.mkdtemp() + + def tearDown(self): + shutil.rmtree(self.dirname) + + def test_still_running_at_exit(self): + script = SCRIPT_THREADED_INTERP.format("""if True: + import time + # Give plenty of time for the main interpreter to finish. + time.sleep(1_000_000) + """) + filename = script_helper.make_script(self.dirname, 'interp', script) + proc = script_helper.spawn_python(filename) + retcode = proc.wait() + + self.assertEqual(retcode, 0) + + class TestBase(unittest.TestCase): def tearDown(self): From c15e79b4e69f405e91f0f7dfc315b12620e01f11 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 3 Jan 2017 17:48:34 -0700 Subject: [PATCH 07/78] Add run_string_unrestricted(). --- Doc/library/_interpreters.rst | 12 +++- Lib/test/test__interpreters.py | 101 ++++++++++++++++++++++++++------- Modules/_interpretersmodule.c | 95 +++++++++++++++++++++++++------ 3 files changed, 168 insertions(+), 40 deletions(-) diff --git a/Doc/library/_interpreters.rst b/Doc/library/_interpreters.rst index 8d05622f5af106..a4f86e38496e5b 100644 --- a/Doc/library/_interpreters.rst +++ b/Doc/library/_interpreters.rst @@ -38,8 +38,6 @@ It defines the following functions: Finalize and destroy the identified interpreter. -.. XXX must not be running? - .. function:: run_string(id, command) @@ -54,10 +52,18 @@ It defines the following functions: how :func:`exec` works. This aligns with how interpreters are not inherently threaded. -.. XXX must not be running already? .. XXX sys.exit() (and SystemExit) is swallowed? +.. function:: run_string_unrestricted(id, command, ns=None) + + Like :c:func:`run_string` but returns the dict in which the code + was executed. It also supports providing a namespace that gets + merged into the execution namespace before execution. Note that + this allows objects to leak between interpreters, which may not + be desirable. + + **Caveats:** * ... diff --git a/Lib/test/test__interpreters.py b/Lib/test/test__interpreters.py index 24724b662dd400..edd279c16c4935 100644 --- a/Lib/test/test__interpreters.py +++ b/Lib/test/test__interpreters.py @@ -105,34 +105,33 @@ def f(): def test_in_subinterpreter(self): main, = interpreters._enumerate() - id = interpreters.create() - interpreters.run_string(id, dedent(""" + id1 = interpreters.create() + ns = interpreters.run_string_unrestricted(id1, dedent(""" import _interpreters id = _interpreters.create() """)) + id2 = ns['id'] - ids = interpreters._enumerate() - self.assertIn(id, ids) - self.assertIn(main, ids) - self.assertEqual(len(ids), 3) + self.assertEqual(set(interpreters._enumerate()), {main, id1, id2}) def test_in_threaded_subinterpreter(self): main, = interpreters._enumerate() - id = interpreters.create() + id1 = interpreters.create() + ns = None + script = dedent(""" + import _interpreters + id = _interpreters.create() + """) def f(): - interpreters.run_string(id, dedent(""" - import _interpreters - _interpreters.create() - """)) + nonlocal ns + ns = interpreters.run_string_unrestricted(id1, script) t = threading.Thread(target=f) t.start() t.join() + id2 = ns['id'] - ids = interpreters._enumerate() - self.assertIn(id, ids) - self.assertIn(main, ids) - self.assertEqual(len(ids), 3) + self.assertEqual(set(interpreters._enumerate()), {main, id1, id2}) def test_after_destroy_all(self): @@ -214,21 +213,27 @@ def test_bad_id(self): interpreters.destroy(-1) def test_from_current(self): + main, = interpreters._enumerate() id = interpreters.create() + script = dedent(""" + import _interpreters + _interpreters.destroy({}) + """).format(id) + with self.assertRaises(RuntimeError): - interpreters.run_string(id, dedent(""" - import _interpreters - _interpreters.destroy({}) - """).format(id)) + interpreters.run_string(id, script) + self.assertEqual(set(interpreters._enumerate()), {main, id}) def test_from_sibling(self): main, = interpreters._enumerate() id1 = interpreters.create() id2 = interpreters.create() - interpreters.run_string(id1, dedent(""" + script = dedent(""" import _interpreters _interpreters.destroy({}) - """).format(id2)) + """).format(id2) + interpreters.run_string(id1, script) + self.assertEqual(set(interpreters._enumerate()), {main, id1}) def test_from_other_thread(self): @@ -241,6 +246,8 @@ def f(): t.join() def test_still_running(self): + # XXX Rewrite this test without files by using + # run_string_unrestricted(). main, = interpreters._enumerate() id = interpreters.create() def f(): @@ -428,5 +435,57 @@ def test_SystemError(self): self.assertEqual(cm.exception.code, 42) +class RunStringUnrestrictedTests(TestBase): + + def setUp(self): + self.id = interpreters.create() + + def test_without_ns(self): + script = dedent(""" + spam = 42 + """) + ns = interpreters.run_string_unrestricted(self.id, script) + + self.assertEqual(ns['spam'], 42) + + def test_with_ns(self): + updates = {'spam': 'ham', 'eggs': -1} + script = dedent(""" + spam = 42 + result = spam + eggs + """) + ns = interpreters.run_string_unrestricted(self.id, script, updates) + + self.assertEqual(ns['spam'], 42) + self.assertEqual(ns['eggs'], -1) + self.assertEqual(ns['result'], 41) + + def test_ns_does_not_overwrite(self): + updates = {'__name__': 'not __main__'} + script = dedent(""" + spam = 42 + """) + ns = interpreters.run_string_unrestricted(self.id, script, updates) + + self.assertEqual(ns['__name__'], '__main__') + + def test_return_execution_namespace(self): + script = dedent(""" + spam = 42 + """) + ns = interpreters.run_string_unrestricted(self.id, script) + + ns.pop('__builtins__') + ns.pop('__loader__') + self.assertEqual(ns, { + '__name__': '__main__', + '__annotations__': {}, + '__doc__': None, + '__package__': None, + '__spec__': None, + 'spam': 42, + }) + + if __name__ == '__main__': unittest.main() diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index a46610e5a10899..47e4c3f1dcee40 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -89,14 +89,11 @@ _ensure_not_running(PyInterpreterState *interp) return 0; } -static int -_run_string(PyInterpreterState *interp, const char *codestr) +static PyObject * +_run_string(PyInterpreterState *interp, const char *codestr, PyObject *updates) { - PyObject *result = NULL; - PyObject *exc = NULL, *value = NULL, *tb = NULL; - if (_ensure_not_running(interp) < 0) - return -1; + return NULL; // Switch to interpreter. PyThreadState *tstate = PyInterpreterState_ThreadHead(interp); @@ -104,19 +101,32 @@ _run_string(PyInterpreterState *interp, const char *codestr) // Run the string (see PyRun_SimpleStringFlags). // XXX How to handle sys.exit()? + PyObject *exc = NULL, *value = NULL, *tb = NULL; + PyObject *ns = NULL; + // XXX Force a fresh __main__ module? PyObject *m = PyImport_AddModule("__main__"); if (m == NULL) { PyErr_Fetch(&exc, &value, &tb); goto done; } - PyObject *d = PyModule_GetDict(m); - result = PyRun_StringFlags(codestr, Py_file_input, d, d, NULL); + ns = PyModule_GetDict(m); + if (ns == NULL) { + PyErr_Fetch(&exc, &value, &tb); + goto done; + } + if (updates != NULL && PyDict_Merge(ns, updates, 0) < 0) { + PyErr_Fetch(&exc, &value, &tb); + goto done; + } + PyObject *result = PyRun_StringFlags(codestr, Py_file_input, ns, ns, NULL); if (result == NULL) { + ns = NULL; // Get the exception from the subinterpreter. PyErr_Fetch(&exc, &value, &tb); goto done; } Py_DECREF(result); // We throw away the result. + Py_INCREF(ns); // It is a borrowed reference. done: // Switch back. @@ -126,7 +136,7 @@ _run_string(PyInterpreterState *interp, const char *codestr) // Propagate any exception out to the caller. PyErr_Restore(exc, value, tb); - return (result == NULL) ? -1 : 0; + return ns; } /* module level code ********************************************************/ @@ -273,32 +283,85 @@ interp_run_string(PyObject *self, PyObject *args) } // Run the code in the interpreter. - if (_run_string(interp, codestr) < 0) + PyObject *ns = _run_string(interp, codestr, NULL); + if (ns == NULL) return NULL; else Py_RETURN_NONE; } PyDoc_STRVAR(run_string_doc, -"run_string(ID, sourcetext) -> run_id\n\ +"run_string(ID, sourcetext)\n\ \n\ Execute the provided string in the identified interpreter.\n\ +\n\ +See PyRun_SimpleStrings."); + + +static PyObject * +interp_run_string_unrestricted(PyObject *self, PyObject *args) +{ + PyObject *id, *code, *ns = NULL; + if (!PyArg_UnpackTuple(args, "run_string_unrestricted", 2, 3, + &id, &code, &ns)) + return NULL; + if (!PyLong_Check(id)) { + PyErr_SetString(PyExc_TypeError, "first arg (ID) must be an int"); + return NULL; + } + if (!PyUnicode_Check(code)) { + PyErr_SetString(PyExc_TypeError, + "second arg (code) must be a string"); + return NULL; + } + if (ns == Py_None) + ns = NULL; + + // Look up the interpreter. + PyInterpreterState *interp = _look_up(id); + if (interp == NULL) + return NULL; + + // Extract code. + Py_ssize_t size; + const char *codestr = PyUnicode_AsUTF8AndSize(code, &size); + if (codestr == NULL) + return NULL; + if (strlen(codestr) != (size_t)size) { + PyErr_SetString(PyExc_ValueError, + "source code string cannot contain null bytes"); + return NULL; + } + + // Run the code in the interpreter. + return _run_string(interp, codestr, ns); +} + +PyDoc_STRVAR(run_string_unrestricted_doc, +"run_string_unrestricted(ID, sourcetext, ns=None) -> main module ns\n\ +\n\ +Execute the provided string in the identified interpreter. Return the\n\ +dict in which the code executed. If the ns arg is provided then it is\n\ +merged into the execution namespace before the code is executed.\n\ +\n\ See PyRun_SimpleStrings."); static PyMethodDef module_functions[] = { - {"create", (PyCFunction)interp_create, + {"create", (PyCFunction)interp_create, METH_VARARGS, create_doc}, - {"destroy", (PyCFunction)interp_destroy, + {"destroy", (PyCFunction)interp_destroy, METH_VARARGS, destroy_doc}, - {"_enumerate", (PyCFunction)interp_enumerate, + {"_enumerate", (PyCFunction)interp_enumerate, METH_NOARGS, NULL}, - {"run_string", (PyCFunction)interp_run_string, + {"run_string", (PyCFunction)interp_run_string, METH_VARARGS, run_string_doc}, + {"run_string_unrestricted", (PyCFunction)interp_run_string_unrestricted, + METH_VARARGS, run_string_unrestricted_doc}, - {NULL, NULL} /* sentinel */ + {NULL, NULL} /* sentinel */ }; From 8d809c7d15408e43f1834f6af66584a4e402157c Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 3 Jan 2017 18:17:18 -0700 Subject: [PATCH 08/78] Exit out of the child process. --- Lib/test/test__interpreters.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Lib/test/test__interpreters.py b/Lib/test/test__interpreters.py index edd279c16c4935..1402872e3412d6 100644 --- a/Lib/test/test__interpreters.py +++ b/Lib/test/test__interpreters.py @@ -357,6 +357,9 @@ def test_fork(self): with open(filename, 'w') as out: out.write('{}') os.write(w, b'done!') + + # Kill the unittest runner in the child process. + os._exit(1) else: import select try: @@ -365,7 +368,6 @@ def test_fork(self): os.close(r) os.close(w) """).format(filename, expected) - # XXX Kill the child process in a unittest-friendly way. interpreters.run_string(self.id, script) self.assert_file_contains(expected) From 55bb53c5234720321bc7191599828368638096bc Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 3 Jan 2017 18:45:52 -0700 Subject: [PATCH 09/78] Resolve several TODOs. --- Doc/library/_interpreters.rst | 6 +++--- Modules/_interpretersmodule.c | 12 +++--------- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/Doc/library/_interpreters.rst b/Doc/library/_interpreters.rst index a4f86e38496e5b..e2d0191ac71703 100644 --- a/Doc/library/_interpreters.rst +++ b/Doc/library/_interpreters.rst @@ -50,9 +50,9 @@ It defines the following functions: threads do. If the code results in an exception then that exception is raised in the thread in which run_string() was called, similar to how :func:`exec` works. This aligns with how interpreters are not - inherently threaded. - -.. XXX sys.exit() (and SystemExit) is swallowed? + inherently threaded. Note that SystemExit (as raised by sys.exit()) + is not treated any differently and will result in the process ending + if not caught explicitly. .. function:: run_string_unrestricted(id, command, ns=None) diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index 47e4c3f1dcee40..a8d5b5391a1a64 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -45,7 +45,7 @@ _look_up(PyObject *requested_id) long long id = PyLong_AsLongLong(requested_id); if (id == -1 && PyErr_Occurred() != NULL) return NULL; - // XXX Fail if larger than INT64_MAX? + assert(id <= INT64_MAX); return _look_up_int64(id); } @@ -100,7 +100,6 @@ _run_string(PyInterpreterState *interp, const char *codestr, PyObject *updates) PyThreadState *save_tstate = PyThreadState_Swap(tstate); // Run the string (see PyRun_SimpleStringFlags). - // XXX How to handle sys.exit()? PyObject *exc = NULL, *value = NULL, *tb = NULL; PyObject *ns = NULL; // XXX Force a fresh __main__ module? @@ -141,8 +140,6 @@ _run_string(PyInterpreterState *interp, const char *codestr, PyObject *updates) /* module level code ********************************************************/ -// XXX track count? - static PyObject * interp_create(PyObject *self, PyObject *args) { @@ -205,10 +202,9 @@ interp_destroy(PyObject *self, PyObject *args) // Destroy the interpreter. //PyInterpreterState_Delete(interp); PyThreadState *tstate, *save_tstate; - tstate = PyInterpreterState_ThreadHead(interp); // XXX Is this the right one? + tstate = PyInterpreterState_ThreadHead(interp); save_tstate = PyThreadState_Swap(tstate); - // XXX Stop current execution? - Py_EndInterpreter(tstate); // XXX Handle possible errors? + Py_EndInterpreter(tstate); PyThreadState_Swap(save_tstate); Py_RETURN_NONE; @@ -229,8 +225,6 @@ interp_enumerate(PyObject *self) PyObject *ids, *id; PyInterpreterState *interp; - // XXX Handle multiple main interpreters. - ids = PyList_New(0); if (ids == NULL) return NULL; From cad0a91efe2e5ce2aba7af5b1f283aa648de16e2 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 3 Jan 2017 19:17:53 -0700 Subject: [PATCH 10/78] Set up the execution namespace before switching threads. --- Modules/_interpretersmodule.c | 66 ++++++++++++++++++++--------------- 1 file changed, 38 insertions(+), 28 deletions(-) diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index a8d5b5391a1a64..6366c7ec5e7611 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -89,45 +89,23 @@ _ensure_not_running(PyInterpreterState *interp) return 0; } -static PyObject * -_run_string(PyInterpreterState *interp, const char *codestr, PyObject *updates) +static int +_run_string(PyInterpreterState *interp, const char *codestr, PyObject *ns) { - if (_ensure_not_running(interp) < 0) - return NULL; - // Switch to interpreter. PyThreadState *tstate = PyInterpreterState_ThreadHead(interp); PyThreadState *save_tstate = PyThreadState_Swap(tstate); // Run the string (see PyRun_SimpleStringFlags). PyObject *exc = NULL, *value = NULL, *tb = NULL; - PyObject *ns = NULL; - // XXX Force a fresh __main__ module? - PyObject *m = PyImport_AddModule("__main__"); - if (m == NULL) { - PyErr_Fetch(&exc, &value, &tb); - goto done; - } - ns = PyModule_GetDict(m); - if (ns == NULL) { - PyErr_Fetch(&exc, &value, &tb); - goto done; - } - if (updates != NULL && PyDict_Merge(ns, updates, 0) < 0) { - PyErr_Fetch(&exc, &value, &tb); - goto done; - } PyObject *result = PyRun_StringFlags(codestr, Py_file_input, ns, ns, NULL); if (result == NULL) { - ns = NULL; // Get the exception from the subinterpreter. PyErr_Fetch(&exc, &value, &tb); - goto done; + } else { + Py_DECREF(result); // We throw away the result. } - Py_DECREF(result); // We throw away the result. - Py_INCREF(ns); // It is a borrowed reference. -done: // Switch back. if (save_tstate != NULL) PyThreadState_Swap(save_tstate); @@ -135,6 +113,38 @@ _run_string(PyInterpreterState *interp, const char *codestr, PyObject *updates) // Propagate any exception out to the caller. PyErr_Restore(exc, value, tb); + return result == NULL ? -1 : 0; +} + +static PyObject * +_run_string_in_main(PyInterpreterState *interp, const char *codestr, + PyObject *updates) +{ + if (_ensure_not_running(interp) < 0) + return NULL; + + // Get the namespace in which to execute. + // XXX Force a fresh __main__ module? + PyObject *m = PyMapping_GetItemString(interp->modules, "__main__"); + if (m == NULL) + return NULL; + PyObject *orig = PyModule_GetDict(m); // borrowed + Py_DECREF(m); + if (orig == NULL) + return NULL; + PyObject *ns = PyDict_Copy(orig); + if (ns == NULL) + return NULL; + if (updates != NULL && PyDict_Merge(ns, updates, 0) < 0) { + Py_DECREF(ns); + return NULL; + } + + if (_run_string(interp, codestr, ns) < 0) { + Py_DECREF(ns); + return NULL; + } + return ns; } @@ -277,7 +287,7 @@ interp_run_string(PyObject *self, PyObject *args) } // Run the code in the interpreter. - PyObject *ns = _run_string(interp, codestr, NULL); + PyObject *ns = _run_string_in_main(interp, codestr, NULL); if (ns == NULL) return NULL; else @@ -328,7 +338,7 @@ interp_run_string_unrestricted(PyObject *self, PyObject *args) } // Run the code in the interpreter. - return _run_string(interp, codestr, ns); + return _run_string_in_main(interp, codestr, ns); } PyDoc_STRVAR(run_string_unrestricted_doc, From 1395c588b882f77e77cf33b3fd6b3cef2fb2ca88 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 3 Jan 2017 23:42:29 -0700 Subject: [PATCH 11/78] Run in a copy of __main__. --- Lib/test/test__interpreters.py | 11 +++++- Modules/_interpretersmodule.c | 70 +++++++++++++++++++++++++++------- 2 files changed, 66 insertions(+), 15 deletions(-) diff --git a/Lib/test/test__interpreters.py b/Lib/test/test__interpreters.py index 1402872e3412d6..3012f4e59ea83d 100644 --- a/Lib/test/test__interpreters.py +++ b/Lib/test/test__interpreters.py @@ -300,7 +300,7 @@ def _empty_file(self): def assert_file_contains(self, expected, filename=None): if filename is None: filename = self.filename - self.assertIsNot(filename, None) + self.assertIsNotNone(filename) with open(filename) as out: content = out.read() self.assertEqual(content, expected) @@ -471,6 +471,15 @@ def test_ns_does_not_overwrite(self): self.assertEqual(ns['__name__'], '__main__') + def test_main_not_shared(self): + ns1 = interpreters.run_string_unrestricted(self.id, 'spam = True') + ns2 = interpreters.run_string_unrestricted(self.id, 'eggs = False') + + self.assertIn('spam', ns1) + self.assertNotIn('eggs', ns1) + self.assertIn('eggs', ns2) + self.assertNotIn('spam', ns2) + def test_return_execution_namespace(self): script = dedent(""" spam = 42 diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index 6366c7ec5e7611..a5146cda48af15 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -89,6 +89,33 @@ _ensure_not_running(PyInterpreterState *interp) return 0; } +static PyObject * +_copy_module(PyObject *m, PyObject *updates) +{ + PyObject *orig = PyModule_GetDict(m); // borrowed + if (orig == NULL) { + return NULL; + } + + PyObject *copy = PyModule_New("__main__"); + if (copy == NULL) { + return NULL; + } + PyObject *ns = PyModule_GetDict(copy); // borrowed + if (ns == NULL) + goto error; + + if (PyDict_Merge(ns, orig, 1) < 0) + goto error; + if (updates != NULL && PyDict_Merge(ns, updates, 0) < 0) + goto error; + return copy; + +error: + Py_DECREF(copy); + return NULL; +} + static int _run_string(PyInterpreterState *interp, const char *codestr, PyObject *ns) { @@ -123,31 +150,46 @@ _run_string_in_main(PyInterpreterState *interp, const char *codestr, if (_ensure_not_running(interp) < 0) return NULL; - // Get the namespace in which to execute. - // XXX Force a fresh __main__ module? - PyObject *m = PyMapping_GetItemString(interp->modules, "__main__"); - if (m == NULL) + // Get the namespace in which to execute. This involves creating + // a new module, updating it from the __main__ module and the given + // updates (if any), replacing the __main__ with the new module in + // sys.modules, and then using the new module's __dict__. At the + // end we restore the original __main__ module. + PyObject *main_mod = PyMapping_GetItemString(interp->modules, "__main__"); + if (main_mod == NULL) return NULL; - PyObject *orig = PyModule_GetDict(m); // borrowed - Py_DECREF(m); - if (orig == NULL) - return NULL; - PyObject *ns = PyDict_Copy(orig); - if (ns == NULL) + PyObject *m = _copy_module(main_mod, updates); + if (m == NULL) { + Py_DECREF(main_mod); return NULL; - if (updates != NULL && PyDict_Merge(ns, updates, 0) < 0) { - Py_DECREF(ns); + } + if (PyMapping_SetItemString(interp->modules, "__main__", m) < 0) { + Py_DECREF(main_mod); + Py_DECREF(m); return NULL; } + PyObject *ns = PyModule_GetDict(m); // borrowed + Py_INCREF(ns); + Py_DECREF(m); + // Run the string. + PyObject *result = ns; if (_run_string(interp, codestr, ns) < 0) { + result = NULL; Py_DECREF(ns); - return NULL; } - return ns; + // Restore __main__. + if (PyMapping_SetItemString(interp->modules, "__main__", main_mod) < 0) { + // XXX Chain exceptions... + //PyErr_Restore(exc, value, tb); + } + Py_DECREF(main_mod); + + return result; } + /* module level code ********************************************************/ static PyObject * From e0e817633bfd824bc3883b38568644393e1cba9b Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 4 Jan 2017 11:58:31 -0700 Subject: [PATCH 12/78] Close stdin and stdout after the proc finishes. --- Lib/test/test__interpreters.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test__interpreters.py b/Lib/test/test__interpreters.py index 3012f4e59ea83d..93c1ee58a7e19f 100644 --- a/Lib/test/test__interpreters.py +++ b/Lib/test/test__interpreters.py @@ -54,8 +54,8 @@ def test_still_running_at_exit(self): time.sleep(1_000_000) """) filename = script_helper.make_script(self.dirname, 'interp', script) - proc = script_helper.spawn_python(filename) - retcode = proc.wait() + with script_helper.spawn_python(filename) as proc: + retcode = proc.wait() self.assertEqual(retcode, 0) From 0eb4acc4bc6af2a4f08774697b3b4589621d28f9 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 4 Jan 2017 12:12:07 -0700 Subject: [PATCH 13/78] Clean up a test. --- Lib/test/test__interpreters.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/Lib/test/test__interpreters.py b/Lib/test/test__interpreters.py index 93c1ee58a7e19f..459ff8650a8470 100644 --- a/Lib/test/test__interpreters.py +++ b/Lib/test/test__interpreters.py @@ -3,7 +3,7 @@ import os.path import shutil import tempfile -from textwrap import dedent +from textwrap import dedent, indent import threading import unittest @@ -14,10 +14,13 @@ SCRIPT_THREADED_INTERP = """\ +from textwrap import dedent import threading import _interpreters def f(): - _interpreters.run_string(id, {!r}) + _interpreters.run_string(id, dedent(''' + {} + ''')) t = threading.Thread(target=f) t.start() @@ -48,11 +51,12 @@ def tearDown(self): shutil.rmtree(self.dirname) def test_still_running_at_exit(self): - script = SCRIPT_THREADED_INTERP.format("""if True: + subscript = dedent(""" import time # Give plenty of time for the main interpreter to finish. time.sleep(1_000_000) """) + script = SCRIPT_THREADED_INTERP.format(indent(subscript, ' ')) filename = script_helper.make_script(self.dirname, 'interp', script) with script_helper.spawn_python(filename) as proc: retcode = proc.wait() From 356208473d53475e89b9dd14ad1f12241c6a965b Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 4 Jan 2017 13:10:53 -0700 Subject: [PATCH 14/78] Chain exceptions during cleanup. --- Modules/_interpretersmodule.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index a5146cda48af15..ff08ae28c449dd 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -180,9 +180,12 @@ _run_string_in_main(PyInterpreterState *interp, const char *codestr, } // Restore __main__. + PyObject *exc = NULL, *value = NULL, *tb = NULL; + PyErr_Fetch(&exc, &value, &tb); if (PyMapping_SetItemString(interp->modules, "__main__", main_mod) < 0) { - // XXX Chain exceptions... - //PyErr_Restore(exc, value, tb); + _PyErr_ChainExceptions(exc, value, tb); + } else { + PyErr_Restore(exc, value, tb); } Py_DECREF(main_mod); From 8e7e724b62054cf77829aca2b0813df07840fe22 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 4 Jan 2017 13:31:50 -0700 Subject: [PATCH 15/78] Finish the module docs. --- Doc/library/_interpreters.rst | 36 +++++++++++++++++++---------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/Doc/library/_interpreters.rst b/Doc/library/_interpreters.rst index e2d0191ac71703..6e79277895936e 100644 --- a/Doc/library/_interpreters.rst +++ b/Doc/library/_interpreters.rst @@ -6,21 +6,30 @@ .. versionadded:: 3,7 - :ref:`_sub-interpreter-support` - -threading - -------------- This module provides low-level primitives for working with multiple -Python interpreters in the same process. - +Python interpreters in the same runtime in the current process. .. XXX The :mod:`interpreters` module provides an easier to use and higher-level API built on top of this module. +More information about (sub)interpreters is found at +:ref:`_sub-interpreter-support`, including what data is shared between +interpreters and what is unique. Note particularly that interpreters +aren't inherently threaded, even though they track and manage Python +threads. To run code in an interpreter in a different OS thread, call +:func:`run_string` in a function that you run in a new Python thread. +For example:: + + id = _interpreters.create() + def f(): + _interpreters.run_string(id, 'print("in a thread")') + + t = threading.Thread(target=f) + t.start() + This module is optional. It is provided by Python implementations which support multiple interpreters. - .. XXX For systems lacking the :mod:`_interpreters` module, the :mod:`_dummy_interpreters` module is available. It duplicates this module's interface and can be used as a drop-in replacement. @@ -42,9 +51,10 @@ It defines the following functions: .. function:: run_string(id, command) A wrapper around :c:func:`PyRun_SimpleString` which runs the provided - Python program using the identified interpreter. Providing an - invalid or unknown ID results in a RuntimeError, likewise if the main - interpreter or any other running interpreter is used. + Python program in the main thread of the identified interpreter. + Providing an invalid or unknown ID results in a RuntimeError, + likewise if the main interpreter or any other running interpreter + is used. Any value returned from the code is thrown away, similar to what threads do. If the code results in an exception then that exception @@ -62,9 +72,3 @@ It defines the following functions: merged into the execution namespace before execution. Note that this allows objects to leak between interpreters, which may not be desirable. - - -**Caveats:** - -* ... - From f8173fb4db0090854c1f0e396f1ac7870915b403 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 23 May 2017 16:41:55 -0700 Subject: [PATCH 16/78] Fix docs. --- Doc/library/_interpreters.rst | 7 +------ Doc/library/concurrency.rst | 1 + 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/Doc/library/_interpreters.rst b/Doc/library/_interpreters.rst index 6e79277895936e..445926ac4b87ba 100644 --- a/Doc/library/_interpreters.rst +++ b/Doc/library/_interpreters.rst @@ -10,11 +10,9 @@ This module provides low-level primitives for working with multiple Python interpreters in the same runtime in the current process. -.. XXX The :mod:`interpreters` module provides an easier to use and - higher-level API built on top of this module. More information about (sub)interpreters is found at -:ref:`_sub-interpreter-support`, including what data is shared between +:ref:`sub-interpreter-support`, including what data is shared between interpreters and what is unique. Note particularly that interpreters aren't inherently threaded, even though they track and manage Python threads. To run code in an interpreter in a different OS thread, call @@ -30,9 +28,6 @@ For example:: This module is optional. It is provided by Python implementations which support multiple interpreters. -.. XXX For systems lacking the :mod:`_interpreters` module, the - :mod:`_dummy_interpreters` module is available. It duplicates this - module's interface and can be used as a drop-in replacement. It defines the following functions: diff --git a/Doc/library/concurrency.rst b/Doc/library/concurrency.rst index 826bf86d081793..fafbf92c6b0181 100644 --- a/Doc/library/concurrency.rst +++ b/Doc/library/concurrency.rst @@ -29,3 +29,4 @@ The following are support modules for some of the above services: _thread.rst _dummy_thread.rst dummy_threading.rst + _interpreters.rst From 84cea38997eeb5882142d041dafcf00660e2b283 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 4 Dec 2017 18:28:54 -0700 Subject: [PATCH 17/78] Fix includes. --- Modules/_interpretersmodule.c | 1 + setup.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index ff08ae28c449dd..64a9e7ff6f6ad7 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -4,6 +4,7 @@ #include "Python.h" #include "frameobject.h" +#include "internal/pystate.h" static PyInterpreterState * diff --git a/setup.py b/setup.py index 2c5cf51bc1ee65..739fac287e7318 100644 --- a/setup.py +++ b/setup.py @@ -746,7 +746,8 @@ def detect_modules(self): ) # Python interface to subinterpreter C-API. - exts.append(Extension('_interpreters', ['_interpretersmodule.c'])) + exts.append(Extension('_interpreters', ['_interpretersmodule.c'], + define_macros=[('Py_BUILD_CORE', '')])) # # Here ends the simple stuff. From here on, modules need certain From 48a1a381e13e4143439a9c88cb6943957ccdd71d Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 28 Nov 2017 12:44:47 -0600 Subject: [PATCH 18/78] Add _interpreters.is_shareable(). --- Modules/_interpretersmodule.c | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index 64a9e7ff6f6ad7..825f987fdf4f2b 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -397,6 +397,26 @@ merged into the execution namespace before the code is executed.\n\ See PyRun_SimpleStrings."); +static PyObject * +object_is_shareable(PyObject *self, PyObject *args) +{ + PyObject *obj; + if (!PyArg_UnpackTuple(args, "is_shareable", 1, 1, &obj)) + return NULL; + + if (PyBytes_CheckExact(obj)) + Py_RETURN_TRUE; + + Py_RETURN_FALSE; +} + +PyDoc_STRVAR(is_shareable_doc, +"is_shareable(obj) -> bool\n\ +\n\ +Return True if the object's data may be shared between interpreters and\n\ +False otherwise."); + + static PyMethodDef module_functions[] = { {"create", (PyCFunction)interp_create, METH_VARARGS, create_doc}, @@ -411,6 +431,9 @@ static PyMethodDef module_functions[] = { {"run_string_unrestricted", (PyCFunction)interp_run_string_unrestricted, METH_VARARGS, run_string_unrestricted_doc}, + {"is_shareable", (PyCFunction)object_is_shareable, + METH_VARARGS, is_shareable_doc}, + {NULL, NULL} /* sentinel */ }; From bbdadb07297ddf976c2e96e99435dd9f2714c5f4 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 4 Dec 2017 15:59:00 -0700 Subject: [PATCH 19/78] Add _PyObject_CheckShareable(). --- Modules/_interpretersmodule.c | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index 825f987fdf4f2b..b47ff6212af17f 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -7,6 +7,20 @@ #include "internal/pystate.h" +/* sharing-specific functions */ + +static int +_PyObject_CheckShareable(PyObject *obj) +{ + if (PyBytes_CheckExact(obj)) + return 0; + PyErr_SetString(PyExc_ValueError, + "obj is not a cross-interpreter shareable type"); + return 1; +} + +/* interpreter-specific functions */ + static PyInterpreterState * _get_current(void) { @@ -403,10 +417,9 @@ object_is_shareable(PyObject *self, PyObject *args) PyObject *obj; if (!PyArg_UnpackTuple(args, "is_shareable", 1, 1, &obj)) return NULL; - - if (PyBytes_CheckExact(obj)) + if (_PyObject_CheckShareable(obj) == 0) Py_RETURN_TRUE; - + PyErr_Clear(); Py_RETURN_FALSE; } From 9a75fd2e50f8b97c49b9e1ecaf04fb62ca42e9c1 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 4 Dec 2017 16:50:55 -0700 Subject: [PATCH 20/78] Add _PyCrossInterpreterData. --- Include/internal/pystate.h | 14 ++++++ Modules/_interpretersmodule.c | 85 +++++++++++++++++++++++++++++++---- 2 files changed, 91 insertions(+), 8 deletions(-) diff --git a/Include/internal/pystate.h b/Include/internal/pystate.h index 0b464bcb2e86de..2c06b48ae36c71 100644 --- a/Include/internal/pystate.h +++ b/Include/internal/pystate.h @@ -65,6 +65,20 @@ PyAPI_FUNC(_PyInitError) _PyPathConfig_Calculate( PyAPI_FUNC(void) _PyPathConfig_Clear(_PyPathConfig *config); +/* Cross-interpreter data sharing */ + +struct _cid; + +typedef struct _cid { + void *data; + PyObject *(*new_object)(struct _cid *); + void (*free)(void *); + + PyInterpreterState *interp; + PyObject *object; +} _PyCrossInterpreterData; + + /* Full Python runtime state */ typedef struct pyruntimestate { diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index b47ff6212af17f..eb7f444cdf6ad1 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -7,6 +7,17 @@ #include "internal/pystate.h" +static PyInterpreterState * +_get_current(void) +{ + PyThreadState *tstate; + + tstate = PyThreadState_Get(); + if (tstate == NULL) + return NULL; + return tstate->interp; +} + /* sharing-specific functions */ static int @@ -19,19 +30,77 @@ _PyObject_CheckShareable(PyObject *obj) return 1; } -/* interpreter-specific functions */ +static PyObject * +_new_bytes_object(_PyCrossInterpreterData *data) +{ + return PyBytes_FromString((char *)(data->data)); +} -static PyInterpreterState * -_get_current(void) +static int +_bytes_shared(PyObject *obj, _PyCrossInterpreterData *data) { - PyThreadState *tstate; + data->data = (void *)(PyBytes_AS_STRING(obj)); + data->new_object = _new_bytes_object; + data->free = NULL; + return 0; +} - tstate = PyThreadState_Get(); - if (tstate == NULL) - return NULL; - return tstate->interp; +static int +_PyObject_GetCrossInterpreterData(PyObject *obj, _PyCrossInterpreterData *data) +{ + Py_INCREF(obj); + + if (_PyObject_CheckShareable(obj) != 0) { + Py_DECREF(obj); + return 1; + } + + data->interp = _get_current(); + data->object = obj; + + if (PyBytes_CheckExact(obj)) { + return _bytes_shared(obj, data); + } + + return 0; +}; + +static void +_PyCrossInterpreterData_Release(_PyCrossInterpreterData *data) +{ + PyThreadState *save_tstate = NULL; + if (data->interp != NULL) { + // Switch to the original interpreter. + PyThreadState *tstate = PyInterpreterState_ThreadHead(data->interp); + save_tstate = PyThreadState_Swap(tstate); + } + + if (data->free != NULL) { + data->free(data->data); + } + Py_XDECREF(data->object); + + // Switch back. + if (save_tstate != NULL) + PyThreadState_Swap(save_tstate); +} + +static PyObject * +_PyCrossInterpreterData_NewObject(_PyCrossInterpreterData *data) +{ + return data->new_object(data); } +static PyObject * +_PyCrossInterpreterData_Use(_PyCrossInterpreterData *data) +{ + PyObject *obj = _PyCrossInterpreterData_NewObject(data); + _PyCrossInterpreterData_Release(data); + return obj; +} + +/* interpreter-specific functions */ + static PyInterpreterState * _look_up_int64(PY_INT64_T requested_id) { From f38b138fd5e80e273190008a31bcf87edc55313c Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 7 Dec 2017 16:28:11 -0700 Subject: [PATCH 21/78] Use the shared data in run() safely. --- Modules/_interpretersmodule.c | 289 ++++++++++++++++++++++++++-------- 1 file changed, 221 insertions(+), 68 deletions(-) diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index eb7f444cdf6ad1..11ea2479a6cf42 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -59,7 +59,10 @@ _PyObject_GetCrossInterpreterData(PyObject *obj, _PyCrossInterpreterData *data) data->object = obj; if (PyBytes_CheckExact(obj)) { - return _bytes_shared(obj, data); + if (_bytes_shared(obj, data) != 0) { + Py_DECREF(obj); + return 1; + } } return 0; @@ -91,14 +94,6 @@ _PyCrossInterpreterData_NewObject(_PyCrossInterpreterData *data) return data->new_object(data); } -static PyObject * -_PyCrossInterpreterData_Use(_PyCrossInterpreterData *data) -{ - PyObject *obj = _PyCrossInterpreterData_NewObject(data); - _PyCrossInterpreterData_Release(data); - return obj; -} - /* interpreter-specific functions */ static PyInterpreterState * @@ -174,14 +169,14 @@ _ensure_not_running(PyInterpreterState *interp) } static PyObject * -_copy_module(PyObject *m, PyObject *updates) +_copy_module(PyObject *m, const char *name) { PyObject *orig = PyModule_GetDict(m); // borrowed if (orig == NULL) { return NULL; } - PyObject *copy = PyModule_New("__main__"); + PyObject *copy = PyModule_New(name); if (copy == NULL) { return NULL; } @@ -191,8 +186,6 @@ _copy_module(PyObject *m, PyObject *updates) if (PyDict_Merge(ns, orig, 1) < 0) goto error; - if (updates != NULL && PyDict_Merge(ns, updates, 0) < 0) - goto error; return copy; error: @@ -200,78 +193,235 @@ _copy_module(PyObject *m, PyObject *updates) return NULL; } -static int -_run_string(PyInterpreterState *interp, const char *codestr, PyObject *ns) -{ - // Switch to interpreter. - PyThreadState *tstate = PyInterpreterState_ThreadHead(interp); - PyThreadState *save_tstate = PyThreadState_Swap(tstate); - - // Run the string (see PyRun_SimpleStringFlags). - PyObject *exc = NULL, *value = NULL, *tb = NULL; - PyObject *result = PyRun_StringFlags(codestr, Py_file_input, ns, ns, NULL); - if (result == NULL) { - // Get the exception from the subinterpreter. - PyErr_Fetch(&exc, &value, &tb); - } else { - Py_DECREF(result); // We throw away the result. - } - - // Switch back. - if (save_tstate != NULL) - PyThreadState_Swap(save_tstate); - - // Propagate any exception out to the caller. - PyErr_Restore(exc, value, tb); - - return result == NULL ? -1 : 0; -} - static PyObject * -_run_string_in_main(PyInterpreterState *interp, const char *codestr, - PyObject *updates) +_copy_module_ns(PyInterpreterState *interp, + const char *name, const char *tempname) { - if (_ensure_not_running(interp) < 0) - return NULL; - // Get the namespace in which to execute. This involves creating // a new module, updating it from the __main__ module and the given // updates (if any), replacing the __main__ with the new module in // sys.modules, and then using the new module's __dict__. At the // end we restore the original __main__ module. - PyObject *main_mod = PyMapping_GetItemString(interp->modules, "__main__"); + PyObject *main_mod = PyMapping_GetItemString(interp->modules, name); if (main_mod == NULL) return NULL; - PyObject *m = _copy_module(main_mod, updates); + PyObject *m = _copy_module(main_mod, name); if (m == NULL) { Py_DECREF(main_mod); return NULL; } - if (PyMapping_SetItemString(interp->modules, "__main__", m) < 0) { - Py_DECREF(main_mod); + if (tempname != NULL) { + if (PyMapping_SetItemString(interp->modules, tempname, main_mod) < 0) { + Py_DECREF(main_mod); + Py_DECREF(m); + return NULL; + } + } + Py_DECREF(main_mod); + if (PyMapping_SetItemString(interp->modules, name, m) < 0) { Py_DECREF(m); return NULL; } PyObject *ns = PyModule_GetDict(m); // borrowed Py_INCREF(ns); Py_DECREF(m); + return ns; +} - // Run the string. - PyObject *result = ns; - if (_run_string(interp, codestr, ns) < 0) { - result = NULL; - Py_DECREF(ns); +static int +_restore_module(PyInterpreterState *interp, + const char *name, const char *tempname) +{ + PyObject *main_mod = PyMapping_GetItemString(interp->modules, tempname); + if (main_mod == NULL) + return -1; + if (PyMapping_SetItemString(interp->modules, name, main_mod) < 0) { + Py_DECREF(main_mod); + return -1; } + Py_DECREF(main_mod); + return 0; +} - // Restore __main__. - PyObject *exc = NULL, *value = NULL, *tb = NULL; - PyErr_Fetch(&exc, &value, &tb); - if (PyMapping_SetItemString(interp->modules, "__main__", main_mod) < 0) { - _PyErr_ChainExceptions(exc, value, tb); +struct _shareditem { + Py_UNICODE *name; + Py_ssize_t namelen; + _PyCrossInterpreterData data; +}; + +void +_sharedns_clear(struct _shareditem *shared) +{ + for (struct _shareditem *item=shared; item->name != NULL; item += 1) { + _PyCrossInterpreterData_Release(&item->data); + } +} + +static struct _shareditem * +_get_shared_ns(PyObject *shareable) +{ + if (shareable == NULL || shareable == Py_None) + return NULL; + Py_ssize_t len = PyDict_Size(shareable); + if (len == 0) + return NULL; + + struct _shareditem *shared = PyMem_NEW(struct _shareditem, len+1); + Py_ssize_t pos = 0; + for (Py_ssize_t i=0; i < len; i++) { + PyObject *key, *value; + if (PyDict_Next(shareable, &pos, &key, &value) == 0) { + break; + } + struct _shareditem *item = shared + i; + + if (_PyObject_GetCrossInterpreterData(value, &item->data) != 0) + break; + item->name = PyUnicode_AsUnicodeAndSize(key, &item->namelen); + if (item->name == NULL) { + _PyCrossInterpreterData_Release(&item->data); + break; + } + (item + 1)->name = NULL; // Mark the next one as the last. + } + if (PyErr_Occurred()) { + _sharedns_clear(shared); + PyMem_Free(shared); + return NULL; + } + return shared; +} + +static int +_shareditem_apply(struct _shareditem *item, PyObject *ns) +{ + PyObject *name = PyUnicode_FromUnicode(item->name, item->namelen); + if (name == NULL) { + return 1; + } + PyObject *value = _PyCrossInterpreterData_NewObject(&item->data); + if (value == NULL) { + Py_DECREF(name); + return 1; + } + int res = PyDict_SetItem(ns, name, value); + Py_DECREF(name); + Py_DECREF(value); + return res; +} + +// XXX This cannot use PyObject fields. + +struct _shared_exception { + PyObject *exc; + PyObject *value; + PyObject *tb; +}; + +static struct _shared_exception * +_get_shared_exception(void) +{ + struct _shared_exception *exc = PyMem_NEW(struct _shared_exception, 1); + // XXX Fatal if NULL? + PyErr_Fetch(&exc->exc, &exc->value, &exc->tb); + return exc; +} + +static void +_apply_shared_exception(struct _shared_exception *exc) +{ + if (PyErr_Occurred()) { + _PyErr_ChainExceptions(exc->exc, exc->value, exc->tb); } else { - PyErr_Restore(exc, value, tb); + PyErr_Restore(exc->exc, exc->value, exc->tb); + } + +} + +// XXX Return int instead. + +static PyObject * +_run_script(PyInterpreterState *interp, const char *codestr, + struct _shareditem *shared, struct _shared_exception **exc) +{ + // XXX Do not copy. + + // Get a copy of the __main__ module. + // + // This involves creating a new module, updating it from the + // __main__ module and the given updates (if any), replacing the + // __main__ with the new module in sys.modules, and then using the + // new module's __dict__. At the end we restore the original + // __main__ module. + PyObject *ns = _copy_module_ns(interp, "__main__", "_orig___main___"); + if (ns == NULL) { + return NULL; + } + + // Apply the cross-interpreter data. + if (shared != NULL) { + for (struct _shareditem *item=shared; item->name != NULL; item += 1) { + if (_shareditem_apply(shared, ns) != 0) { + Py_DECREF(ns); + ns = NULL; + goto done; + } + } + } + + // Run the string (see PyRun_SimpleStringFlags). + PyObject *result = PyRun_StringFlags(codestr, Py_file_input, ns, ns, NULL); + if (result == NULL) { + // Get the exception from the subinterpreter. + *exc = _get_shared_exception(); + // XXX Clear the exception? + } else { + Py_DECREF(result); // We throw away the result. + } + +done: + // Restore __main__. + if (_restore_module(interp, "__main__", "_orig___main___") != 0) { + // XXX How to propagate this exception... + //_PyErr_ChainExceptions(exc, value, tb); + } + return ns; +} + +static PyObject * +_run_script_in_interpreter(PyInterpreterState *interp, const char *codestr, + PyObject *shareable) +{ + // XXX lock? + if (_ensure_not_running(interp) < 0) + return NULL; + + struct _shareditem *shared = _get_shared_ns(shareable); + if (shared == NULL && PyErr_Occurred()) + return NULL; + + // Switch to interpreter. + PyThreadState *tstate = PyInterpreterState_ThreadHead(interp); + PyThreadState *save_tstate = PyThreadState_Swap(tstate); + + // Run the script. + struct _shared_exception *exc = NULL; + PyObject *result = _run_script(interp, codestr, shared, &exc); + // XXX What to do if result is NULL? + + // Switch back. + if (save_tstate != NULL) + PyThreadState_Swap(save_tstate); + + // Propagate any exception out to the caller. + if (exc != NULL) { + _apply_shared_exception(exc); + } + + if (shared != NULL) { + _sharedns_clear(shared); + PyMem_Free(shared); } - Py_DECREF(main_mod); return result; } @@ -387,7 +537,8 @@ static PyObject * interp_run_string(PyObject *self, PyObject *args) { PyObject *id, *code; - if (!PyArg_UnpackTuple(args, "run_string", 2, 2, &id, &code)) + PyObject *shared = NULL; + if (!PyArg_UnpackTuple(args, "run_string", 2, 3, &id, &code, &shared)) return NULL; if (!PyLong_Check(id)) { PyErr_SetString(PyExc_TypeError, "first arg (ID) must be an int"); @@ -416,7 +567,7 @@ interp_run_string(PyObject *self, PyObject *args) } // Run the code in the interpreter. - PyObject *ns = _run_string_in_main(interp, codestr, NULL); + PyObject *ns = _run_script_in_interpreter(interp, codestr, shared); if (ns == NULL) return NULL; else @@ -431,12 +582,14 @@ Execute the provided string in the identified interpreter.\n\ See PyRun_SimpleStrings."); +/* XXX Drop run_string_unrestricted(). */ + static PyObject * interp_run_string_unrestricted(PyObject *self, PyObject *args) { - PyObject *id, *code, *ns = NULL; + PyObject *id, *code, *shared = NULL; if (!PyArg_UnpackTuple(args, "run_string_unrestricted", 2, 3, - &id, &code, &ns)) + &id, &code, &shared)) return NULL; if (!PyLong_Check(id)) { PyErr_SetString(PyExc_TypeError, "first arg (ID) must be an int"); @@ -447,8 +600,8 @@ interp_run_string_unrestricted(PyObject *self, PyObject *args) "second arg (code) must be a string"); return NULL; } - if (ns == Py_None) - ns = NULL; + if (shared == Py_None) + shared = NULL; // Look up the interpreter. PyInterpreterState *interp = _look_up(id); @@ -467,7 +620,7 @@ interp_run_string_unrestricted(PyObject *self, PyObject *args) } // Run the code in the interpreter. - return _run_string_in_main(interp, codestr, ns); + return _run_script_in_interpreter(interp, codestr, shared); } PyDoc_STRVAR(run_string_unrestricted_doc, From 0055e808b88dc94b7c2fb87a49373e73acd9b210 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 7 Dec 2017 16:56:35 -0700 Subject: [PATCH 22/78] Do not use a copy of the __main__ ns. --- Modules/_interpretersmodule.c | 102 +++------------------------------- 1 file changed, 7 insertions(+), 95 deletions(-) diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index 11ea2479a6cf42..26b7f69ed71940 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -168,81 +168,6 @@ _ensure_not_running(PyInterpreterState *interp) return 0; } -static PyObject * -_copy_module(PyObject *m, const char *name) -{ - PyObject *orig = PyModule_GetDict(m); // borrowed - if (orig == NULL) { - return NULL; - } - - PyObject *copy = PyModule_New(name); - if (copy == NULL) { - return NULL; - } - PyObject *ns = PyModule_GetDict(copy); // borrowed - if (ns == NULL) - goto error; - - if (PyDict_Merge(ns, orig, 1) < 0) - goto error; - return copy; - -error: - Py_DECREF(copy); - return NULL; -} - -static PyObject * -_copy_module_ns(PyInterpreterState *interp, - const char *name, const char *tempname) -{ - // Get the namespace in which to execute. This involves creating - // a new module, updating it from the __main__ module and the given - // updates (if any), replacing the __main__ with the new module in - // sys.modules, and then using the new module's __dict__. At the - // end we restore the original __main__ module. - PyObject *main_mod = PyMapping_GetItemString(interp->modules, name); - if (main_mod == NULL) - return NULL; - PyObject *m = _copy_module(main_mod, name); - if (m == NULL) { - Py_DECREF(main_mod); - return NULL; - } - if (tempname != NULL) { - if (PyMapping_SetItemString(interp->modules, tempname, main_mod) < 0) { - Py_DECREF(main_mod); - Py_DECREF(m); - return NULL; - } - } - Py_DECREF(main_mod); - if (PyMapping_SetItemString(interp->modules, name, m) < 0) { - Py_DECREF(m); - return NULL; - } - PyObject *ns = PyModule_GetDict(m); // borrowed - Py_INCREF(ns); - Py_DECREF(m); - return ns; -} - -static int -_restore_module(PyInterpreterState *interp, - const char *name, const char *tempname) -{ - PyObject *main_mod = PyMapping_GetItemString(interp->modules, tempname); - if (main_mod == NULL) - return -1; - if (PyMapping_SetItemString(interp->modules, name, main_mod) < 0) { - Py_DECREF(main_mod); - return -1; - } - Py_DECREF(main_mod); - return 0; -} - struct _shareditem { Py_UNICODE *name; Py_ssize_t namelen; @@ -344,27 +269,20 @@ static PyObject * _run_script(PyInterpreterState *interp, const char *codestr, struct _shareditem *shared, struct _shared_exception **exc) { - // XXX Do not copy. - - // Get a copy of the __main__ module. - // - // This involves creating a new module, updating it from the - // __main__ module and the given updates (if any), replacing the - // __main__ with the new module in sys.modules, and then using the - // new module's __dict__. At the end we restore the original - // __main__ module. - PyObject *ns = _copy_module_ns(interp, "__main__", "_orig___main___"); - if (ns == NULL) { + PyObject *main_mod = PyMapping_GetItemString(interp->modules, "__main__"); + if (main_mod == NULL) + return NULL; + PyObject *ns = PyModule_GetDict(main_mod); // borrowed + Py_DECREF(main_mod); + if (ns == NULL) return NULL; - } // Apply the cross-interpreter data. if (shared != NULL) { for (struct _shareditem *item=shared; item->name != NULL; item += 1) { if (_shareditem_apply(shared, ns) != 0) { Py_DECREF(ns); - ns = NULL; - goto done; + return NULL; } } } @@ -379,12 +297,6 @@ _run_script(PyInterpreterState *interp, const char *codestr, Py_DECREF(result); // We throw away the result. } -done: - // Restore __main__. - if (_restore_module(interp, "__main__", "_orig___main___") != 0) { - // XXX How to propagate this exception... - //_PyErr_ChainExceptions(exc, value, tb); - } return ns; } From 9030634bc42ea68f865bc7f4ae35145436091c95 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 7 Dec 2017 17:04:35 -0700 Subject: [PATCH 23/78] Never return the execution namespace. --- Modules/_interpretersmodule.c | 85 +++++++---------------------------- 1 file changed, 15 insertions(+), 70 deletions(-) diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index 26b7f69ed71940..324b25f6a187dc 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -263,32 +263,31 @@ _apply_shared_exception(struct _shared_exception *exc) } -// XXX Return int instead. - -static PyObject * +static int _run_script(PyInterpreterState *interp, const char *codestr, struct _shareditem *shared, struct _shared_exception **exc) { PyObject *main_mod = PyMapping_GetItemString(interp->modules, "__main__"); if (main_mod == NULL) - return NULL; + return -1; PyObject *ns = PyModule_GetDict(main_mod); // borrowed Py_DECREF(main_mod); if (ns == NULL) - return NULL; + return -1; // Apply the cross-interpreter data. if (shared != NULL) { for (struct _shareditem *item=shared; item->name != NULL; item += 1) { if (_shareditem_apply(shared, ns) != 0) { Py_DECREF(ns); - return NULL; + return -1; } } } // Run the string (see PyRun_SimpleStringFlags). PyObject *result = PyRun_StringFlags(codestr, Py_file_input, ns, ns, NULL); + Py_DECREF(ns); if (result == NULL) { // Get the exception from the subinterpreter. *exc = _get_shared_exception(); @@ -297,20 +296,20 @@ _run_script(PyInterpreterState *interp, const char *codestr, Py_DECREF(result); // We throw away the result. } - return ns; + return 0; } -static PyObject * +static int _run_script_in_interpreter(PyInterpreterState *interp, const char *codestr, PyObject *shareable) { // XXX lock? if (_ensure_not_running(interp) < 0) - return NULL; + return -1; struct _shareditem *shared = _get_shared_ns(shareable); if (shared == NULL && PyErr_Occurred()) - return NULL; + return -1; // Switch to interpreter. PyThreadState *tstate = PyInterpreterState_ThreadHead(interp); @@ -318,8 +317,9 @@ _run_script_in_interpreter(PyInterpreterState *interp, const char *codestr, // Run the script. struct _shared_exception *exc = NULL; - PyObject *result = _run_script(interp, codestr, shared, &exc); - // XXX What to do if result is NULL? + if (_run_script(interp, codestr, shared, &exc) != 0) { + // XXX What to do if the the result isn't 0? + } // Switch back. if (save_tstate != NULL) @@ -335,7 +335,7 @@ _run_script_in_interpreter(PyInterpreterState *interp, const char *codestr, PyMem_Free(shared); } - return result; + return 0; } @@ -479,11 +479,9 @@ interp_run_string(PyObject *self, PyObject *args) } // Run the code in the interpreter. - PyObject *ns = _run_script_in_interpreter(interp, codestr, shared); - if (ns == NULL) + if (_run_script_in_interpreter(interp, codestr, shared) != 0) return NULL; - else - Py_RETURN_NONE; + Py_RETURN_NONE; } PyDoc_STRVAR(run_string_doc, @@ -494,57 +492,6 @@ Execute the provided string in the identified interpreter.\n\ See PyRun_SimpleStrings."); -/* XXX Drop run_string_unrestricted(). */ - -static PyObject * -interp_run_string_unrestricted(PyObject *self, PyObject *args) -{ - PyObject *id, *code, *shared = NULL; - if (!PyArg_UnpackTuple(args, "run_string_unrestricted", 2, 3, - &id, &code, &shared)) - return NULL; - if (!PyLong_Check(id)) { - PyErr_SetString(PyExc_TypeError, "first arg (ID) must be an int"); - return NULL; - } - if (!PyUnicode_Check(code)) { - PyErr_SetString(PyExc_TypeError, - "second arg (code) must be a string"); - return NULL; - } - if (shared == Py_None) - shared = NULL; - - // Look up the interpreter. - PyInterpreterState *interp = _look_up(id); - if (interp == NULL) - return NULL; - - // Extract code. - Py_ssize_t size; - const char *codestr = PyUnicode_AsUTF8AndSize(code, &size); - if (codestr == NULL) - return NULL; - if (strlen(codestr) != (size_t)size) { - PyErr_SetString(PyExc_ValueError, - "source code string cannot contain null bytes"); - return NULL; - } - - // Run the code in the interpreter. - return _run_script_in_interpreter(interp, codestr, shared); -} - -PyDoc_STRVAR(run_string_unrestricted_doc, -"run_string_unrestricted(ID, sourcetext, ns=None) -> main module ns\n\ -\n\ -Execute the provided string in the identified interpreter. Return the\n\ -dict in which the code executed. If the ns arg is provided then it is\n\ -merged into the execution namespace before the code is executed.\n\ -\n\ -See PyRun_SimpleStrings."); - - static PyObject * object_is_shareable(PyObject *self, PyObject *args) { @@ -575,8 +522,6 @@ static PyMethodDef module_functions[] = { {"run_string", (PyCFunction)interp_run_string, METH_VARARGS, run_string_doc}, - {"run_string_unrestricted", (PyCFunction)interp_run_string_unrestricted, - METH_VARARGS, run_string_unrestricted_doc}, {"is_shareable", (PyCFunction)object_is_shareable, METH_VARARGS, is_shareable_doc}, From df53c6cd611499e22a6944c12076e552430f10f8 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 7 Dec 2017 17:09:03 -0700 Subject: [PATCH 24/78] Group sharing-related code. --- Modules/_interpretersmodule.c | 150 +++++++++++++++++----------------- 1 file changed, 75 insertions(+), 75 deletions(-) diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index 324b25f6a187dc..7909cef65c0b7e 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -18,7 +18,7 @@ _get_current(void) return tstate->interp; } -/* sharing-specific functions */ +/* sharing-specific functions and structs */ static int _PyObject_CheckShareable(PyObject *obj) @@ -94,80 +94,6 @@ _PyCrossInterpreterData_NewObject(_PyCrossInterpreterData *data) return data->new_object(data); } -/* interpreter-specific functions */ - -static PyInterpreterState * -_look_up_int64(PY_INT64_T requested_id) -{ - if (requested_id < 0) - goto error; - - PyInterpreterState *interp = PyInterpreterState_Head(); - while (interp != NULL) { - PY_INT64_T id = PyInterpreterState_GetID(interp); - if (id < 0) - return NULL; - if (requested_id == id) - return interp; - interp = PyInterpreterState_Next(interp); - } - -error: - PyErr_Format(PyExc_RuntimeError, - "unrecognized interpreter ID %lld", requested_id); - return NULL; -} - -static PyInterpreterState * -_look_up(PyObject *requested_id) -{ - long long id = PyLong_AsLongLong(requested_id); - if (id == -1 && PyErr_Occurred() != NULL) - return NULL; - assert(id <= INT64_MAX); - return _look_up_int64(id); -} - -static PyObject * -_get_id(PyInterpreterState *interp) -{ - PY_INT64_T id = PyInterpreterState_GetID(interp); - if (id < 0) - return NULL; - return PyLong_FromLongLong(id); -} - -static int -_is_running(PyInterpreterState *interp) -{ - PyThreadState *tstate = PyInterpreterState_ThreadHead(interp); - if (PyThreadState_Next(tstate) != NULL) { - PyErr_SetString(PyExc_RuntimeError, - "interpreter has more than one thread"); - return -1; - } - PyFrameObject *frame = _PyThreadState_GetFrame(tstate); - if (frame == NULL) { - if (PyErr_Occurred() != NULL) - return -1; - return 0; - } - return (int)(frame->f_executing); -} - -static int -_ensure_not_running(PyInterpreterState *interp) -{ - int is_running = _is_running(interp); - if (is_running < 0) - return -1; - if (is_running) { - PyErr_Format(PyExc_RuntimeError, "interpreter already running"); - return -1; - } - return 0; -} - struct _shareditem { Py_UNICODE *name; Py_ssize_t namelen; @@ -263,6 +189,80 @@ _apply_shared_exception(struct _shared_exception *exc) } +/* interpreter-specific functions */ + +static PyInterpreterState * +_look_up_int64(PY_INT64_T requested_id) +{ + if (requested_id < 0) + goto error; + + PyInterpreterState *interp = PyInterpreterState_Head(); + while (interp != NULL) { + PY_INT64_T id = PyInterpreterState_GetID(interp); + if (id < 0) + return NULL; + if (requested_id == id) + return interp; + interp = PyInterpreterState_Next(interp); + } + +error: + PyErr_Format(PyExc_RuntimeError, + "unrecognized interpreter ID %lld", requested_id); + return NULL; +} + +static PyInterpreterState * +_look_up(PyObject *requested_id) +{ + long long id = PyLong_AsLongLong(requested_id); + if (id == -1 && PyErr_Occurred() != NULL) + return NULL; + assert(id <= INT64_MAX); + return _look_up_int64(id); +} + +static PyObject * +_get_id(PyInterpreterState *interp) +{ + PY_INT64_T id = PyInterpreterState_GetID(interp); + if (id < 0) + return NULL; + return PyLong_FromLongLong(id); +} + +static int +_is_running(PyInterpreterState *interp) +{ + PyThreadState *tstate = PyInterpreterState_ThreadHead(interp); + if (PyThreadState_Next(tstate) != NULL) { + PyErr_SetString(PyExc_RuntimeError, + "interpreter has more than one thread"); + return -1; + } + PyFrameObject *frame = _PyThreadState_GetFrame(tstate); + if (frame == NULL) { + if (PyErr_Occurred() != NULL) + return -1; + return 0; + } + return (int)(frame->f_executing); +} + +static int +_ensure_not_running(PyInterpreterState *interp) +{ + int is_running = _is_running(interp); + if (is_running < 0) + return -1; + if (is_running) { + PyErr_Format(PyExc_RuntimeError, "interpreter already running"); + return -1; + } + return 0; +} + static int _run_script(PyInterpreterState *interp, const char *codestr, struct _shareditem *shared, struct _shared_exception **exc) From b27597fa66bf1da1cd9e168f8426636eca5a26a0 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 7 Dec 2017 17:10:45 -0700 Subject: [PATCH 25/78] Fix a refcount. --- Modules/_interpretersmodule.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index 7909cef65c0b7e..299e71e620ddc3 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -271,6 +271,7 @@ _run_script(PyInterpreterState *interp, const char *codestr, if (main_mod == NULL) return -1; PyObject *ns = PyModule_GetDict(main_mod); // borrowed + Py_INCREF(ns); Py_DECREF(main_mod); if (ns == NULL) return -1; From b69a6438c64a2092b51efaef24d074bf141cf20e Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 29 Dec 2016 15:30:43 -0700 Subject: [PATCH 26/78] Add get_current() and enumerate(). --- Doc/library/_interpreters.rst | 10 ++++ Lib/test/test__interpreters.py | 88 ++++++++++++++++++++++++---------- Modules/_interpretersmodule.c | 27 ++++++++++- 3 files changed, 97 insertions(+), 28 deletions(-) diff --git a/Doc/library/_interpreters.rst b/Doc/library/_interpreters.rst index 445926ac4b87ba..8c58d22cca79df 100644 --- a/Doc/library/_interpreters.rst +++ b/Doc/library/_interpreters.rst @@ -31,6 +31,16 @@ support multiple interpreters. It defines the following functions: +.. function:: enumerate() + + Return a list of the IDs of every existing interpreter. + + +.. function:: get_current() + + Return the ID of the currently running interpreter. + + .. function:: create() Initialize a new Python interpreter and return its identifier. The diff --git a/Lib/test/test__interpreters.py b/Lib/test/test__interpreters.py index 459ff8650a8470..c0836299b83117 100644 --- a/Lib/test/test__interpreters.py +++ b/Lib/test/test__interpreters.py @@ -67,7 +67,7 @@ def test_still_running_at_exit(self): class TestBase(unittest.TestCase): def tearDown(self): - for id in interpreters._enumerate(): + for id in interpreters.enumerate(): if id == 0: # main continue try: @@ -76,13 +76,49 @@ def tearDown(self): pass # already destroyed +class EnumerateTests(TestBase): + + def test_multiple(self): + main, = interpreters.enumerate() + id1 = interpreters.create() + id2 = interpreters.create() + ids = interpreters.enumerate() + + self.assertEqual(set(ids), {main, id1, id2}) + + def test_main_only(self): + main, = interpreters.enumerate() + + self.assertEqual(main, 0) + + +class GetCurrentTests(TestBase): + + def test_main(self): + main, = interpreters.enumerate() + id = interpreters.get_current() + + self.assertEqual(id, main) + + def test_sub(self): + id1 = interpreters.create() + ns = interpreters.run_string_unrestricted(id1, dedent(""" + import _interpreters + id = _interpreters.get_current() + """)) + id2 = ns['id'] + + self.assertEqual(id2, id1) + + class CreateTests(TestBase): def test_in_main(self): id = interpreters.create() - self.assertIn(id, interpreters._enumerate()) + self.assertIn(id, interpreters.enumerate()) + @unittest.skip('enable this test when working on pystate.c') def test_unique_id(self): seen = set() for _ in range(100): @@ -105,10 +141,10 @@ def f(): with lock: t.start() t.join() - self.assertIn(id, interpreters._enumerate()) + self.assertIn(id, interpreters.enumerate()) def test_in_subinterpreter(self): - main, = interpreters._enumerate() + main, = interpreters.enumerate() id1 = interpreters.create() ns = interpreters.run_string_unrestricted(id1, dedent(""" import _interpreters @@ -116,10 +152,10 @@ def test_in_subinterpreter(self): """)) id2 = ns['id'] - self.assertEqual(set(interpreters._enumerate()), {main, id1, id2}) + self.assertEqual(set(interpreters.enumerate()), {main, id1, id2}) def test_in_threaded_subinterpreter(self): - main, = interpreters._enumerate() + main, = interpreters.enumerate() id1 = interpreters.create() ns = None script = dedent(""" @@ -135,11 +171,11 @@ def f(): t.join() id2 = ns['id'] - self.assertEqual(set(interpreters._enumerate()), {main, id1, id2}) + self.assertEqual(set(interpreters.enumerate()), {main, id1, id2}) def test_after_destroy_all(self): - before = set(interpreters._enumerate()) + before = set(interpreters.enumerate()) # Create 3 subinterpreters. ids = [] for _ in range(3): @@ -150,10 +186,10 @@ def test_after_destroy_all(self): interpreters.destroy(id) # Finally, create another. id = interpreters.create() - self.assertEqual(set(interpreters._enumerate()), before | {id}) + self.assertEqual(set(interpreters.enumerate()), before | {id}) def test_after_destroy_some(self): - before = set(interpreters._enumerate()) + before = set(interpreters.enumerate()) # Create 3 subinterpreters. id1 = interpreters.create() id2 = interpreters.create() @@ -163,7 +199,7 @@ def test_after_destroy_some(self): interpreters.destroy(id3) # Finally, create another. id = interpreters.create() - self.assertEqual(set(interpreters._enumerate()), before | {id, id2}) + self.assertEqual(set(interpreters.enumerate()), before | {id, id2}) class DestroyTests(TestBase): @@ -172,25 +208,25 @@ def test_one(self): id1 = interpreters.create() id2 = interpreters.create() id3 = interpreters.create() - self.assertIn(id2, interpreters._enumerate()) + self.assertIn(id2, interpreters.enumerate()) interpreters.destroy(id2) - self.assertNotIn(id2, interpreters._enumerate()) - self.assertIn(id1, interpreters._enumerate()) - self.assertIn(id3, interpreters._enumerate()) + self.assertNotIn(id2, interpreters.enumerate()) + self.assertIn(id1, interpreters.enumerate()) + self.assertIn(id3, interpreters.enumerate()) def test_all(self): - before = set(interpreters._enumerate()) + before = set(interpreters.enumerate()) ids = set() for _ in range(3): id = interpreters.create() ids.add(id) - self.assertEqual(set(interpreters._enumerate()), before | ids) + self.assertEqual(set(interpreters.enumerate()), before | ids) for id in ids: interpreters.destroy(id) - self.assertEqual(set(interpreters._enumerate()), before) + self.assertEqual(set(interpreters.enumerate()), before) def test_main(self): - main, = interpreters._enumerate() + main, = interpreters.enumerate() with self.assertRaises(RuntimeError): interpreters.destroy(main) @@ -217,7 +253,7 @@ def test_bad_id(self): interpreters.destroy(-1) def test_from_current(self): - main, = interpreters._enumerate() + main, = interpreters.enumerate() id = interpreters.create() script = dedent(""" import _interpreters @@ -226,10 +262,10 @@ def test_from_current(self): with self.assertRaises(RuntimeError): interpreters.run_string(id, script) - self.assertEqual(set(interpreters._enumerate()), {main, id}) + self.assertEqual(set(interpreters.enumerate()), {main, id}) def test_from_sibling(self): - main, = interpreters._enumerate() + main, = interpreters.enumerate() id1 = interpreters.create() id2 = interpreters.create() script = dedent(""" @@ -238,7 +274,7 @@ def test_from_sibling(self): """).format(id2) interpreters.run_string(id1, script) - self.assertEqual(set(interpreters._enumerate()), {main, id1}) + self.assertEqual(set(interpreters.enumerate()), {main, id1}) def test_from_other_thread(self): id = interpreters.create() @@ -252,7 +288,7 @@ def f(): def test_still_running(self): # XXX Rewrite this test without files by using # run_string_unrestricted(). - main, = interpreters._enumerate() + main, = interpreters.enumerate() id = interpreters.create() def f(): interpreters.run_string(id, wait_script) @@ -265,7 +301,7 @@ def f(): interpreters.destroy(id) t.join() - self.assertEqual(set(interpreters._enumerate()), {main, id}) + self.assertEqual(set(interpreters.enumerate()), {main, id}) class RunStringTests(TestBase): @@ -389,7 +425,7 @@ def f(): def test_does_not_exist(self): id = 0 - while id in interpreters._enumerate(): + while id in interpreters.enumerate(): id += 1 with self.assertRaises(RuntimeError): interpreters.run_string(id, 'print("spam")') diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index 299e71e620ddc3..a29333ac621df6 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -446,6 +446,27 @@ interp_enumerate(PyObject *self) return ids; } +PyDoc_STRVAR(enumerate_doc, +"enumerate() -> [ID]\n\ +\n\ +Return a list containing the ID of every existing interpreter."); + + +static PyObject * +interp_get_current(PyObject *self) +{ + PyInterpreterState *interp =_get_current(); + if (interp == NULL) + return NULL; + return _get_id(interp); +} + +PyDoc_STRVAR(get_current_doc, +"get_current() -> ID\n\ +\n\ +Return the ID of current interpreter."); + + static PyObject * interp_run_string(PyObject *self, PyObject *args) { @@ -518,8 +539,10 @@ static PyMethodDef module_functions[] = { {"destroy", (PyCFunction)interp_destroy, METH_VARARGS, destroy_doc}, - {"_enumerate", (PyCFunction)interp_enumerate, - METH_NOARGS, NULL}, + {"enumerate", (PyCFunction)interp_enumerate, + METH_NOARGS, enumerate_doc}, + {"get_current", (PyCFunction)interp_get_current, + METH_NOARGS, get_current_doc}, {"run_string", (PyCFunction)interp_run_string, METH_VARARGS, run_string_doc}, From dc21da6ec052aad9d028a0ed88e20b0346f596a2 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 29 Dec 2016 15:33:17 -0700 Subject: [PATCH 27/78] Add is_running(). --- Doc/library/_interpreters.rst | 6 ++++++ Lib/test/test__interpreters.py | 30 ++++++++++++++++++++++++++++++ Modules/_interpretersmodule.c | 30 ++++++++++++++++++++++++++++++ 3 files changed, 66 insertions(+) diff --git a/Doc/library/_interpreters.rst b/Doc/library/_interpreters.rst index 8c58d22cca79df..1bb107e500e4a5 100644 --- a/Doc/library/_interpreters.rst +++ b/Doc/library/_interpreters.rst @@ -41,6 +41,12 @@ It defines the following functions: Return the ID of the currently running interpreter. +.. function:: is_running(id) + + Return whether or not the identified interpreter is currently + running any code. + + .. function:: create() Initialize a new Python interpreter and return its identifier. The diff --git a/Lib/test/test__interpreters.py b/Lib/test/test__interpreters.py index c0836299b83117..49ed4444be84ee 100644 --- a/Lib/test/test__interpreters.py +++ b/Lib/test/test__interpreters.py @@ -111,6 +111,36 @@ def test_sub(self): self.assertEqual(id2, id1) +class IsRunningTests(TestBase): + + def test_main_running(self): + main, = interpreters.enumerate() + sub = interpreters.create() + main_running = interpreters.is_running(main) + sub_running = interpreters.is_running(sub) + + self.assertTrue(main_running) + self.assertFalse(sub_running) + + def test_sub_running(self): + main, = interpreters.enumerate() + sub1 = interpreters.create() + sub2 = interpreters.create() + ns = interpreters.run_string_unrestricted(sub1, dedent(f""" + import _interpreters + main = _interpreters.is_running({main}) + sub1 = _interpreters.is_running({sub1}) + sub2 = _interpreters.is_running({sub2}) + """)) + main_running = ns['main'] + sub1_running = ns['sub1'] + sub2_running = ns['sub2'] + + self.assertTrue(main_running) + self.assertTrue(sub1_running) + self.assertFalse(sub2_running) + + class CreateTests(TestBase): def test_in_main(self): diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index a29333ac621df6..375388fb75390f 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -533,6 +533,34 @@ Return True if the object's data may be shared between interpreters and\n\ False otherwise."); +static PyObject * +interp_is_running(PyObject *self, PyObject *args) +{ + PyObject *id; + if (!PyArg_UnpackTuple(args, "is_running", 1, 1, &id)) + return NULL; + if (!PyLong_Check(id)) { + PyErr_SetString(PyExc_TypeError, "ID must be an int"); + return NULL; + } + + PyInterpreterState *interp = _look_up(id); + if (interp == NULL) + return NULL; + int is_running = _is_running(interp); + if (is_running < 0) + return NULL; + if (is_running) + Py_RETURN_TRUE; + Py_RETURN_FALSE; +} + +PyDoc_STRVAR(is_running_doc, +"is_running(id) -> bool\n\ +\n\ +Return whether or not the identified interpreter is running."); + + static PyMethodDef module_functions[] = { {"create", (PyCFunction)interp_create, METH_VARARGS, create_doc}, @@ -543,6 +571,8 @@ static PyMethodDef module_functions[] = { METH_NOARGS, enumerate_doc}, {"get_current", (PyCFunction)interp_get_current, METH_NOARGS, get_current_doc}, + {"is_running", (PyCFunction)interp_is_running, + METH_VARARGS, is_running_doc}, {"run_string", (PyCFunction)interp_run_string, METH_VARARGS, run_string_doc}, From a4dad0888219d1c645ddd5c95ffadc075a04b393 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 4 Jan 2017 14:41:58 -0700 Subject: [PATCH 28/78] Add get_main(). --- Doc/library/_interpreters.rst | 5 +++++ Lib/test/test__interpreters.py | 10 ++++++++++ Modules/_interpretersmodule.c | 15 +++++++++++++++ 3 files changed, 30 insertions(+) diff --git a/Doc/library/_interpreters.rst b/Doc/library/_interpreters.rst index 1bb107e500e4a5..bbc945016e2880 100644 --- a/Doc/library/_interpreters.rst +++ b/Doc/library/_interpreters.rst @@ -41,6 +41,11 @@ It defines the following functions: Return the ID of the currently running interpreter. +.. function:: get_main() + + Return the ID of the main interpreter. + + .. function:: is_running(id) Return whether or not the identified interpreter is currently diff --git a/Lib/test/test__interpreters.py b/Lib/test/test__interpreters.py index 49ed4444be84ee..29d04aa4d00cfb 100644 --- a/Lib/test/test__interpreters.py +++ b/Lib/test/test__interpreters.py @@ -111,6 +111,16 @@ def test_sub(self): self.assertEqual(id2, id1) +class GetMainTests(TestBase): + + def test_main(self): + expected, = interpreters.enumerate() + main = interpreters.get_main() + + self.assertEqual(main, 0) + self.assertEqual(main, expected) + + class IsRunningTests(TestBase): def test_main_running(self): diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index 375388fb75390f..f9b800cb1ace35 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -467,6 +467,19 @@ PyDoc_STRVAR(get_current_doc, Return the ID of current interpreter."); +static PyObject * +interp_get_main(PyObject *self) +{ + // Currently, 0 is always the main interpreter. + return PyLong_FromLongLong(0); +} + +PyDoc_STRVAR(get_main_doc, +"get_main() -> ID\n\ +\n\ +Return the ID of main interpreter."); + + static PyObject * interp_run_string(PyObject *self, PyObject *args) { @@ -571,6 +584,8 @@ static PyMethodDef module_functions[] = { METH_NOARGS, enumerate_doc}, {"get_current", (PyCFunction)interp_get_current, METH_NOARGS, get_current_doc}, + {"get_main", (PyCFunction)interp_get_main, + METH_NOARGS, get_main_doc}, {"is_running", (PyCFunction)interp_is_running, METH_VARARGS, is_running_doc}, From bc818e50fc28a61578eff0e174d1f8e06e03a48a Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 11 Dec 2017 19:10:47 -0700 Subject: [PATCH 29/78] Fix an INCREF placement. --- Modules/_interpretersmodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index f9b800cb1ace35..36996d52fd6b17 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -271,10 +271,10 @@ _run_script(PyInterpreterState *interp, const char *codestr, if (main_mod == NULL) return -1; PyObject *ns = PyModule_GetDict(main_mod); // borrowed - Py_INCREF(ns); Py_DECREF(main_mod); if (ns == NULL) return -1; + Py_INCREF(ns); // Apply the cross-interpreter data. if (shared != NULL) { From d80ffdbe43e20fdec4389074bf4c1d48094a4cfe Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 11 Dec 2017 19:16:56 -0700 Subject: [PATCH 30/78] Fix error propagation. --- Modules/_interpretersmodule.c | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index 36996d52fd6b17..4395279c35e0d2 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -269,11 +269,11 @@ _run_script(PyInterpreterState *interp, const char *codestr, { PyObject *main_mod = PyMapping_GetItemString(interp->modules, "__main__"); if (main_mod == NULL) - return -1; + goto error; PyObject *ns = PyModule_GetDict(main_mod); // borrowed Py_DECREF(main_mod); if (ns == NULL) - return -1; + goto error; Py_INCREF(ns); // Apply the cross-interpreter data. @@ -281,7 +281,7 @@ _run_script(PyInterpreterState *interp, const char *codestr, for (struct _shareditem *item=shared; item->name != NULL; item += 1) { if (_shareditem_apply(shared, ns) != 0) { Py_DECREF(ns); - return -1; + goto error; } } } @@ -290,14 +290,17 @@ _run_script(PyInterpreterState *interp, const char *codestr, PyObject *result = PyRun_StringFlags(codestr, Py_file_input, ns, ns, NULL); Py_DECREF(ns); if (result == NULL) { - // Get the exception from the subinterpreter. - *exc = _get_shared_exception(); - // XXX Clear the exception? + goto error; } else { Py_DECREF(result); // We throw away the result. } return 0; + +error: + *exc = _get_shared_exception(); + PyErr_Clear(); + return -1; } static int @@ -318,9 +321,7 @@ _run_script_in_interpreter(PyInterpreterState *interp, const char *codestr, // Run the script. struct _shared_exception *exc = NULL; - if (_run_script(interp, codestr, shared, &exc) != 0) { - // XXX What to do if the the result isn't 0? - } + int result = _run_script(interp, codestr, shared, &exc); // Switch back. if (save_tstate != NULL) @@ -336,7 +337,7 @@ _run_script_in_interpreter(PyInterpreterState *interp, const char *codestr, PyMem_Free(shared); } - return 0; + return result; } From f4b8819dcee59830d998b1d56efab37e3c0d8bc7 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 11 Dec 2017 19:18:14 -0700 Subject: [PATCH 31/78] enumerate -> list_all. --- Lib/test/test__interpreters.py | 66 +++++++++++++++++----------------- Modules/_interpretersmodule.c | 10 +++--- 2 files changed, 38 insertions(+), 38 deletions(-) diff --git a/Lib/test/test__interpreters.py b/Lib/test/test__interpreters.py index 29d04aa4d00cfb..7609caee7fdc75 100644 --- a/Lib/test/test__interpreters.py +++ b/Lib/test/test__interpreters.py @@ -67,7 +67,7 @@ def test_still_running_at_exit(self): class TestBase(unittest.TestCase): def tearDown(self): - for id in interpreters.enumerate(): + for id in interpreters.list_all(): if id == 0: # main continue try: @@ -79,15 +79,15 @@ def tearDown(self): class EnumerateTests(TestBase): def test_multiple(self): - main, = interpreters.enumerate() + main, = interpreters.list_all() id1 = interpreters.create() id2 = interpreters.create() - ids = interpreters.enumerate() + ids = interpreters.list_all() self.assertEqual(set(ids), {main, id1, id2}) def test_main_only(self): - main, = interpreters.enumerate() + main, = interpreters.list_all() self.assertEqual(main, 0) @@ -95,7 +95,7 @@ def test_main_only(self): class GetCurrentTests(TestBase): def test_main(self): - main, = interpreters.enumerate() + main, = interpreters.list_all() id = interpreters.get_current() self.assertEqual(id, main) @@ -114,7 +114,7 @@ def test_sub(self): class GetMainTests(TestBase): def test_main(self): - expected, = interpreters.enumerate() + expected, = interpreters.list_all() main = interpreters.get_main() self.assertEqual(main, 0) @@ -124,7 +124,7 @@ def test_main(self): class IsRunningTests(TestBase): def test_main_running(self): - main, = interpreters.enumerate() + main, = interpreters.list_all() sub = interpreters.create() main_running = interpreters.is_running(main) sub_running = interpreters.is_running(sub) @@ -133,7 +133,7 @@ def test_main_running(self): self.assertFalse(sub_running) def test_sub_running(self): - main, = interpreters.enumerate() + main, = interpreters.list_all() sub1 = interpreters.create() sub2 = interpreters.create() ns = interpreters.run_string_unrestricted(sub1, dedent(f""" @@ -156,7 +156,7 @@ class CreateTests(TestBase): def test_in_main(self): id = interpreters.create() - self.assertIn(id, interpreters.enumerate()) + self.assertIn(id, interpreters.list_all()) @unittest.skip('enable this test when working on pystate.c') def test_unique_id(self): @@ -181,10 +181,10 @@ def f(): with lock: t.start() t.join() - self.assertIn(id, interpreters.enumerate()) + self.assertIn(id, interpreters.list_all()) def test_in_subinterpreter(self): - main, = interpreters.enumerate() + main, = interpreters.list_all() id1 = interpreters.create() ns = interpreters.run_string_unrestricted(id1, dedent(""" import _interpreters @@ -192,10 +192,10 @@ def test_in_subinterpreter(self): """)) id2 = ns['id'] - self.assertEqual(set(interpreters.enumerate()), {main, id1, id2}) + self.assertEqual(set(interpreters.list_all()), {main, id1, id2}) def test_in_threaded_subinterpreter(self): - main, = interpreters.enumerate() + main, = interpreters.list_all() id1 = interpreters.create() ns = None script = dedent(""" @@ -211,11 +211,11 @@ def f(): t.join() id2 = ns['id'] - self.assertEqual(set(interpreters.enumerate()), {main, id1, id2}) + self.assertEqual(set(interpreters.list_all()), {main, id1, id2}) def test_after_destroy_all(self): - before = set(interpreters.enumerate()) + before = set(interpreters.list_all()) # Create 3 subinterpreters. ids = [] for _ in range(3): @@ -226,10 +226,10 @@ def test_after_destroy_all(self): interpreters.destroy(id) # Finally, create another. id = interpreters.create() - self.assertEqual(set(interpreters.enumerate()), before | {id}) + self.assertEqual(set(interpreters.list_all()), before | {id}) def test_after_destroy_some(self): - before = set(interpreters.enumerate()) + before = set(interpreters.list_all()) # Create 3 subinterpreters. id1 = interpreters.create() id2 = interpreters.create() @@ -239,7 +239,7 @@ def test_after_destroy_some(self): interpreters.destroy(id3) # Finally, create another. id = interpreters.create() - self.assertEqual(set(interpreters.enumerate()), before | {id, id2}) + self.assertEqual(set(interpreters.list_all()), before | {id, id2}) class DestroyTests(TestBase): @@ -248,25 +248,25 @@ def test_one(self): id1 = interpreters.create() id2 = interpreters.create() id3 = interpreters.create() - self.assertIn(id2, interpreters.enumerate()) + self.assertIn(id2, interpreters.list_all()) interpreters.destroy(id2) - self.assertNotIn(id2, interpreters.enumerate()) - self.assertIn(id1, interpreters.enumerate()) - self.assertIn(id3, interpreters.enumerate()) + self.assertNotIn(id2, interpreters.list_all()) + self.assertIn(id1, interpreters.list_all()) + self.assertIn(id3, interpreters.list_all()) def test_all(self): - before = set(interpreters.enumerate()) + before = set(interpreters.list_all()) ids = set() for _ in range(3): id = interpreters.create() ids.add(id) - self.assertEqual(set(interpreters.enumerate()), before | ids) + self.assertEqual(set(interpreters.list_all()), before | ids) for id in ids: interpreters.destroy(id) - self.assertEqual(set(interpreters.enumerate()), before) + self.assertEqual(set(interpreters.list_all()), before) def test_main(self): - main, = interpreters.enumerate() + main, = interpreters.list_all() with self.assertRaises(RuntimeError): interpreters.destroy(main) @@ -293,7 +293,7 @@ def test_bad_id(self): interpreters.destroy(-1) def test_from_current(self): - main, = interpreters.enumerate() + main, = interpreters.list_all() id = interpreters.create() script = dedent(""" import _interpreters @@ -302,10 +302,10 @@ def test_from_current(self): with self.assertRaises(RuntimeError): interpreters.run_string(id, script) - self.assertEqual(set(interpreters.enumerate()), {main, id}) + self.assertEqual(set(interpreters.list_all()), {main, id}) def test_from_sibling(self): - main, = interpreters.enumerate() + main, = interpreters.list_all() id1 = interpreters.create() id2 = interpreters.create() script = dedent(""" @@ -314,7 +314,7 @@ def test_from_sibling(self): """).format(id2) interpreters.run_string(id1, script) - self.assertEqual(set(interpreters.enumerate()), {main, id1}) + self.assertEqual(set(interpreters.list_all()), {main, id1}) def test_from_other_thread(self): id = interpreters.create() @@ -328,7 +328,7 @@ def f(): def test_still_running(self): # XXX Rewrite this test without files by using # run_string_unrestricted(). - main, = interpreters.enumerate() + main, = interpreters.list_all() id = interpreters.create() def f(): interpreters.run_string(id, wait_script) @@ -341,7 +341,7 @@ def f(): interpreters.destroy(id) t.join() - self.assertEqual(set(interpreters.enumerate()), {main, id}) + self.assertEqual(set(interpreters.list_all()), {main, id}) class RunStringTests(TestBase): @@ -465,7 +465,7 @@ def f(): def test_does_not_exist(self): id = 0 - while id in interpreters.enumerate(): + while id in interpreters.list_all(): id += 1 with self.assertRaises(RuntimeError): interpreters.run_string(id, 'print("spam")') diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index 4395279c35e0d2..e0cfc69ab8b9d3 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -423,7 +423,7 @@ So does an unrecognized ID."); static PyObject * -interp_enumerate(PyObject *self) +interp_list_all(PyObject *self) { PyObject *ids, *id; PyInterpreterState *interp; @@ -447,8 +447,8 @@ interp_enumerate(PyObject *self) return ids; } -PyDoc_STRVAR(enumerate_doc, -"enumerate() -> [ID]\n\ +PyDoc_STRVAR(list_all_doc, +"list_all() -> [ID]\n\ \n\ Return a list containing the ID of every existing interpreter."); @@ -581,8 +581,8 @@ static PyMethodDef module_functions[] = { {"destroy", (PyCFunction)interp_destroy, METH_VARARGS, destroy_doc}, - {"enumerate", (PyCFunction)interp_enumerate, - METH_NOARGS, enumerate_doc}, + {"list_all", (PyCFunction)interp_list_all, + METH_NOARGS, list_all_doc}, {"get_current", (PyCFunction)interp_get_current, METH_NOARGS, get_current_doc}, {"get_main", (PyCFunction)interp_get_main, From c1b2a7931734bc6f220b86bae28ef3091843dee3 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 11 Dec 2017 19:30:07 -0700 Subject: [PATCH 32/78] Correctly handle a failed _shared_exception allocation. --- Modules/_interpretersmodule.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index e0cfc69ab8b9d3..899db17208ab6d 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -173,7 +173,8 @@ static struct _shared_exception * _get_shared_exception(void) { struct _shared_exception *exc = PyMem_NEW(struct _shared_exception, 1); - // XXX Fatal if NULL? + if (exc == NULL) + return NULL; PyErr_Fetch(&exc->exc, &exc->value, &exc->tb); return exc; } @@ -330,6 +331,11 @@ _run_script_in_interpreter(PyInterpreterState *interp, const char *codestr, // Propagate any exception out to the caller. if (exc != NULL) { _apply_shared_exception(exc); + PyMem_Free(exc); + } + else if (result != 0) { + // We were unable to allocate a shared exception. + PyErr_NoMemory(); } if (shared != NULL) { From 406d1eccf4a6f6ba7199ee5a8c5e1f01dc87f320 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 11 Dec 2017 20:04:34 -0700 Subject: [PATCH 33/78] Add _interpreters.RunFailedError. --- Modules/_interpretersmodule.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index 899db17208ab6d..b75440b01bb74b 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -18,6 +18,8 @@ _get_current(void) return tstate->interp; } +static PyObject * _interp_failed_error; + /* sharing-specific functions and structs */ static int @@ -628,12 +630,16 @@ static struct PyModuleDef interpretersmodule = { PyMODINIT_FUNC PyInit__interpreters(void) { - PyObject *module; - - module = PyModule_Create(&interpretersmodule); + PyObject *module = PyModule_Create(&interpretersmodule); if (module == NULL) return NULL; + PyObject *ns = PyModule_GetDict(module); // borrowed + _interp_failed_error = PyErr_NewException("_interpreters.RunFailedError", + PyExc_RuntimeError, NULL); + if (_interp_failed_error == NULL) + return NULL; + PyDict_SetItemString(ns, "RunFailedError", _interp_failed_error); return module; } From 6a53db3a955f201f96830da0a04d736012e01665 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 11 Dec 2017 21:59:19 -0700 Subject: [PATCH 34/78] Return a simple RunFailedError. --- Modules/_interpretersmodule.c | 40 +++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index b75440b01bb74b..413573525a915b 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -18,8 +18,6 @@ _get_current(void) return tstate->interp; } -static PyObject * _interp_failed_error; - /* sharing-specific functions and structs */ static int @@ -163,33 +161,43 @@ _shareditem_apply(struct _shareditem *item, PyObject *ns) return res; } -// XXX This cannot use PyObject fields. +// Ultimately we'd like to preserve enough information about the +// exception and traceback that we could re-constitute (or at least +// simulate, a la traceback.TracebackException), and even chain, a copy +// of the exception in the calling interpreter. struct _shared_exception { - PyObject *exc; - PyObject *value; - PyObject *tb; + char *msg; }; static struct _shared_exception * _get_shared_exception(void) { - struct _shared_exception *exc = PyMem_NEW(struct _shared_exception, 1); - if (exc == NULL) + struct _shared_exception *err = PyMem_NEW(struct _shared_exception, 1); + if (err == NULL) + return NULL; + PyObject *exc; + PyObject *value; + PyObject *tb; + PyErr_Fetch(&exc, &value, &tb); + PyObject *msg = PyUnicode_FromFormat("%S: %S", exc, value); + if (msg == NULL) { + err->msg = "unable to format exception"; return NULL; - PyErr_Fetch(&exc->exc, &exc->value, &exc->tb); - return exc; + } + err->msg = (char *)PyUnicode_AsUTF8(msg); + if (err->msg == NULL) { + err->msg = "unable to encode exception"; + } + return err; } +static PyObject * _interp_failed_error; + static void _apply_shared_exception(struct _shared_exception *exc) { - if (PyErr_Occurred()) { - _PyErr_ChainExceptions(exc->exc, exc->value, exc->tb); - } else { - PyErr_Restore(exc->exc, exc->value, exc->tb); - } - + PyErr_SetString(_interp_failed_error, exc->msg); } /* interpreter-specific functions */ From f1e91b029bb36bcd12eae9ed94301491ba6e094c Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 22 Dec 2017 16:30:06 -0700 Subject: [PATCH 35/78] (mostly) Fix the unit tests. --- Lib/test/test__interpreters.py | 513 +++++++++++++++++---------------- Modules/_interpretersmodule.c | 7 +- 2 files changed, 268 insertions(+), 252 deletions(-) diff --git a/Lib/test/test__interpreters.py b/Lib/test/test__interpreters.py index 7609caee7fdc75..914ef59e10484c 100644 --- a/Lib/test/test__interpreters.py +++ b/Lib/test/test__interpreters.py @@ -1,8 +1,6 @@ import contextlib import os -import os.path -import shutil -import tempfile +import pickle from textwrap import dedent, indent import threading import unittest @@ -13,55 +11,43 @@ interpreters = support.import_module('_interpreters') -SCRIPT_THREADED_INTERP = """\ -from textwrap import dedent -import threading -import _interpreters -def f(): - _interpreters.run_string(id, dedent(''' - {} - ''')) +def _captured_script(script): + r, w = os.pipe() + indented = script.replace('\n', '\n ') + wrapped = dedent(f""" + import contextlib + with open({w}, 'w') as chan: + with contextlib.redirect_stdout(chan): + {indented} + """) + return wrapped, open(r) -t = threading.Thread(target=f) -t.start() -""" +def _run_output(interp, request): + script, chan = _captured_script(request) + with chan: + interpreters.run_string(interp, script) + return chan.read() -@contextlib.contextmanager -def _blocked(dirname): - filename = os.path.join(dirname, '.lock') - wait_script = dedent(""" - import os.path - import time - while not os.path.exists('{}'): - time.sleep(0.1) - """).format(filename) - try: - yield wait_script - finally: - support.create_empty_file(filename) - - -class InterpreterTests(unittest.TestCase): - def setUp(self): - self.dirname = tempfile.mkdtemp() +@contextlib.contextmanager +def _running(interp): + r, w = os.pipe() + def run(): + interpreters.run_string(interp, dedent(f""" + # wait for "signal" + with open({r}) as chan: + chan.read() + """)) - def tearDown(self): - shutil.rmtree(self.dirname) + t = threading.Thread(target=run) + t.start() - def test_still_running_at_exit(self): - subscript = dedent(""" - import time - # Give plenty of time for the main interpreter to finish. - time.sleep(1_000_000) - """) - script = SCRIPT_THREADED_INTERP.format(indent(subscript, ' ')) - filename = script_helper.make_script(self.dirname, 'interp', script) - with script_helper.spawn_python(filename) as proc: - retcode = proc.wait() + yield - self.assertEqual(retcode, 0) + with open(w, 'w') as chan: + chan.write('done') + t.join() class TestBase(unittest.TestCase): @@ -76,79 +62,105 @@ def tearDown(self): pass # already destroyed -class EnumerateTests(TestBase): +class ListAllTests(TestBase): - def test_multiple(self): - main, = interpreters.list_all() - id1 = interpreters.create() - id2 = interpreters.create() + def test_initial(self): + main = interpreters.get_main() ids = interpreters.list_all() + self.assertEqual(ids, [main]) - self.assertEqual(set(ids), {main, id1, id2}) - - def test_main_only(self): - main, = interpreters.list_all() + def test_after_creating(self): + main = interpreters.get_main() + first = interpreters.create() + second = interpreters.create() + ids = interpreters.list_all() + self.assertEqual(ids, [main, first, second]) - self.assertEqual(main, 0) + def test_after_destroying(self): + main = interpreters.get_main() + first = interpreters.create() + second = interpreters.create() + interpreters.destroy(first) + ids = interpreters.list_all() + self.assertEqual(ids, [main, second]) class GetCurrentTests(TestBase): def test_main(self): - main, = interpreters.list_all() - id = interpreters.get_current() - - self.assertEqual(id, main) + main = interpreters.get_main() + cur = interpreters.get_current() + self.assertEqual(cur, main) - def test_sub(self): - id1 = interpreters.create() - ns = interpreters.run_string_unrestricted(id1, dedent(""" + def test_subinterpreter(self): + main = interpreters.get_main() + interp = interpreters.create() + out = _run_output(interp, dedent(""" import _interpreters - id = _interpreters.get_current() + print(_interpreters.get_current()) """)) - id2 = ns['id'] - - self.assertEqual(id2, id1) + cur = int(out.strip()) + _, expected = interpreters.list_all() + self.assertEqual(cur, expected) + self.assertNotEqual(cur, main) class GetMainTests(TestBase): - def test_main(self): - expected, = interpreters.list_all() + def test_from_main(self): + [expected] = interpreters.list_all() main = interpreters.get_main() + self.assertEqual(main, expected) - self.assertEqual(main, 0) + def test_from_subinterpreter(self): + [expected] = interpreters.list_all() + interp = interpreters.create() + out = _run_output(interp, dedent(""" + import _interpreters + print(_interpreters.get_main()) + """)) + main = int(out.strip()) self.assertEqual(main, expected) class IsRunningTests(TestBase): - def test_main_running(self): - main, = interpreters.list_all() - sub = interpreters.create() - main_running = interpreters.is_running(main) - sub_running = interpreters.is_running(sub) + def test_main(self): + main = interpreters.get_main() + self.assertTrue(interpreters.is_running(main)) - self.assertTrue(main_running) - self.assertFalse(sub_running) + def test_subinterpreter(self): + interp = interpreters.create() + self.assertFalse(interpreters.is_running(interp)) - def test_sub_running(self): - main, = interpreters.list_all() - sub1 = interpreters.create() - sub2 = interpreters.create() - ns = interpreters.run_string_unrestricted(sub1, dedent(f""" + with _running(interp): + self.assertTrue(interpreters.is_running(interp)) + self.assertFalse(interpreters.is_running(interp)) + + def test_from_subinterpreter(self): + interp = interpreters.create() + out = _run_output(interp, dedent(f""" import _interpreters - main = _interpreters.is_running({main}) - sub1 = _interpreters.is_running({sub1}) - sub2 = _interpreters.is_running({sub2}) + if _interpreters.is_running({interp}): + print(True) + else: + print(False) """)) - main_running = ns['main'] - sub1_running = ns['sub1'] - sub2_running = ns['sub2'] + self.assertEqual(out.strip(), 'True') + + def test_already_destroyed(self): + interp = interpreters.create() + interpreters.destroy(interp) + with self.assertRaises(RuntimeError): + interpreters.is_running(interp) + + def test_does_not_exist(self): + with self.assertRaises(RuntimeError): + interpreters.is_running(1_000_000) - self.assertTrue(main_running) - self.assertTrue(sub1_running) - self.assertFalse(sub2_running) + def test_bad_id(self): + with self.assertRaises(RuntimeError): + interpreters.is_running(-1) class CreateTests(TestBase): @@ -186,34 +198,34 @@ def f(): def test_in_subinterpreter(self): main, = interpreters.list_all() id1 = interpreters.create() - ns = interpreters.run_string_unrestricted(id1, dedent(""" + out = _run_output(id1, dedent(""" import _interpreters id = _interpreters.create() + print(id) """)) - id2 = ns['id'] + id2 = int(out.strip()) self.assertEqual(set(interpreters.list_all()), {main, id1, id2}) def test_in_threaded_subinterpreter(self): main, = interpreters.list_all() id1 = interpreters.create() - ns = None - script = dedent(""" - import _interpreters - id = _interpreters.create() - """) + id2 = None def f(): - nonlocal ns - ns = interpreters.run_string_unrestricted(id1, script) + nonlocal id2 + out = _run_output(id1, dedent(""" + import _interpreters + id = _interpreters.create() + print(id) + """)) + id2 = int(out.strip()) t = threading.Thread(target=f) t.start() t.join() - id2 = ns['id'] self.assertEqual(set(interpreters.list_all()), {main, id1, id2}) - def test_after_destroy_all(self): before = set(interpreters.list_all()) # Create 3 subinterpreters. @@ -326,22 +338,12 @@ def f(): t.join() def test_still_running(self): - # XXX Rewrite this test without files by using - # run_string_unrestricted(). main, = interpreters.list_all() - id = interpreters.create() - def f(): - interpreters.run_string(id, wait_script) - - dirname = tempfile.mkdtemp() - t = threading.Thread(target=f) - with _blocked(dirname) as wait_script: - t.start() + interp = interpreters.create() + with _running(interp): with self.assertRaises(RuntimeError): - interpreters.destroy(id) - - t.join() - self.assertEqual(set(interpreters.list_all()), {main, id}) + interpreters.destroy(interp) + self.assertTrue(interpreters.is_running(interp)) class RunStringTests(TestBase): @@ -353,115 +355,96 @@ class RunStringTests(TestBase): FILENAME = 'spam' def setUp(self): + super().setUp() self.id = interpreters.create() - self.dirname = None - self.filename = None + self._fs = None def tearDown(self): - if self.dirname is not None: - try: - shutil.rmtree(self.dirname) - except FileNotFoundError: - pass # already deleted + if self._fs is not None: + self._fs.close() super().tearDown() - def _resolve_filename(self, name=None): - if name is None: - name = self.FILENAME - if self.dirname is None: - self.dirname = tempfile.mkdtemp() - return os.path.join(self.dirname, name) - - def _empty_file(self): - self.filename = self._resolve_filename() - support.create_empty_file(self.filename) - return self.filename - - def assert_file_contains(self, expected, filename=None): - if filename is None: - filename = self.filename - self.assertIsNotNone(filename) - with open(filename) as out: - content = out.read() - self.assertEqual(content, expected) + @property + def fs(self): + if self._fs is None: + self._fs = FSFixture(self) + return self._fs def test_success(self): - filename = self._empty_file() - expected = 'spam spam spam spam spam' - script = self.SCRIPT.format(filename, expected) - interpreters.run_string(self.id, script) + script, file = _captured_script('print("it worked!", end="")') + with file: + interpreters.run_string(self.id, script) + out = file.read() - self.assert_file_contains(expected) + self.assertEqual(out, 'it worked!') def test_in_thread(self): - filename = self._empty_file() - expected = 'spam spam spam spam spam' - script = self.SCRIPT.format(filename, expected) - def f(): - interpreters.run_string(self.id, script) + script, file = _captured_script('print("it worked!", end="")') + with file: + def f(): + interpreters.run_string(self.id, script) - t = threading.Thread(target=f) - t.start() - t.join() + t = threading.Thread(target=f) + t.start() + t.join() + out = file.read() - self.assert_file_contains(expected) + self.assertEqual(out, 'it worked!') def test_create_thread(self): - filename = self._empty_file() - expected = 'spam spam spam spam spam' - script = dedent(""" + script, file = _captured_script(""" import threading def f(): - with open('{}', 'w') as out: - out.write('{}') + print('it worked!', end='') t = threading.Thread(target=f) t.start() t.join() - """).format(filename, expected) - interpreters.run_string(self.id, script) + """) + with file: + interpreters.run_string(self.id, script) + out = file.read() - self.assert_file_contains(expected) + self.assertEqual(out, 'it worked!') @unittest.skipUnless(hasattr(os, 'fork'), "test needs os.fork()") def test_fork(self): - filename = self._empty_file() - expected = 'spam spam spam spam spam' - script = dedent(""" - import os - r, w = os.pipe() - pid = os.fork() - if pid == 0: # child - import sys - filename = '{}' - with open(filename, 'w') as out: - out.write('{}') - os.write(w, b'done!') + import tempfile + with tempfile.NamedTemporaryFile('w+') as file: + file.write('') + file.flush() + + expected = 'spam spam spam spam spam' + # XXX leaking a process... + script = dedent(f""" + # (inspired by Lib/test/test_fork.py) + import os + pid = os.fork() + if pid == 0: # child + with open('{file.name}', 'w') as out: + out.write('{expected}') + # Kill the unittest runner in the child process. + os._exit(1) + else: + SHORT_SLEEP = 0.1 + import time + for _ in range(10): + spid, status = os.waitpid(pid, os.WNOHANG) + if spid == pid: + break + time.sleep(SHORT_SLEEP) + assert(spid == pid) + """) + interpreters.run_string(self.id, script) - # Kill the unittest runner in the child process. - os._exit(1) - else: - import select - try: - select.select([r], [], []) - finally: - os.close(r) - os.close(w) - """).format(filename, expected) - interpreters.run_string(self.id, script) - self.assert_file_contains(expected) + file.seek(0) + content = file.read() + self.assertEqual(content, expected) def test_already_running(self): - def f(): - interpreters.run_string(self.id, wait_script) - - t = threading.Thread(target=f) - dirname = tempfile.mkdtemp() - with _blocked(dirname) as wait_script: - t.start() + with _running(self.id): with self.assertRaises(RuntimeError): interpreters.run_string(self.id, 'print("spam")') - t.join() def test_does_not_exist(self): id = 0 @@ -478,93 +461,99 @@ def test_bad_id(self): with self.assertRaises(TypeError): interpreters.run_string('spam', 'print("spam")') - def test_bad_code(self): + def test_bad_script(self): with self.assertRaises(TypeError): interpreters.run_string(self.id, 10) - def test_bytes_for_code(self): + def test_bytes_for_script(self): with self.assertRaises(TypeError): interpreters.run_string(self.id, b'print("spam")') + @contextlib.contextmanager + def assert_run_failed(self, exctype, msg=None): + with self.assertRaises(interpreters.RunFailedError) as caught: + yield + if msg is None: + self.assertEqual(str(caught.exception).split(':')[0], + str(exctype)) + else: + self.assertEqual(str(caught.exception), + "{}: {}".format(exctype, msg)) + def test_invalid_syntax(self): - with self.assertRaises(SyntaxError): + with self.assert_run_failed(SyntaxError): # missing close paren interpreters.run_string(self.id, 'print("spam"') def test_failure(self): - with self.assertRaises(Exception) as caught: + with self.assert_run_failed(Exception, 'spam'): interpreters.run_string(self.id, 'raise Exception("spam")') - self.assertEqual(str(caught.exception), 'spam') + + def test_SystemExit(self): + with self.assert_run_failed(SystemExit, '42'): + interpreters.run_string(self.id, 'raise SystemExit(42)') def test_sys_exit(self): - with self.assertRaises(SystemExit) as cm: + with self.assert_run_failed(SystemExit): interpreters.run_string(self.id, dedent(""" import sys sys.exit() """)) - self.assertIsNone(cm.exception.code) - with self.assertRaises(SystemExit) as cm: + with self.assert_run_failed(SystemExit, '42'): interpreters.run_string(self.id, dedent(""" import sys sys.exit(42) """)) - self.assertEqual(cm.exception.code, 42) - - def test_SystemError(self): - with self.assertRaises(SystemExit) as cm: - interpreters.run_string(self.id, 'raise SystemExit(42)') - self.assertEqual(cm.exception.code, 42) - - -class RunStringUnrestrictedTests(TestBase): - - def setUp(self): - self.id = interpreters.create() - - def test_without_ns(self): - script = dedent(""" - spam = 42 - """) - ns = interpreters.run_string_unrestricted(self.id, script) - - self.assertEqual(ns['spam'], 42) - - def test_with_ns(self): - updates = {'spam': 'ham', 'eggs': -1} - script = dedent(""" - spam = 42 - result = spam + eggs - """) - ns = interpreters.run_string_unrestricted(self.id, script, updates) - - self.assertEqual(ns['spam'], 42) - self.assertEqual(ns['eggs'], -1) - self.assertEqual(ns['result'], 41) - - def test_ns_does_not_overwrite(self): - updates = {'__name__': 'not __main__'} - script = dedent(""" - spam = 42 - """) - ns = interpreters.run_string_unrestricted(self.id, script, updates) - - self.assertEqual(ns['__name__'], '__main__') - def test_main_not_shared(self): - ns1 = interpreters.run_string_unrestricted(self.id, 'spam = True') - ns2 = interpreters.run_string_unrestricted(self.id, 'eggs = False') + def test_main_shared(self): + r, w = os.pipe() + interpreters.run_string(self.id, dedent(f""" + spam = True + + ns = dict(vars()) + del ns['__builtins__'] + import pickle + with open({w}, 'wb') as chan: + pickle.dump(ns, chan) + del ns, pickle, chan + """)) + with open(r, 'rb') as chan: + ns1 = pickle.load(chan) + + r, w = os.pipe() + interpreters.run_string(self.id, dedent(f""" + eggs = False + + ns = dict(vars()) + del ns['__builtins__'] + import pickle + with open({w}, 'wb') as chan: + pickle.dump(ns, chan) + """)) + with open(r, 'rb') as chan: + ns2 = pickle.load(chan) self.assertIn('spam', ns1) self.assertNotIn('eggs', ns1) self.assertIn('eggs', ns2) - self.assertNotIn('spam', ns2) + self.assertIn('spam', ns2) - def test_return_execution_namespace(self): - script = dedent(""" + def test_execution_namespace_is_main(self): + r, w = os.pipe() + + script = dedent(f""" spam = 42 + + ns = dict(vars()) + ns['__builtins__'] = str(ns['__builtins__']) + import pickle + with open({w}, 'wb') as chan: + pickle.dump(ns, chan) """) - ns = interpreters.run_string_unrestricted(self.id, script) + interpreters.run_string(self.id, script) + with open(r, 'rb') as chan: + ns = pickle.load(chan) ns.pop('__builtins__') ns.pop('__loader__') @@ -577,6 +566,28 @@ def test_return_execution_namespace(self): 'spam': 42, }) + def test_still_running_at_exit(self): + script = dedent(f""" + from textwrap import dedent + import threading + import _interpreters + def f(): + _interpreters.run_string(id, dedent(''' + import time + # Give plenty of time for the main interpreter to finish. + time.sleep(1_000_000) + ''')) + + t = threading.Thread(target=f) + t.start() + """) + with support.temp_dir() as dirname: + filename = script_helper.make_script(dirname, 'interp', script) + with script_helper.spawn_python(filename) as proc: + retcode = proc.wait() + + self.assertEqual(retcode, 0) + if __name__ == '__main__': unittest.main() diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index 413573525a915b..c94ac4c3b89d70 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -180,7 +180,11 @@ _get_shared_exception(void) PyObject *value; PyObject *tb; PyErr_Fetch(&exc, &value, &tb); - PyObject *msg = PyUnicode_FromFormat("%S: %S", exc, value); + PyObject *msg; + if (value == NULL) + msg = PyUnicode_FromFormat("%S", exc); + else + msg = PyUnicode_FromFormat("%S: %S", exc, value); if (msg == NULL) { err->msg = "unable to format exception"; return NULL; @@ -609,6 +613,7 @@ static PyMethodDef module_functions[] = { {"run_string", (PyCFunction)interp_run_string, METH_VARARGS, run_string_doc}, + // XXX untested {"is_shareable", (PyCFunction)object_is_shareable, METH_VARARGS, is_shareable_doc}, From e7c581a25243f22920097777858b730b86f9e141 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 8 Jan 2018 22:30:59 +0000 Subject: [PATCH 36/78] Add tests for shareable objects. --- Lib/test/test__interpreters.py | 125 ++++++++++++++++++++++++++++++++- Modules/_interpretersmodule.c | 19 ++--- 2 files changed, 135 insertions(+), 9 deletions(-) diff --git a/Lib/test/test__interpreters.py b/Lib/test/test__interpreters.py index 914ef59e10484c..747f1538d200e4 100644 --- a/Lib/test/test__interpreters.py +++ b/Lib/test/test__interpreters.py @@ -50,6 +50,54 @@ def run(): t.join() +class IsShareableTests(unittest.TestCase): + + def test_default_shareables(self): + shareables = [ + # builtin objects + b'spam', + ] + for obj in shareables: + with self.subTest(obj): + self.assertTrue( + interpreters.is_shareable(obj)) + + def test_not_shareable(self): + class Cheese: + def __init__(self, name): + self.name = name + def __str__(self): + return self.name + + class SubBytes(bytes): + """A subclass of a shareable type.""" + + not_shareables = [ + # singletons + None, + True, + False, + NotImplemented, + ..., + # builtin types and objects + type, + object, + object(), + Exception(), + 42, + 100.0, + 'spam', + # user-defined types and objects + Cheese, + Cheese('Wensleydale'), + SubBytes(b'spam'), + ] + for obj in not_shareables: + with self.subTest(obj): + self.assertFalse( + interpreters.is_shareable(obj)) + + class TestBase(unittest.TestCase): def tearDown(self): @@ -506,7 +554,82 @@ def test_sys_exit(self): sys.exit(42) """)) - def test_main_shared(self): + def test_with_shared(self): + r, w = os.pipe() + + shared = { + 'spam': b'ham', + 'eggs': b'-1', + } + script = dedent(f""" + eggs = int(eggs) + spam = 42 + result = spam + eggs + + ns = dict(vars()) + del ns['__builtins__'] + import pickle + with open({w}, 'wb') as chan: + pickle.dump(ns, chan) + """) + interpreters.run_string(self.id, script, shared) + with open(r, 'rb') as chan: + ns = pickle.load(chan) + + self.assertEqual(ns['spam'], 42) + self.assertEqual(ns['eggs'], -1) + self.assertEqual(ns['result'], 41) + + def test_shared_overwrites(self): + interpreters.run_string(self.id, dedent(""" + spam = 'eggs' + ns1 = dict(vars()) + del ns1['__builtins__'] + """)) + + shared = {'spam': b'ham'} + script = dedent(f""" + ns2 = dict(vars()) + del ns2['__builtins__'] + """) + interpreters.run_string(self.id, script, shared) + + r, w = os.pipe() + script = dedent(f""" + ns = dict(vars()) + del ns['__builtins__'] + import pickle + with open({w}, 'wb') as chan: + pickle.dump(ns, chan) + """) + interpreters.run_string(self.id, script) + with open(r, 'rb') as chan: + ns = pickle.load(chan) + + self.assertEqual(ns['ns1']['spam'], 'eggs') + self.assertEqual(ns['ns2']['spam'], b'ham') + self.assertEqual(ns['spam'], b'ham') + + def test_shared_overwrites_default_vars(self): + r, w = os.pipe() + + shared = {'__name__': b'not __main__'} + script = dedent(f""" + spam = 42 + + ns = dict(vars()) + del ns['__builtins__'] + import pickle + with open({w}, 'wb') as chan: + pickle.dump(ns, chan) + """) + interpreters.run_string(self.id, script, shared) + with open(r, 'rb') as chan: + ns = pickle.load(chan) + + self.assertEqual(ns['__name__'], b'not __main__') + + def test_main_reused(self): r, w = os.pipe() interpreters.run_string(self.id, dedent(f""" spam = True diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index c94ac4c3b89d70..9555a3abca9db2 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -109,11 +109,12 @@ _sharedns_clear(struct _shareditem *shared) } static struct _shareditem * -_get_shared_ns(PyObject *shareable) +_get_shared_ns(PyObject *shareable, Py_ssize_t *lenp) { if (shareable == NULL || shareable == Py_None) return NULL; Py_ssize_t len = PyDict_Size(shareable); + *lenp = len; if (len == 0) return NULL; @@ -280,7 +281,8 @@ _ensure_not_running(PyInterpreterState *interp) static int _run_script(PyInterpreterState *interp, const char *codestr, - struct _shareditem *shared, struct _shared_exception **exc) + struct _shareditem *shared, Py_ssize_t num_shared, + struct _shared_exception **exc) { PyObject *main_mod = PyMapping_GetItemString(interp->modules, "__main__"); if (main_mod == NULL) @@ -293,8 +295,9 @@ _run_script(PyInterpreterState *interp, const char *codestr, // Apply the cross-interpreter data. if (shared != NULL) { - for (struct _shareditem *item=shared; item->name != NULL; item += 1) { - if (_shareditem_apply(shared, ns) != 0) { + for (Py_ssize_t i=0; i < num_shared; i++) { + struct _shareditem *item = &shared[i]; + if (_shareditem_apply(item, ns) != 0) { Py_DECREF(ns); goto error; } @@ -320,13 +323,14 @@ _run_script(PyInterpreterState *interp, const char *codestr, static int _run_script_in_interpreter(PyInterpreterState *interp, const char *codestr, - PyObject *shareable) + PyObject *shareables) { // XXX lock? if (_ensure_not_running(interp) < 0) return -1; - struct _shareditem *shared = _get_shared_ns(shareable); + Py_ssize_t num_shared; + struct _shareditem *shared = _get_shared_ns(shareables, &num_shared); if (shared == NULL && PyErr_Occurred()) return -1; @@ -336,7 +340,7 @@ _run_script_in_interpreter(PyInterpreterState *interp, const char *codestr, // Run the script. struct _shared_exception *exc = NULL; - int result = _run_script(interp, codestr, shared, &exc); + int result = _run_script(interp, codestr, shared, num_shared, &exc); // Switch back. if (save_tstate != NULL) @@ -613,7 +617,6 @@ static PyMethodDef module_functions[] = { {"run_string", (PyCFunction)interp_run_string, METH_VARARGS, run_string_doc}, - // XXX untested {"is_shareable", (PyCFunction)object_is_shareable, METH_VARARGS, is_shareable_doc}, From 24095ab401996e139a0417523c996bd37fbdce3c Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 13 Dec 2017 19:01:17 -0700 Subject: [PATCH 37/78] Add None as shareable. --- Lib/test/test__interpreters.py | 5 ++++- Modules/_interpretersmodule.c | 28 +++++++++++++++++++++++++++- 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/Lib/test/test__interpreters.py b/Lib/test/test__interpreters.py index 747f1538d200e4..13ccfa001594ef 100644 --- a/Lib/test/test__interpreters.py +++ b/Lib/test/test__interpreters.py @@ -54,6 +54,8 @@ class IsShareableTests(unittest.TestCase): def test_default_shareables(self): shareables = [ + # singletons + None, # builtin objects b'spam', ] @@ -74,7 +76,6 @@ class SubBytes(bytes): not_shareables = [ # singletons - None, True, False, NotImplemented, @@ -560,6 +561,7 @@ def test_with_shared(self): shared = { 'spam': b'ham', 'eggs': b'-1', + 'cheddar': None, } script = dedent(f""" eggs = int(eggs) @@ -579,6 +581,7 @@ def test_with_shared(self): self.assertEqual(ns['spam'], 42) self.assertEqual(ns['eggs'], -1) self.assertEqual(ns['result'], 41) + self.assertIsNone(ns['cheddar']) def test_shared_overwrites(self): interpreters.run_string(self.id, dedent(""" diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index 9555a3abca9db2..5c0d13197dd618 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -18,13 +18,15 @@ _get_current(void) return tstate->interp; } -/* sharing-specific functions and structs */ +/* cross-interpreter data */ static int _PyObject_CheckShareable(PyObject *obj) { if (PyBytes_CheckExact(obj)) return 0; + if (obj == Py_None) + return 0; PyErr_SetString(PyExc_ValueError, "obj is not a cross-interpreter shareable type"); return 1; @@ -45,6 +47,23 @@ _bytes_shared(PyObject *obj, _PyCrossInterpreterData *data) return 0; } +static PyObject * +_new_none_object(_PyCrossInterpreterData *data) +{ + // XXX Singleton refcounts are problematic across interpreters... + Py_INCREF(Py_None); + return Py_None; +} + +static int +_none_shared(PyObject *obj, _PyCrossInterpreterData *data) +{ + data->data = NULL; + data->new_object = _new_none_object; + data->free = NULL; + return 0; +} + static int _PyObject_GetCrossInterpreterData(PyObject *obj, _PyCrossInterpreterData *data) { @@ -63,6 +82,11 @@ _PyObject_GetCrossInterpreterData(PyObject *obj, _PyCrossInterpreterData *data) Py_DECREF(obj); return 1; } + } else if (obj == Py_None) { + if (_none_shared(obj, data) != 0) { + Py_DECREF(obj); + return 1; + } } return 0; @@ -94,6 +118,8 @@ _PyCrossInterpreterData_NewObject(_PyCrossInterpreterData *data) return data->new_object(data); } +/* other sharing-specific functions and structs */ + struct _shareditem { Py_UNICODE *name; Py_ssize_t namelen; From 7ebae2fdb49b8483d7863236366542809a40be61 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 11 Jan 2018 00:12:28 +0000 Subject: [PATCH 38/78] Add a registry for shareable types. --- Include/internal/pystate.h | 18 +++++ Modules/_interpretersmodule.c | 87 +++++++----------------- Python/pystate.c | 124 +++++++++++++++++++++++++++++++++- 3 files changed, 167 insertions(+), 62 deletions(-) diff --git a/Include/internal/pystate.h b/Include/internal/pystate.h index 2c06b48ae36c71..e58641e267c0c1 100644 --- a/Include/internal/pystate.h +++ b/Include/internal/pystate.h @@ -67,6 +67,7 @@ PyAPI_FUNC(void) _PyPathConfig_Clear(_PyPathConfig *config); /* Cross-interpreter data sharing */ +struct pyruntimestate; struct _cid; typedef struct _cid { @@ -78,6 +79,19 @@ typedef struct _cid { PyObject *object; } _PyCrossInterpreterData; +typedef int (*crossinterpdatafunc)(PyObject *, _PyCrossInterpreterData *); + +PyAPI_FUNC(int) _PyCrossInterpreterData_Register_Class( + PyTypeObject *, crossinterpdatafunc); +PyAPI_FUNC(crossinterpdatafunc) _PyCrossInterpreterData_Lookup(PyObject *); + +struct _cidclass; + +struct _cidclass { + const char *classname; + crossinterpdatafunc getdata; + struct _cidclass *next; +}; /* Full Python runtime state */ @@ -100,6 +114,10 @@ typedef struct pyruntimestate { using a Python int. */ int64_t next_id; } interpreters; + /* For now we use a global registry of shareable classes. An + alternative would be to add a tp_* slot for a class's + crossinterpdatafunc. It would be simpler and more efficient. */ + struct _cidclass *crossinterpclasses; #define NEXITFUNCS 32 void (*exitfuncs[NEXITFUNCS])(void); diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index 5c0d13197dd618..28ca029d786624 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -20,77 +20,31 @@ _get_current(void) /* cross-interpreter data */ -static int -_PyObject_CheckShareable(PyObject *obj) -{ - if (PyBytes_CheckExact(obj)) - return 0; - if (obj == Py_None) - return 0; - PyErr_SetString(PyExc_ValueError, - "obj is not a cross-interpreter shareable type"); - return 1; -} - -static PyObject * -_new_bytes_object(_PyCrossInterpreterData *data) -{ - return PyBytes_FromString((char *)(data->data)); -} - -static int -_bytes_shared(PyObject *obj, _PyCrossInterpreterData *data) -{ - data->data = (void *)(PyBytes_AS_STRING(obj)); - data->new_object = _new_bytes_object; - data->free = NULL; - return 0; -} - -static PyObject * -_new_none_object(_PyCrossInterpreterData *data) +static crossinterpdatafunc +_lookup_getdata(PyObject *obj) { - // XXX Singleton refcounts are problematic across interpreters... - Py_INCREF(Py_None); - return Py_None; -} - -static int -_none_shared(PyObject *obj, _PyCrossInterpreterData *data) -{ - data->data = NULL; - data->new_object = _new_none_object; - data->free = NULL; - return 0; + crossinterpdatafunc getdata = _PyCrossInterpreterData_Lookup(obj); + if (getdata == NULL && PyErr_Occurred() == 0) + PyErr_Format(PyExc_ValueError, + "%S is not a cross-interpreter shareable type", obj); + return getdata; } static int _PyObject_GetCrossInterpreterData(PyObject *obj, _PyCrossInterpreterData *data) { Py_INCREF(obj); - - if (_PyObject_CheckShareable(obj) != 0) { + crossinterpdatafunc getdata = _lookup_getdata(obj); + if (getdata == NULL) { Py_DECREF(obj); - return 1; + return -1; } - - data->interp = _get_current(); - data->object = obj; - - if (PyBytes_CheckExact(obj)) { - if (_bytes_shared(obj, data) != 0) { - Py_DECREF(obj); - return 1; - } - } else if (obj == Py_None) { - if (_none_shared(obj, data) != 0) { - Py_DECREF(obj); - return 1; - } + int res = getdata(obj, data); + if (res != 0) { + Py_DECREF(obj); } - - return 0; -}; + return res; +} static void _PyCrossInterpreterData_Release(_PyCrossInterpreterData *data) @@ -120,6 +74,14 @@ _PyCrossInterpreterData_NewObject(_PyCrossInterpreterData *data) /* other sharing-specific functions and structs */ +static int +_PyObject_CheckShareable(PyObject *obj) +{ + if (_lookup_getdata(obj) == NULL) + return -1; + return 0; +} + struct _shareditem { Py_UNICODE *name; Py_ssize_t namelen; @@ -145,6 +107,9 @@ _get_shared_ns(PyObject *shareable, Py_ssize_t *lenp) return NULL; struct _shareditem *shared = PyMem_NEW(struct _shareditem, len+1); + for (Py_ssize_t i=0; i < len; i++) { + *(shared + i) = (struct _shareditem){0}; + } Py_ssize_t pos = 0; for (Py_ssize_t i=0; i < len; i++) { PyObject *key, *value; diff --git a/Python/pystate.c b/Python/pystate.c index 909d831465d4e6..272120d654eaf4 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -54,8 +54,8 @@ _PyRuntimeState_Init_impl(_PyRuntimeState *runtime) if (runtime->interpreters.mutex == NULL) { return _Py_INIT_ERR("Can't initialize threads for interpreter"); } - runtime->interpreters.next_id = -1; + return _Py_INIT_OK(); } @@ -1024,6 +1024,128 @@ PyGILState_Release(PyGILState_STATE oldstate) } +/* cross-interpreter data */ + +static const char * +_get_qualname(PyObject *obj) +{ + PyObject *qualname = PyObject_GetAttrString(obj, "__qualname__"); + if (qualname == NULL) + return NULL; + Py_ssize_t name_size; + const char *encoded = PyUnicode_AsUTF8AndSize(qualname, &name_size); + if (encoded == NULL) + return NULL; + if (strlen(encoded) != (size_t)name_size) { + PyErr_SetString(PyExc_ValueError, + "qualname must not contain null characters"); + return NULL; + } + return encoded; +} + +int +_PyCrossInterpreterData_Register_Class(PyTypeObject *cls, + crossinterpdatafunc getdata) +{ + if (!PyType_Check(cls)) { + PyErr_Format(PyExc_ValueError, "only classes may be registered"); + return -1; + } + if (getdata == NULL) { + PyErr_Format(PyExc_ValueError, "missing 'getdata' func"); + return -1; + } + // XXX lock? + + const char *classname = _get_qualname((PyObject *)cls); + if (classname == NULL) + return -1; + + // XXX Fail if already registered (instead of effectively replacing)? + struct _cidclass *newhead = PyMem_NEW(struct _cidclass, 1); + newhead->classname = classname; + newhead->getdata = getdata; + _PyRuntimeState *runtime = &_PyRuntime; + newhead->next = runtime->crossinterpclasses; + runtime->crossinterpclasses = newhead; + return 0; +} + +static void _register_builtins_for_crossinterpreter_data(void); + +crossinterpdatafunc +_PyCrossInterpreterData_Lookup(PyObject *obj) +{ + const char *classname = _get_qualname(PyObject_Type(obj)); + if (classname == NULL) + return NULL; + + _PyRuntimeState *runtime = &_PyRuntime; + struct _cidclass *cur = runtime->crossinterpclasses; + if (cur == NULL) { + _register_builtins_for_crossinterpreter_data(); + cur = runtime->crossinterpclasses; + } + for(; cur != NULL; cur = cur->next) { + // XXX Be more strict (e.g. Py*_CheckExact)? + if (strcmp(cur->classname, classname) == 0) + return cur->getdata; + } + return NULL; +} + +/* cross-interpreter data for builtin types */ + +static PyObject * +_new_bytes_object(_PyCrossInterpreterData *data) +{ + return PyBytes_FromString((char *)(data->data)); +} + +static int +_bytes_shared(PyObject *obj, _PyCrossInterpreterData *data) +{ + data->data = (void *)(PyBytes_AS_STRING(obj)); + data->new_object = _new_bytes_object; + data->free = NULL; + return 0; +} + +static PyObject * +_new_none_object(_PyCrossInterpreterData *data) +{ + // XXX Singleton refcounts are problematic across interpreters... + Py_INCREF(Py_None); + return Py_None; +} + +static int +_none_shared(PyObject *obj, _PyCrossInterpreterData *data) +{ + data->data = NULL; + data->new_object = _new_none_object; + data->free = NULL; + return 0; +} + +static void +_register_builtins_for_crossinterpreter_data(void) +{ + // None + if (_PyCrossInterpreterData_Register_Class( + (PyTypeObject *)PyObject_Type(Py_None), _none_shared) != 0) { + Py_FatalError("could not register None for X-interpreter sharing"); + } + + // bytes + if (_PyCrossInterpreterData_Register_Class( + &PyBytes_Type, _bytes_shared) != 0) { + Py_FatalError("could not register bytes for X-interpreter sharing"); + } +} + + #ifdef __cplusplus } #endif From 6d562bbc39c37e44607ffe20dfd23b1a644e068f Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 12 Jan 2018 17:22:57 +0000 Subject: [PATCH 39/78] Fix (and clean up) the cross-interpreter-data code. --- Include/internal/pystate.h | 65 ++++++++-- Modules/_interpretersmodule.c | 90 +------------- Python/pystate.c | 219 ++++++++++++++++++++++++++++++---- 3 files changed, 255 insertions(+), 119 deletions(-) diff --git a/Include/internal/pystate.h b/Include/internal/pystate.h index e58641e267c0c1..65828a56b12eb7 100644 --- a/Include/internal/pystate.h +++ b/Include/internal/pystate.h @@ -65,34 +65,80 @@ PyAPI_FUNC(_PyInitError) _PyPathConfig_Calculate( PyAPI_FUNC(void) _PyPathConfig_Clear(_PyPathConfig *config); -/* Cross-interpreter data sharing */ +/* interpreter state */ + +PyAPI_FUNC(PyInterpreterState *) _PyInterpreterState_LookUpID(PY_INT64_T); + + +/* cross-interpreter data */ -struct pyruntimestate; struct _cid; +// _PyCrossInterpreterData is similar to Py_buffer as an effectively +// opaque struct that holds data outside the object machinery. This +// is necessary to pass between interpreters in the same process. typedef struct _cid { + // data is the cross-interpreter-safe derivation of a Python object + // (see _PyObject_GetCrossInterpreterData). It will be NULL if the + // new_object func (below) encodes the data. void *data; + // obj is the Python object from which the data was derived. This + // is non-NULL only if the data remains bound to the object in some + // way, such that the object must be "released" (via a decref) when + // the data is released. In that case it is automatically + // incref'ed (to match the automatic decref when releaed). + PyObject *obj; + // interp is the ID of the owning interpreter of the original + // object. It corresponds to the active interpreter when + // _PyObject_GetCrossInterpreterData() was called. This should only + // be set by the cross-interpreter machinery. + // + // We use the ID rather than the PyInterpreterState to avoid issues + // with deleted interpreters. + int64_t interp; + // new_object is a function that returns a new object in the current + // interpreter given the data. The resulting object will be + // equivalent to the original object. This field is required. PyObject *(*new_object)(struct _cid *); + // free is called when the data is released. If it is NULL then + // nothing will be done to free the data. For some types this is + // okay (e.g. bytes) and for those types this field should be set + // to NULL. However, for most the data was allocated just for + // cross-interpreter use, so it must be freed when + // _PyCrossInterpreterData_Release is called or the memory will + // leak. In that case, at the very least this field should be set + // to PyMem_RawFree (the default if not explicitly set to NULL). + // The call will happen with the original interpreter activated. void (*free)(void *); - - PyInterpreterState *interp; - PyObject *object; } _PyCrossInterpreterData; typedef int (*crossinterpdatafunc)(PyObject *, _PyCrossInterpreterData *); +PyAPI_FUNC(int) _PyObject_CheckCrossInterpreterData(PyObject *); -PyAPI_FUNC(int) _PyCrossInterpreterData_Register_Class( - PyTypeObject *, crossinterpdatafunc); +PyAPI_FUNC(int) _PyObject_GetCrossInterpreterData(PyObject *, _PyCrossInterpreterData *); +PyAPI_FUNC(PyObject *) _PyCrossInterpreterData_NewObject(_PyCrossInterpreterData *); +PyAPI_FUNC(void) _PyCrossInterpreterData_Release(_PyCrossInterpreterData *); + +/* cross-interpreter data registry */ + +/* For now we use a global registry of shareable classes. An + alternative would be to add a tp_* slot for a class's + crossinterpdatafunc. It would be simpler and more efficient. */ + +PyAPI_FUNC(int) _PyCrossInterpreterData_Register_Class(PyTypeObject *, crossinterpdatafunc); PyAPI_FUNC(crossinterpdatafunc) _PyCrossInterpreterData_Lookup(PyObject *); struct _cidclass; struct _cidclass { + // XXX Do we really need classname? const char *classname; + PyTypeObject *cls; // XXX only safe for static types? crossinterpdatafunc getdata; struct _cidclass *next; }; + /* Full Python runtime state */ typedef struct pyruntimestate { @@ -114,9 +160,8 @@ typedef struct pyruntimestate { using a Python int. */ int64_t next_id; } interpreters; - /* For now we use a global registry of shareable classes. An - alternative would be to add a tp_* slot for a class's - crossinterpdatafunc. It would be simpler and more efficient. */ + // XXX Move this to the module of the _intepreters module? + // XXX Remove this field once we have a tp_* slot. struct _cidclass *crossinterpclasses; #define NEXITFUNCS 32 diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index 28ca029d786624..0640bdf9997e0a 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -18,69 +18,7 @@ _get_current(void) return tstate->interp; } -/* cross-interpreter data */ - -static crossinterpdatafunc -_lookup_getdata(PyObject *obj) -{ - crossinterpdatafunc getdata = _PyCrossInterpreterData_Lookup(obj); - if (getdata == NULL && PyErr_Occurred() == 0) - PyErr_Format(PyExc_ValueError, - "%S is not a cross-interpreter shareable type", obj); - return getdata; -} - -static int -_PyObject_GetCrossInterpreterData(PyObject *obj, _PyCrossInterpreterData *data) -{ - Py_INCREF(obj); - crossinterpdatafunc getdata = _lookup_getdata(obj); - if (getdata == NULL) { - Py_DECREF(obj); - return -1; - } - int res = getdata(obj, data); - if (res != 0) { - Py_DECREF(obj); - } - return res; -} - -static void -_PyCrossInterpreterData_Release(_PyCrossInterpreterData *data) -{ - PyThreadState *save_tstate = NULL; - if (data->interp != NULL) { - // Switch to the original interpreter. - PyThreadState *tstate = PyInterpreterState_ThreadHead(data->interp); - save_tstate = PyThreadState_Swap(tstate); - } - - if (data->free != NULL) { - data->free(data->data); - } - Py_XDECREF(data->object); - - // Switch back. - if (save_tstate != NULL) - PyThreadState_Swap(save_tstate); -} - -static PyObject * -_PyCrossInterpreterData_NewObject(_PyCrossInterpreterData *data) -{ - return data->new_object(data); -} - -/* other sharing-specific functions and structs */ - -static int -_PyObject_CheckShareable(PyObject *obj) -{ - if (_lookup_getdata(obj) == NULL) - return -1; - return 0; -} +/* data-sharing-specific functions and structs */ struct _shareditem { Py_UNICODE *name; @@ -198,28 +136,6 @@ _apply_shared_exception(struct _shared_exception *exc) /* interpreter-specific functions */ -static PyInterpreterState * -_look_up_int64(PY_INT64_T requested_id) -{ - if (requested_id < 0) - goto error; - - PyInterpreterState *interp = PyInterpreterState_Head(); - while (interp != NULL) { - PY_INT64_T id = PyInterpreterState_GetID(interp); - if (id < 0) - return NULL; - if (requested_id == id) - return interp; - interp = PyInterpreterState_Next(interp); - } - -error: - PyErr_Format(PyExc_RuntimeError, - "unrecognized interpreter ID %lld", requested_id); - return NULL; -} - static PyInterpreterState * _look_up(PyObject *requested_id) { @@ -227,7 +143,7 @@ _look_up(PyObject *requested_id) if (id == -1 && PyErr_Occurred() != NULL) return NULL; assert(id <= INT64_MAX); - return _look_up_int64(id); + return _PyInterpreterState_LookUpID(id); } static PyObject * @@ -549,7 +465,7 @@ object_is_shareable(PyObject *self, PyObject *args) PyObject *obj; if (!PyArg_UnpackTuple(args, "is_shareable", 1, 1, &obj)) return NULL; - if (_PyObject_CheckShareable(obj) == 0) + if (_PyObject_CheckCrossInterpreterData(obj) == 0) Py_RETURN_TRUE; PyErr_Clear(); Py_RETURN_FALSE; diff --git a/Python/pystate.c b/Python/pystate.c index 272120d654eaf4..d950350fc957ca 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -166,6 +166,7 @@ PyInterpreterState_New(void) /* overflow or Py_Initialize() not called! */ PyErr_SetString(PyExc_RuntimeError, "failed to get an interpreter ID"); + /* XXX deallocate! */ interp = NULL; } else { interp->id = _PyRuntime.interpreters.next_id; @@ -256,6 +257,28 @@ PyInterpreterState_GetID(PyInterpreterState *interp) } +PyInterpreterState * +_PyInterpreterState_LookUpID(PY_INT64_T requested_id) +{ + if (requested_id < 0) + goto error; + + PyInterpreterState *interp = PyInterpreterState_Head(); + while (interp != NULL) { + PY_INT64_T id = PyInterpreterState_GetID(interp); + if (id < 0) + return NULL; + if (requested_id == id) + return interp; + interp = PyInterpreterState_Next(interp); + } + +error: + PyErr_Format(PyExc_RuntimeError, + "unrecognized interpreter ID %lld", requested_id); + return NULL; +} + /* Default implementation for _PyThreadState_GetFrame */ static struct _frame * threadstate_getframe(PyThreadState *self) @@ -1024,8 +1047,139 @@ PyGILState_Release(PyGILState_STATE oldstate) } +/**************************/ /* cross-interpreter data */ +/**************************/ +/* cross-interpreter data */ + +crossinterpdatafunc _PyCrossInterpreterData_Lookup(PyObject *); + +/* This is a separate func from _PyCrossInterpreterData_Lookup in order + to keep the registry code separate. */ +static crossinterpdatafunc +_lookup_getdata(PyObject *obj) +{ + crossinterpdatafunc getdata = _PyCrossInterpreterData_Lookup(obj); + if (getdata == NULL && PyErr_Occurred() == 0) + PyErr_Format(PyExc_ValueError, + "%S does not support cross-interpreter data", obj); + return getdata; +} + +int +_PyObject_CheckCrossInterpreterData(PyObject *obj) +{ + crossinterpdatafunc getdata = _lookup_getdata(obj); + if (getdata == NULL) { + return -1; + } + return 0; +} + +static int +_check_xidata(_PyCrossInterpreterData *data) +{ + // data->data can be anything, including NULL, so we don't check it. + + // data->obj may be NULL, so we don't check it. + + if (data->interp < 0) { + PyErr_SetString(PyExc_SystemError, "missing interp"); + return -1; + } + + if (data->new_object == NULL) { + PyErr_SetString(PyExc_SystemError, "missing new_object func"); + return -1; + } + + // data->free may be NULL, so we don't check it. + + return 0; +} + +int +_PyObject_GetCrossInterpreterData(PyObject *obj, _PyCrossInterpreterData *data) +{ + PyThreadState *tstate = PyThreadState_Get(); + if (tstate == NULL) + return -1; + PyInterpreterState *interp = tstate->interp; + + // Reset data before re-populating. + *data = (_PyCrossInterpreterData){0}; + data->free = PyMem_RawFree; // Set a default that may be overridden. + + // Call the "getdata" func for the object. + Py_INCREF(obj); + crossinterpdatafunc getdata = _lookup_getdata(obj); + if (getdata == NULL) { + Py_DECREF(obj); + return -1; + } + int res = getdata(obj, data); + Py_DECREF(obj); + if (res != 0) { + return -1; + } + // XXX Make sure they didn't set data->interp? + + // Fill in the blanks and validate the result. + Py_XINCREF(data->obj); + data->interp = interp->id; + if (_check_xidata(data) != 0) { + _PyCrossInterpreterData_Release(data); + return -1; + } + + return 0; +} + +void +_PyCrossInterpreterData_Release(_PyCrossInterpreterData *data) +{ + if (data->data == NULL && data->obj == NULL) { + // Nothing to release! + return; + } + + // Switch to the original interpreter. + PyInterpreterState *interp = _PyInterpreterState_LookUpID(data->interp); + if (interp == NULL) { + // The intepreter was already destroyed. + if (data->free != NULL) { + // XXX Someone leaked some memory... + } + return; + } + PyThreadState *tstate = PyInterpreterState_ThreadHead(interp); + PyThreadState *save_tstate = PyThreadState_Swap(tstate); + + // "Release" the data and/or the object. + if (data->free != NULL) { + data->free(data->data); + } + Py_XDECREF(data->obj); + + // Switch back. + if (save_tstate != NULL) + PyThreadState_Swap(save_tstate); +} + +PyObject * +_PyCrossInterpreterData_NewObject(_PyCrossInterpreterData *data) +{ + return data->new_object(data); +} + +/* registry of {type -> crossinterpdatafunc} */ + +/* For now we use a global registry of shareable classes. An + alternative would be to add a tp_* slot for a class's + crossinterpdatafunc. It would be simpler and more efficient. */ + +/* XXX Rename to _PyObject_GetQualname? */ static const char * _get_qualname(PyObject *obj) { @@ -1044,6 +1198,28 @@ _get_qualname(PyObject *obj) return encoded; } +static int +_register_xidata(PyTypeObject *cls, crossinterpdatafunc getdata) +{ + const char *classname = _get_qualname((PyObject *)cls); + if (classname == NULL) + return -1; + + // XXX Fail if already registered (instead of effectively replacing)? + + // XXX lock + struct _cidclass *newhead = PyMem_NEW(struct _cidclass, 1); + newhead->classname = classname; + newhead->cls = cls; + newhead->getdata = getdata; + newhead->next = _PyRuntime.crossinterpclasses; + _PyRuntime.crossinterpclasses = newhead; + // XXX unlock + return 0; +} + +static void _register_builtins_for_crossinterpreter_data(void); + int _PyCrossInterpreterData_Register_Class(PyTypeObject *cls, crossinterpdatafunc getdata) @@ -1056,28 +1232,22 @@ _PyCrossInterpreterData_Register_Class(PyTypeObject *cls, PyErr_Format(PyExc_ValueError, "missing 'getdata' func"); return -1; } - // XXX lock? - - const char *classname = _get_qualname((PyObject *)cls); - if (classname == NULL) - return -1; - // XXX Fail if already registered (instead of effectively replacing)? - struct _cidclass *newhead = PyMem_NEW(struct _cidclass, 1); - newhead->classname = classname; - newhead->getdata = getdata; + // XXX lock _PyRuntimeState *runtime = &_PyRuntime; - newhead->next = runtime->crossinterpclasses; - runtime->crossinterpclasses = newhead; - return 0; + if (runtime->crossinterpclasses == NULL) { + _register_builtins_for_crossinterpreter_data(); + } + int res = _register_xidata(cls, getdata); + // XXX unlock + return res; } -static void _register_builtins_for_crossinterpreter_data(void); - crossinterpdatafunc _PyCrossInterpreterData_Lookup(PyObject *obj) { - const char *classname = _get_qualname(PyObject_Type(obj)); + PyObject *cls = PyObject_Type(obj); + const char *classname = _get_qualname(cls); if (classname == NULL) return NULL; @@ -1089,8 +1259,13 @@ _PyCrossInterpreterData_Lookup(PyObject *obj) } for(; cur != NULL; cur = cur->next) { // XXX Be more strict (e.g. Py*_CheckExact)? - if (strcmp(cur->classname, classname) == 0) + if (strcmp(cur->classname, classname) == 0) { + if (cur->cls != (PyTypeObject *)cls) { + // oops! + break; + } return cur->getdata; + } } return NULL; } @@ -1107,8 +1282,9 @@ static int _bytes_shared(PyObject *obj, _PyCrossInterpreterData *data) { data->data = (void *)(PyBytes_AS_STRING(obj)); + data->obj = obj; // Will be "released" (decref'ed) when data released. data->new_object = _new_bytes_object; - data->free = NULL; + data->free = NULL; // Do not free the data (it belongs to the object). return 0; } @@ -1124,8 +1300,9 @@ static int _none_shared(PyObject *obj, _PyCrossInterpreterData *data) { data->data = NULL; + // data->obj remains NULL data->new_object = _new_none_object; - data->free = NULL; + data->free = NULL; // There is nothing to free. return 0; } @@ -1133,14 +1310,12 @@ static void _register_builtins_for_crossinterpreter_data(void) { // None - if (_PyCrossInterpreterData_Register_Class( - (PyTypeObject *)PyObject_Type(Py_None), _none_shared) != 0) { + if (_register_xidata((PyTypeObject *)PyObject_Type(Py_None), _none_shared) != 0) { Py_FatalError("could not register None for X-interpreter sharing"); } // bytes - if (_PyCrossInterpreterData_Register_Class( - &PyBytes_Type, _bytes_shared) != 0) { + if (_register_xidata(&PyBytes_Type, _bytes_shared) != 0) { Py_FatalError("could not register bytes for X-interpreter sharing"); } } From 237f743344e3860d3c788dc159b63308876017c3 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 12 Jan 2018 17:25:28 +0000 Subject: [PATCH 40/78] Add basic channel code. --- Lib/test/test__interpreters.py | 71 +++++++ Modules/_interpretersmodule.c | 360 ++++++++++++++++++++++++++++++++- Python/pystate.c | 2 +- 3 files changed, 422 insertions(+), 11 deletions(-) diff --git a/Lib/test/test__interpreters.py b/Lib/test/test__interpreters.py index 13ccfa001594ef..41c41ff72f23c4 100644 --- a/Lib/test/test__interpreters.py +++ b/Lib/test/test__interpreters.py @@ -715,5 +715,76 @@ def f(): self.assertEqual(retcode, 0) +class ChannelTests(TestBase): + + def test_sequential_ids(self): + id1 = interpreters.channel_create() + id2 = interpreters.channel_create() + id3 = interpreters.channel_create() + + self.assertEqual(id2, id1 + 1) + self.assertEqual(id3, id2 + 1) + + def test_ids_global(self): + id1 = interpreters.create() + out = _run_output(id1, dedent(""" + import _interpreters + cid = _interpreters.channel_create() + print(cid) + """)) + cid1 = int(out.strip()) + + id2 = interpreters.create() + out = _run_output(id2, dedent(""" + import _interpreters + cid = _interpreters.channel_create() + print(cid) + """)) + cid2 = int(out.strip()) + + self.assertEqual(cid2, cid1 + 1) + + def test_send_recv_main(self): + cid = interpreters.channel_create() + orig = b'spam' + interpreters.channel_send(cid, orig) + obj = interpreters.channel_recv(cid) + + self.assertEqual(obj, orig) + self.assertIsNot(obj, orig) + + def test_send_recv_same_interpreter(self): + id1 = interpreters.create() + out = _run_output(id1, dedent(""" + import _interpreters + cid = _interpreters.channel_create() + orig = b'spam' + _interpreters.channel_send(cid, orig) + obj = _interpreters.channel_recv(cid) + assert obj is not orig + assert obj == orig + """)) + + def test_send_recv_different_interpreters(self): + cid = interpreters.channel_create() + id1 = interpreters.create() + out = _run_output(id1, dedent(f""" + import _interpreters + _interpreters.channel_send({cid}, b'spam') + """)) + obj = interpreters.channel_recv(cid) + + self.assertEqual(obj, b'spam') + + def test_recv_empty(self): + cid = interpreters.channel_create() + with self.assertRaises(RuntimeError): + interpreters.channel_recv(cid) + + @unittest.skip('not implemented yet') + def test_run_string_arg(self): + raise NotImplementedError + + if __name__ == '__main__': unittest.main() diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index 0640bdf9997e0a..e3ae1987844786 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -7,6 +7,21 @@ #include "internal/pystate.h" +struct _channels; + +/* globals is the process-global state for the module. It holds all + the data that we need to share between interpreters, so it cannot + hold PyObject values. */ +static struct globals { + struct _channels { + //PyThread_type_lock mutex; + struct _channel *head; + int64_t count; + int64_t next_id; + } channels; +} _globals; + + static PyInterpreterState * _get_current(void) { @@ -18,7 +33,7 @@ _get_current(void) return tstate->interp; } -/* data-sharing-specific functions and structs */ +/* data-sharing-specific code ***********************************************/ struct _shareditem { Py_UNICODE *name; @@ -134,7 +149,254 @@ _apply_shared_exception(struct _shared_exception *exc) PyErr_SetString(_interp_failed_error, exc->msg); } -/* interpreter-specific functions */ +/* channel-specific code */ + +struct _channelitem; + +struct _channelitem { + _PyCrossInterpreterData *data; + struct _channelitem *next; +}; + +struct _channel; + +typedef struct _channel { + struct _channel *next; + + int64_t id; + + //PyThread_type_lock mutex; + + int64_t count; + struct _channelitem *first; + struct _channelitem *last; +} _PyChannelState; + +static _PyChannelState * +_channel_new(void) +{ + _PyChannelState *chan = PyMem_Malloc(sizeof(_PyChannelState)); + if (chan == NULL) + return NULL; + chan->next = NULL; + chan->id = -1; + chan->count = 0; + chan->first = NULL; + chan->last = NULL; + return chan; +} + +static int +_channel_add(_PyChannelState *chan, _PyCrossInterpreterData *data) +{ + struct _channelitem *item = PyMem_Malloc(sizeof(struct _channelitem)); + if (item == NULL) + return -1; + item->data = data; + item->next = NULL; + + /* XXX lock */ + chan->count += 1; + if (chan->first == NULL) + chan->first = item; + chan->last = item; + /* XXX unlock */ + + return 0; +} + +static _PyCrossInterpreterData * +_channel_next(_PyChannelState *chan) +{ + /* XXX lock */ + struct _channelitem *item = chan->first; + if (item == NULL) { + /* XXX unlock */ + return NULL; + } + + chan->first = item->next; + if (chan->last == item) + chan->last = NULL; + chan->count -= 1; + /* XXX unlock */ + + _PyCrossInterpreterData *data = item->data; + PyMem_Free(item); + return data; +} + +static void +_channel_clear(_PyChannelState *chan) +{ + /* XXX lock */ + _PyCrossInterpreterData *data = _channel_next(chan); + for (; data != NULL; data = _channel_next(chan)) { + _PyCrossInterpreterData_Release(data); + PyMem_Free(data); + } + /* XXX unlock */ +} + +static void +_channel_free(_PyChannelState *chan) +{ + _channel_clear(chan); + PyMem_Free(chan); +} + +static int64_t +_channels_next_id(void) +{ + int64_t id = _globals.channels.next_id; + if (id < 0) { + /* overflow */ + PyErr_SetString(PyExc_RuntimeError, + "failed to get a channel ID"); + return -1; + } + _globals.channels.next_id += 1; + return id; +} + +_PyChannelState * +_channels_lookup(int64_t id) +{ + /* XXX lock */ + _PyChannelState *chan = _globals.channels.head; + for (;chan != NULL; chan = chan->next) { + if (chan->id == id) + break; + } + /* XXX unlock */ + if (chan == NULL) { + PyErr_Format(PyExc_RuntimeError, "channel %d not found", id); + } + return chan; +} + +static int +_channels_add(_PyChannelState *chan) +{ + /* XXX lock */ + int64_t id = _channels_next_id(); + if (id < 0) { + /* XXX unlock */ + return -1; + } + if (_globals.channels.head != NULL) { + chan->next = _globals.channels.head; + } + _globals.channels.head = chan; + _globals.channels.count += 1; + chan->id = id; + /* XXX unlock */ + + return 0; +} + +_PyChannelState * +_channels_remove(int64_t id) +{ + /* XXX lock */ + if (_globals.channels.head == NULL) { + /* XXX unlock */ + return NULL; + } + + _PyChannelState *prev = NULL; + _PyChannelState *chan = _globals.channels.head; + for (;chan != NULL; chan = chan->next) { + if (chan->id == id) + break; + prev = chan; + } + if (chan == NULL) { + PyErr_Format(PyExc_RuntimeError, "channel %d not found", id); + /* XXX unlock */ + return NULL; + } + + if (chan == _globals.channels.head) { + _globals.channels.head = chan->next; + } else { + prev->next = chan->next; + } + chan->next = NULL; + + _globals.channels.count -= 1; + /* XXX unlock */ + return chan; +} + +int64_t * +_channels_list_all(int64_t *count) +{ + int64_t *ids = PyMem_Malloc(sizeof(int64_t) * _globals.channels.count); + if (ids == NULL) + return NULL; + _PyChannelState *chan = _globals.channels.head; + for (int64_t i=0; chan != NULL; chan = chan->next, i++) { + ids[i] = chan->id; + } + *count = _globals.channels.count; + return ids; +} + +/* +int +channel_list_interpreters(int id) +{ +} + +int +channel_close(void) +{ +} +*/ + +int +_channel_send(int64_t id, PyObject *obj) +{ + _PyCrossInterpreterData *data = PyMem_Malloc(sizeof(_PyCrossInterpreterData)); + if (_PyObject_GetCrossInterpreterData(obj, data) != 0) + return -1; + + // XXX lock _PyChannelState to avoid race on destroy here? + _PyChannelState *chan = _channels_lookup(id); + if (chan == NULL) { + PyMem_Free(data); + return -1; + } + if (_channel_add(chan, data) != 0) { + PyMem_Free(data); + return -1; + } + + return 0; +} + +PyObject * +_channel_recv(int64_t id) +{ + // XXX lock _PyChannelState to avoid race on destroy here? + _PyChannelState *chan = _channels_lookup(id); + if (chan == NULL) + return NULL; + _PyCrossInterpreterData *data = _channel_next(chan); + if (data == NULL) { + PyErr_SetString(PyExc_RuntimeError, "empty channel"); + return NULL; + } + + PyObject *obj = _PyCrossInterpreterData_NewObject(data); + if (obj == NULL) + return NULL; + _PyCrossInterpreterData_Release(data); + return obj; +} + +/* interpreter-specific functions *******************************************/ static PyInterpreterState * _look_up(PyObject *requested_id) @@ -505,6 +767,77 @@ PyDoc_STRVAR(is_running_doc, \n\ Return whether or not the identified interpreter is running."); +static PyObject * +channel_create(PyObject *self) +{ + _PyChannelState *chan = _channel_new(); + if (chan == NULL) + return NULL; + if (_channels_add(chan) != 0) { + _channel_free(chan); + return NULL; + } + PyObject *id = PyLong_FromLongLong(chan->id); + if (id == NULL) { + _channel_free(chan); + return NULL; + } + return id; +} + +PyDoc_STRVAR(channel_create_doc, +"channel_create() -> ID\n\ +\n\ +Create a new cross-interpreter channel and return a unique generated ID."); + +static PyObject * +channel_send(PyObject *self, PyObject *args) +{ + PyObject *id; + PyObject *obj; + if (!PyArg_UnpackTuple(args, "channel_send", 2, 2, &id, &obj)) + return NULL; + if (!PyLong_Check(id)) { + PyErr_SetString(PyExc_TypeError, "ID must be an int"); + return NULL; + } + + long long cid = PyLong_AsLongLong(id); + if (cid == -1 && PyErr_Occurred() != NULL) + return NULL; + assert(cid <= INT64_MAX); + if (_channel_send(cid, obj) != 0) + return NULL; + Py_RETURN_NONE; +} + +PyDoc_STRVAR(channel_send_doc, +"channel_send(ID, obj)\n\ +\n\ +Add the object's data to the channel's queue."); + +static PyObject * +channel_recv(PyObject *self, PyObject *args) +{ + PyObject *id; + if (!PyArg_UnpackTuple(args, "channel_recv", 1, 1, &id)) + return NULL; + if (!PyLong_Check(id)) { + PyErr_SetString(PyExc_TypeError, "ID must be an int"); + return NULL; + } + + long long cid = PyLong_AsLongLong(id); + if (cid == -1 && PyErr_Occurred() != NULL) + return NULL; + assert(cid <= INT64_MAX); + return _channel_recv(cid); +} + +PyDoc_STRVAR(channel_recv_doc, +"channel_recv(ID) -> obj\n\ +\n\ +Return a new object from the data at the from of the channel's queue."); static PyMethodDef module_functions[] = { {"create", (PyCFunction)interp_create, @@ -527,6 +860,13 @@ static PyMethodDef module_functions[] = { {"is_shareable", (PyCFunction)object_is_shareable, METH_VARARGS, is_shareable_doc}, + {"channel_create", (PyCFunction)channel_create, + METH_NOARGS, channel_create_doc}, + {"channel_send", (PyCFunction)channel_send, + METH_VARARGS, channel_send_doc}, + {"channel_recv", (PyCFunction)channel_recv, + METH_VARARGS, channel_recv_doc}, + {NULL, NULL} /* sentinel */ }; @@ -539,14 +879,14 @@ The 'interpreters' module provides a more convenient interface."); static struct PyModuleDef interpretersmodule = { PyModuleDef_HEAD_INIT, - "_interpreters", - module_doc, - -1, - module_functions, - NULL, - NULL, - NULL, - NULL + "_interpreters", /* m_name */ + module_doc, /* m_doc */ + -1, /* m_size */ + module_functions, /* m_methods */ + NULL, /* m_slots */ + NULL, /* m_traverse */ + NULL, /* m_clear */ + NULL /* m_free */ }; diff --git a/Python/pystate.c b/Python/pystate.c index d950350fc957ca..7b5fb93050d469 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1208,7 +1208,7 @@ _register_xidata(PyTypeObject *cls, crossinterpdatafunc getdata) // XXX Fail if already registered (instead of effectively replacing)? // XXX lock - struct _cidclass *newhead = PyMem_NEW(struct _cidclass, 1); + struct _cidclass *newhead = PyMem_RawMalloc(sizeof(struct _cidclass)); newhead->classname = classname; newhead->cls = cls; newhead->getdata = getdata; From 3b0ca21f49f29656d7ff0d82152f35ddc8dfbb9f Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Sat, 13 Jan 2018 01:08:05 +0000 Subject: [PATCH 41/78] Stub out the other channel-related module functions. --- Modules/_interpretersmodule.c | 86 ++++++++++++++++++++++++++++++----- 1 file changed, 75 insertions(+), 11 deletions(-) diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index e3ae1987844786..b973cac209dc44 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -767,6 +767,19 @@ PyDoc_STRVAR(is_running_doc, \n\ Return whether or not the identified interpreter is running."); +static PyObject * +channel_list_all(PyObject *self) +{ + // XXX finish + PyErr_SetString(PyExc_NotImplementedError, "not implemented yet"); + Py_RETURN_NONE; +} + +PyDoc_STRVAR(channel_list_all_doc, +"channel_list_all() -> [ID]\n\ +\n\ +Return the list of all IDs for active channels."); + static PyObject * channel_create(PyObject *self) { @@ -839,33 +852,84 @@ PyDoc_STRVAR(channel_recv_doc, \n\ Return a new object from the data at the from of the channel's queue."); +static PyObject * +channel_close(PyObject *self, PyObject *args) +{ + PyObject *id; + if (!PyArg_UnpackTuple(args, "channel_close", 1, 1, &id)) + return NULL; + if (!PyLong_Check(id)) { + PyErr_SetString(PyExc_TypeError, "ID must be an int"); + return NULL; + } + + // XXX finish + PyErr_SetString(PyExc_NotImplementedError, "not implemented yet"); + Py_RETURN_NONE; +} + +PyDoc_STRVAR(channel_close_doc, +"channel_close(ID, *, send=None, recv=None)\n\ +\n\ +Close the channel for the current interpreter. 'send' and 'receive'\n\ +(bool) may be used to indicate the ends to close. By default both\n\ +ends are closed. Closing an already closed end is a noop."); + +static PyObject * +channel_list_interpreters(PyObject *self, PyObject *args) +{ + PyObject *id; + if (!PyArg_UnpackTuple(args, "...", 1, 1, &id)) + return NULL; + if (!PyLong_Check(id)) { + PyErr_SetString(PyExc_TypeError, "ID must be an int"); + return NULL; + } + + // XXX finish + PyErr_SetString(PyExc_NotImplementedError, "not implemented yet"); + Py_RETURN_NONE; +} + +PyDoc_STRVAR(channel_list_interpreters_doc, +"channel_list_interpreters(ID) -> ([ID], [ID])\n\ +\n\ +Return the list of intepreter IDs associated with the channel\n\ +on the recv and send ends, respectively."); + static PyMethodDef module_functions[] = { - {"create", (PyCFunction)interp_create, + {"create", (PyCFunction)interp_create, METH_VARARGS, create_doc}, - {"destroy", (PyCFunction)interp_destroy, + {"destroy", (PyCFunction)interp_destroy, METH_VARARGS, destroy_doc}, - {"list_all", (PyCFunction)interp_list_all, + {"list_all", (PyCFunction)interp_list_all, METH_NOARGS, list_all_doc}, - {"get_current", (PyCFunction)interp_get_current, + {"get_current", (PyCFunction)interp_get_current, METH_NOARGS, get_current_doc}, - {"get_main", (PyCFunction)interp_get_main, + {"get_main", (PyCFunction)interp_get_main, METH_NOARGS, get_main_doc}, - {"is_running", (PyCFunction)interp_is_running, + {"is_running", (PyCFunction)interp_is_running, METH_VARARGS, is_running_doc}, - {"run_string", (PyCFunction)interp_run_string, + {"run_string", (PyCFunction)interp_run_string, METH_VARARGS, run_string_doc}, - {"is_shareable", (PyCFunction)object_is_shareable, + {"is_shareable", (PyCFunction)object_is_shareable, METH_VARARGS, is_shareable_doc}, - {"channel_create", (PyCFunction)channel_create, + {"channel_list_all", (PyCFunction)channel_list_all, + METH_NOARGS, channel_list_all_doc}, + {"channel_create", (PyCFunction)channel_create, METH_NOARGS, channel_create_doc}, - {"channel_send", (PyCFunction)channel_send, + {"channel_send", (PyCFunction)channel_send, METH_VARARGS, channel_send_doc}, - {"channel_recv", (PyCFunction)channel_recv, + {"channel_recv", (PyCFunction)channel_recv, METH_VARARGS, channel_recv_doc}, + {"channel_close", (PyCFunction)channel_close, + METH_VARARGS, channel_close_doc}, + {"channel_list_interpreters", (PyCFunction)channel_list_interpreters, + METH_VARARGS, channel_list_interpreters_doc}, {NULL, NULL} /* sentinel */ }; From 064f710cf840c6245437194c5c3e99d7d6ffcac1 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Sat, 13 Jan 2018 20:01:14 +0000 Subject: [PATCH 42/78] Drop _cidclass.classname. --- Include/internal/pystate.h | 4 +-- Python/pystate.c | 60 ++++++++++---------------------------- 2 files changed, 16 insertions(+), 48 deletions(-) diff --git a/Include/internal/pystate.h b/Include/internal/pystate.h index 65828a56b12eb7..4c6c3f96d60ef1 100644 --- a/Include/internal/pystate.h +++ b/Include/internal/pystate.h @@ -131,9 +131,7 @@ PyAPI_FUNC(crossinterpdatafunc) _PyCrossInterpreterData_Lookup(PyObject *); struct _cidclass; struct _cidclass { - // XXX Do we really need classname? - const char *classname; - PyTypeObject *cls; // XXX only safe for static types? + PyTypeObject *cls; crossinterpdatafunc getdata; struct _cidclass *next; }; diff --git a/Python/pystate.c b/Python/pystate.c index 7b5fb93050d469..2808807a43ef46 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1179,42 +1179,16 @@ _PyCrossInterpreterData_NewObject(_PyCrossInterpreterData *data) alternative would be to add a tp_* slot for a class's crossinterpdatafunc. It would be simpler and more efficient. */ -/* XXX Rename to _PyObject_GetQualname? */ -static const char * -_get_qualname(PyObject *obj) -{ - PyObject *qualname = PyObject_GetAttrString(obj, "__qualname__"); - if (qualname == NULL) - return NULL; - Py_ssize_t name_size; - const char *encoded = PyUnicode_AsUTF8AndSize(qualname, &name_size); - if (encoded == NULL) - return NULL; - if (strlen(encoded) != (size_t)name_size) { - PyErr_SetString(PyExc_ValueError, - "qualname must not contain null characters"); - return NULL; - } - return encoded; -} - static int _register_xidata(PyTypeObject *cls, crossinterpdatafunc getdata) { - const char *classname = _get_qualname((PyObject *)cls); - if (classname == NULL) - return -1; - - // XXX Fail if already registered (instead of effectively replacing)? - - // XXX lock + // Note that we effectively replace already registered classes + // rather than failing. struct _cidclass *newhead = PyMem_RawMalloc(sizeof(struct _cidclass)); - newhead->classname = classname; newhead->cls = cls; newhead->getdata = getdata; newhead->next = _PyRuntime.crossinterpclasses; _PyRuntime.crossinterpclasses = newhead; - // XXX unlock return 0; } @@ -1233,9 +1207,11 @@ _PyCrossInterpreterData_Register_Class(PyTypeObject *cls, return -1; } + // Make sure the class isn't ever deallocated. + Py_INCREF((PyObject *)cls); + // XXX lock - _PyRuntimeState *runtime = &_PyRuntime; - if (runtime->crossinterpclasses == NULL) { + if (_PyRuntime.crossinterpclasses == NULL) { _register_builtins_for_crossinterpreter_data(); } int res = _register_xidata(cls, getdata); @@ -1247,27 +1223,21 @@ crossinterpdatafunc _PyCrossInterpreterData_Lookup(PyObject *obj) { PyObject *cls = PyObject_Type(obj); - const char *classname = _get_qualname(cls); - if (classname == NULL) - return NULL; - - _PyRuntimeState *runtime = &_PyRuntime; - struct _cidclass *cur = runtime->crossinterpclasses; + crossinterpdatafunc getdata = NULL; + // XXX lock + struct _cidclass *cur = _PyRuntime.crossinterpclasses; if (cur == NULL) { _register_builtins_for_crossinterpreter_data(); - cur = runtime->crossinterpclasses; + cur = _PyRuntime.crossinterpclasses; } for(; cur != NULL; cur = cur->next) { - // XXX Be more strict (e.g. Py*_CheckExact)? - if (strcmp(cur->classname, classname) == 0) { - if (cur->cls != (PyTypeObject *)cls) { - // oops! - break; - } - return cur->getdata; + if (cur->cls == (PyTypeObject *)cls) { + getdata = cur->getdata; + break; } } - return NULL; + // XXX unlock + return getdata; } /* cross-interpreter data for builtin types */ From 703ac97919bfbd8a63a1401fe7c649090cf03c06 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Sat, 13 Jan 2018 20:06:28 +0000 Subject: [PATCH 43/78] Drop some outdated TODO comments. --- Include/internal/pystate.h | 1 - Lib/test/test__interpreters.py | 1 - Python/pystate.c | 1 - 3 files changed, 3 deletions(-) diff --git a/Include/internal/pystate.h b/Include/internal/pystate.h index 4c6c3f96d60ef1..5704bcea4a7580 100644 --- a/Include/internal/pystate.h +++ b/Include/internal/pystate.h @@ -158,7 +158,6 @@ typedef struct pyruntimestate { using a Python int. */ int64_t next_id; } interpreters; - // XXX Move this to the module of the _intepreters module? // XXX Remove this field once we have a tp_* slot. struct _cidclass *crossinterpclasses; diff --git a/Lib/test/test__interpreters.py b/Lib/test/test__interpreters.py index 41c41ff72f23c4..5ad6e9b39067df 100644 --- a/Lib/test/test__interpreters.py +++ b/Lib/test/test__interpreters.py @@ -464,7 +464,6 @@ def test_fork(self): file.flush() expected = 'spam spam spam spam spam' - # XXX leaking a process... script = dedent(f""" # (inspired by Lib/test/test_fork.py) import os diff --git a/Python/pystate.c b/Python/pystate.c index 2808807a43ef46..3edb913fe673bd 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1123,7 +1123,6 @@ _PyObject_GetCrossInterpreterData(PyObject *obj, _PyCrossInterpreterData *data) if (res != 0) { return -1; } - // XXX Make sure they didn't set data->interp? // Fill in the blanks and validate the result. Py_XINCREF(data->obj); From e36e4d466e0577f209d39e3157bf37a0eb221f6e Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Sat, 13 Jan 2018 20:36:23 +0000 Subject: [PATCH 44/78] Add a mutex for the registry. --- Include/internal/pystate.h | 17 ++++++++++------- Modules/_interpretersmodule.c | 2 +- Python/pystate.c | 31 +++++++++++++++++++------------ 3 files changed, 30 insertions(+), 20 deletions(-) diff --git a/Include/internal/pystate.h b/Include/internal/pystate.h index 5704bcea4a7580..edf41ca752ba73 100644 --- a/Include/internal/pystate.h +++ b/Include/internal/pystate.h @@ -72,12 +72,12 @@ PyAPI_FUNC(PyInterpreterState *) _PyInterpreterState_LookUpID(PY_INT64_T); /* cross-interpreter data */ -struct _cid; +struct _xid; // _PyCrossInterpreterData is similar to Py_buffer as an effectively // opaque struct that holds data outside the object machinery. This // is necessary to pass between interpreters in the same process. -typedef struct _cid { +typedef struct _xid { // data is the cross-interpreter-safe derivation of a Python object // (see _PyObject_GetCrossInterpreterData). It will be NULL if the // new_object func (below) encodes the data. @@ -99,7 +99,7 @@ typedef struct _cid { // new_object is a function that returns a new object in the current // interpreter given the data. The resulting object will be // equivalent to the original object. This field is required. - PyObject *(*new_object)(struct _cid *); + PyObject *(*new_object)(struct _xid *); // free is called when the data is released. If it is NULL then // nothing will be done to free the data. For some types this is // okay (e.g. bytes) and for those types this field should be set @@ -128,12 +128,12 @@ PyAPI_FUNC(void) _PyCrossInterpreterData_Release(_PyCrossInterpreterData *); PyAPI_FUNC(int) _PyCrossInterpreterData_Register_Class(PyTypeObject *, crossinterpdatafunc); PyAPI_FUNC(crossinterpdatafunc) _PyCrossInterpreterData_Lookup(PyObject *); -struct _cidclass; +struct _xidregitem; -struct _cidclass { +struct _xidregitem { PyTypeObject *cls; crossinterpdatafunc getdata; - struct _cidclass *next; + struct _xidregitem *next; }; @@ -159,7 +159,10 @@ typedef struct pyruntimestate { int64_t next_id; } interpreters; // XXX Remove this field once we have a tp_* slot. - struct _cidclass *crossinterpclasses; + struct _xidregistry { + PyThread_type_lock mutex; + struct _xidregitem *head; + } xidregistry; #define NEXITFUNCS 32 void (*exitfuncs[NEXITFUNCS])(void); diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index b973cac209dc44..23db0f115ea28e 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -426,7 +426,7 @@ _is_running(PyInterpreterState *interp) "interpreter has more than one thread"); return -1; } - PyFrameObject *frame = _PyThreadState_GetFrame(tstate); + PyFrameObject *frame = tstate->frame; if (frame == NULL) { if (PyErr_Occurred() != NULL) return -1; diff --git a/Python/pystate.c b/Python/pystate.c index 3edb913fe673bd..22bda90b53b59f 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -56,6 +56,11 @@ _PyRuntimeState_Init_impl(_PyRuntimeState *runtime) } runtime->interpreters.next_id = -1; + runtime->xidregistry.mutex = PyThread_allocate_lock(); + if (runtime->xidregistry.mutex == NULL) { + return _Py_INIT_ERR("Can't initialize threads for cross-interpreter data registry"); + } + return _Py_INIT_OK(); } @@ -1183,11 +1188,13 @@ _register_xidata(PyTypeObject *cls, crossinterpdatafunc getdata) { // Note that we effectively replace already registered classes // rather than failing. - struct _cidclass *newhead = PyMem_RawMalloc(sizeof(struct _cidclass)); + struct _xidregitem *newhead = PyMem_RawMalloc(sizeof(struct _xidregitem)); + if (newhead == NULL) + return -1; newhead->cls = cls; newhead->getdata = getdata; - newhead->next = _PyRuntime.crossinterpclasses; - _PyRuntime.crossinterpclasses = newhead; + newhead->next = _PyRuntime.xidregistry.head; + _PyRuntime.xidregistry.head = newhead; return 0; } @@ -1209,12 +1216,12 @@ _PyCrossInterpreterData_Register_Class(PyTypeObject *cls, // Make sure the class isn't ever deallocated. Py_INCREF((PyObject *)cls); - // XXX lock - if (_PyRuntime.crossinterpclasses == NULL) { + PyThread_acquire_lock(_PyRuntime.xidregistry.mutex, WAIT_LOCK); + if (_PyRuntime.xidregistry.head == NULL) { _register_builtins_for_crossinterpreter_data(); } int res = _register_xidata(cls, getdata); - // XXX unlock + PyThread_release_lock(_PyRuntime.xidregistry.mutex); return res; } @@ -1223,11 +1230,11 @@ _PyCrossInterpreterData_Lookup(PyObject *obj) { PyObject *cls = PyObject_Type(obj); crossinterpdatafunc getdata = NULL; - // XXX lock - struct _cidclass *cur = _PyRuntime.crossinterpclasses; + PyThread_acquire_lock(_PyRuntime.xidregistry.mutex, WAIT_LOCK); + struct _xidregitem *cur = _PyRuntime.xidregistry.head; if (cur == NULL) { _register_builtins_for_crossinterpreter_data(); - cur = _PyRuntime.crossinterpclasses; + cur = _PyRuntime.xidregistry.head; } for(; cur != NULL; cur = cur->next) { if (cur->cls == (PyTypeObject *)cls) { @@ -1235,7 +1242,7 @@ _PyCrossInterpreterData_Lookup(PyObject *obj) break; } } - // XXX unlock + PyThread_release_lock(_PyRuntime.xidregistry.mutex); return getdata; } @@ -1280,12 +1287,12 @@ _register_builtins_for_crossinterpreter_data(void) { // None if (_register_xidata((PyTypeObject *)PyObject_Type(Py_None), _none_shared) != 0) { - Py_FatalError("could not register None for X-interpreter sharing"); + Py_FatalError("could not register None for cross-interpreter sharing"); } // bytes if (_register_xidata(&PyBytes_Type, _bytes_shared) != 0) { - Py_FatalError("could not register bytes for X-interpreter sharing"); + Py_FatalError("could not register bytes for cross-interpreter sharing"); } } From b973cc47e6856d61e2fb2c3008017238e4deb331 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Sat, 13 Jan 2018 20:44:34 +0000 Subject: [PATCH 45/78] Add _PyChannelState.mutex. --- Modules/_interpretersmodule.c | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index 23db0f115ea28e..f6e335e04898ae 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -165,7 +165,7 @@ typedef struct _channel { int64_t id; - //PyThread_type_lock mutex; + PyThread_type_lock mutex; int64_t count; struct _channelitem *first; @@ -180,6 +180,11 @@ _channel_new(void) return NULL; chan->next = NULL; chan->id = -1; + chan->mutex = PyThread_allocate_lock(); + if (chan->mutex == NULL) { + PyErr_SetString(PyExc_RuntimeError, "can't initialize mutex for new channel"); + return NULL; + } chan->count = 0; chan->first = NULL; chan->last = NULL; @@ -195,12 +200,12 @@ _channel_add(_PyChannelState *chan, _PyCrossInterpreterData *data) item->data = data; item->next = NULL; - /* XXX lock */ + PyThread_acquire_lock(chan->mutex, WAIT_LOCK); chan->count += 1; if (chan->first == NULL) chan->first = item; chan->last = item; - /* XXX unlock */ + PyThread_release_lock(chan->mutex); return 0; } @@ -208,10 +213,10 @@ _channel_add(_PyChannelState *chan, _PyCrossInterpreterData *data) static _PyCrossInterpreterData * _channel_next(_PyChannelState *chan) { - /* XXX lock */ + PyThread_acquire_lock(chan->mutex, WAIT_LOCK); struct _channelitem *item = chan->first; if (item == NULL) { - /* XXX unlock */ + PyThread_release_lock(chan->mutex); return NULL; } @@ -219,7 +224,7 @@ _channel_next(_PyChannelState *chan) if (chan->last == item) chan->last = NULL; chan->count -= 1; - /* XXX unlock */ + PyThread_release_lock(chan->mutex); _PyCrossInterpreterData *data = item->data; PyMem_Free(item); @@ -229,19 +234,20 @@ _channel_next(_PyChannelState *chan) static void _channel_clear(_PyChannelState *chan) { - /* XXX lock */ + PyThread_acquire_lock(chan->mutex, WAIT_LOCK); _PyCrossInterpreterData *data = _channel_next(chan); for (; data != NULL; data = _channel_next(chan)) { _PyCrossInterpreterData_Release(data); PyMem_Free(data); } - /* XXX unlock */ + PyThread_release_lock(chan->mutex); } static void _channel_free(_PyChannelState *chan) { _channel_clear(chan); + PyThread_free_lock(chan->mutex); PyMem_Free(chan); } From 6b9e45774575acf1b23a80280355d0098a47c39d Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Sat, 13 Jan 2018 21:51:57 +0000 Subject: [PATCH 46/78] Add a mutex for operating on the global list of channels. --- Modules/_interpretersmodule.c | 170 ++++++++++++++++++++++------------ 1 file changed, 110 insertions(+), 60 deletions(-) diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index f6e335e04898ae..60df8e90a6d3b3 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -7,21 +7,6 @@ #include "internal/pystate.h" -struct _channels; - -/* globals is the process-global state for the module. It holds all - the data that we need to share between interpreters, so it cannot - hold PyObject values. */ -static struct globals { - struct _channels { - //PyThread_type_lock mutex; - struct _channel *head; - int64_t count; - int64_t next_id; - } channels; -} _globals; - - static PyInterpreterState * _get_current(void) { @@ -251,30 +236,52 @@ _channel_free(_PyChannelState *chan) PyMem_Free(chan); } +struct _channels { + PyThread_type_lock mutex; + _PyChannelState *head; + int64_t count; + int64_t next_id; +}; + +static int +_channels_init(struct _channels *channels) +{ + if (channels->mutex == NULL) { + channels->mutex = PyThread_allocate_lock(); + if (channels->mutex == NULL) { + PyMem_Free(channels); + PyErr_SetString(PyExc_RuntimeError, "can't initialize mutex for new channel"); + return -1; + } + } + channels->head = NULL; + return 0; +} + static int64_t -_channels_next_id(void) +_channels_next_id(struct _channels *channels) { - int64_t id = _globals.channels.next_id; + int64_t id = channels->next_id; if (id < 0) { /* overflow */ PyErr_SetString(PyExc_RuntimeError, "failed to get a channel ID"); return -1; } - _globals.channels.next_id += 1; + channels->next_id += 1; return id; } _PyChannelState * -_channels_lookup(int64_t id) +_channels_lookup(struct _channels *channels, int64_t id) { - /* XXX lock */ - _PyChannelState *chan = _globals.channels.head; + PyThread_acquire_lock(channels->mutex, WAIT_LOCK); + _PyChannelState *chan = channels->head; for (;chan != NULL; chan = chan->next) { if (chan->id == id) break; } - /* XXX unlock */ + PyThread_release_lock(channels->mutex); if (chan == NULL) { PyErr_Format(PyExc_RuntimeError, "channel %d not found", id); } @@ -282,36 +289,36 @@ _channels_lookup(int64_t id) } static int -_channels_add(_PyChannelState *chan) +_channels_add(struct _channels *channels, _PyChannelState *chan) { - /* XXX lock */ - int64_t id = _channels_next_id(); + PyThread_acquire_lock(channels->mutex, WAIT_LOCK); + int64_t id = _channels_next_id(channels); if (id < 0) { - /* XXX unlock */ + PyThread_release_lock(channels->mutex); return -1; } - if (_globals.channels.head != NULL) { - chan->next = _globals.channels.head; + if (channels->head != NULL) { + chan->next = channels->head; } - _globals.channels.head = chan; - _globals.channels.count += 1; + channels->head = chan; + channels->count += 1; chan->id = id; - /* XXX unlock */ + PyThread_release_lock(channels->mutex); return 0; } _PyChannelState * -_channels_remove(int64_t id) +_channels_remove(struct _channels *channels, int64_t id) { - /* XXX lock */ - if (_globals.channels.head == NULL) { - /* XXX unlock */ + PyThread_acquire_lock(channels->mutex, WAIT_LOCK); + if (channels->head == NULL) { + PyThread_release_lock(channels->mutex); return NULL; } _PyChannelState *prev = NULL; - _PyChannelState *chan = _globals.channels.head; + _PyChannelState *chan = channels->head; for (;chan != NULL; chan = chan->next) { if (chan->id == id) break; @@ -319,33 +326,37 @@ _channels_remove(int64_t id) } if (chan == NULL) { PyErr_Format(PyExc_RuntimeError, "channel %d not found", id); - /* XXX unlock */ + PyThread_release_lock(channels->mutex); return NULL; } - if (chan == _globals.channels.head) { - _globals.channels.head = chan->next; + if (chan == channels->head) { + channels->head = chan->next; } else { prev->next = chan->next; } chan->next = NULL; - _globals.channels.count -= 1; - /* XXX unlock */ + channels->count -= 1; + PyThread_release_lock(channels->mutex); return chan; } int64_t * -_channels_list_all(int64_t *count) +_channels_list_all(struct _channels *channels, int64_t *count) { - int64_t *ids = PyMem_Malloc(sizeof(int64_t) * _globals.channels.count); - if (ids == NULL) + PyThread_acquire_lock(channels->mutex, WAIT_LOCK); + int64_t *ids = PyMem_Malloc(sizeof(int64_t) * channels->count); + if (ids == NULL) { + PyThread_release_lock(channels->mutex); return NULL; - _PyChannelState *chan = _globals.channels.head; + } + _PyChannelState *chan = channels->head; for (int64_t i=0; chan != NULL; chan = chan->next, i++) { ids[i] = chan->id; } - *count = _globals.channels.count; + *count = channels->count; + PyThread_release_lock(channels->mutex); return ids; } @@ -361,15 +372,40 @@ channel_close(void) } */ +/* "high"-level channel-related functions */ + +static int64_t +_channel_create(struct _channels *channels) +{ + _PyChannelState *chan = _channel_new(); + if (chan == NULL) + return -1; + if (_channels_add(channels, chan) != 0) { + _channel_free(chan); + return -1; + } + return chan->id; +} + +static int +_channel_destroy(struct _channels *channels, int64_t id) +{ + _PyChannelState *chan = _channels_remove(channels, id); + if (chan == NULL) + return -1; + _channel_free(chan); + return 0; +} + int -_channel_send(int64_t id, PyObject *obj) +_channel_send(struct _channels *channels, int64_t id, PyObject *obj) { _PyCrossInterpreterData *data = PyMem_Malloc(sizeof(_PyCrossInterpreterData)); if (_PyObject_GetCrossInterpreterData(obj, data) != 0) return -1; // XXX lock _PyChannelState to avoid race on destroy here? - _PyChannelState *chan = _channels_lookup(id); + _PyChannelState *chan = _channels_lookup(channels, id); if (chan == NULL) { PyMem_Free(data); return -1; @@ -383,10 +419,10 @@ _channel_send(int64_t id, PyObject *obj) } PyObject * -_channel_recv(int64_t id) +_channel_recv(struct _channels *channels, int64_t id) { // XXX lock _PyChannelState to avoid race on destroy here? - _PyChannelState *chan = _channels_lookup(id); + _PyChannelState *chan = _channels_lookup(channels, id); if (chan == NULL) return NULL; _PyCrossInterpreterData *data = _channel_next(chan); @@ -542,6 +578,21 @@ _run_script_in_interpreter(PyInterpreterState *interp, const char *codestr, /* module level code ********************************************************/ +/* globals is the process-global state for the module. It holds all + the data that we need to share between interpreters, so it cannot + hold PyObject values. */ +static struct globals { + struct _channels channels; +} _globals = {0}; + +static int +_init_globals(void) +{ + if (_channels_init(&_globals.channels) != 0) + return -1; + return 0; +} + static PyObject * interp_create(PyObject *self, PyObject *args) { @@ -789,16 +840,12 @@ Return the list of all IDs for active channels."); static PyObject * channel_create(PyObject *self) { - _PyChannelState *chan = _channel_new(); - if (chan == NULL) - return NULL; - if (_channels_add(chan) != 0) { - _channel_free(chan); + int64_t cid = _channel_create(&_globals.channels); + if (cid < 0) return NULL; - } - PyObject *id = PyLong_FromLongLong(chan->id); + PyObject *id = PyLong_FromLongLong(cid); if (id == NULL) { - _channel_free(chan); + _channel_destroy(&_globals.channels, cid); return NULL; } return id; @@ -825,7 +872,7 @@ channel_send(PyObject *self, PyObject *args) if (cid == -1 && PyErr_Occurred() != NULL) return NULL; assert(cid <= INT64_MAX); - if (_channel_send(cid, obj) != 0) + if (_channel_send(&_globals.channels, cid, obj) != 0) return NULL; Py_RETURN_NONE; } @@ -850,7 +897,7 @@ channel_recv(PyObject *self, PyObject *args) if (cid == -1 && PyErr_Occurred() != NULL) return NULL; assert(cid <= INT64_MAX); - return _channel_recv(cid); + return _channel_recv(&_globals.channels, cid); } PyDoc_STRVAR(channel_recv_doc, @@ -963,6 +1010,9 @@ static struct PyModuleDef interpretersmodule = { PyMODINIT_FUNC PyInit__interpreters(void) { + if (_init_globals() != 0) + return NULL; + PyObject *module = PyModule_Create(&interpretersmodule); if (module == NULL) return NULL; From 12efb004efe267a7f7206a1fb3bd2b0614c99af3 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Sat, 13 Jan 2018 22:34:28 +0000 Subject: [PATCH 47/78] Clean up the locking a little. --- Modules/_interpretersmodule.c | 50 ++++++++++++++++++++++------------- 1 file changed, 32 insertions(+), 18 deletions(-) diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index 60df8e90a6d3b3..88aa81383aa313 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -204,7 +204,6 @@ _channel_next(_PyChannelState *chan) PyThread_release_lock(chan->mutex); return NULL; } - chan->first = item->next; if (chan->last == item) chan->last = NULL; @@ -272,29 +271,34 @@ _channels_next_id(struct _channels *channels) return id; } -_PyChannelState * -_channels_lookup(struct _channels *channels, int64_t id) +static _PyChannelState * +_channels_lookup_locked(struct _channels *channels, int64_t id) { - PyThread_acquire_lock(channels->mutex, WAIT_LOCK); _PyChannelState *chan = channels->head; for (;chan != NULL; chan = chan->next) { if (chan->id == id) break; } - PyThread_release_lock(channels->mutex); if (chan == NULL) { PyErr_Format(PyExc_RuntimeError, "channel %d not found", id); } return chan; } -static int -_channels_add(struct _channels *channels, _PyChannelState *chan) +static _PyChannelState * +_channels_lookup(struct _channels *channels, int64_t id) { PyThread_acquire_lock(channels->mutex, WAIT_LOCK); + _PyChannelState *chan = _channels_lookup_locked(channels, id); + PyThread_release_lock(channels->mutex); + return chan; +} + +static int +_channels_add_locked(struct _channels *channels, _PyChannelState *chan) +{ int64_t id = _channels_next_id(channels); if (id < 0) { - PyThread_release_lock(channels->mutex); return -1; } if (channels->head != NULL) { @@ -303,17 +307,22 @@ _channels_add(struct _channels *channels, _PyChannelState *chan) channels->head = chan; channels->count += 1; chan->id = id; - PyThread_release_lock(channels->mutex); - return 0; } -_PyChannelState * -_channels_remove(struct _channels *channels, int64_t id) +static int +_channels_add(struct _channels *channels, _PyChannelState *chan) { PyThread_acquire_lock(channels->mutex, WAIT_LOCK); + int res = _channels_add_locked(channels, chan); + PyThread_release_lock(channels->mutex); + return res; +} + +static _PyChannelState * +_channels_remove_locked(struct _channels *channels, int64_t id) +{ if (channels->head == NULL) { - PyThread_release_lock(channels->mutex); return NULL; } @@ -326,7 +335,6 @@ _channels_remove(struct _channels *channels, int64_t id) } if (chan == NULL) { PyErr_Format(PyExc_RuntimeError, "channel %d not found", id); - PyThread_release_lock(channels->mutex); return NULL; } @@ -338,6 +346,14 @@ _channels_remove(struct _channels *channels, int64_t id) chan->next = NULL; channels->count -= 1; + return chan; +} + +static _PyChannelState * +_channels_remove(struct _channels *channels, int64_t id) +{ + PyThread_acquire_lock(channels->mutex, WAIT_LOCK); + _PyChannelState *chan = _channels_remove_locked(channels, id); PyThread_release_lock(channels->mutex); return chan; } @@ -404,7 +420,6 @@ _channel_send(struct _channels *channels, int64_t id, PyObject *obj) if (_PyObject_GetCrossInterpreterData(obj, data) != 0) return -1; - // XXX lock _PyChannelState to avoid race on destroy here? _PyChannelState *chan = _channels_lookup(channels, id); if (chan == NULL) { PyMem_Free(data); @@ -421,7 +436,6 @@ _channel_send(struct _channels *channels, int64_t id, PyObject *obj) PyObject * _channel_recv(struct _channels *channels, int64_t id) { - // XXX lock _PyChannelState to avoid race on destroy here? _PyChannelState *chan = _channels_lookup(channels, id); if (chan == NULL) return NULL; @@ -432,8 +446,9 @@ _channel_recv(struct _channels *channels, int64_t id) } PyObject *obj = _PyCrossInterpreterData_NewObject(data); - if (obj == NULL) + if (obj == NULL) { return NULL; + } _PyCrossInterpreterData_Release(data); return obj; } @@ -536,7 +551,6 @@ static int _run_script_in_interpreter(PyInterpreterState *interp, const char *codestr, PyObject *shareables) { - // XXX lock? if (_ensure_not_running(interp) < 0) return -1; From 310d0ac0e44f0f04dc004441cd39d65e72531c89 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Sat, 13 Jan 2018 23:28:46 +0000 Subject: [PATCH 48/78] Implement _interpreters.channel_list_all(). --- Lib/test/test__interpreters.py | 3 +++ Modules/_interpretersmodule.c | 26 +++++++++++++++++++++++--- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/Lib/test/test__interpreters.py b/Lib/test/test__interpreters.py index 5ad6e9b39067df..b3041a3799e9c8 100644 --- a/Lib/test/test__interpreters.py +++ b/Lib/test/test__interpreters.py @@ -717,12 +717,15 @@ def f(): class ChannelTests(TestBase): def test_sequential_ids(self): + before = interpreters.channel_list_all() id1 = interpreters.channel_create() id2 = interpreters.channel_create() id3 = interpreters.channel_create() + after = interpreters.channel_list_all() self.assertEqual(id2, id1 + 1) self.assertEqual(id3, id2 + 1) + self.assertEqual(set(after) - set(before), {id1, id2, id3}) def test_ids_global(self): id1 = interpreters.create() diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index 88aa81383aa313..e262ec686e5daa 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -841,9 +841,29 @@ Return whether or not the identified interpreter is running."); static PyObject * channel_list_all(PyObject *self) { - // XXX finish - PyErr_SetString(PyExc_NotImplementedError, "not implemented yet"); - Py_RETURN_NONE; + int64_t count = 0; + int64_t *cids = _channels_list_all(&_globals.channels, &count); + if (cids == NULL) { + if (count == 0) + return PyList_New(0); + return NULL; + } + PyObject *ids = PyList_New((Py_ssize_t)count); + if (ids == NULL) { + // XXX free cids + return NULL; + } + for (int64_t i=0; i < count; cids++, i++) { + PyObject *id = PyLong_FromLongLong(*cids); + if (id == NULL) { + Py_DECREF(ids); + ids = NULL; + break; + } + PyList_SET_ITEM(ids, i, id); + } + // XXX free cids + return ids; } PyDoc_STRVAR(channel_list_all_doc, From 8a242208b4e4e27e31082373ff2381599569d2f8 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Sun, 14 Jan 2018 02:20:12 +0000 Subject: [PATCH 49/78] Implement _interpreters.channel_close(). --- Lib/test/test__interpreters.py | 73 +++++++++ Modules/_interpretersmodule.c | 269 ++++++++++++++++++++++++++++----- 2 files changed, 304 insertions(+), 38 deletions(-) diff --git a/Lib/test/test__interpreters.py b/Lib/test/test__interpreters.py index b3041a3799e9c8..970dff17ea5a4b 100644 --- a/Lib/test/test__interpreters.py +++ b/Lib/test/test__interpreters.py @@ -746,6 +746,79 @@ def test_ids_global(self): self.assertEqual(cid2, cid1 + 1) + def test_close_single_user(self): + cid = interpreters.channel_create() + interpreters.channel_send(cid, b'spam') + interpreters.channel_recv(cid) + interpreters.channel_close(cid, send=True, recv=True) + + with self.assertRaises(RuntimeError): + interpreters.channel_send(cid, b'eggs') + with self.assertRaises(RuntimeError): + interpreters.channel_recv(cid) + + def test_close_multiple_users(self): + raise NotImplementedError + + def test_close_no_kwargs(self): + cid = interpreters.channel_create() + interpreters.channel_send(cid, b'spam') + interpreters.channel_recv(cid) + interpreters.channel_close(cid) + + with self.assertRaises(RuntimeError): + interpreters.channel_send(cid, b'eggs') + with self.assertRaises(RuntimeError): + interpreters.channel_recv(cid) + + def test_close_multiple_times(self): + cid = interpreters.channel_create() + interpreters.channel_send(cid, b'spam') + interpreters.channel_recv(cid) + interpreters.channel_close(cid, send=True, recv=True) + + with self.assertRaises(RuntimeError): + interpreters.channel_close(cid, send=True, recv=True) + + def test_close_with_unused_items(self): + cid = interpreters.channel_create() + interpreters.channel_send(cid, b'spam') + interpreters.channel_send(cid, b'ham') + interpreters.channel_close(cid, send=True, recv=True) + + with self.assertRaises(RuntimeError): + interpreters.channel_send(cid, b'eggs') + with self.assertRaises(RuntimeError): + interpreters.channel_recv(cid) + + def test_close_never_used(self): + cid = interpreters.channel_create() + interpreters.channel_close(cid) + # XXX Do we want this to fail? + interpreters.channel_send(cid, b'spam') + obj = interpreters.channel_recv(cid) + + self.assertEqual(obj, b'spam') + + def test_close_by_unassociated_interp(self): + raise NotImplementedError + + def test_close_partially(self): + raise NotImplementedError + + def test_close_used_multiple_times_by_single_user(self): + cid = interpreters.channel_create() + interpreters.channel_send(cid, b'spam') + interpreters.channel_send(cid, b'spam') + interpreters.channel_send(cid, b'spam') + interpreters.channel_recv(cid) + interpreters.channel_close(cid, send=True, recv=True) + + with self.assertRaises(RuntimeError): + interpreters.channel_send(cid, b'eggs') + with self.assertRaises(RuntimeError): + interpreters.channel_recv(cid) + def test_send_recv_main(self): cid = interpreters.channel_create() orig = b'spam' diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index e262ec686e5daa..b434adda62f172 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -136,6 +136,38 @@ _apply_shared_exception(struct _shared_exception *exc) /* channel-specific code */ +struct _channelend; + +struct _channelend { + int64_t interp; + struct _channelend *next; +}; + +static void +_channelend_free_all(struct _channelend *end) { + while (end != NULL) { + struct _channelend *last = end; + end = end->next; + PyMem_Free(last); + } +} + +static struct _channelend * +_channelend_find(struct _channelend *first, int64_t interp, struct _channelend **pprev) +{ + struct _channelend *prev = NULL; + struct _channelend *end = first; + while (end != NULL) { + if (end->interp == interp) + break; + prev = end; + end = end->next; + } + if (pprev != NULL) + *pprev = prev; + return end; +} + struct _channelitem; struct _channelitem { @@ -155,6 +187,17 @@ typedef struct _channel { int64_t count; struct _channelitem *first; struct _channelitem *last; + + // Note that the list entries are never removed for interpreter + // for which the channel is closed. This should be a problem in + // practice. Also, a channel isn't automatically closed when an + // interpreter is destroyed. + int64_t sendcount; + int64_t recvcount; + struct _channelend *send; + struct _channelend *recv; + struct _channelend *sendclosed; + struct _channelend *recvclosed; } _PyChannelState; static _PyChannelState * @@ -173,19 +216,121 @@ _channel_new(void) chan->count = 0; chan->first = NULL; chan->last = NULL; + chan->sendcount = 0; + chan->recvcount = 0; + chan->send = NULL; + chan->recv = NULL; + chan->sendclosed = NULL; + chan->recvclosed = NULL; return chan; } static int -_channel_add(_PyChannelState *chan, _PyCrossInterpreterData *data) +_channel_associate_end(_PyChannelState *chan, int64_t interp, int send) +{ + struct _channelend *prev; + struct _channelend *end = _channelend_find(send ? chan->send : chan->recv, interp, &prev); + if (end != NULL) { + // already associated + return 0; + } + end = _channelend_find(send ? chan->sendclosed : chan->recvclosed, interp, &prev); + if (end != NULL) { + PyErr_SetString(PyExc_RuntimeError, "channel already closed"); + return 1; + } + + end = PyMem_Malloc(sizeof(struct _channelend)); + if (end == NULL) + return -1; + end->interp = interp; + end->next = NULL; + if (prev == NULL) { + if (send) { + chan->send = end; + } else { + chan->recv = end; + } + } else { + prev->next = end; + } + if (send) + chan->sendcount += 1; + else + chan->recvcount += 1; + return 0; +} + +static int +_channel_close_end(_PyChannelState *chan, int64_t interp, int send) +{ + PyThread_acquire_lock(chan->mutex, WAIT_LOCK); + + struct _channelend *prev; + struct _channelend *end = _channelend_find(send ? chan->send : chan->recv, interp, &prev); + if (end == NULL) { + // never associated or already closed + PyThread_release_lock(chan->mutex); + return 0; + } + + // Drop it from the "open" list. + if (prev == NULL) { + if (send) + chan->send = NULL; + else + chan->recv = NULL; + } else { + prev->next = end->next; + } + if (send) + chan->sendcount -= 1; + else + chan->recvcount -= 1; + + // Add it to the "closed" list. + if (send) { + end->next = chan->sendclosed; + chan->sendclosed = end; + } else { + end->next = chan->recvclosed; + chan->recvclosed = end; + } + + PyThread_release_lock(chan->mutex); + return 0; +} + +static int +_channel_is_closed(_PyChannelState *chan) { + PyThread_acquire_lock(chan->mutex, WAIT_LOCK); + int res = 0; + if (chan->sendcount == 0 && chan->recvcount == 0) { + if (chan->sendclosed != NULL || chan->recvclosed != NULL) + res = 1; + } + PyThread_release_lock(chan->mutex); + return res; +} + +static int +_channel_add(_PyChannelState *chan, int64_t interp, _PyCrossInterpreterData *data) +{ + PyThread_acquire_lock(chan->mutex, WAIT_LOCK); + if (_channel_associate_end(chan, interp, 1) != 0) { + PyThread_release_lock(chan->mutex); + return -1; + } + struct _channelitem *item = PyMem_Malloc(sizeof(struct _channelitem)); - if (item == NULL) + if (item == NULL) { + PyThread_release_lock(chan->mutex); return -1; + } item->data = data; item->next = NULL; - PyThread_acquire_lock(chan->mutex, WAIT_LOCK); chan->count += 1; if (chan->first == NULL) chan->first = item; @@ -196,9 +341,14 @@ _channel_add(_PyChannelState *chan, _PyCrossInterpreterData *data) } static _PyCrossInterpreterData * -_channel_next(_PyChannelState *chan) +_channel_next(_PyChannelState *chan, int64_t interp) { PyThread_acquire_lock(chan->mutex, WAIT_LOCK); + if (_channel_associate_end(chan, interp, 0) != 0) { + PyThread_release_lock(chan->mutex); + return NULL; + } + struct _channelitem *item = chan->first; if (item == NULL) { PyThread_release_lock(chan->mutex); @@ -218,19 +368,29 @@ _channel_next(_PyChannelState *chan) static void _channel_clear(_PyChannelState *chan) { - PyThread_acquire_lock(chan->mutex, WAIT_LOCK); - _PyCrossInterpreterData *data = _channel_next(chan); - for (; data != NULL; data = _channel_next(chan)) { - _PyCrossInterpreterData_Release(data); - PyMem_Free(data); + struct _channelitem *item = chan->first; + while (item != NULL) { + _PyCrossInterpreterData_Release(item->data); + PyMem_Free(item->data); + struct _channelitem *last = item; + item = item->next; + PyMem_Free(last); } - PyThread_release_lock(chan->mutex); + chan->first = NULL; + chan->last = NULL; } static void _channel_free(_PyChannelState *chan) { + PyThread_acquire_lock(chan->mutex, WAIT_LOCK); _channel_clear(chan); + _channelend_free_all(chan->send); + _channelend_free_all(chan->recv); + _channelend_free_all(chan->sendclosed); + _channelend_free_all(chan->recvclosed); + PyThread_release_lock(chan->mutex); + PyThread_free_lock(chan->mutex); PyMem_Free(chan); } @@ -376,18 +536,6 @@ _channels_list_all(struct _channels *channels, int64_t *count) return ids; } -/* -int -channel_list_interpreters(int id) -{ -} - -int -channel_close(void) -{ -} -*/ - /* "high"-level channel-related functions */ static int64_t @@ -413,9 +561,13 @@ _channel_destroy(struct _channels *channels, int64_t id) return 0; } -int +static int _channel_send(struct _channels *channels, int64_t id, PyObject *obj) { + PyInterpreterState *interp = _get_current(); + if (interp == NULL) + return -1; + _PyCrossInterpreterData *data = PyMem_Malloc(sizeof(_PyCrossInterpreterData)); if (_PyObject_GetCrossInterpreterData(obj, data) != 0) return -1; @@ -425,7 +577,7 @@ _channel_send(struct _channels *channels, int64_t id, PyObject *obj) PyMem_Free(data); return -1; } - if (_channel_add(chan, data) != 0) { + if (_channel_add(chan, interp->id, data) != 0) { PyMem_Free(data); return -1; } @@ -433,13 +585,17 @@ _channel_send(struct _channels *channels, int64_t id, PyObject *obj) return 0; } -PyObject * +static PyObject * _channel_recv(struct _channels *channels, int64_t id) { + PyInterpreterState *interp = _get_current(); + if (interp == NULL) + return NULL; + _PyChannelState *chan = _channels_lookup(channels, id); if (chan == NULL) return NULL; - _PyCrossInterpreterData *data = _channel_next(chan); + _PyCrossInterpreterData *data = _channel_next(chan, interp->id); if (data == NULL) { PyErr_SetString(PyExc_RuntimeError, "empty channel"); return NULL; @@ -453,6 +609,39 @@ _channel_recv(struct _channels *channels, int64_t id) return obj; } +static int +_channel_close(struct _channels *channels, int64_t id, int send, int recv) +{ + PyInterpreterState *interp = _get_current(); + if (interp == NULL) + return -1; + _PyChannelState *chan = _channels_lookup(channels, id); + if (chan == NULL) + return -1; + + if (send > 0) { + if (_channel_close_end(chan, interp->id, 1) != 0) + return -1; + } + if (recv > 0) { + if (_channel_close_end(chan, interp->id, 0) != 0) + return -1; + } + if (_channel_is_closed(chan)) { + if (_channel_destroy(channels, chan->id) != 0) + return -1; + } + + return 0; +} + +/* +int +channel_list_interpreters(int id) +{ +} +*/ + /* interpreter-specific functions *******************************************/ static PyInterpreterState * @@ -940,25 +1129,29 @@ PyDoc_STRVAR(channel_recv_doc, Return a new object from the data at the from of the channel's queue."); static PyObject * -channel_close(PyObject *self, PyObject *args) +channel_close(PyObject *self, PyObject *args, PyObject *kwds) { - PyObject *id; - if (!PyArg_UnpackTuple(args, "channel_close", 1, 1, &id)) + static char *kwlist[] = {"id", "send", "recv"}; + long long cid; + int send = -1; + int recv = -1; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "n|$pp:channel_close", kwlist, + &cid, &send, &recv)) return NULL; - if (!PyLong_Check(id)) { - PyErr_SetString(PyExc_TypeError, "ID must be an int"); - return NULL; - } - // XXX finish - PyErr_SetString(PyExc_NotImplementedError, "not implemented yet"); + if (send < 0 && recv < 0) { + send = 1; + recv = 1; + } + if (_channel_close(&_globals.channels, cid, send, recv) != 0) + return NULL; Py_RETURN_NONE; } PyDoc_STRVAR(channel_close_doc, "channel_close(ID, *, send=None, recv=None)\n\ \n\ -Close the channel for the current interpreter. 'send' and 'receive'\n\ +Close the channel for the current interpreter. 'send' and 'recv'\n\ (bool) may be used to indicate the ends to close. By default both\n\ ends are closed. Closing an already closed end is a noop."); @@ -1013,8 +1206,8 @@ static PyMethodDef module_functions[] = { METH_VARARGS, channel_send_doc}, {"channel_recv", (PyCFunction)channel_recv, METH_VARARGS, channel_recv_doc}, - {"channel_close", (PyCFunction)channel_close, - METH_VARARGS, channel_close_doc}, + {"channel_close", (PyCFunctionWithKeywords)channel_close, + METH_VARARGS|METH_KEYWORDS, channel_close_doc}, {"channel_list_interpreters", (PyCFunction)channel_list_interpreters, METH_VARARGS, channel_list_interpreters_doc}, From 7c8ede9484fd3009d31a1a39c1650b7c27d76829 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 15 Jan 2018 17:51:10 +0000 Subject: [PATCH 50/78] Add channel-related exceptions. --- Lib/test/test__interpreters.py | 20 ++++----- Modules/_interpretersmodule.c | 81 ++++++++++++++++++++++++++++------ 2 files changed, 77 insertions(+), 24 deletions(-) diff --git a/Lib/test/test__interpreters.py b/Lib/test/test__interpreters.py index 970dff17ea5a4b..fab6d62d45c50f 100644 --- a/Lib/test/test__interpreters.py +++ b/Lib/test/test__interpreters.py @@ -752,9 +752,9 @@ def test_close_single_user(self): interpreters.channel_recv(cid) interpreters.channel_close(cid, send=True, recv=True) - with self.assertRaises(RuntimeError): + with self.assertRaises(interpreters.ChannelClosedError): interpreters.channel_send(cid, b'eggs') - with self.assertRaises(RuntimeError): + with self.assertRaises(interpreters.ChannelClosedError): interpreters.channel_recv(cid) def test_close_multiple_users(self): @@ -766,9 +766,9 @@ def test_close_no_kwargs(self): interpreters.channel_recv(cid) interpreters.channel_close(cid) - with self.assertRaises(RuntimeError): + with self.assertRaises(interpreters.ChannelClosedError): interpreters.channel_send(cid, b'eggs') - with self.assertRaises(RuntimeError): + with self.assertRaises(interpreters.ChannelClosedError): interpreters.channel_recv(cid) def test_close_multiple_times(self): @@ -777,7 +777,7 @@ def test_close_multiple_times(self): interpreters.channel_recv(cid) interpreters.channel_close(cid, send=True, recv=True) - with self.assertRaises(RuntimeError): + with self.assertRaises(interpreters.ChannelClosedError): interpreters.channel_close(cid, send=True, recv=True) def test_close_with_unused_items(self): @@ -786,9 +786,7 @@ def test_close_with_unused_items(self): interpreters.channel_send(cid, b'ham') interpreters.channel_close(cid, send=True, recv=True) - with self.assertRaises(RuntimeError): - interpreters.channel_send(cid, b'eggs') - with self.assertRaises(RuntimeError): + with self.assertRaises(interpreters.ChannelClosedError): interpreters.channel_recv(cid) def test_close_never_used(self): @@ -814,9 +812,9 @@ def test_close_used_multiple_times_by_single_user(self): interpreters.channel_recv(cid) interpreters.channel_close(cid, send=True, recv=True) - with self.assertRaises(RuntimeError): + with self.assertRaises(interpreters.ChannelClosedError): interpreters.channel_send(cid, b'eggs') - with self.assertRaises(RuntimeError): + with self.assertRaises(interpreters.ChannelClosedError): interpreters.channel_recv(cid) def test_send_recv_main(self): @@ -853,7 +851,7 @@ def test_send_recv_different_interpreters(self): def test_recv_empty(self): cid = interpreters.channel_create() - with self.assertRaises(RuntimeError): + with self.assertRaises(interpreters.ChannelEmptyError): interpreters.channel_recv(cid) @unittest.skip('not implemented yet') diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index b434adda62f172..3e01e404b51b9a 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -126,16 +126,72 @@ _get_shared_exception(void) return err; } -static PyObject * _interp_failed_error; +static PyObject * RunFailedError; + +static int +interp_exceptions_init(PyObject *ns) +{ + // XXX Move the exceptions into per-module memory? + + // An uncaught exception came out of interp_run_string(). + RunFailedError = PyErr_NewException("_interpreters.RunFailedError", + PyExc_RuntimeError, NULL); + if (RunFailedError == NULL) + return -1; + PyDict_SetItemString(ns, "RunFailedError", RunFailedError); + + return 0; +} static void _apply_shared_exception(struct _shared_exception *exc) { - PyErr_SetString(_interp_failed_error, exc->msg); + PyErr_SetString(RunFailedError, exc->msg); } /* channel-specific code */ +static PyObject *ChannelError; +static PyObject *ChannelNotFoundError; +static PyObject *ChannelClosedError; +static PyObject *ChannelEmptyError; + +static int +channel_exceptions_init(PyObject *ns) +{ + // XXX Move the exceptions into per-module memory? + + // A channel-related operation failed. + ChannelError = PyErr_NewException("_interpreters.ChannelError", + PyExc_RuntimeError, NULL); + if (ChannelError == NULL) + return -1; + PyDict_SetItemString(ns, "ChannelError", ChannelError); + + // An operation tried to use a channel that doesn't exist. + ChannelNotFoundError = PyErr_NewException("_interpreters.ChannelNotFoundError", + ChannelError, NULL); + if (ChannelNotFoundError == NULL) + return -1; + PyDict_SetItemString(ns, "ChannelNotFoundError", ChannelNotFoundError); + + // An operation tried to use a closed channel. + ChannelClosedError = PyErr_NewException("_interpreters.ChannelClosedError", + ChannelError, NULL); + if (ChannelClosedError == NULL) + return -1; + PyDict_SetItemString(ns, "ChannelClosedError", ChannelClosedError); + + // An operation tried to pop from an empty channel. + ChannelEmptyError = PyErr_NewException("_interpreters.ChannelEmptyError", + ChannelError, NULL); + if (ChannelEmptyError == NULL) + return -1; + PyDict_SetItemString(ns, "ChannelEmptyError", ChannelEmptyError); + + return 0; +} + struct _channelend; struct _channelend { @@ -210,7 +266,7 @@ _channel_new(void) chan->id = -1; chan->mutex = PyThread_allocate_lock(); if (chan->mutex == NULL) { - PyErr_SetString(PyExc_RuntimeError, "can't initialize mutex for new channel"); + PyErr_SetString(ChannelError, "can't initialize mutex for new channel"); return NULL; } chan->count = 0; @@ -236,7 +292,7 @@ _channel_associate_end(_PyChannelState *chan, int64_t interp, int send) } end = _channelend_find(send ? chan->sendclosed : chan->recvclosed, interp, &prev); if (end != NULL) { - PyErr_SetString(PyExc_RuntimeError, "channel already closed"); + PyErr_SetString(ChannelClosedError, "channel already closed"); return 1; } @@ -409,7 +465,7 @@ _channels_init(struct _channels *channels) channels->mutex = PyThread_allocate_lock(); if (channels->mutex == NULL) { PyMem_Free(channels); - PyErr_SetString(PyExc_RuntimeError, "can't initialize mutex for new channel"); + PyErr_SetString(ChannelError, "can't initialize mutex for channel management"); return -1; } } @@ -423,7 +479,7 @@ _channels_next_id(struct _channels *channels) int64_t id = channels->next_id; if (id < 0) { /* overflow */ - PyErr_SetString(PyExc_RuntimeError, + PyErr_SetString(ChannelError, "failed to get a channel ID"); return -1; } @@ -440,7 +496,7 @@ _channels_lookup_locked(struct _channels *channels, int64_t id) break; } if (chan == NULL) { - PyErr_Format(PyExc_RuntimeError, "channel %d not found", id); + PyErr_Format(ChannelNotFoundError, "channel %d not found", id); } return chan; } @@ -494,7 +550,7 @@ _channels_remove_locked(struct _channels *channels, int64_t id) prev = chan; } if (chan == NULL) { - PyErr_Format(PyExc_RuntimeError, "channel %d not found", id); + PyErr_Format(ChannelNotFoundError, "channel %d not found", id); return NULL; } @@ -597,7 +653,7 @@ _channel_recv(struct _channels *channels, int64_t id) return NULL; _PyCrossInterpreterData *data = _channel_next(chan, interp->id); if (data == NULL) { - PyErr_SetString(PyExc_RuntimeError, "empty channel"); + PyErr_SetString(ChannelEmptyError, "empty channel"); return NULL; } @@ -1245,11 +1301,10 @@ PyInit__interpreters(void) return NULL; PyObject *ns = PyModule_GetDict(module); // borrowed - _interp_failed_error = PyErr_NewException("_interpreters.RunFailedError", - PyExc_RuntimeError, NULL); - if (_interp_failed_error == NULL) + if (interp_exceptions_init(ns) != 0) + return NULL; + if (channel_exceptions_init(ns) != 0) return NULL; - PyDict_SetItemString(ns, "RunFailedError", _interp_failed_error); return module; } From 47d75def8200f3da83fdd56e7f5bf7957d94fba6 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 16 Jan 2018 02:51:42 +0000 Subject: [PATCH 51/78] Add _interpreters.ChannelID. --- Lib/test/test__interpreters.py | 93 +++++++++- Modules/_interpretersmodule.c | 319 +++++++++++++++++++++++++++++++-- 2 files changed, 393 insertions(+), 19 deletions(-) diff --git a/Lib/test/test__interpreters.py b/Lib/test/test__interpreters.py index fab6d62d45c50f..79daa5463ad77b 100644 --- a/Lib/test/test__interpreters.py +++ b/Lib/test/test__interpreters.py @@ -714,6 +714,87 @@ def f(): self.assertEqual(retcode, 0) +class ChannelIDTests(unittest.TestCase): + + def test_default_kwargs(self): + cid = interpreters.ChannelID(10) + + self.assertEqual(int(cid), 10) + self.assertEqual(cid.end, 'both') + + def test_with_kwargs(self): + cid = interpreters.ChannelID(10, send=True) + self.assertEqual(cid.end, 'send') + + cid = interpreters.ChannelID(10, send=True, recv=False) + self.assertEqual(cid.end, 'send') + + cid = interpreters.ChannelID(10, recv=True) + self.assertEqual(cid.end, 'recv') + + cid = interpreters.ChannelID(10, recv=True, send=False) + self.assertEqual(cid.end, 'recv') + + cid = interpreters.ChannelID(10, send=True, recv=True) + self.assertEqual(cid.end, 'both') + + def test_coerce_id(self): + cid = interpreters.ChannelID('10') + self.assertEqual(int(cid), 10) + + cid = interpreters.ChannelID(10.0) + self.assertEqual(int(cid), 10) + + class Int(str): + def __init__(self, value): + self._value = value + def __int__(self): + return self._value + + cid = interpreters.ChannelID(Int(10)) + self.assertEqual(int(cid), 10) + + def test_bad_id(self): + ids = [-1, 2**64, "spam"] + for cid in ids: + with self.subTest(cid): + with self.assertRaises(ValueError): + interpreters.ChannelID(cid) + + with self.assertRaises(TypeError): + interpreters.ChannelID(object()) + + def test_bad_kwargs(self): + with self.assertRaises(ValueError): + interpreters.ChannelID(10, send=False, recv=False) + + def test_repr(self): + cid = interpreters.ChannelID(10) + self.assertEqual(repr(cid), 'ChannelID(10)') + + cid = interpreters.ChannelID(10, send=True) + self.assertEqual(repr(cid), 'ChannelID(10, send=True)') + + cid = interpreters.ChannelID(10, recv=True) + self.assertEqual(repr(cid), 'ChannelID(10, recv=True)') + + cid = interpreters.ChannelID(10, send=True, recv=True) + self.assertEqual(repr(cid), 'ChannelID(10)') + + def test_equality(self): + cid1 = interpreters.ChannelID(10) + cid2 = interpreters.ChannelID(10) + cid3 = interpreters.ChannelID(20) + + self.assertTrue(cid1 == cid1) + self.assertTrue(cid1 == cid2) + self.assertFalse(cid1 == cid3) + + self.assertFalse(cid1 != cid1) + self.assertFalse(cid1 != cid2) + self.assertTrue(cid1 != cid3) + + class ChannelTests(TestBase): def test_sequential_ids(self): @@ -723,8 +804,8 @@ def test_sequential_ids(self): id3 = interpreters.channel_create() after = interpreters.channel_list_all() - self.assertEqual(id2, id1 + 1) - self.assertEqual(id3, id2 + 1) + self.assertEqual(id2, int(id1) + 1) + self.assertEqual(id3, int(id2) + 1) self.assertEqual(set(after) - set(before), {id1, id2, id3}) def test_ids_global(self): @@ -732,7 +813,7 @@ def test_ids_global(self): out = _run_output(id1, dedent(""" import _interpreters cid = _interpreters.channel_create() - print(cid) + print(int(cid)) """)) cid1 = int(out.strip()) @@ -740,11 +821,11 @@ def test_ids_global(self): out = _run_output(id2, dedent(""" import _interpreters cid = _interpreters.channel_create() - print(cid) + print(int(cid)) """)) cid2 = int(out.strip()) - self.assertEqual(cid2, cid1 + 1) + self.assertEqual(cid2, int(cid1) + 1) def test_close_single_user(self): cid = interpreters.channel_create() @@ -843,7 +924,7 @@ def test_send_recv_different_interpreters(self): id1 = interpreters.create() out = _run_output(id1, dedent(f""" import _interpreters - _interpreters.channel_send({cid}, b'spam') + _interpreters.channel_send({int(cid)}, b'spam') """)) obj = interpreters.channel_recv(cid) diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index 3e01e404b51b9a..48f7319e8fbedf 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -18,6 +18,33 @@ _get_current(void) return tstate->interp; } +static int64_t +_coerce_id(PyObject *id) +{ + id = PyNumber_Long(id); + if (id == NULL) { + if (PyErr_ExceptionMatches(PyExc_TypeError)) + PyErr_SetString(PyExc_TypeError, "'id' must be a non-negative int"); + else + PyErr_SetString(PyExc_ValueError, "'id' must be a non-negative int"); + return -1; + } + long long cid = PyLong_AsLongLong(id); + if (cid == -1 && PyErr_Occurred() != NULL) { + PyErr_SetString(PyExc_ValueError, "'id' must be a non-negative int"); + return -1; + } + if (cid < 0) { + PyErr_SetString(PyExc_ValueError, "'id' must be a non-negative int"); + return -1; + } + if (cid > INT64_MAX) { + PyErr_SetString(PyExc_ValueError, "'id' too large (must be 64-bit int)"); + return -1; + } + return cid; +} + /* data-sharing-specific code ***********************************************/ struct _shareditem { @@ -698,6 +725,258 @@ channel_list_interpreters(int id) } */ +/* ChannelID class */ + +#define CHANNEL_SEND 1 +#define CHANNEL_RECV -1 + +static PyTypeObject ChannelIDtype; + +typedef struct channelid { + PyObject_HEAD + int64_t id; + int end; +} channelid; + +static channelid * +newchannelid(PyTypeObject *cls, int64_t id, int end) +{ + channelid *self = PyObject_New(channelid, cls); + if (self == NULL) + return NULL; + self->id = id; + self->end = end; + return self; +} + +static PyObject * +channelid_new(PyTypeObject *cls, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"id", "send", "recv"}; + PyObject *id; + int send = -1; + int recv = -1; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|$pp:ChannelID.__init__", kwlist, + &id, &send, &recv)) + return NULL; + + // Coerce and check the ID. + int64_t cid; + if (PyObject_TypeCheck(id, &ChannelIDtype)) { + cid = ((channelid *)id)->id; + } + else { + cid = _coerce_id(id); + if (cid < 0) + return NULL; + } + + // Handle "send" and "recv". + if (send == 0 && recv == 0) { + PyErr_SetString(PyExc_ValueError, "'send' and 'recv' cannot both be False"); + return NULL; + } + int end = 0; + if (send == 1) { + if (recv == 0 || recv == -1) + end = CHANNEL_SEND; + } + else if (recv == 1) { + end = CHANNEL_RECV; + } + + return (PyObject *)newchannelid(cls, cid, end); +} + +static void +channelid_dealloc(PyObject *v) +{ + Py_TYPE(v)->tp_free(v); +} + +static PyObject * +channelid_repr(PyObject *self) +{ + PyTypeObject *type = Py_TYPE(self); + // XXX Use the qualname? + const char *name = _PyType_Name(type); + + channelid *cid = (channelid *)self; + const char *fmt; + if (cid->end == CHANNEL_SEND) + fmt = "%s(%d, send=True)"; + else if (cid->end == CHANNEL_RECV) + fmt = "%s(%d, recv=True)"; + else + fmt = "%s(%d)"; + return PyUnicode_FromFormat(fmt, name, cid->id); +} + +PyObject * +channelid_int(PyObject *self) +{ + channelid *cid = (channelid *)self; + return PyLong_FromLongLong(cid->id); +} + +static PyNumberMethods channelid_as_number = { + 0, /* nb_add */ + 0, /* nb_subtract */ + 0, /* nb_multiply */ + 0, /* nb_remainder */ + 0, /* nb_divmod */ + 0, /* nb_power */ + 0, /* nb_negative */ + 0, /* nb_positive */ + 0, /* nb_absolute */ + 0, /* nb_bool */ + 0, /* nb_invert */ + 0, /* nb_lshift */ + 0, /* nb_rshift */ + 0, /* nb_and */ + 0, /* nb_xor */ + 0, /* nb_or */ + (unaryfunc)channelid_int, /* nb_int */ + 0, /* nb_reserved */ + 0, /* nb_float */ + + 0, /* nb_inplace_add */ + 0, /* nb_inplace_subtract */ + 0, /* nb_inplace_multiply */ + 0, /* nb_inplace_remainder */ + 0, /* nb_inplace_power */ + 0, /* nb_inplace_lshift */ + 0, /* nb_inplace_rshift */ + 0, /* nb_inplace_and */ + 0, /* nb_inplace_xor */ + 0, /* nb_inplace_or */ + + 0, /* nb_floor_divide */ + 0, /* nb_true_divide */ + 0, /* nb_inplace_floor_divide */ + 0, /* nb_inplace_true_divide */ + + (unaryfunc)channelid_int, /* nb_index */ +}; + +static Py_hash_t +channelid_hash(PyObject *self) +{ + channelid *cid = (channelid *)self; + PyObject *id = PyLong_FromLongLong(cid->id); + if (id == NULL) + return -1; + return PyObject_Hash(id); +} + +static PyObject * +channelid_richcompare(PyObject *self, PyObject *other, int op) +{ + switch(op) { + case Py_LT: + case Py_LE: + case Py_GT: + case Py_GE: + Py_RETURN_NOTIMPLEMENTED; + } + + if (!PyObject_TypeCheck(self, &ChannelIDtype)) + Py_RETURN_NOTIMPLEMENTED; + + int64_t cid = ((channelid *)self)->id; + int64_t othercid; + if (PyObject_TypeCheck(other, &ChannelIDtype)) { + othercid = ((channelid *)other)->id; + if (((channelid *)other)->end != ((channelid *)self)->end) + Py_RETURN_FALSE; + } + else { + other = PyNumber_Long(other); + if (other == NULL) { + PyErr_Clear(); + Py_RETURN_NOTIMPLEMENTED; + } + othercid = PyLong_AsLongLong(other); + if (othercid == -1 && PyErr_Occurred() != NULL) + return NULL; + if (othercid < 0 || othercid > INT64_MAX) + Py_RETURN_FALSE; + } + Py_RETURN_RICHCOMPARE(othercid != cid, 0, op); +} + +static PyObject * +channelid_end(PyObject *self, void *end) +{ + channelid *cid = (channelid *)self; + if (end != NULL) + return (PyObject *)newchannelid(Py_TYPE(self), cid->id, *(int *)end); + + if (cid->end == CHANNEL_SEND) + return PyUnicode_InternFromString("send"); + if (cid->end == CHANNEL_RECV) + return PyUnicode_InternFromString("recv"); + return PyUnicode_InternFromString("both"); +} + +static int _channelid_end_send = CHANNEL_SEND; +static int _channelid_end_recv = CHANNEL_RECV; + +static PyGetSetDef channelid_getsets[] = { + {"end", (getter)channelid_end, NULL, + PyDoc_STR("'send', 'recv', or 'both'")}, + {"send", (getter)channelid_end, NULL, + PyDoc_STR("the 'send' end of the channel"), &_channelid_end_send}, + {"recv", (getter)channelid_end, NULL, + PyDoc_STR("the 'recv' end of the channel"), &_channelid_end_recv}, + {NULL} +}; + +PyDoc_STRVAR(channelid_doc, +"A channel ID identifies a channel and may be used as an int."); + +static PyTypeObject ChannelIDtype = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "_interpreters.ChannelID", /* tp_name */ + sizeof(channelid), /* tp_size */ + 0, /* tp_itemsize */ + (destructor)channelid_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_as_async */ + (reprfunc)channelid_repr, /* tp_repr */ + &channelid_as_number, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + channelid_hash, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | + Py_TPFLAGS_LONG_SUBCLASS, /* tp_flags */ + channelid_doc, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + channelid_richcompare, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + 0, /* tp_methods */ + 0, /* tp_members */ + channelid_getsets, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + (newfunc)channelid_new, /* tp_new */ +}; + /* interpreter-specific functions *******************************************/ static PyInterpreterState * @@ -1099,7 +1378,7 @@ channel_list_all(PyObject *self) return NULL; } for (int64_t i=0; i < count; cids++, i++) { - PyObject *id = PyLong_FromLongLong(*cids); + PyObject *id = (PyObject *)newchannelid(&ChannelIDtype, *cids, 0); if (id == NULL) { Py_DECREF(ids); ids = NULL; @@ -1122,7 +1401,7 @@ channel_create(PyObject *self) int64_t cid = _channel_create(&_globals.channels); if (cid < 0) return NULL; - PyObject *id = PyLong_FromLongLong(cid); + PyObject *id = (PyObject *)newchannelid(&ChannelIDtype, cid, 0); if (id == NULL) { _channel_destroy(&_globals.channels, cid); return NULL; @@ -1147,10 +1426,9 @@ channel_send(PyObject *self, PyObject *args) return NULL; } - long long cid = PyLong_AsLongLong(id); - if (cid == -1 && PyErr_Occurred() != NULL) + int64_t cid = _coerce_id(id); + if (cid < 0) return NULL; - assert(cid <= INT64_MAX); if (_channel_send(&_globals.channels, cid, obj) != 0) return NULL; Py_RETURN_NONE; @@ -1172,8 +1450,8 @@ channel_recv(PyObject *self, PyObject *args) return NULL; } - long long cid = PyLong_AsLongLong(id); - if (cid == -1 && PyErr_Occurred() != NULL) + int64_t cid = _coerce_id(id); + if (cid < 0) return NULL; assert(cid <= INT64_MAX); return _channel_recv(&_globals.channels, cid); @@ -1188,17 +1466,21 @@ static PyObject * channel_close(PyObject *self, PyObject *args, PyObject *kwds) { static char *kwlist[] = {"id", "send", "recv"}; - long long cid; + PyObject *id; int send = -1; int recv = -1; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "n|$pp:channel_close", kwlist, - &cid, &send, &recv)) + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|$pp:channel_close", kwlist, + &id, &send, &recv)) return NULL; + int64_t cid = _coerce_id(id); + if (cid < 0) + return NULL; if (send < 0 && recv < 0) { send = 1; recv = 1; } + if (_channel_close(&_globals.channels, cid, send, recv) != 0) return NULL; Py_RETURN_NONE; @@ -1262,8 +1544,8 @@ static PyMethodDef module_functions[] = { METH_VARARGS, channel_send_doc}, {"channel_recv", (PyCFunction)channel_recv, METH_VARARGS, channel_recv_doc}, - {"channel_close", (PyCFunctionWithKeywords)channel_close, - METH_VARARGS|METH_KEYWORDS, channel_close_doc}, + {"channel_close", (PyCFunction)channel_close, + METH_VARARGS | METH_KEYWORDS, channel_close_doc}, {"channel_list_interpreters", (PyCFunction)channel_list_interpreters, METH_VARARGS, channel_list_interpreters_doc}, @@ -1296,15 +1578,26 @@ PyInit__interpreters(void) if (_init_globals() != 0) return NULL; + /* Initialize types */ + ChannelIDtype.tp_base = &PyLong_Type; + if (PyType_Ready(&ChannelIDtype) != 0) + return NULL; + + /* Create the module */ PyObject *module = PyModule_Create(&interpretersmodule); if (module == NULL) return NULL; - PyObject *ns = PyModule_GetDict(module); // borrowed + /* Add exception types */ + PyObject *ns = PyModule_GetDict(module); // borrowed if (interp_exceptions_init(ns) != 0) return NULL; if (channel_exceptions_init(ns) != 0) return NULL; + /* Add other types */ + Py_INCREF(&ChannelIDtype); + PyDict_SetItemString(ns, "ChannelID", (PyObject *)&ChannelIDtype); + return module; } From 153bbec758c68d73652418826bad35044119a2b6 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 16 Jan 2018 04:59:11 +0000 Subject: [PATCH 52/78] Solidify the "close" story. --- Include/internal/pystate.h | 5 +- Lib/test/test__interpreters.py | 237 ++++++++-- Modules/_interpretersmodule.c | 774 +++++++++++++++++++++++---------- 3 files changed, 736 insertions(+), 280 deletions(-) diff --git a/Include/internal/pystate.h b/Include/internal/pystate.h index edf41ca752ba73..2b60b25c19b1ee 100644 --- a/Include/internal/pystate.h +++ b/Include/internal/pystate.h @@ -97,8 +97,9 @@ typedef struct _xid { // with deleted interpreters. int64_t interp; // new_object is a function that returns a new object in the current - // interpreter given the data. The resulting object will be - // equivalent to the original object. This field is required. + // interpreter given the data. The resulting object (a new + // reference) will be equivalent to the original object. This field + // is required. PyObject *(*new_object)(struct _xid *); // free is called when the data is released. If it is NULL then // nothing will be done to free the data. For some types this is diff --git a/Lib/test/test__interpreters.py b/Lib/test/test__interpreters.py index 79daa5463ad77b..8658723fe40b60 100644 --- a/Lib/test/test__interpreters.py +++ b/Lib/test/test__interpreters.py @@ -110,6 +110,12 @@ def tearDown(self): except RuntimeError: pass # already destroyed + for cid in interpreters.channel_list_all(): + try: + interpreters.channel_destroy(cid) + except interpreters.ChannelNotFoundError: + pass # already destroyed + class ListAllTests(TestBase): @@ -714,35 +720,35 @@ def f(): self.assertEqual(retcode, 0) -class ChannelIDTests(unittest.TestCase): +class ChannelIDTests(TestBase): def test_default_kwargs(self): - cid = interpreters.ChannelID(10) + cid = interpreters._channel_id(10, force=True) self.assertEqual(int(cid), 10) self.assertEqual(cid.end, 'both') def test_with_kwargs(self): - cid = interpreters.ChannelID(10, send=True) + cid = interpreters._channel_id(10, send=True, force=True) self.assertEqual(cid.end, 'send') - cid = interpreters.ChannelID(10, send=True, recv=False) + cid = interpreters._channel_id(10, send=True, recv=False, force=True) self.assertEqual(cid.end, 'send') - cid = interpreters.ChannelID(10, recv=True) + cid = interpreters._channel_id(10, recv=True, force=True) self.assertEqual(cid.end, 'recv') - cid = interpreters.ChannelID(10, recv=True, send=False) + cid = interpreters._channel_id(10, recv=True, send=False, force=True) self.assertEqual(cid.end, 'recv') - cid = interpreters.ChannelID(10, send=True, recv=True) + cid = interpreters._channel_id(10, send=True, recv=True, force=True) self.assertEqual(cid.end, 'both') def test_coerce_id(self): - cid = interpreters.ChannelID('10') + cid = interpreters._channel_id('10', force=True) self.assertEqual(int(cid), 10) - cid = interpreters.ChannelID(10.0) + cid = interpreters._channel_id(10.0, force=True) self.assertEqual(int(cid), 10) class Int(str): @@ -751,7 +757,7 @@ def __init__(self, value): def __int__(self): return self._value - cid = interpreters.ChannelID(Int(10)) + cid = interpreters._channel_id(Int(10), force=True) self.assertEqual(int(cid), 10) def test_bad_id(self): @@ -759,35 +765,41 @@ def test_bad_id(self): for cid in ids: with self.subTest(cid): with self.assertRaises(ValueError): - interpreters.ChannelID(cid) + interpreters._channel_id(cid) with self.assertRaises(TypeError): - interpreters.ChannelID(object()) + interpreters._channel_id(object()) def test_bad_kwargs(self): with self.assertRaises(ValueError): - interpreters.ChannelID(10, send=False, recv=False) + interpreters._channel_id(10, send=False, recv=False) + + def test_does_not_exist(self): + cid = interpreters.channel_create() + with self.assertRaises(interpreters.ChannelNotFoundError): + interpreters._channel_id(int(cid) + 1) # unforced def test_repr(self): - cid = interpreters.ChannelID(10) + cid = interpreters._channel_id(10, force=True) self.assertEqual(repr(cid), 'ChannelID(10)') - cid = interpreters.ChannelID(10, send=True) + cid = interpreters._channel_id(10, send=True, force=True) self.assertEqual(repr(cid), 'ChannelID(10, send=True)') - cid = interpreters.ChannelID(10, recv=True) + cid = interpreters._channel_id(10, recv=True, force=True) self.assertEqual(repr(cid), 'ChannelID(10, recv=True)') - cid = interpreters.ChannelID(10, send=True, recv=True) + cid = interpreters._channel_id(10, send=True, recv=True, force=True) self.assertEqual(repr(cid), 'ChannelID(10)') def test_equality(self): - cid1 = interpreters.ChannelID(10) - cid2 = interpreters.ChannelID(10) - cid3 = interpreters.ChannelID(20) + cid1 = interpreters.channel_create() + cid2 = interpreters._channel_id(int(cid1)) + cid3 = interpreters.channel_create() self.assertTrue(cid1 == cid1) self.assertTrue(cid1 == cid2) + self.assertTrue(cid1 == int(cid1)) self.assertFalse(cid1 == cid3) self.assertFalse(cid1 != cid1) @@ -827,21 +839,131 @@ def test_ids_global(self): self.assertEqual(cid2, int(cid1) + 1) - def test_close_single_user(self): + #################### + + def test_drop_single_user(self): cid = interpreters.channel_create() interpreters.channel_send(cid, b'spam') interpreters.channel_recv(cid) - interpreters.channel_close(cid, send=True, recv=True) + interpreters.channel_drop_interpreter(cid, send=True, recv=True) with self.assertRaises(interpreters.ChannelClosedError): interpreters.channel_send(cid, b'eggs') with self.assertRaises(interpreters.ChannelClosedError): interpreters.channel_recv(cid) - def test_close_multiple_users(self): - raise NotImplementedError + def test_drop_multiple_users(self): + cid = interpreters.channel_create() + id1 = interpreters.create() + id2 = interpreters.create() + interpreters.run_string(id1, dedent(f""" + import _interpreters + _interpreters.channel_send({int(cid)}, b'spam') + """)) + out = _run_output(id2, dedent(f""" + import _interpreters + obj = _interpreters.channel_recv({int(cid)}) + _interpreters.channel_drop_interpreter({int(cid)}) + print(obj) + """)) + interpreters.run_string(id1, dedent(f""" + _interpreters.channel_drop_interpreter({int(cid)}) + """)) + + self.assertEqual(out.strip(), "b'spam'") + + def test_drop_no_kwargs(self): + cid = interpreters.channel_create() + interpreters.channel_send(cid, b'spam') + interpreters.channel_recv(cid) + interpreters.channel_drop_interpreter(cid) + + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_send(cid, b'eggs') + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_recv(cid) + + def test_drop_multiple_times(self): + cid = interpreters.channel_create() + interpreters.channel_send(cid, b'spam') + interpreters.channel_recv(cid) + interpreters.channel_drop_interpreter(cid, send=True, recv=True) + + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_drop_interpreter(cid, send=True, recv=True) + + def test_drop_with_unused_items(self): + cid = interpreters.channel_create() + interpreters.channel_send(cid, b'spam') + interpreters.channel_send(cid, b'ham') + interpreters.channel_drop_interpreter(cid, send=True, recv=True) + + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_recv(cid) + + def test_drop_never_used(self): + cid = interpreters.channel_create() + interpreters.channel_drop_interpreter(cid) + + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_send(cid, b'spam') + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_recv(cid) + + def test_drop_by_unassociated_interp(self): + cid = interpreters.channel_create() + interpreters.channel_send(cid, b'spam') + interp = interpreters.create() + interpreters.run_string(interp, dedent(f""" + import _interpreters + _interpreters.channel_drop_interpreter({int(cid)}) + """)) + obj = interpreters.channel_recv(cid) + interpreters.channel_drop_interpreter(cid) + + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_send(cid, b'eggs') + self.assertEqual(obj, b'spam') + + def test_drop_close_if_unassociated(self): + cid = interpreters.channel_create() + interp = interpreters.create() + interpreters.run_string(interp, dedent(f""" + import _interpreters + obj = _interpreters.channel_send({int(cid)}, b'spam') + _interpreters.channel_drop_interpreter({int(cid)}) + """)) + + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_recv(cid) + + def test_drop_partially(self): + # XXX Is partial close too wierd/confusing? + cid = interpreters.channel_create() + interpreters.channel_send(cid, None) + interpreters.channel_recv(cid) + interpreters.channel_send(cid, b'spam') + interpreters.channel_drop_interpreter(cid, send=True) + obj = interpreters.channel_recv(cid) + + self.assertEqual(obj, b'spam') + + def test_drop_used_multiple_times_by_single_user(self): + cid = interpreters.channel_create() + interpreters.channel_send(cid, b'spam') + interpreters.channel_send(cid, b'spam') + interpreters.channel_send(cid, b'spam') + interpreters.channel_recv(cid) + interpreters.channel_drop_interpreter(cid, send=True, recv=True) + + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_send(cid, b'eggs') + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_recv(cid) - def test_close_no_kwargs(self): + #################### + + def test_close_single_user(self): cid = interpreters.channel_create() interpreters.channel_send(cid, b'spam') interpreters.channel_recv(cid) @@ -852,20 +974,44 @@ def test_close_no_kwargs(self): with self.assertRaises(interpreters.ChannelClosedError): interpreters.channel_recv(cid) + def test_close_multiple_users(self): + cid = interpreters.channel_create() + id1 = interpreters.create() + id2 = interpreters.create() + interpreters.run_string(id1, dedent(f""" + import _interpreters + _interpreters.channel_send({int(cid)}, b'spam') + """)) + interpreters.run_string(id2, dedent(f""" + import _interpreters + _interpreters.channel_recv({int(cid)}) + """)) + interpreters.channel_close(cid) + with self.assertRaises(interpreters.RunFailedError) as cm: + interpreters.run_string(id1, dedent(f""" + _interpreters.channel_send({int(cid)}, b'spam') + """)) + self.assertIn('ChannelClosedError', str(cm.exception)) + with self.assertRaises(interpreters.RunFailedError) as cm: + interpreters.run_string(id2, dedent(f""" + _interpreters.channel_send({int(cid)}, b'spam') + """)) + self.assertIn('ChannelClosedError', str(cm.exception)) + def test_close_multiple_times(self): cid = interpreters.channel_create() interpreters.channel_send(cid, b'spam') interpreters.channel_recv(cid) - interpreters.channel_close(cid, send=True, recv=True) + interpreters.channel_close(cid) with self.assertRaises(interpreters.ChannelClosedError): - interpreters.channel_close(cid, send=True, recv=True) + interpreters.channel_close(cid) def test_close_with_unused_items(self): cid = interpreters.channel_create() interpreters.channel_send(cid, b'spam') interpreters.channel_send(cid, b'ham') - interpreters.channel_close(cid, send=True, recv=True) + interpreters.channel_close(cid) with self.assertRaises(interpreters.ChannelClosedError): interpreters.channel_recv(cid) @@ -873,17 +1019,24 @@ def test_close_with_unused_items(self): def test_close_never_used(self): cid = interpreters.channel_create() interpreters.channel_close(cid) - # XXX Do we want this to fail? - interpreters.channel_send(cid, b'spam') - obj = interpreters.channel_recv(cid) - self.assertEqual(obj, b'spam') + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_send(cid, b'spam') + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_recv(cid) def test_close_by_unassociated_interp(self): - raise NotImplementedError - - def test_close_partially(self): - raise NotImplementedError + cid = interpreters.channel_create() + interpreters.channel_send(cid, b'spam') + interp = interpreters.create() + interpreters.run_string(interp, dedent(f""" + import _interpreters + _interpreters.channel_close({int(cid)}) + """)) + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_recv(cid) + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_close(cid) def test_close_used_multiple_times_by_single_user(self): cid = interpreters.channel_create() @@ -891,13 +1044,15 @@ def test_close_used_multiple_times_by_single_user(self): interpreters.channel_send(cid, b'spam') interpreters.channel_send(cid, b'spam') interpreters.channel_recv(cid) - interpreters.channel_close(cid, send=True, recv=True) + interpreters.channel_close(cid) with self.assertRaises(interpreters.ChannelClosedError): interpreters.channel_send(cid, b'eggs') with self.assertRaises(interpreters.ChannelClosedError): interpreters.channel_recv(cid) + #################### + def test_send_recv_main(self): cid = interpreters.channel_create() orig = b'spam' @@ -930,6 +1085,14 @@ def test_send_recv_different_interpreters(self): self.assertEqual(obj, b'spam') + def test_send_not_found(self): + with self.assertRaises(interpreters.ChannelNotFoundError): + interpreters.channel_send(10, b'spam') + + def test_recv_not_found(self): + with self.assertRaises(interpreters.ChannelNotFoundError): + interpreters.channel_recv(10) + def test_recv_empty(self): cid = interpreters.channel_create() with self.assertRaises(interpreters.ChannelEmptyError): diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index 48f7319e8fbedf..49247bed9912b5 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -222,10 +222,26 @@ channel_exceptions_init(PyObject *ns) struct _channelend; struct _channelend { - int64_t interp; struct _channelend *next; + int64_t interp; + int open; }; +static struct _channelend * +_channelend_new(int64_t interp) +{ + struct _channelend *end = PyMem_Malloc(sizeof(struct _channelend)); + if (end == NULL) + return NULL; + + end->next = NULL; + end->interp = interp; + + end->open = 1; + + return end; +} + static void _channelend_free_all(struct _channelend *end) { while (end != NULL) { @@ -261,12 +277,10 @@ struct _channelitem { struct _channel; typedef struct _channel { - struct _channel *next; - - int64_t id; - PyThread_type_lock mutex; + int open; + int64_t count; struct _channelitem *first; struct _channelitem *last; @@ -275,12 +289,10 @@ typedef struct _channel { // for which the channel is closed. This should be a problem in // practice. Also, a channel isn't automatically closed when an // interpreter is destroyed. - int64_t sendcount; - int64_t recvcount; + int64_t numsendopen; + int64_t numrecvopen; struct _channelend *send; struct _channelend *recv; - struct _channelend *sendclosed; - struct _channelend *recvclosed; } _PyChannelState; static _PyChannelState * @@ -289,110 +301,157 @@ _channel_new(void) _PyChannelState *chan = PyMem_Malloc(sizeof(_PyChannelState)); if (chan == NULL) return NULL; - chan->next = NULL; - chan->id = -1; chan->mutex = PyThread_allocate_lock(); if (chan->mutex == NULL) { PyErr_SetString(ChannelError, "can't initialize mutex for new channel"); return NULL; } + + chan->open = 1; + chan->count = 0; chan->first = NULL; chan->last = NULL; - chan->sendcount = 0; - chan->recvcount = 0; + + chan->numsendopen = 0; + chan->numrecvopen = 0; chan->send = NULL; chan->recv = NULL; - chan->sendclosed = NULL; - chan->recvclosed = NULL; + return chan; } -static int -_channel_associate_end(_PyChannelState *chan, int64_t interp, int send) +static struct _channelend * +_channel_add_end(_PyChannelState *chan, struct _channelend *prev, int64_t interp, int send) { - struct _channelend *prev; - struct _channelend *end = _channelend_find(send ? chan->send : chan->recv, interp, &prev); - if (end != NULL) { - // already associated - return 0; - } - end = _channelend_find(send ? chan->sendclosed : chan->recvclosed, interp, &prev); - if (end != NULL) { - PyErr_SetString(ChannelClosedError, "channel already closed"); - return 1; - } - - end = PyMem_Malloc(sizeof(struct _channelend)); + struct _channelend *end = _channelend_new(interp); if (end == NULL) - return -1; - end->interp = interp; - end->next = NULL; + return NULL; + if (prev == NULL) { if (send) { chan->send = end; } else { chan->recv = end; } - } else { + } + else { prev->next = end; } + if (send) { + chan->numsendopen += 1; + } + else { + chan->numrecvopen += 1; + } + return end; +} + +static struct _channelend * +_channel_associate_end(_PyChannelState *chan, int64_t interp, int send) +{ + if (!chan->open) { + PyErr_SetString(ChannelClosedError, "channel closed"); + return NULL; + } + + struct _channelend *prev; + struct _channelend *end = _channelend_find(send ? chan->send : chan->recv, interp, &prev); + if (end != NULL) { + if (!end->open) { + PyErr_SetString(ChannelClosedError, "channel already closed"); + return NULL; + } + // already associated + return end; + } + return _channel_add_end(chan, prev, interp, send); +} + +static void +_channel_close_channelend(_PyChannelState *chan, struct _channelend *end, int send) +{ + end->open = 0; if (send) - chan->sendcount += 1; + chan->numsendopen -= 1; else - chan->recvcount += 1; - return 0; + chan->numrecvopen -= 1; } static int -_channel_close_end(_PyChannelState *chan, int64_t interp, int send) +_channel_close_interpreter(_PyChannelState *chan, int64_t interp, int which) { PyThread_acquire_lock(chan->mutex, WAIT_LOCK); - struct _channelend *prev; - struct _channelend *end = _channelend_find(send ? chan->send : chan->recv, interp, &prev); - if (end == NULL) { - // never associated or already closed - PyThread_release_lock(chan->mutex); - return 0; + int res = -1; + if (!chan->open) { + PyErr_SetString(ChannelClosedError, "channel already closed"); + goto done; } - // Drop it from the "open" list. - if (prev == NULL) { - if (send) - chan->send = NULL; - else - chan->recv = NULL; - } else { - prev->next = end->next; + struct _channelend *prev; + struct _channelend *end; + if (which >= 0) { // send/both + end = _channelend_find(chan->send, interp, &prev); + if (end == NULL) { + // never associated so add it + end = _channel_add_end(chan, prev, interp, 1); + if (end == NULL) + goto done; + } + _channel_close_channelend(chan, end, 1); + } + if (which <= 0) { // recv/both + end = _channelend_find(chan->recv, interp, &prev); + if (end == NULL) { + // never associated so add it + end = _channel_add_end(chan, prev, interp, 0); + if (end == NULL) + goto done; + } + _channel_close_channelend(chan, end, 0); } - if (send) - chan->sendcount -= 1; - else - chan->recvcount -= 1; - // Add it to the "closed" list. - if (send) { - end->next = chan->sendclosed; - chan->sendclosed = end; - } else { - end->next = chan->recvclosed; - chan->recvclosed = end; + if (chan->numsendopen == 0 && chan->numrecvopen == 0) { + if (chan->send != NULL || chan->recv != NULL) + chan->open = 0; } + res = 0; +done: PyThread_release_lock(chan->mutex); - return 0; + return res; } static int -_channel_is_closed(_PyChannelState *chan) +_channel_close_all(_PyChannelState *chan) { + int res = -1; PyThread_acquire_lock(chan->mutex, WAIT_LOCK); - int res = 0; - if (chan->sendcount == 0 && chan->recvcount == 0) { - if (chan->sendclosed != NULL || chan->recvclosed != NULL) - res = 1; + + if (!chan->open) { + PyErr_SetString(ChannelClosedError, "channel already closed"); + goto done; } + + chan->open = 0; + + // We *could* also just leave these in place, since we've marked + // the channel as closed already. + + // Ensure all the "send"-associated interpreters are closed. + struct _channelend *end; + for (end = chan->send; end != NULL; end = end->next) { + _channel_close_channelend(chan, end, 1); + } + + // Ensure all the "recv"-associated interpreters are closed. + for (end = chan->recv; end != NULL; end = end->next) { + _channel_close_channelend(chan, end, 0); + } + + res = 0; +done: PyThread_release_lock(chan->mutex); return res; } @@ -400,16 +459,16 @@ _channel_is_closed(_PyChannelState *chan) static int _channel_add(_PyChannelState *chan, int64_t interp, _PyCrossInterpreterData *data) { + int res = -1; + PyThread_acquire_lock(chan->mutex, WAIT_LOCK); - if (_channel_associate_end(chan, interp, 1) != 0) { - PyThread_release_lock(chan->mutex); - return -1; + if (_channel_associate_end(chan, interp, 1) == NULL) { + goto done; } struct _channelitem *item = PyMem_Malloc(sizeof(struct _channelitem)); if (item == NULL) { - PyThread_release_lock(chan->mutex); - return -1; + goto done; } item->data = data; item->next = NULL; @@ -418,33 +477,36 @@ _channel_add(_PyChannelState *chan, int64_t interp, _PyCrossInterpreterData *dat if (chan->first == NULL) chan->first = item; chan->last = item; - PyThread_release_lock(chan->mutex); - return 0; + res = 0; +done: + PyThread_release_lock(chan->mutex); + return res; } static _PyCrossInterpreterData * _channel_next(_PyChannelState *chan, int64_t interp) { + _PyCrossInterpreterData *data = NULL; PyThread_acquire_lock(chan->mutex, WAIT_LOCK); - if (_channel_associate_end(chan, interp, 0) != 0) { - PyThread_release_lock(chan->mutex); - return NULL; + if (_channel_associate_end(chan, interp, 0) == NULL) { + goto done; } struct _channelitem *item = chan->first; if (item == NULL) { - PyThread_release_lock(chan->mutex); - return NULL; + goto done; } chan->first = item->next; if (chan->last == item) chan->last = NULL; chan->count -= 1; - PyThread_release_lock(chan->mutex); - _PyCrossInterpreterData *data = item->data; + data = item->data; PyMem_Free(item); + +done: + PyThread_release_lock(chan->mutex); return data; } @@ -470,18 +532,54 @@ _channel_free(_PyChannelState *chan) _channel_clear(chan); _channelend_free_all(chan->send); _channelend_free_all(chan->recv); - _channelend_free_all(chan->sendclosed); - _channelend_free_all(chan->recvclosed); PyThread_release_lock(chan->mutex); PyThread_free_lock(chan->mutex); PyMem_Free(chan); } +struct _channelref; + +struct _channelref { + int64_t id; + _PyChannelState *chan; + struct _channelref *next; + Py_ssize_t objcount; +}; + +static struct _channelref * +_channelref_new(int64_t id, _PyChannelState *chan) +{ + struct _channelref *ref = PyMem_Malloc(sizeof(struct _channelref)); + if (ref == NULL) + return NULL; + ref->id = id; + ref->chan = chan; + ref->next = NULL; + ref->objcount = 0; + return ref; +} + +static struct _channelref * +_channelref_find(struct _channelref *first, int64_t id, struct _channelref **pprev) +{ + struct _channelref *prev = NULL; + struct _channelref *ref = first; + while (ref != NULL) { + if (ref->id == id) + break; + prev = ref; + ref = ref->next; + } + if (pprev != NULL) + *pprev = prev; + return ref; +} + struct _channels { PyThread_type_lock mutex; - _PyChannelState *head; - int64_t count; + struct _channelref *head; + int64_t numopen; int64_t next_id; }; @@ -497,11 +595,13 @@ _channels_init(struct _channels *channels) } } channels->head = NULL; + channels->numopen = 0; + channels->next_id = 0; return 0; } static int64_t -_channels_next_id(struct _channels *channels) +_channels_next_id(struct _channels *channels) // needs lock { int64_t id = channels->next_id; if (id < 0) { @@ -515,108 +615,199 @@ _channels_next_id(struct _channels *channels) } static _PyChannelState * -_channels_lookup_locked(struct _channels *channels, int64_t id) +_channels_lookup(struct _channels *channels, int64_t id, PyThread_type_lock *pmutex) { - _PyChannelState *chan = channels->head; - for (;chan != NULL; chan = chan->next) { - if (chan->id == id) - break; - } - if (chan == NULL) { + _PyChannelState *chan = NULL; + PyThread_acquire_lock(channels->mutex, WAIT_LOCK); + if (pmutex != NULL) + *pmutex = NULL; + + struct _channelref *ref = _channelref_find(channels->head, id, NULL); + if (ref == NULL) { PyErr_Format(ChannelNotFoundError, "channel %d not found", id); + goto done; + } + if (ref->chan == NULL || !ref->chan->open) { + PyErr_Format(ChannelClosedError, "channel %d closed", id); + goto done; + } + + if (pmutex != NULL) { + // The mutex will be closed by the caller. + *pmutex = channels->mutex; + } + + chan = ref->chan; +done: + if (pmutex == NULL || *pmutex == NULL) { + PyThread_release_lock(channels->mutex); } return chan; } -static _PyChannelState * -_channels_lookup(struct _channels *channels, int64_t id) +static int64_t +_channels_add(struct _channels *channels, _PyChannelState *chan) { + int64_t cid = -1; PyThread_acquire_lock(channels->mutex, WAIT_LOCK); - _PyChannelState *chan = _channels_lookup_locked(channels, id); - PyThread_release_lock(channels->mutex); - return chan; -} -static int -_channels_add_locked(struct _channels *channels, _PyChannelState *chan) -{ + // Create a new ref. int64_t id = _channels_next_id(channels); if (id < 0) { - return -1; + goto done; } - if (channels->head != NULL) { - chan->next = channels->head; + struct _channelref *ref = _channelref_new(id, chan); + if (ref == NULL) { + goto done; } - channels->head = chan; - channels->count += 1; - chan->id = id; - return 0; + + // Add it to the list. + // We assume that the channel is a new one (not already in the list). + ref->next = channels->head; + channels->head = ref; + channels->numopen += 1; + + cid = id; +done: + PyThread_release_lock(channels->mutex); + return cid; } static int -_channels_add(struct _channels *channels, _PyChannelState *chan) +_channels_close(struct _channels *channels, int64_t cid, _PyChannelState **pchan) { + int res = -1; PyThread_acquire_lock(channels->mutex, WAIT_LOCK); - int res = _channels_add_locked(channels, chan); + if (pchan != NULL) + *pchan = NULL; + + struct _channelref *ref = _channelref_find(channels->head, cid, NULL); + if (ref == NULL) { + PyErr_Format(ChannelNotFoundError, "channel %d not found", cid); + goto done; + } + + if (ref->chan == NULL) { + PyErr_Format(ChannelClosedError, "channel %d closed", cid); + goto done; + } + else { + if (_channel_close_all(ref->chan) != 0) { + goto done; + } + if (pchan != NULL) + *pchan = ref->chan; + ref->chan = NULL; + } + + res = 0; +done: PyThread_release_lock(channels->mutex); return res; } -static _PyChannelState * -_channels_remove_locked(struct _channels *channels, int64_t id) +static void +_channels_remove_ref(struct _channels *channels, struct _channelref *ref, struct _channelref *prev, _PyChannelState **pchan) { - if (channels->head == NULL) { - return NULL; + if (ref == channels->head) { + channels->head = ref->next; + } else { + prev->next = ref->next; } + channels->numopen -= 1; - _PyChannelState *prev = NULL; - _PyChannelState *chan = channels->head; - for (;chan != NULL; chan = chan->next) { - if (chan->id == id) - break; - prev = chan; - } - if (chan == NULL) { + if (pchan != NULL) + *pchan = ref->chan; + PyMem_Free(ref); +} + +static int +_channels_remove(struct _channels *channels, int64_t id, _PyChannelState **pchan) +{ + int res = -1; + PyThread_acquire_lock(channels->mutex, WAIT_LOCK); + + if (pchan != NULL) + *pchan = NULL; + + struct _channelref *prev = NULL; + struct _channelref *ref = _channelref_find(channels->head, id, &prev); + if (ref == NULL) { PyErr_Format(ChannelNotFoundError, "channel %d not found", id); - return NULL; + goto done; } - if (chan == channels->head) { - channels->head = chan->next; - } else { - prev->next = chan->next; + _channels_remove_ref(channels, ref, prev, pchan); + + res = 0; +done: + PyThread_release_lock(channels->mutex); + return res; +} + +static int +_channels_add_id_object(struct _channels *channels, int64_t id) +{ + int res = -1; + PyThread_acquire_lock(channels->mutex, WAIT_LOCK); + + struct _channelref *ref = _channelref_find(channels->head, id, NULL); + if (ref == NULL) { + PyErr_Format(ChannelNotFoundError, "channel %d not found", id); + goto done; } - chan->next = NULL; + ref->objcount += 1; - channels->count -= 1; - return chan; + res = 0; +done: + PyThread_release_lock(channels->mutex); + return res; } -static _PyChannelState * -_channels_remove(struct _channels *channels, int64_t id) +static void +_channels_drop_id_object(struct _channels *channels, int64_t id) { PyThread_acquire_lock(channels->mutex, WAIT_LOCK); - _PyChannelState *chan = _channels_remove_locked(channels, id); + + struct _channelref *prev = NULL; + struct _channelref *ref = _channelref_find(channels->head, id, &prev); + if (ref == NULL) { + // Already destroyed. + goto done; + } + ref->objcount -= 1; + + // Destroy if no longer used. + if (ref->objcount == 0) { + _PyChannelState *chan = NULL; + _channels_remove_ref(channels, ref, prev, &chan); + if (chan != NULL) + _channel_free(chan); + } + +done: PyThread_release_lock(channels->mutex); - return chan; } int64_t * _channels_list_all(struct _channels *channels, int64_t *count) { + int64_t *cids = NULL; PyThread_acquire_lock(channels->mutex, WAIT_LOCK); - int64_t *ids = PyMem_Malloc(sizeof(int64_t) * channels->count); + int64_t *ids = PyMem_Malloc(sizeof(int64_t) * channels->numopen); if (ids == NULL) { - PyThread_release_lock(channels->mutex); - return NULL; + goto done; } - _PyChannelState *chan = channels->head; - for (int64_t i=0; chan != NULL; chan = chan->next, i++) { - ids[i] = chan->id; + struct _channelref *ref = channels->head; + for (int64_t i=0; ref != NULL; ref = ref->next, i++) { + ids[i] = ref->id; } - *count = channels->count; + *count = channels->numopen; + + cids = ids; +done: PyThread_release_lock(channels->mutex); - return ids; + return cids; } /* "high"-level channel-related functions */ @@ -627,20 +818,22 @@ _channel_create(struct _channels *channels) _PyChannelState *chan = _channel_new(); if (chan == NULL) return -1; - if (_channels_add(channels, chan) != 0) { + int64_t id = _channels_add(channels, chan); + if (id < 0) { _channel_free(chan); return -1; } - return chan->id; + return id; } static int _channel_destroy(struct _channels *channels, int64_t id) { - _PyChannelState *chan = _channels_remove(channels, id); - if (chan == NULL) + _PyChannelState *chan = NULL; + if (_channels_remove(channels, id, &chan) != 0) return -1; - _channel_free(chan); + if (chan != NULL) + _channel_free(chan); return 0; } @@ -651,16 +844,26 @@ _channel_send(struct _channels *channels, int64_t id, PyObject *obj) if (interp == NULL) return -1; - _PyCrossInterpreterData *data = PyMem_Malloc(sizeof(_PyCrossInterpreterData)); - if (_PyObject_GetCrossInterpreterData(obj, data) != 0) + // Look up the channel. + PyThread_type_lock mutex = NULL; + _PyChannelState *chan = _channels_lookup(channels, id, &mutex); + if (chan == NULL) { return -1; + } + // Past this point we are responsible for releasing the mutex. - _PyChannelState *chan = _channels_lookup(channels, id); - if (chan == NULL) { - PyMem_Free(data); + // Convert the object to cross-interpreter data. + _PyCrossInterpreterData *data = PyMem_Malloc(sizeof(_PyCrossInterpreterData)); + if (_PyObject_GetCrossInterpreterData(obj, data) != 0) { + PyThread_release_lock(mutex); return -1; } - if (_channel_add(chan, interp->id, data) != 0) { + + // Add the data to the channel. + int res = _channel_add(chan, interp->id, data); + PyThread_release_lock(mutex); + if (res != 0) { + _PyCrossInterpreterData_Release(data); PyMem_Free(data); return -1; } @@ -675,47 +878,56 @@ _channel_recv(struct _channels *channels, int64_t id) if (interp == NULL) return NULL; - _PyChannelState *chan = _channels_lookup(channels, id); + // Look up the channel. + PyThread_type_lock mutex = NULL; + _PyChannelState *chan = _channels_lookup(channels, id, &mutex); if (chan == NULL) return NULL; + // Past this point we are responsible for releasing the mutex. + + // Pop off the next item from the channel. _PyCrossInterpreterData *data = _channel_next(chan, interp->id); + PyThread_release_lock(mutex); if (data == NULL) { - PyErr_SetString(ChannelEmptyError, "empty channel"); + PyErr_Format(ChannelEmptyError, "channel %d is empty", id); return NULL; } + // Convert the data back to an object. PyObject *obj = _PyCrossInterpreterData_NewObject(data); if (obj == NULL) { return NULL; } _PyCrossInterpreterData_Release(data); + return obj; } static int -_channel_close(struct _channels *channels, int64_t id, int send, int recv) +_channel_drop(struct _channels *channels, int64_t id, int send, int recv) { PyInterpreterState *interp = _get_current(); if (interp == NULL) return -1; - _PyChannelState *chan = _channels_lookup(channels, id); - if (chan == NULL) - return -1; - if (send > 0) { - if (_channel_close_end(chan, interp->id, 1) != 0) - return -1; - } - if (recv > 0) { - if (_channel_close_end(chan, interp->id, 0) != 0) - return -1; - } - if (_channel_is_closed(chan)) { - if (_channel_destroy(channels, chan->id) != 0) - return -1; + // Look up the channel. + PyThread_type_lock mutex = NULL; + _PyChannelState *chan = _channels_lookup(channels, id, &mutex); + if (chan == NULL) { + return -1; } + // Past this point we are responsible for releasing the mutex. - return 0; + // Close one or both of the two ends. + int res =_channel_close_interpreter(chan, interp->id, send-recv); + PyThread_release_lock(mutex); + return res; +} + +static int +_channel_close(struct _channels *channels, int64_t id) +{ + return _channels_close(channels, id, NULL); } /* @@ -736,28 +948,44 @@ typedef struct channelid { PyObject_HEAD int64_t id; int end; + struct _channels *channels; } channelid; static channelid * -newchannelid(PyTypeObject *cls, int64_t id, int end) +newchannelid(PyTypeObject *cls, int64_t cid, int end, struct _channels *channels, int force) { channelid *self = PyObject_New(channelid, cls); if (self == NULL) return NULL; - self->id = id; + self->id = cid; self->end = end; + self->channels = channels; + + if (_channels_add_id_object(channels, cid) != 0) { + if (force && PyErr_ExceptionMatches(ChannelNotFoundError)) { + PyErr_Clear(); + } + else { + Py_DECREF((PyObject *)self); + return NULL; + } + } + return self; } +static struct _channels * _global_channels(void); + static PyObject * channelid_new(PyTypeObject *cls, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"id", "send", "recv"}; + static char *kwlist[] = {"id", "send", "recv", "force"}; PyObject *id; int send = -1; int recv = -1; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|$pp:ChannelID.__init__", kwlist, - &id, &send, &recv)) + int force = 0; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|$ppp:ChannelID.__init__", kwlist, + &id, &send, &recv, &force)) return NULL; // Coerce and check the ID. @@ -785,13 +1013,18 @@ channelid_new(PyTypeObject *cls, PyObject *args, PyObject *kwds) end = CHANNEL_RECV; } - return (PyObject *)newchannelid(cls, cid, end); + id = (PyObject *)newchannelid(cls, cid, end, _global_channels(), force); + return id; } static void channelid_dealloc(PyObject *v) { + int64_t cid = ((channelid *)v)->id; + struct _channels *channels = ((channelid *)v)->channels; Py_TYPE(v)->tp_free(v); + + _channels_drop_id_object(channels, cid); } static PyObject * @@ -908,9 +1141,10 @@ channelid_richcompare(PyObject *self, PyObject *other, int op) static PyObject * channelid_end(PyObject *self, void *end) { + int force = 1; channelid *cid = (channelid *)self; if (end != NULL) - return (PyObject *)newchannelid(Py_TYPE(self), cid->id, *(int *)end); + return (PyObject *)newchannelid(Py_TYPE(self), cid->id, *(int *)end, cid->channels, force); if (cid->end == CHANNEL_SEND) return PyUnicode_InternFromString("send"); @@ -974,7 +1208,8 @@ static PyTypeObject ChannelIDtype = { 0, /* tp_dictoffset */ 0, /* tp_init */ 0, /* tp_alloc */ - (newfunc)channelid_new, /* tp_new */ + NULL, /* tp_new */ + //(newfunc)channelid_new, /* tp_new */ }; /* interpreter-specific functions *******************************************/ @@ -1131,6 +1366,11 @@ _init_globals(void) return 0; } +static struct _channels * +_global_channels(void) { + return &_globals.channels; +} + static PyObject * interp_create(PyObject *self, PyObject *args) { @@ -1362,6 +1602,49 @@ PyDoc_STRVAR(is_running_doc, \n\ Return whether or not the identified interpreter is running."); +static PyObject * +channel_create(PyObject *self) +{ + int64_t cid = _channel_create(&_globals.channels); + if (cid < 0) + return NULL; + PyObject *id = (PyObject *)newchannelid(&ChannelIDtype, cid, 0, &_globals.channels, 0); + if (id == NULL) { + if (_channel_destroy(&_globals.channels, cid) != 0) { + // XXX issue a warning? + } + return NULL; + } + assert(((channelid *)id)->channels != NULL); + return id; +} + +PyDoc_STRVAR(channel_create_doc, +"channel_create() -> ID\n\ +\n\ +Create a new cross-interpreter channel and return a unique generated ID."); + +static PyObject * +channel_destroy(PyObject *self, PyObject *args) +{ + PyObject *id; + if (!PyArg_UnpackTuple(args, "channel_destroy", 1, 1, &id)) + return NULL; + int64_t cid = _coerce_id(id); + if (cid < 0) + return NULL; + + if (_channel_destroy(&_globals.channels, cid) != 0) + return NULL; + Py_RETURN_NONE; +} + +PyDoc_STRVAR(channel_destroy_doc, +"channel_destroy(ID)\n\ +\n\ +Close and finalize the channel. Afterward attempts to use the channel\n\ +will behave as though it never existed."); + static PyObject * channel_list_all(PyObject *self) { @@ -1378,7 +1661,7 @@ channel_list_all(PyObject *self) return NULL; } for (int64_t i=0; i < count; cids++, i++) { - PyObject *id = (PyObject *)newchannelid(&ChannelIDtype, *cids, 0); + PyObject *id = (PyObject *)newchannelid(&ChannelIDtype, *cids, 0, &_globals.channels, 0); if (id == NULL) { Py_DECREF(ids); ids = NULL; @@ -1395,25 +1678,6 @@ PyDoc_STRVAR(channel_list_all_doc, \n\ Return the list of all IDs for active channels."); -static PyObject * -channel_create(PyObject *self) -{ - int64_t cid = _channel_create(&_globals.channels); - if (cid < 0) - return NULL; - PyObject *id = (PyObject *)newchannelid(&ChannelIDtype, cid, 0); - if (id == NULL) { - _channel_destroy(&_globals.channels, cid); - return NULL; - } - return id; -} - -PyDoc_STRVAR(channel_create_doc, -"channel_create() -> ID\n\ -\n\ -Create a new cross-interpreter channel and return a unique generated ID."); - static PyObject * channel_send(PyObject *self, PyObject *args) { @@ -1421,14 +1685,10 @@ channel_send(PyObject *self, PyObject *args) PyObject *obj; if (!PyArg_UnpackTuple(args, "channel_send", 2, 2, &id, &obj)) return NULL; - if (!PyLong_Check(id)) { - PyErr_SetString(PyExc_TypeError, "ID must be an int"); - return NULL; - } - int64_t cid = _coerce_id(id); if (cid < 0) return NULL; + if (_channel_send(&_globals.channels, cid, obj) != 0) return NULL; Py_RETURN_NONE; @@ -1445,15 +1705,10 @@ channel_recv(PyObject *self, PyObject *args) PyObject *id; if (!PyArg_UnpackTuple(args, "channel_recv", 1, 1, &id)) return NULL; - if (!PyLong_Check(id)) { - PyErr_SetString(PyExc_TypeError, "ID must be an int"); - return NULL; - } - int64_t cid = _coerce_id(id); if (cid < 0) return NULL; - assert(cid <= INT64_MAX); + return _channel_recv(&_globals.channels, cid); } @@ -1465,11 +1720,33 @@ Return a new object from the data at the from of the channel's queue."); static PyObject * channel_close(PyObject *self, PyObject *args, PyObject *kwds) { + PyObject *id; + if (!PyArg_UnpackTuple(args, "channel_recv", 1, 1, &id)) + return NULL; + int64_t cid = _coerce_id(id); + if (cid < 0) + return NULL; + + if (_channel_close(&_globals.channels, cid) != 0) + return NULL; + Py_RETURN_NONE; +} + +PyDoc_STRVAR(channel_close_doc, +"channel_close(ID)\n\ +\n\ +Close the channel for all interpreters. Once the channel's ID has\n\ +no more ref counts the channel will be destroyed."); + +static PyObject * +channel_drop_interpreter(PyObject *self, PyObject *args, PyObject *kwds) +{ + // Note that only the current interpreter is affected. static char *kwlist[] = {"id", "send", "recv"}; PyObject *id; int send = -1; int recv = -1; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|$pp:channel_close", kwlist, + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|$pp:channel_drop_interpreter", kwlist, &id, &send, &recv)) return NULL; @@ -1480,14 +1757,19 @@ channel_close(PyObject *self, PyObject *args, PyObject *kwds) send = 1; recv = 1; } - - if (_channel_close(&_globals.channels, cid, send, recv) != 0) + else { + if (send < 0) + send = 0; + if (recv < 0) + recv = 0; + } + if (_channel_drop(&_globals.channels, cid, send, recv) != 0) return NULL; Py_RETURN_NONE; } -PyDoc_STRVAR(channel_close_doc, -"channel_close(ID, *, send=None, recv=None)\n\ +PyDoc_STRVAR(channel_drop_interpreter_doc, +"channel_drop_interpreter(ID, *, send=None, recv=None)\n\ \n\ Close the channel for the current interpreter. 'send' and 'recv'\n\ (bool) may be used to indicate the ends to close. By default both\n\ @@ -1497,12 +1779,12 @@ static PyObject * channel_list_interpreters(PyObject *self, PyObject *args) { PyObject *id; - if (!PyArg_UnpackTuple(args, "...", 1, 1, &id)) + if (!PyArg_UnpackTuple(args, "channel_list_interpreters", 1, 1, &id)) return NULL; - if (!PyLong_Check(id)) { - PyErr_SetString(PyExc_TypeError, "ID must be an int"); + + int64_t cid = _coerce_id(id); + if (cid < 0) return NULL; - } // XXX finish PyErr_SetString(PyExc_NotImplementedError, "not implemented yet"); @@ -1515,41 +1797,51 @@ PyDoc_STRVAR(channel_list_interpreters_doc, Return the list of intepreter IDs associated with the channel\n\ on the recv and send ends, respectively."); +static PyObject * +channel__channel_id(PyObject *self, PyObject *args, PyObject *kwds) +{ + return channelid_new(&ChannelIDtype, args, kwds); +} + static PyMethodDef module_functions[] = { - {"create", (PyCFunction)interp_create, + {"create", (PyCFunction)interp_create, METH_VARARGS, create_doc}, - {"destroy", (PyCFunction)interp_destroy, + {"destroy", (PyCFunction)interp_destroy, METH_VARARGS, destroy_doc}, - - {"list_all", (PyCFunction)interp_list_all, + {"list_all", (PyCFunction)interp_list_all, METH_NOARGS, list_all_doc}, - {"get_current", (PyCFunction)interp_get_current, + {"get_current", (PyCFunction)interp_get_current, METH_NOARGS, get_current_doc}, - {"get_main", (PyCFunction)interp_get_main, + {"get_main", (PyCFunction)interp_get_main, METH_NOARGS, get_main_doc}, - {"is_running", (PyCFunction)interp_is_running, + {"is_running", (PyCFunction)interp_is_running, METH_VARARGS, is_running_doc}, - - {"run_string", (PyCFunction)interp_run_string, + {"run_string", (PyCFunction)interp_run_string, METH_VARARGS, run_string_doc}, - {"is_shareable", (PyCFunction)object_is_shareable, + {"is_shareable", (PyCFunction)object_is_shareable, METH_VARARGS, is_shareable_doc}, - {"channel_list_all", (PyCFunction)channel_list_all, - METH_NOARGS, channel_list_all_doc}, - {"channel_create", (PyCFunction)channel_create, + {"channel_create", (PyCFunction)channel_create, METH_NOARGS, channel_create_doc}, - {"channel_send", (PyCFunction)channel_send, + {"channel_destroy", (PyCFunction)channel_destroy, + METH_VARARGS, channel_destroy_doc}, + {"channel_list_all", (PyCFunction)channel_list_all, + METH_NOARGS, channel_list_all_doc}, + {"channel_send", (PyCFunction)channel_send, METH_VARARGS, channel_send_doc}, - {"channel_recv", (PyCFunction)channel_recv, + {"channel_recv", (PyCFunction)channel_recv, METH_VARARGS, channel_recv_doc}, - {"channel_close", (PyCFunction)channel_close, - METH_VARARGS | METH_KEYWORDS, channel_close_doc}, - {"channel_list_interpreters", (PyCFunction)channel_list_interpreters, + {"channel_close", (PyCFunction)channel_close, + METH_VARARGS, channel_close_doc}, + {"channel_drop_interpreter", (PyCFunction)channel_drop_interpreter, + METH_VARARGS | METH_KEYWORDS, channel_drop_interpreter_doc}, + {"channel_list_interpreters", (PyCFunction)channel_list_interpreters, METH_VARARGS, channel_list_interpreters_doc}, + {"_channel_id", (PyCFunction)channel__channel_id, + METH_VARARGS | METH_KEYWORDS, NULL}, - {NULL, NULL} /* sentinel */ + {NULL, NULL} /* sentinel */ }; From 79d31cd1f72ac48b552ce37b3ab053035c8365bc Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Sat, 20 Jan 2018 01:14:24 +0000 Subject: [PATCH 53/78] Drop channel_list_interpreters(). --- Modules/_interpretersmodule.c | 31 ------------------------------- 1 file changed, 31 deletions(-) diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index 49247bed9912b5..5bc7d2e04edf65 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -930,13 +930,6 @@ _channel_close(struct _channels *channels, int64_t id) return _channels_close(channels, id, NULL); } -/* -int -channel_list_interpreters(int id) -{ -} -*/ - /* ChannelID class */ #define CHANNEL_SEND 1 @@ -1775,28 +1768,6 @@ Close the channel for the current interpreter. 'send' and 'recv'\n\ (bool) may be used to indicate the ends to close. By default both\n\ ends are closed. Closing an already closed end is a noop."); -static PyObject * -channel_list_interpreters(PyObject *self, PyObject *args) -{ - PyObject *id; - if (!PyArg_UnpackTuple(args, "channel_list_interpreters", 1, 1, &id)) - return NULL; - - int64_t cid = _coerce_id(id); - if (cid < 0) - return NULL; - - // XXX finish - PyErr_SetString(PyExc_NotImplementedError, "not implemented yet"); - Py_RETURN_NONE; -} - -PyDoc_STRVAR(channel_list_interpreters_doc, -"channel_list_interpreters(ID) -> ([ID], [ID])\n\ -\n\ -Return the list of intepreter IDs associated with the channel\n\ -on the recv and send ends, respectively."); - static PyObject * channel__channel_id(PyObject *self, PyObject *args, PyObject *kwds) { @@ -1836,8 +1807,6 @@ static PyMethodDef module_functions[] = { METH_VARARGS, channel_close_doc}, {"channel_drop_interpreter", (PyCFunction)channel_drop_interpreter, METH_VARARGS | METH_KEYWORDS, channel_drop_interpreter_doc}, - {"channel_list_interpreters", (PyCFunction)channel_list_interpreters, - METH_VARARGS, channel_list_interpreters_doc}, {"_channel_id", (PyCFunction)channel__channel_id, METH_VARARGS | METH_KEYWORDS, NULL}, From 3aad28b3532cd44568edf1932bbdb9cf3703c43a Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Sat, 20 Jan 2018 01:43:26 +0000 Subject: [PATCH 54/78] Register ChannelID as a cross-intepreter type. --- Lib/test/test__interpreters.py | 19 +++++++++++++++---- Modules/_interpretersmodule.c | 32 ++++++++++++++++++++++++++++++-- 2 files changed, 45 insertions(+), 6 deletions(-) diff --git a/Lib/test/test__interpreters.py b/Lib/test/test__interpreters.py index 8658723fe40b60..a98c8d05814acf 100644 --- a/Lib/test/test__interpreters.py +++ b/Lib/test/test__interpreters.py @@ -23,10 +23,10 @@ def _captured_script(script): return wrapped, open(r) -def _run_output(interp, request): +def _run_output(interp, request, shared=None): script, chan = _captured_script(request) with chan: - interpreters.run_string(interp, script) + interpreters.run_string(interp, script, shared) return chan.read() @@ -1098,9 +1098,20 @@ def test_recv_empty(self): with self.assertRaises(interpreters.ChannelEmptyError): interpreters.channel_recv(cid) - @unittest.skip('not implemented yet') def test_run_string_arg(self): - raise NotImplementedError + cid = interpreters.channel_create() + interp = interpreters.create() + + out = _run_output(interp, dedent(""" + import _interpreters + print(cid.end) + _interpreters.channel_send(cid, b'spam') + """), + dict(cid=cid.send)) + obj = interpreters.channel_recv(cid) + + self.assertEqual(obj, b'spam') + self.assertEqual(out.strip(), 'send') if __name__ == '__main__': diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index 5bc7d2e04edf65..c3541105b9d42d 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -1006,8 +1006,7 @@ channelid_new(PyTypeObject *cls, PyObject *args, PyObject *kwds) end = CHANNEL_RECV; } - id = (PyObject *)newchannelid(cls, cid, end, _global_channels(), force); - return id; + return (PyObject *)newchannelid(cls, cid, end, _global_channels(), force); } static void @@ -1131,6 +1130,32 @@ channelid_richcompare(PyObject *self, PyObject *other, int op) Py_RETURN_RICHCOMPARE(othercid != cid, 0, op); } +struct _channelid_xid { + int64_t id; + int end; +}; + +static PyObject * +_channelid_from_xid(_PyCrossInterpreterData *data) +{ + struct _channelid_xid *xid = (struct _channelid_xid *)data->data; + return (PyObject *)newchannelid(&ChannelIDtype, xid->id, xid->end, _global_channels(), 0); +} + +static int +_channelid_shared(PyObject *obj, _PyCrossInterpreterData *data) +{ + struct _channelid_xid *xid = PyMem_Malloc(sizeof(struct _channelid_xid)); + xid->id = ((channelid *)obj)->id; + xid->end = ((channelid *)obj)->end; + + data->data = xid; + data->obj = obj; + data->new_object = _channelid_from_xid; + data->free = PyMem_Free; + return 0; +} + static PyObject * channelid_end(PyObject *self, void *end) { @@ -1860,5 +1885,8 @@ PyInit__interpreters(void) Py_INCREF(&ChannelIDtype); PyDict_SetItemString(ns, "ChannelID", (PyObject *)&ChannelIDtype); + if (_PyCrossInterpreterData_Register_Class(&ChannelIDtype, _channelid_shared)) + return NULL; + return module; } From db72825ef7ceeb4368584103328460e2ee494f32 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 26 Jan 2018 17:06:23 +0000 Subject: [PATCH 55/78] _interpreters -> _xxsubinterpreters. --- ...rpreters.py => test__xxsubinterpreters.py} | 42 ++++---- ...ersmodule.c => _xxsubinterpretersmodule.c} | 102 +++++++++--------- setup.py | 2 +- 3 files changed, 73 insertions(+), 73 deletions(-) rename Lib/test/{test__interpreters.py => test__xxsubinterpreters.py} (96%) rename Modules/{_interpretersmodule.c => _xxsubinterpretersmodule.c} (94%) diff --git a/Lib/test/test__interpreters.py b/Lib/test/test__xxsubinterpreters.py similarity index 96% rename from Lib/test/test__interpreters.py rename to Lib/test/test__xxsubinterpreters.py index a98c8d05814acf..144b90d42fc96f 100644 --- a/Lib/test/test__interpreters.py +++ b/Lib/test/test__xxsubinterpreters.py @@ -8,7 +8,7 @@ from test import support from test.support import script_helper -interpreters = support.import_module('_interpreters') +interpreters = support.import_module('_xxsubinterpreters') def _captured_script(script): @@ -151,7 +151,7 @@ def test_subinterpreter(self): main = interpreters.get_main() interp = interpreters.create() out = _run_output(interp, dedent(""" - import _interpreters + import _xxsubinterpreters as _interpreters print(_interpreters.get_current()) """)) cur = int(out.strip()) @@ -171,7 +171,7 @@ def test_from_subinterpreter(self): [expected] = interpreters.list_all() interp = interpreters.create() out = _run_output(interp, dedent(""" - import _interpreters + import _xxsubinterpreters as _interpreters print(_interpreters.get_main()) """)) main = int(out.strip()) @@ -195,7 +195,7 @@ def test_subinterpreter(self): def test_from_subinterpreter(self): interp = interpreters.create() out = _run_output(interp, dedent(f""" - import _interpreters + import _xxsubinterpreters as _interpreters if _interpreters.is_running({interp}): print(True) else: @@ -254,7 +254,7 @@ def test_in_subinterpreter(self): main, = interpreters.list_all() id1 = interpreters.create() out = _run_output(id1, dedent(""" - import _interpreters + import _xxsubinterpreters as _interpreters id = _interpreters.create() print(id) """)) @@ -269,7 +269,7 @@ def test_in_threaded_subinterpreter(self): def f(): nonlocal id2 out = _run_output(id1, dedent(""" - import _interpreters + import _xxsubinterpreters as _interpreters id = _interpreters.create() print(id) """)) @@ -363,7 +363,7 @@ def test_from_current(self): main, = interpreters.list_all() id = interpreters.create() script = dedent(""" - import _interpreters + import _xxsubinterpreters as _interpreters _interpreters.destroy({}) """).format(id) @@ -376,7 +376,7 @@ def test_from_sibling(self): id1 = interpreters.create() id2 = interpreters.create() script = dedent(""" - import _interpreters + import _xxsubinterpreters as _interpreters _interpreters.destroy({}) """).format(id2) interpreters.run_string(id1, script) @@ -701,7 +701,7 @@ def test_still_running_at_exit(self): script = dedent(f""" from textwrap import dedent import threading - import _interpreters + import _xxsubinterpreters as _interpreters def f(): _interpreters.run_string(id, dedent(''' import time @@ -823,7 +823,7 @@ def test_sequential_ids(self): def test_ids_global(self): id1 = interpreters.create() out = _run_output(id1, dedent(""" - import _interpreters + import _xxsubinterpreters as _interpreters cid = _interpreters.channel_create() print(int(cid)) """)) @@ -831,7 +831,7 @@ def test_ids_global(self): id2 = interpreters.create() out = _run_output(id2, dedent(""" - import _interpreters + import _xxsubinterpreters as _interpreters cid = _interpreters.channel_create() print(int(cid)) """)) @@ -857,11 +857,11 @@ def test_drop_multiple_users(self): id1 = interpreters.create() id2 = interpreters.create() interpreters.run_string(id1, dedent(f""" - import _interpreters + import _xxsubinterpreters as _interpreters _interpreters.channel_send({int(cid)}, b'spam') """)) out = _run_output(id2, dedent(f""" - import _interpreters + import _xxsubinterpreters as _interpreters obj = _interpreters.channel_recv({int(cid)}) _interpreters.channel_drop_interpreter({int(cid)}) print(obj) @@ -915,7 +915,7 @@ def test_drop_by_unassociated_interp(self): interpreters.channel_send(cid, b'spam') interp = interpreters.create() interpreters.run_string(interp, dedent(f""" - import _interpreters + import _xxsubinterpreters as _interpreters _interpreters.channel_drop_interpreter({int(cid)}) """)) obj = interpreters.channel_recv(cid) @@ -929,7 +929,7 @@ def test_drop_close_if_unassociated(self): cid = interpreters.channel_create() interp = interpreters.create() interpreters.run_string(interp, dedent(f""" - import _interpreters + import _xxsubinterpreters as _interpreters obj = _interpreters.channel_send({int(cid)}, b'spam') _interpreters.channel_drop_interpreter({int(cid)}) """)) @@ -979,11 +979,11 @@ def test_close_multiple_users(self): id1 = interpreters.create() id2 = interpreters.create() interpreters.run_string(id1, dedent(f""" - import _interpreters + import _xxsubinterpreters as _interpreters _interpreters.channel_send({int(cid)}, b'spam') """)) interpreters.run_string(id2, dedent(f""" - import _interpreters + import _xxsubinterpreters as _interpreters _interpreters.channel_recv({int(cid)}) """)) interpreters.channel_close(cid) @@ -1030,7 +1030,7 @@ def test_close_by_unassociated_interp(self): interpreters.channel_send(cid, b'spam') interp = interpreters.create() interpreters.run_string(interp, dedent(f""" - import _interpreters + import _xxsubinterpreters as _interpreters _interpreters.channel_close({int(cid)}) """)) with self.assertRaises(interpreters.ChannelClosedError): @@ -1065,7 +1065,7 @@ def test_send_recv_main(self): def test_send_recv_same_interpreter(self): id1 = interpreters.create() out = _run_output(id1, dedent(""" - import _interpreters + import _xxsubinterpreters as _interpreters cid = _interpreters.channel_create() orig = b'spam' _interpreters.channel_send(cid, orig) @@ -1078,7 +1078,7 @@ def test_send_recv_different_interpreters(self): cid = interpreters.channel_create() id1 = interpreters.create() out = _run_output(id1, dedent(f""" - import _interpreters + import _xxsubinterpreters as _interpreters _interpreters.channel_send({int(cid)}, b'spam') """)) obj = interpreters.channel_recv(cid) @@ -1103,7 +1103,7 @@ def test_run_string_arg(self): interp = interpreters.create() out = _run_output(interp, dedent(""" - import _interpreters + import _xxsubinterpreters as _interpreters print(cid.end) _interpreters.channel_send(cid, b'spam') """), diff --git a/Modules/_interpretersmodule.c b/Modules/_xxsubinterpretersmodule.c similarity index 94% rename from Modules/_interpretersmodule.c rename to Modules/_xxsubinterpretersmodule.c index c3541105b9d42d..51904ace0423a6 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -161,7 +161,7 @@ interp_exceptions_init(PyObject *ns) // XXX Move the exceptions into per-module memory? // An uncaught exception came out of interp_run_string(). - RunFailedError = PyErr_NewException("_interpreters.RunFailedError", + RunFailedError = PyErr_NewException("_xxsubinterpreters.RunFailedError", PyExc_RuntimeError, NULL); if (RunFailedError == NULL) return -1; @@ -189,28 +189,28 @@ channel_exceptions_init(PyObject *ns) // XXX Move the exceptions into per-module memory? // A channel-related operation failed. - ChannelError = PyErr_NewException("_interpreters.ChannelError", + ChannelError = PyErr_NewException("_xxsubinterpreters.ChannelError", PyExc_RuntimeError, NULL); if (ChannelError == NULL) return -1; PyDict_SetItemString(ns, "ChannelError", ChannelError); // An operation tried to use a channel that doesn't exist. - ChannelNotFoundError = PyErr_NewException("_interpreters.ChannelNotFoundError", + ChannelNotFoundError = PyErr_NewException("_xxsubinterpreters.ChannelNotFoundError", ChannelError, NULL); if (ChannelNotFoundError == NULL) return -1; PyDict_SetItemString(ns, "ChannelNotFoundError", ChannelNotFoundError); // An operation tried to use a closed channel. - ChannelClosedError = PyErr_NewException("_interpreters.ChannelClosedError", + ChannelClosedError = PyErr_NewException("_xxsubinterpreters.ChannelClosedError", ChannelError, NULL); if (ChannelClosedError == NULL) return -1; PyDict_SetItemString(ns, "ChannelClosedError", ChannelClosedError); // An operation tried to pop from an empty channel. - ChannelEmptyError = PyErr_NewException("_interpreters.ChannelEmptyError", + ChannelEmptyError = PyErr_NewException("_xxsubinterpreters.ChannelEmptyError", ChannelError, NULL); if (ChannelEmptyError == NULL) return -1; @@ -1189,44 +1189,44 @@ PyDoc_STRVAR(channelid_doc, static PyTypeObject ChannelIDtype = { PyVarObject_HEAD_INIT(&PyType_Type, 0) - "_interpreters.ChannelID", /* tp_name */ - sizeof(channelid), /* tp_size */ - 0, /* tp_itemsize */ - (destructor)channelid_dealloc, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - (reprfunc)channelid_repr, /* tp_repr */ - &channelid_as_number, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - channelid_hash, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ + "_xxsubinterpreters.ChannelID", /* tp_name */ + sizeof(channelid), /* tp_size */ + 0, /* tp_itemsize */ + (destructor)channelid_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_as_async */ + (reprfunc)channelid_repr, /* tp_repr */ + &channelid_as_number, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + channelid_hash, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | - Py_TPFLAGS_LONG_SUBCLASS, /* tp_flags */ - channelid_doc, /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - channelid_richcompare, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - 0, /* tp_methods */ - 0, /* tp_members */ - channelid_getsets, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - NULL, /* tp_new */ + Py_TPFLAGS_LONG_SUBCLASS, /* tp_flags */ + channelid_doc, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + channelid_richcompare, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + 0, /* tp_methods */ + 0, /* tp_members */ + channelid_getsets, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + NULL, /* tp_new */ //(newfunc)channelid_new, /* tp_new */ }; @@ -1847,19 +1847,19 @@ The 'interpreters' module provides a more convenient interface."); static struct PyModuleDef interpretersmodule = { PyModuleDef_HEAD_INIT, - "_interpreters", /* m_name */ - module_doc, /* m_doc */ - -1, /* m_size */ - module_functions, /* m_methods */ - NULL, /* m_slots */ - NULL, /* m_traverse */ - NULL, /* m_clear */ - NULL /* m_free */ + "_xxsubinterpreters", /* m_name */ + module_doc, /* m_doc */ + -1, /* m_size */ + module_functions, /* m_methods */ + NULL, /* m_slots */ + NULL, /* m_traverse */ + NULL, /* m_clear */ + NULL /* m_free */ }; PyMODINIT_FUNC -PyInit__interpreters(void) +PyInit__xxsubinterpreters(void) { if (_init_globals() != 0) return NULL; diff --git a/setup.py b/setup.py index 739fac287e7318..bfa3ed23d20232 100644 --- a/setup.py +++ b/setup.py @@ -746,7 +746,7 @@ def detect_modules(self): ) # Python interface to subinterpreter C-API. - exts.append(Extension('_interpreters', ['_interpretersmodule.c'], + exts.append(Extension('_xxsubinterpreters', ['_xxsubinterpretersmodule.c'], define_macros=[('Py_BUILD_CORE', '')])) # From 6a0c0ff75f470f8b2f661601b383765d3d2e5325 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 26 Jan 2018 17:08:21 +0000 Subject: [PATCH 56/78] Fix an un-initialized variable. --- Modules/_xxsubinterpretersmodule.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 51904ace0423a6..cf52bb547a50b3 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -64,8 +64,10 @@ _sharedns_clear(struct _shareditem *shared) static struct _shareditem * _get_shared_ns(PyObject *shareable, Py_ssize_t *lenp) { - if (shareable == NULL || shareable == Py_None) + if (shareable == NULL || shareable == Py_None) { + *lenp = 0; return NULL; + } Py_ssize_t len = PyDict_Size(shareable); *lenp = len; if (len == 0) @@ -1287,6 +1289,7 @@ _run_script(PyInterpreterState *interp, const char *codestr, struct _shareditem *shared, Py_ssize_t num_shared, struct _shared_exception **exc) { + assert(num_shared >= 0); PyObject *main_mod = PyMapping_GetItemString(interp->modules, "__main__"); if (main_mod == NULL) goto error; @@ -1331,7 +1334,7 @@ _run_script_in_interpreter(PyInterpreterState *interp, const char *codestr, if (_ensure_not_running(interp) < 0) return -1; - Py_ssize_t num_shared; + Py_ssize_t num_shared = -1; struct _shareditem *shared = _get_shared_ns(shareables, &num_shared); if (shared == NULL && PyErr_Occurred()) return -1; From 9d50730826589297a372c54e3224fdecc032b363 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 26 Jan 2018 17:23:28 +0000 Subject: [PATCH 57/78] NULL-terminate kwlist for PyArg_ParseTupleAndKeywords(). --- Modules/_xxsubinterpretersmodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index cf52bb547a50b3..9e8041db7ae1e6 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -974,7 +974,7 @@ static struct _channels * _global_channels(void); static PyObject * channelid_new(PyTypeObject *cls, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"id", "send", "recv", "force"}; + static char *kwlist[] = {"id", "send", "recv", "force", NULL}; PyObject *id; int send = -1; int recv = -1; From b31d130067572752da2d2ba2a31a3ee8d56faf56 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 26 Jan 2018 19:59:48 +0000 Subject: [PATCH 58/78] Drop docs for now. --- Doc/library/_interpreters.rst | 90 ----------------------------------- Doc/library/concurrency.rst | 1 - 2 files changed, 91 deletions(-) delete mode 100644 Doc/library/_interpreters.rst diff --git a/Doc/library/_interpreters.rst b/Doc/library/_interpreters.rst deleted file mode 100644 index bbc945016e2880..00000000000000 --- a/Doc/library/_interpreters.rst +++ /dev/null @@ -1,90 +0,0 @@ -:mod:`_interpreters` --- Low-level interpreters API -=================================================== - -.. module:: _interpreters - :synopsis: Low-level interpreters API. - -.. versionadded:: 3,7 - --------------- - -This module provides low-level primitives for working with multiple -Python interpreters in the same runtime in the current process. - -More information about (sub)interpreters is found at -:ref:`sub-interpreter-support`, including what data is shared between -interpreters and what is unique. Note particularly that interpreters -aren't inherently threaded, even though they track and manage Python -threads. To run code in an interpreter in a different OS thread, call -:func:`run_string` in a function that you run in a new Python thread. -For example:: - - id = _interpreters.create() - def f(): - _interpreters.run_string(id, 'print("in a thread")') - - t = threading.Thread(target=f) - t.start() - -This module is optional. It is provided by Python implementations which -support multiple interpreters. - -It defines the following functions: - -.. function:: enumerate() - - Return a list of the IDs of every existing interpreter. - - -.. function:: get_current() - - Return the ID of the currently running interpreter. - - -.. function:: get_main() - - Return the ID of the main interpreter. - - -.. function:: is_running(id) - - Return whether or not the identified interpreter is currently - running any code. - - -.. function:: create() - - Initialize a new Python interpreter and return its identifier. The - interpreter will be created in the current thread and will remain - idle until something is run in it. - - -.. function:: destroy(id) - - Finalize and destroy the identified interpreter. - - -.. function:: run_string(id, command) - - A wrapper around :c:func:`PyRun_SimpleString` which runs the provided - Python program in the main thread of the identified interpreter. - Providing an invalid or unknown ID results in a RuntimeError, - likewise if the main interpreter or any other running interpreter - is used. - - Any value returned from the code is thrown away, similar to what - threads do. If the code results in an exception then that exception - is raised in the thread in which run_string() was called, similar to - how :func:`exec` works. This aligns with how interpreters are not - inherently threaded. Note that SystemExit (as raised by sys.exit()) - is not treated any differently and will result in the process ending - if not caught explicitly. - - -.. function:: run_string_unrestricted(id, command, ns=None) - - Like :c:func:`run_string` but returns the dict in which the code - was executed. It also supports providing a namespace that gets - merged into the execution namespace before execution. Note that - this allows objects to leak between interpreters, which may not - be desirable. diff --git a/Doc/library/concurrency.rst b/Doc/library/concurrency.rst index fafbf92c6b0181..826bf86d081793 100644 --- a/Doc/library/concurrency.rst +++ b/Doc/library/concurrency.rst @@ -29,4 +29,3 @@ The following are support modules for some of the above services: _thread.rst _dummy_thread.rst dummy_threading.rst - _interpreters.rst From 91e7c8b8d8196ca8c4617906d8190a953a1150f1 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 26 Jan 2018 20:15:10 +0000 Subject: [PATCH 59/78] Do not check for NULL from PyThreadState_Get(). --- Modules/_xxsubinterpretersmodule.c | 8 +++----- Python/pystate.c | 4 ++-- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 9e8041db7ae1e6..7cdcfbf187045c 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -10,11 +10,9 @@ static PyInterpreterState * _get_current(void) { - PyThreadState *tstate; - - tstate = PyThreadState_Get(); - if (tstate == NULL) - return NULL; + PyThreadState *tstate = PyThreadState_Get(); + // PyThreadState_Get() aborts if lookup fails, so we don't need + // to check the result for NULL. return tstate->interp; } diff --git a/Python/pystate.c b/Python/pystate.c index 22bda90b53b59f..a474549a8c730d 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1108,8 +1108,8 @@ int _PyObject_GetCrossInterpreterData(PyObject *obj, _PyCrossInterpreterData *data) { PyThreadState *tstate = PyThreadState_Get(); - if (tstate == NULL) - return -1; + // PyThreadState_Get() aborts if lookup fails, so we don't need + // to check the result for NULL. PyInterpreterState *interp = tstate->interp; // Reset data before re-populating. From 7d2cfea1f30e80c0fb7a52468521dce007f567d3 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 26 Jan 2018 20:19:18 +0000 Subject: [PATCH 60/78] Format "else" statements properly. --- Modules/_xxsubinterpretersmodule.c | 36 ++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 7cdcfbf187045c..1cb67fe7ba2ae5 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -21,10 +21,12 @@ _coerce_id(PyObject *id) { id = PyNumber_Long(id); if (id == NULL) { - if (PyErr_ExceptionMatches(PyExc_TypeError)) + if (PyErr_ExceptionMatches(PyExc_TypeError)) { PyErr_SetString(PyExc_TypeError, "'id' must be a non-negative int"); - else + } + else { PyErr_SetString(PyExc_ValueError, "'id' must be a non-negative int"); + } return -1; } long long cid = PyLong_AsLongLong(id); @@ -138,10 +140,12 @@ _get_shared_exception(void) PyObject *tb; PyErr_Fetch(&exc, &value, &tb); PyObject *msg; - if (value == NULL) + if (value == NULL) { msg = PyUnicode_FromFormat("%S", exc); - else + } + else { msg = PyUnicode_FromFormat("%S: %S", exc, value); + } if (msg == NULL) { err->msg = "unable to format exception"; return NULL; @@ -331,7 +335,8 @@ _channel_add_end(_PyChannelState *chan, struct _channelend *prev, int64_t interp if (prev == NULL) { if (send) { chan->send = end; - } else { + } + else { chan->recv = end; } } @@ -372,10 +377,12 @@ static void _channel_close_channelend(_PyChannelState *chan, struct _channelend *end, int send) { end->open = 0; - if (send) + if (send) { chan->numsendopen -= 1; - else + } + else { chan->numrecvopen -= 1; + } } static int @@ -711,7 +718,8 @@ _channels_remove_ref(struct _channels *channels, struct _channelref *ref, struct { if (ref == channels->head) { channels->head = ref->next; - } else { + } + else { prev->next = ref->next; } channels->numopen -= 1; @@ -1028,12 +1036,15 @@ channelid_repr(PyObject *self) channelid *cid = (channelid *)self; const char *fmt; - if (cid->end == CHANNEL_SEND) + if (cid->end == CHANNEL_SEND) { fmt = "%s(%d, send=True)"; - else if (cid->end == CHANNEL_RECV) + } + else if (cid->end == CHANNEL_RECV) { fmt = "%s(%d, recv=True)"; - else + } + else { fmt = "%s(%d)"; + } return PyUnicode_FromFormat(fmt, name, cid->id); } @@ -1313,7 +1324,8 @@ _run_script(PyInterpreterState *interp, const char *codestr, Py_DECREF(ns); if (result == NULL) { goto error; - } else { + } + else { Py_DECREF(result); // We throw away the result. } From 73dad339d5c964736683420335a083f00c55a444 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 26 Jan 2018 20:32:41 +0000 Subject: [PATCH 61/78] Always use braces with if statements. --- Modules/_xxsubinterpretersmodule.c | 291 +++++++++++++++++++---------- 1 file changed, 194 insertions(+), 97 deletions(-) diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 1cb67fe7ba2ae5..cdd2da4301f89b 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -70,8 +70,9 @@ _get_shared_ns(PyObject *shareable, Py_ssize_t *lenp) } Py_ssize_t len = PyDict_Size(shareable); *lenp = len; - if (len == 0) + if (len == 0) { return NULL; + } struct _shareditem *shared = PyMem_NEW(struct _shareditem, len+1); for (Py_ssize_t i=0; i < len; i++) { @@ -85,8 +86,9 @@ _get_shared_ns(PyObject *shareable, Py_ssize_t *lenp) } struct _shareditem *item = shared + i; - if (_PyObject_GetCrossInterpreterData(value, &item->data) != 0) + if (_PyObject_GetCrossInterpreterData(value, &item->data) != 0) { break; + } item->name = PyUnicode_AsUnicodeAndSize(key, &item->namelen); if (item->name == NULL) { _PyCrossInterpreterData_Release(&item->data); @@ -133,8 +135,9 @@ static struct _shared_exception * _get_shared_exception(void) { struct _shared_exception *err = PyMem_NEW(struct _shared_exception, 1); - if (err == NULL) + if (err == NULL) { return NULL; + } PyObject *exc; PyObject *value; PyObject *tb; @@ -167,8 +170,9 @@ interp_exceptions_init(PyObject *ns) // An uncaught exception came out of interp_run_string(). RunFailedError = PyErr_NewException("_xxsubinterpreters.RunFailedError", PyExc_RuntimeError, NULL); - if (RunFailedError == NULL) + if (RunFailedError == NULL) { return -1; + } PyDict_SetItemString(ns, "RunFailedError", RunFailedError); return 0; @@ -195,29 +199,33 @@ channel_exceptions_init(PyObject *ns) // A channel-related operation failed. ChannelError = PyErr_NewException("_xxsubinterpreters.ChannelError", PyExc_RuntimeError, NULL); - if (ChannelError == NULL) + if (ChannelError == NULL) { return -1; + } PyDict_SetItemString(ns, "ChannelError", ChannelError); // An operation tried to use a channel that doesn't exist. ChannelNotFoundError = PyErr_NewException("_xxsubinterpreters.ChannelNotFoundError", ChannelError, NULL); - if (ChannelNotFoundError == NULL) + if (ChannelNotFoundError == NULL) { return -1; + } PyDict_SetItemString(ns, "ChannelNotFoundError", ChannelNotFoundError); // An operation tried to use a closed channel. ChannelClosedError = PyErr_NewException("_xxsubinterpreters.ChannelClosedError", ChannelError, NULL); - if (ChannelClosedError == NULL) + if (ChannelClosedError == NULL) { return -1; + } PyDict_SetItemString(ns, "ChannelClosedError", ChannelClosedError); // An operation tried to pop from an empty channel. ChannelEmptyError = PyErr_NewException("_xxsubinterpreters.ChannelEmptyError", ChannelError, NULL); - if (ChannelEmptyError == NULL) + if (ChannelEmptyError == NULL) { return -1; + } PyDict_SetItemString(ns, "ChannelEmptyError", ChannelEmptyError); return 0; @@ -235,8 +243,9 @@ static struct _channelend * _channelend_new(int64_t interp) { struct _channelend *end = PyMem_Malloc(sizeof(struct _channelend)); - if (end == NULL) + if (end == NULL) { return NULL; + } end->next = NULL; end->interp = interp; @@ -261,13 +270,15 @@ _channelend_find(struct _channelend *first, int64_t interp, struct _channelend * struct _channelend *prev = NULL; struct _channelend *end = first; while (end != NULL) { - if (end->interp == interp) + if (end->interp == interp) { break; + } prev = end; end = end->next; } - if (pprev != NULL) + if (pprev != NULL) { *pprev = prev; + } return end; } @@ -303,8 +314,9 @@ static _PyChannelState * _channel_new(void) { _PyChannelState *chan = PyMem_Malloc(sizeof(_PyChannelState)); - if (chan == NULL) + if (chan == NULL) { return NULL; + } chan->mutex = PyThread_allocate_lock(); if (chan->mutex == NULL) { PyErr_SetString(ChannelError, "can't initialize mutex for new channel"); @@ -329,8 +341,9 @@ static struct _channelend * _channel_add_end(_PyChannelState *chan, struct _channelend *prev, int64_t interp, int send) { struct _channelend *end = _channelend_new(interp); - if (end == NULL) + if (end == NULL) { return NULL; + } if (prev == NULL) { if (send) { @@ -403,8 +416,9 @@ _channel_close_interpreter(_PyChannelState *chan, int64_t interp, int which) if (end == NULL) { // never associated so add it end = _channel_add_end(chan, prev, interp, 1); - if (end == NULL) + if (end == NULL) { goto done; + } } _channel_close_channelend(chan, end, 1); } @@ -413,15 +427,17 @@ _channel_close_interpreter(_PyChannelState *chan, int64_t interp, int which) if (end == NULL) { // never associated so add it end = _channel_add_end(chan, prev, interp, 0); - if (end == NULL) + if (end == NULL) { goto done; + } } _channel_close_channelend(chan, end, 0); } if (chan->numsendopen == 0 && chan->numrecvopen == 0) { - if (chan->send != NULL || chan->recv != NULL) + if (chan->send != NULL || chan->recv != NULL) { chan->open = 0; + } } res = 0; @@ -481,8 +497,9 @@ _channel_add(_PyChannelState *chan, int64_t interp, _PyCrossInterpreterData *dat item->next = NULL; chan->count += 1; - if (chan->first == NULL) + if (chan->first == NULL) { chan->first = item; + } chan->last = item; res = 0; @@ -505,8 +522,9 @@ _channel_next(_PyChannelState *chan, int64_t interp) goto done; } chan->first = item->next; - if (chan->last == item) + if (chan->last == item) { chan->last = NULL; + } chan->count -= 1; data = item->data; @@ -558,8 +576,9 @@ static struct _channelref * _channelref_new(int64_t id, _PyChannelState *chan) { struct _channelref *ref = PyMem_Malloc(sizeof(struct _channelref)); - if (ref == NULL) + if (ref == NULL) { return NULL; + } ref->id = id; ref->chan = chan; ref->next = NULL; @@ -573,13 +592,15 @@ _channelref_find(struct _channelref *first, int64_t id, struct _channelref **ppr struct _channelref *prev = NULL; struct _channelref *ref = first; while (ref != NULL) { - if (ref->id == id) + if (ref->id == id) { break; + } prev = ref; ref = ref->next; } - if (pprev != NULL) + if (pprev != NULL) { *pprev = prev; + } return ref; } @@ -626,8 +647,9 @@ _channels_lookup(struct _channels *channels, int64_t id, PyThread_type_lock *pmu { _PyChannelState *chan = NULL; PyThread_acquire_lock(channels->mutex, WAIT_LOCK); - if (pmutex != NULL) + if (pmutex != NULL) { *pmutex = NULL; + } struct _channelref *ref = _channelref_find(channels->head, id, NULL); if (ref == NULL) { @@ -685,8 +707,9 @@ _channels_close(struct _channels *channels, int64_t cid, _PyChannelState **pchan { int res = -1; PyThread_acquire_lock(channels->mutex, WAIT_LOCK); - if (pchan != NULL) + if (pchan != NULL) { *pchan = NULL; + } struct _channelref *ref = _channelref_find(channels->head, cid, NULL); if (ref == NULL) { @@ -702,8 +725,9 @@ _channels_close(struct _channels *channels, int64_t cid, _PyChannelState **pchan if (_channel_close_all(ref->chan) != 0) { goto done; } - if (pchan != NULL) + if (pchan != NULL) { *pchan = ref->chan; + } ref->chan = NULL; } @@ -724,8 +748,9 @@ _channels_remove_ref(struct _channels *channels, struct _channelref *ref, struct } channels->numopen -= 1; - if (pchan != NULL) + if (pchan != NULL) { *pchan = ref->chan; + } PyMem_Free(ref); } @@ -735,8 +760,9 @@ _channels_remove(struct _channels *channels, int64_t id, _PyChannelState **pchan int res = -1; PyThread_acquire_lock(channels->mutex, WAIT_LOCK); - if (pchan != NULL) + if (pchan != NULL) { *pchan = NULL; + } struct _channelref *prev = NULL; struct _channelref *ref = _channelref_find(channels->head, id, &prev); @@ -789,8 +815,9 @@ _channels_drop_id_object(struct _channels *channels, int64_t id) if (ref->objcount == 0) { _PyChannelState *chan = NULL; _channels_remove_ref(channels, ref, prev, &chan); - if (chan != NULL) + if (chan != NULL) { _channel_free(chan); + } } done: @@ -824,8 +851,9 @@ static int64_t _channel_create(struct _channels *channels) { _PyChannelState *chan = _channel_new(); - if (chan == NULL) + if (chan == NULL) { return -1; + } int64_t id = _channels_add(channels, chan); if (id < 0) { _channel_free(chan); @@ -838,10 +866,12 @@ static int _channel_destroy(struct _channels *channels, int64_t id) { _PyChannelState *chan = NULL; - if (_channels_remove(channels, id, &chan) != 0) + if (_channels_remove(channels, id, &chan) != 0) { return -1; - if (chan != NULL) + } + if (chan != NULL) { _channel_free(chan); + } return 0; } @@ -849,8 +879,9 @@ static int _channel_send(struct _channels *channels, int64_t id, PyObject *obj) { PyInterpreterState *interp = _get_current(); - if (interp == NULL) + if (interp == NULL) { return -1; + } // Look up the channel. PyThread_type_lock mutex = NULL; @@ -883,14 +914,16 @@ static PyObject * _channel_recv(struct _channels *channels, int64_t id) { PyInterpreterState *interp = _get_current(); - if (interp == NULL) + if (interp == NULL) { return NULL; + } // Look up the channel. PyThread_type_lock mutex = NULL; _PyChannelState *chan = _channels_lookup(channels, id, &mutex); - if (chan == NULL) + if (chan == NULL) { return NULL; + } // Past this point we are responsible for releasing the mutex. // Pop off the next item from the channel. @@ -915,8 +948,9 @@ static int _channel_drop(struct _channels *channels, int64_t id, int send, int recv) { PyInterpreterState *interp = _get_current(); - if (interp == NULL) + if (interp == NULL) { return -1; + } // Look up the channel. PyThread_type_lock mutex = NULL; @@ -956,8 +990,9 @@ static channelid * newchannelid(PyTypeObject *cls, int64_t cid, int end, struct _channels *channels, int force) { channelid *self = PyObject_New(channelid, cls); - if (self == NULL) + if (self == NULL) { return NULL; + } self->id = cid; self->end = end; self->channels = channels; @@ -996,8 +1031,9 @@ channelid_new(PyTypeObject *cls, PyObject *args, PyObject *kwds) } else { cid = _coerce_id(id); - if (cid < 0) + if (cid < 0) { return NULL; + } } // Handle "send" and "recv". @@ -1007,8 +1043,9 @@ channelid_new(PyTypeObject *cls, PyObject *args, PyObject *kwds) } int end = 0; if (send == 1) { - if (recv == 0 || recv == -1) + if (recv == 0 || recv == -1) { end = CHANNEL_SEND; + } } else if (recv == 1) { end = CHANNEL_RECV; @@ -1100,8 +1137,9 @@ channelid_hash(PyObject *self) { channelid *cid = (channelid *)self; PyObject *id = PyLong_FromLongLong(cid->id); - if (id == NULL) + if (id == NULL) { return -1; + } return PyObject_Hash(id); } @@ -1116,15 +1154,17 @@ channelid_richcompare(PyObject *self, PyObject *other, int op) Py_RETURN_NOTIMPLEMENTED; } - if (!PyObject_TypeCheck(self, &ChannelIDtype)) + if (!PyObject_TypeCheck(self, &ChannelIDtype)) { Py_RETURN_NOTIMPLEMENTED; + } int64_t cid = ((channelid *)self)->id; int64_t othercid; if (PyObject_TypeCheck(other, &ChannelIDtype)) { othercid = ((channelid *)other)->id; - if (((channelid *)other)->end != ((channelid *)self)->end) + if (((channelid *)other)->end != ((channelid *)self)->end) { Py_RETURN_FALSE; + } } else { other = PyNumber_Long(other); @@ -1133,10 +1173,12 @@ channelid_richcompare(PyObject *self, PyObject *other, int op) Py_RETURN_NOTIMPLEMENTED; } othercid = PyLong_AsLongLong(other); - if (othercid == -1 && PyErr_Occurred() != NULL) + if (othercid == -1 && PyErr_Occurred() != NULL) { return NULL; - if (othercid < 0 || othercid > INT64_MAX) + } + if (othercid < 0 || othercid > INT64_MAX) { Py_RETURN_FALSE; + } } Py_RETURN_RICHCOMPARE(othercid != cid, 0, op); } @@ -1172,13 +1214,16 @@ channelid_end(PyObject *self, void *end) { int force = 1; channelid *cid = (channelid *)self; - if (end != NULL) + if (end != NULL) { return (PyObject *)newchannelid(Py_TYPE(self), cid->id, *(int *)end, cid->channels, force); + } - if (cid->end == CHANNEL_SEND) + if (cid->end == CHANNEL_SEND) { return PyUnicode_InternFromString("send"); - if (cid->end == CHANNEL_RECV) + } + if (cid->end == CHANNEL_RECV) { return PyUnicode_InternFromString("recv"); + } return PyUnicode_InternFromString("both"); } @@ -1247,8 +1292,9 @@ static PyInterpreterState * _look_up(PyObject *requested_id) { long long id = PyLong_AsLongLong(requested_id); - if (id == -1 && PyErr_Occurred() != NULL) + if (id == -1 && PyErr_Occurred() != NULL) { return NULL; + } assert(id <= INT64_MAX); return _PyInterpreterState_LookUpID(id); } @@ -1257,8 +1303,9 @@ static PyObject * _get_id(PyInterpreterState *interp) { PY_INT64_T id = PyInterpreterState_GetID(interp); - if (id < 0) + if (id < 0) { return NULL; + } return PyLong_FromLongLong(id); } @@ -1273,8 +1320,9 @@ _is_running(PyInterpreterState *interp) } PyFrameObject *frame = tstate->frame; if (frame == NULL) { - if (PyErr_Occurred() != NULL) + if (PyErr_Occurred() != NULL) { return -1; + } return 0; } return (int)(frame->f_executing); @@ -1284,8 +1332,9 @@ static int _ensure_not_running(PyInterpreterState *interp) { int is_running = _is_running(interp); - if (is_running < 0) + if (is_running < 0) { return -1; + } if (is_running) { PyErr_Format(PyExc_RuntimeError, "interpreter already running"); return -1; @@ -1300,12 +1349,14 @@ _run_script(PyInterpreterState *interp, const char *codestr, { assert(num_shared >= 0); PyObject *main_mod = PyMapping_GetItemString(interp->modules, "__main__"); - if (main_mod == NULL) + if (main_mod == NULL) { goto error; + } PyObject *ns = PyModule_GetDict(main_mod); // borrowed Py_DECREF(main_mod); - if (ns == NULL) + if (ns == NULL) { goto error; + } Py_INCREF(ns); // Apply the cross-interpreter data. @@ -1341,13 +1392,15 @@ static int _run_script_in_interpreter(PyInterpreterState *interp, const char *codestr, PyObject *shareables) { - if (_ensure_not_running(interp) < 0) + if (_ensure_not_running(interp) < 0) { return -1; + } Py_ssize_t num_shared = -1; struct _shareditem *shared = _get_shared_ns(shareables, &num_shared); - if (shared == NULL && PyErr_Occurred()) + if (shared == NULL && PyErr_Occurred()) { return -1; + } // Switch to interpreter. PyThreadState *tstate = PyInterpreterState_ThreadHead(interp); @@ -1358,8 +1411,9 @@ _run_script_in_interpreter(PyInterpreterState *interp, const char *codestr, int result = _run_script(interp, codestr, shared, num_shared, &exc); // Switch back. - if (save_tstate != NULL) + if (save_tstate != NULL) { PyThreadState_Swap(save_tstate); + } // Propagate any exception out to the caller. if (exc != NULL) { @@ -1392,8 +1446,9 @@ static struct globals { static int _init_globals(void) { - if (_channels_init(&_globals.channels) != 0) + if (_channels_init(&_globals.channels) != 0) { return -1; + } return 0; } @@ -1405,8 +1460,9 @@ _global_channels(void) { static PyObject * interp_create(PyObject *self, PyObject *args) { - if (!PyArg_UnpackTuple(args, "create", 0, 0)) + if (!PyArg_UnpackTuple(args, "create", 0, 0)) { return NULL; + } // Create and initialize the new interpreter. PyThreadState *tstate, *save_tstate; @@ -1433,8 +1489,9 @@ static PyObject * interp_destroy(PyObject *self, PyObject *args) { PyObject *id; - if (!PyArg_UnpackTuple(args, "destroy", 1, 1, &id)) + if (!PyArg_UnpackTuple(args, "destroy", 1, 1, &id)) { return NULL; + } if (!PyLong_Check(id)) { PyErr_SetString(PyExc_TypeError, "ID must be an int"); return NULL; @@ -1442,13 +1499,15 @@ interp_destroy(PyObject *self, PyObject *args) // Look up the interpreter. PyInterpreterState *interp = _look_up(id); - if (interp == NULL) + if (interp == NULL) { return NULL; + } // Ensure we don't try to destroy the current interpreter. PyInterpreterState *current = _get_current(); - if (current == NULL) + if (current == NULL) { return NULL; + } if (interp == current) { PyErr_SetString(PyExc_RuntimeError, "cannot destroy the current interpreter"); @@ -1458,8 +1517,9 @@ interp_destroy(PyObject *self, PyObject *args) // Ensure the interpreter isn't running. /* XXX We *could* support destroying a running interpreter but aren't going to worry about it for now. */ - if (_ensure_not_running(interp) < 0) + if (_ensure_not_running(interp) < 0) { return NULL; + } // Destroy the interpreter. //PyInterpreterState_Delete(interp); @@ -1488,17 +1548,20 @@ interp_list_all(PyObject *self) PyInterpreterState *interp; ids = PyList_New(0); - if (ids == NULL) + if (ids == NULL) { return NULL; + } interp = PyInterpreterState_Head(); while (interp != NULL) { id = _get_id(interp); - if (id == NULL) + if (id == NULL) { return NULL; + } // insert at front of list - if (PyList_Insert(ids, 0, id) < 0) + if (PyList_Insert(ids, 0, id) < 0) { return NULL; + } interp = PyInterpreterState_Next(interp); } @@ -1516,8 +1579,9 @@ static PyObject * interp_get_current(PyObject *self) { PyInterpreterState *interp =_get_current(); - if (interp == NULL) + if (interp == NULL) { return NULL; + } return _get_id(interp); } @@ -1545,8 +1609,9 @@ interp_run_string(PyObject *self, PyObject *args) { PyObject *id, *code; PyObject *shared = NULL; - if (!PyArg_UnpackTuple(args, "run_string", 2, 3, &id, &code, &shared)) + if (!PyArg_UnpackTuple(args, "run_string", 2, 3, &id, &code, &shared)) { return NULL; + } if (!PyLong_Check(id)) { PyErr_SetString(PyExc_TypeError, "first arg (ID) must be an int"); return NULL; @@ -1559,14 +1624,16 @@ interp_run_string(PyObject *self, PyObject *args) // Look up the interpreter. PyInterpreterState *interp = _look_up(id); - if (interp == NULL) + if (interp == NULL) { return NULL; + } // Extract code. Py_ssize_t size; const char *codestr = PyUnicode_AsUTF8AndSize(code, &size); - if (codestr == NULL) + if (codestr == NULL) { return NULL; + } if (strlen(codestr) != (size_t)size) { PyErr_SetString(PyExc_ValueError, "source code string cannot contain null bytes"); @@ -1574,8 +1641,9 @@ interp_run_string(PyObject *self, PyObject *args) } // Run the code in the interpreter. - if (_run_script_in_interpreter(interp, codestr, shared) != 0) + if (_run_script_in_interpreter(interp, codestr, shared) != 0) { return NULL; + } Py_RETURN_NONE; } @@ -1591,10 +1659,12 @@ static PyObject * object_is_shareable(PyObject *self, PyObject *args) { PyObject *obj; - if (!PyArg_UnpackTuple(args, "is_shareable", 1, 1, &obj)) + if (!PyArg_UnpackTuple(args, "is_shareable", 1, 1, &obj)) { return NULL; - if (_PyObject_CheckCrossInterpreterData(obj) == 0) + } + if (_PyObject_CheckCrossInterpreterData(obj) == 0) { Py_RETURN_TRUE; + } PyErr_Clear(); Py_RETURN_FALSE; } @@ -1610,21 +1680,25 @@ static PyObject * interp_is_running(PyObject *self, PyObject *args) { PyObject *id; - if (!PyArg_UnpackTuple(args, "is_running", 1, 1, &id)) + if (!PyArg_UnpackTuple(args, "is_running", 1, 1, &id)) { return NULL; + } if (!PyLong_Check(id)) { PyErr_SetString(PyExc_TypeError, "ID must be an int"); return NULL; } PyInterpreterState *interp = _look_up(id); - if (interp == NULL) + if (interp == NULL) { return NULL; + } int is_running = _is_running(interp); - if (is_running < 0) + if (is_running < 0) { return NULL; - if (is_running) + } + if (is_running) { Py_RETURN_TRUE; + } Py_RETURN_FALSE; } @@ -1637,8 +1711,9 @@ static PyObject * channel_create(PyObject *self) { int64_t cid = _channel_create(&_globals.channels); - if (cid < 0) + if (cid < 0) { return NULL; + } PyObject *id = (PyObject *)newchannelid(&ChannelIDtype, cid, 0, &_globals.channels, 0); if (id == NULL) { if (_channel_destroy(&_globals.channels, cid) != 0) { @@ -1659,14 +1734,17 @@ static PyObject * channel_destroy(PyObject *self, PyObject *args) { PyObject *id; - if (!PyArg_UnpackTuple(args, "channel_destroy", 1, 1, &id)) + if (!PyArg_UnpackTuple(args, "channel_destroy", 1, 1, &id)) { return NULL; + } int64_t cid = _coerce_id(id); - if (cid < 0) + if (cid < 0) { return NULL; + } - if (_channel_destroy(&_globals.channels, cid) != 0) + if (_channel_destroy(&_globals.channels, cid) != 0) { return NULL; + } Py_RETURN_NONE; } @@ -1682,8 +1760,9 @@ channel_list_all(PyObject *self) int64_t count = 0; int64_t *cids = _channels_list_all(&_globals.channels, &count); if (cids == NULL) { - if (count == 0) + if (count == 0) { return PyList_New(0); + } return NULL; } PyObject *ids = PyList_New((Py_ssize_t)count); @@ -1714,14 +1793,17 @@ channel_send(PyObject *self, PyObject *args) { PyObject *id; PyObject *obj; - if (!PyArg_UnpackTuple(args, "channel_send", 2, 2, &id, &obj)) + if (!PyArg_UnpackTuple(args, "channel_send", 2, 2, &id, &obj)) { return NULL; + } int64_t cid = _coerce_id(id); - if (cid < 0) + if (cid < 0) { return NULL; + } - if (_channel_send(&_globals.channels, cid, obj) != 0) + if (_channel_send(&_globals.channels, cid, obj) != 0) { return NULL; + } Py_RETURN_NONE; } @@ -1734,11 +1816,13 @@ static PyObject * channel_recv(PyObject *self, PyObject *args) { PyObject *id; - if (!PyArg_UnpackTuple(args, "channel_recv", 1, 1, &id)) + if (!PyArg_UnpackTuple(args, "channel_recv", 1, 1, &id)) { return NULL; + } int64_t cid = _coerce_id(id); - if (cid < 0) + if (cid < 0) { return NULL; + } return _channel_recv(&_globals.channels, cid); } @@ -1752,14 +1836,17 @@ static PyObject * channel_close(PyObject *self, PyObject *args, PyObject *kwds) { PyObject *id; - if (!PyArg_UnpackTuple(args, "channel_recv", 1, 1, &id)) + if (!PyArg_UnpackTuple(args, "channel_recv", 1, 1, &id)) { return NULL; + } int64_t cid = _coerce_id(id); - if (cid < 0) + if (cid < 0) { return NULL; + } - if (_channel_close(&_globals.channels, cid) != 0) + if (_channel_close(&_globals.channels, cid) != 0) { return NULL; + } Py_RETURN_NONE; } @@ -1782,20 +1869,24 @@ channel_drop_interpreter(PyObject *self, PyObject *args, PyObject *kwds) return NULL; int64_t cid = _coerce_id(id); - if (cid < 0) + if (cid < 0) { return NULL; + } if (send < 0 && recv < 0) { send = 1; recv = 1; } else { - if (send < 0) + if (send < 0) { send = 0; - if (recv < 0) + } + if (recv < 0) { recv = 0; + } } - if (_channel_drop(&_globals.channels, cid, send, recv) != 0) + if (_channel_drop(&_globals.channels, cid, send, recv) != 0) { return NULL; + } Py_RETURN_NONE; } @@ -1874,32 +1965,38 @@ static struct PyModuleDef interpretersmodule = { PyMODINIT_FUNC PyInit__xxsubinterpreters(void) { - if (_init_globals() != 0) + if (_init_globals() != 0) { return NULL; + } /* Initialize types */ ChannelIDtype.tp_base = &PyLong_Type; - if (PyType_Ready(&ChannelIDtype) != 0) + if (PyType_Ready(&ChannelIDtype) != 0) { return NULL; + } /* Create the module */ PyObject *module = PyModule_Create(&interpretersmodule); - if (module == NULL) + if (module == NULL) { return NULL; + } /* Add exception types */ PyObject *ns = PyModule_GetDict(module); // borrowed - if (interp_exceptions_init(ns) != 0) + if (interp_exceptions_init(ns) != 0) { return NULL; - if (channel_exceptions_init(ns) != 0) + } + if (channel_exceptions_init(ns) != 0) { return NULL; + } /* Add other types */ Py_INCREF(&ChannelIDtype); PyDict_SetItemString(ns, "ChannelID", (PyObject *)&ChannelIDtype); - if (_PyCrossInterpreterData_Register_Class(&ChannelIDtype, _channelid_shared)) + if (_PyCrossInterpreterData_Register_Class(&ChannelIDtype, _channelid_shared)) { return NULL; + } return module; } From ee7141c3cc58e0cacbd00e5448589287327b9ccf Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 26 Jan 2018 20:34:40 +0000 Subject: [PATCH 62/78] Check for a NULL return from PyMem_NEW(). --- Modules/_xxsubinterpretersmodule.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index cdd2da4301f89b..76c9cbd36400a9 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -75,6 +75,9 @@ _get_shared_ns(PyObject *shareable, Py_ssize_t *lenp) } struct _shareditem *shared = PyMem_NEW(struct _shareditem, len+1); + if (shared == NULL) { + return NULL; + } for (Py_ssize_t i=0; i < len; i++) { *(shared + i) = (struct _shareditem){0}; } @@ -893,6 +896,10 @@ _channel_send(struct _channels *channels, int64_t id, PyObject *obj) // Convert the object to cross-interpreter data. _PyCrossInterpreterData *data = PyMem_Malloc(sizeof(_PyCrossInterpreterData)); + if (data == NULL) { + PyThread_release_lock(mutex); + return -1; + } if (_PyObject_GetCrossInterpreterData(obj, data) != 0) { PyThread_release_lock(mutex); return -1; @@ -1199,6 +1206,9 @@ static int _channelid_shared(PyObject *obj, _PyCrossInterpreterData *data) { struct _channelid_xid *xid = PyMem_Malloc(sizeof(struct _channelid_xid)); + if (xid == NULL) { + return -1; + } xid->id = ((channelid *)obj)->id; xid->end = ((channelid *)obj)->end; From 8ec5137fd1fbc737229878d769c441a383889904 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 26 Jan 2018 20:45:19 +0000 Subject: [PATCH 63/78] Return the error. --- Modules/_xxsubinterpretersmodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 76c9cbd36400a9..9727261c4a8bcb 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -154,7 +154,7 @@ _get_shared_exception(void) } if (msg == NULL) { err->msg = "unable to format exception"; - return NULL; + return err; } err->msg = (char *)PyUnicode_AsUTF8(msg); if (err->msg == NULL) { From 65d822eba480fee5a7fba180014fbcde6fd16629 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 26 Jan 2018 20:50:41 +0000 Subject: [PATCH 64/78] Always check the result of PyDict_SetItemString(). --- Modules/_xxsubinterpretersmodule.c | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 9727261c4a8bcb..542af5791d791a 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -176,7 +176,9 @@ interp_exceptions_init(PyObject *ns) if (RunFailedError == NULL) { return -1; } - PyDict_SetItemString(ns, "RunFailedError", RunFailedError); + if (PyDict_SetItemString(ns, "RunFailedError", RunFailedError) != 0) { + return -1; + } return 0; } @@ -205,7 +207,9 @@ channel_exceptions_init(PyObject *ns) if (ChannelError == NULL) { return -1; } - PyDict_SetItemString(ns, "ChannelError", ChannelError); + if (PyDict_SetItemString(ns, "ChannelError", ChannelError) != 0) { + return -1; + } // An operation tried to use a channel that doesn't exist. ChannelNotFoundError = PyErr_NewException("_xxsubinterpreters.ChannelNotFoundError", @@ -213,7 +217,9 @@ channel_exceptions_init(PyObject *ns) if (ChannelNotFoundError == NULL) { return -1; } - PyDict_SetItemString(ns, "ChannelNotFoundError", ChannelNotFoundError); + if (PyDict_SetItemString(ns, "ChannelNotFoundError", ChannelNotFoundError) != 0) { + return -1; + } // An operation tried to use a closed channel. ChannelClosedError = PyErr_NewException("_xxsubinterpreters.ChannelClosedError", @@ -221,7 +227,9 @@ channel_exceptions_init(PyObject *ns) if (ChannelClosedError == NULL) { return -1; } - PyDict_SetItemString(ns, "ChannelClosedError", ChannelClosedError); + if (PyDict_SetItemString(ns, "ChannelClosedError", ChannelClosedError) != 0) { + return -1; + } // An operation tried to pop from an empty channel. ChannelEmptyError = PyErr_NewException("_xxsubinterpreters.ChannelEmptyError", @@ -229,7 +237,9 @@ channel_exceptions_init(PyObject *ns) if (ChannelEmptyError == NULL) { return -1; } - PyDict_SetItemString(ns, "ChannelEmptyError", ChannelEmptyError); + if (PyDict_SetItemString(ns, "ChannelEmptyError", ChannelEmptyError) != 0) { + return -1; + } return 0; } @@ -2002,7 +2012,9 @@ PyInit__xxsubinterpreters(void) /* Add other types */ Py_INCREF(&ChannelIDtype); - PyDict_SetItemString(ns, "ChannelID", (PyObject *)&ChannelIDtype); + if (PyDict_SetItemString(ns, "ChannelID", (PyObject *)&ChannelIDtype) != 0) { + return NULL; + } if (_PyCrossInterpreterData_Register_Class(&ChannelIDtype, _channelid_shared)) { return NULL; From 2df5affc8d93adde70eb8d4764e55cf3e296399a Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 26 Jan 2018 21:04:59 +0000 Subject: [PATCH 65/78] Add typedefs. --- Modules/_xxsubinterpretersmodule.c | 182 ++++++++++++++--------------- 1 file changed, 91 insertions(+), 91 deletions(-) diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 542af5791d791a..21b0bcb8ed42e9 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -47,21 +47,21 @@ _coerce_id(PyObject *id) /* data-sharing-specific code ***********************************************/ -struct _shareditem { +typedef struct _shareditem { Py_UNICODE *name; Py_ssize_t namelen; _PyCrossInterpreterData data; -}; +} _shareditem; void -_sharedns_clear(struct _shareditem *shared) +_sharedns_clear(_shareditem *shared) { - for (struct _shareditem *item=shared; item->name != NULL; item += 1) { + for (_shareditem *item=shared; item->name != NULL; item += 1) { _PyCrossInterpreterData_Release(&item->data); } } -static struct _shareditem * +static _shareditem * _get_shared_ns(PyObject *shareable, Py_ssize_t *lenp) { if (shareable == NULL || shareable == Py_None) { @@ -74,12 +74,12 @@ _get_shared_ns(PyObject *shareable, Py_ssize_t *lenp) return NULL; } - struct _shareditem *shared = PyMem_NEW(struct _shareditem, len+1); + _shareditem *shared = PyMem_NEW(_shareditem, len+1); if (shared == NULL) { return NULL; } for (Py_ssize_t i=0; i < len; i++) { - *(shared + i) = (struct _shareditem){0}; + *(shared + i) = (_shareditem){0}; } Py_ssize_t pos = 0; for (Py_ssize_t i=0; i < len; i++) { @@ -87,7 +87,7 @@ _get_shared_ns(PyObject *shareable, Py_ssize_t *lenp) if (PyDict_Next(shareable, &pos, &key, &value) == 0) { break; } - struct _shareditem *item = shared + i; + _shareditem *item = shared + i; if (_PyObject_GetCrossInterpreterData(value, &item->data) != 0) { break; @@ -108,7 +108,7 @@ _get_shared_ns(PyObject *shareable, Py_ssize_t *lenp) } static int -_shareditem_apply(struct _shareditem *item, PyObject *ns) +_shareditem_apply(_shareditem *item, PyObject *ns) { PyObject *name = PyUnicode_FromUnicode(item->name, item->namelen); if (name == NULL) { @@ -130,14 +130,14 @@ _shareditem_apply(struct _shareditem *item, PyObject *ns) // simulate, a la traceback.TracebackException), and even chain, a copy // of the exception in the calling interpreter. -struct _shared_exception { +typedef struct _shared_exception { char *msg; -}; +} _shared_exception; -static struct _shared_exception * +static _shared_exception * _get_shared_exception(void) { - struct _shared_exception *err = PyMem_NEW(struct _shared_exception, 1); + _shared_exception *err = PyMem_NEW(_shared_exception, 1); if (err == NULL) { return NULL; } @@ -184,7 +184,7 @@ interp_exceptions_init(PyObject *ns) } static void -_apply_shared_exception(struct _shared_exception *exc) +_apply_shared_exception(_shared_exception *exc) { PyErr_SetString(RunFailedError, exc->msg); } @@ -246,16 +246,16 @@ channel_exceptions_init(PyObject *ns) struct _channelend; -struct _channelend { +typedef struct _channelend { struct _channelend *next; int64_t interp; int open; -}; +} _channelend; -static struct _channelend * +static _channelend * _channelend_new(int64_t interp) { - struct _channelend *end = PyMem_Malloc(sizeof(struct _channelend)); + _channelend *end = PyMem_Malloc(sizeof(_channelend)); if (end == NULL) { return NULL; } @@ -269,19 +269,19 @@ _channelend_new(int64_t interp) } static void -_channelend_free_all(struct _channelend *end) { +_channelend_free_all(_channelend *end) { while (end != NULL) { - struct _channelend *last = end; + _channelend *last = end; end = end->next; PyMem_Free(last); } } -static struct _channelend * -_channelend_find(struct _channelend *first, int64_t interp, struct _channelend **pprev) +static _channelend * +_channelend_find(_channelend *first, int64_t interp, _channelend **pprev) { - struct _channelend *prev = NULL; - struct _channelend *end = first; + _channelend *prev = NULL; + _channelend *end = first; while (end != NULL) { if (end->interp == interp) { break; @@ -297,10 +297,10 @@ _channelend_find(struct _channelend *first, int64_t interp, struct _channelend * struct _channelitem; -struct _channelitem { +typedef struct _channelitem { _PyCrossInterpreterData *data; struct _channelitem *next; -}; +} _channelitem; struct _channel; @@ -310,8 +310,8 @@ typedef struct _channel { int open; int64_t count; - struct _channelitem *first; - struct _channelitem *last; + _channelitem *first; + _channelitem *last; // Note that the list entries are never removed for interpreter // for which the channel is closed. This should be a problem in @@ -319,8 +319,8 @@ typedef struct _channel { // interpreter is destroyed. int64_t numsendopen; int64_t numrecvopen; - struct _channelend *send; - struct _channelend *recv; + _channelend *send; + _channelend *recv; } _PyChannelState; static _PyChannelState * @@ -350,10 +350,10 @@ _channel_new(void) return chan; } -static struct _channelend * -_channel_add_end(_PyChannelState *chan, struct _channelend *prev, int64_t interp, int send) +static _channelend * +_channel_add_end(_PyChannelState *chan, _channelend *prev, int64_t interp, int send) { - struct _channelend *end = _channelend_new(interp); + _channelend *end = _channelend_new(interp); if (end == NULL) { return NULL; } @@ -378,7 +378,7 @@ _channel_add_end(_PyChannelState *chan, struct _channelend *prev, int64_t interp return end; } -static struct _channelend * +static _channelend * _channel_associate_end(_PyChannelState *chan, int64_t interp, int send) { if (!chan->open) { @@ -386,8 +386,8 @@ _channel_associate_end(_PyChannelState *chan, int64_t interp, int send) return NULL; } - struct _channelend *prev; - struct _channelend *end = _channelend_find(send ? chan->send : chan->recv, interp, &prev); + _channelend *prev; + _channelend *end = _channelend_find(send ? chan->send : chan->recv, interp, &prev); if (end != NULL) { if (!end->open) { PyErr_SetString(ChannelClosedError, "channel already closed"); @@ -400,7 +400,7 @@ _channel_associate_end(_PyChannelState *chan, int64_t interp, int send) } static void -_channel_close_channelend(_PyChannelState *chan, struct _channelend *end, int send) +_channel_close_channelend(_PyChannelState *chan, _channelend *end, int send) { end->open = 0; if (send) { @@ -422,8 +422,8 @@ _channel_close_interpreter(_PyChannelState *chan, int64_t interp, int which) goto done; } - struct _channelend *prev; - struct _channelend *end; + _channelend *prev; + _channelend *end; if (which >= 0) { // send/both end = _channelend_find(chan->send, interp, &prev); if (end == NULL) { @@ -476,7 +476,7 @@ _channel_close_all(_PyChannelState *chan) // the channel as closed already. // Ensure all the "send"-associated interpreters are closed. - struct _channelend *end; + _channelend *end; for (end = chan->send; end != NULL; end = end->next) { _channel_close_channelend(chan, end, 1); } @@ -502,7 +502,7 @@ _channel_add(_PyChannelState *chan, int64_t interp, _PyCrossInterpreterData *dat goto done; } - struct _channelitem *item = PyMem_Malloc(sizeof(struct _channelitem)); + _channelitem *item = PyMem_Malloc(sizeof(_channelitem)); if (item == NULL) { goto done; } @@ -530,7 +530,7 @@ _channel_next(_PyChannelState *chan, int64_t interp) goto done; } - struct _channelitem *item = chan->first; + _channelitem *item = chan->first; if (item == NULL) { goto done; } @@ -551,11 +551,11 @@ _channel_next(_PyChannelState *chan, int64_t interp) static void _channel_clear(_PyChannelState *chan) { - struct _channelitem *item = chan->first; + _channelitem *item = chan->first; while (item != NULL) { _PyCrossInterpreterData_Release(item->data); PyMem_Free(item->data); - struct _channelitem *last = item; + _channelitem *last = item; item = item->next; PyMem_Free(last); } @@ -578,17 +578,17 @@ _channel_free(_PyChannelState *chan) struct _channelref; -struct _channelref { +typedef struct _channelref { int64_t id; _PyChannelState *chan; struct _channelref *next; Py_ssize_t objcount; -}; +} _channelref; -static struct _channelref * +static _channelref * _channelref_new(int64_t id, _PyChannelState *chan) { - struct _channelref *ref = PyMem_Malloc(sizeof(struct _channelref)); + _channelref *ref = PyMem_Malloc(sizeof(_channelref)); if (ref == NULL) { return NULL; } @@ -599,11 +599,11 @@ _channelref_new(int64_t id, _PyChannelState *chan) return ref; } -static struct _channelref * -_channelref_find(struct _channelref *first, int64_t id, struct _channelref **pprev) +static _channelref * +_channelref_find(_channelref *first, int64_t id, _channelref **pprev) { - struct _channelref *prev = NULL; - struct _channelref *ref = first; + _channelref *prev = NULL; + _channelref *ref = first; while (ref != NULL) { if (ref->id == id) { break; @@ -617,15 +617,15 @@ _channelref_find(struct _channelref *first, int64_t id, struct _channelref **ppr return ref; } -struct _channels { +typedef struct _channels { PyThread_type_lock mutex; - struct _channelref *head; + _channelref *head; int64_t numopen; int64_t next_id; -}; +} _channels; static int -_channels_init(struct _channels *channels) +_channels_init(_channels *channels) { if (channels->mutex == NULL) { channels->mutex = PyThread_allocate_lock(); @@ -642,7 +642,7 @@ _channels_init(struct _channels *channels) } static int64_t -_channels_next_id(struct _channels *channels) // needs lock +_channels_next_id(_channels *channels) // needs lock { int64_t id = channels->next_id; if (id < 0) { @@ -656,7 +656,7 @@ _channels_next_id(struct _channels *channels) // needs lock } static _PyChannelState * -_channels_lookup(struct _channels *channels, int64_t id, PyThread_type_lock *pmutex) +_channels_lookup(_channels *channels, int64_t id, PyThread_type_lock *pmutex) { _PyChannelState *chan = NULL; PyThread_acquire_lock(channels->mutex, WAIT_LOCK); @@ -664,7 +664,7 @@ _channels_lookup(struct _channels *channels, int64_t id, PyThread_type_lock *pmu *pmutex = NULL; } - struct _channelref *ref = _channelref_find(channels->head, id, NULL); + _channelref *ref = _channelref_find(channels->head, id, NULL); if (ref == NULL) { PyErr_Format(ChannelNotFoundError, "channel %d not found", id); goto done; @@ -688,7 +688,7 @@ _channels_lookup(struct _channels *channels, int64_t id, PyThread_type_lock *pmu } static int64_t -_channels_add(struct _channels *channels, _PyChannelState *chan) +_channels_add(_channels *channels, _PyChannelState *chan) { int64_t cid = -1; PyThread_acquire_lock(channels->mutex, WAIT_LOCK); @@ -698,7 +698,7 @@ _channels_add(struct _channels *channels, _PyChannelState *chan) if (id < 0) { goto done; } - struct _channelref *ref = _channelref_new(id, chan); + _channelref *ref = _channelref_new(id, chan); if (ref == NULL) { goto done; } @@ -716,7 +716,7 @@ _channels_add(struct _channels *channels, _PyChannelState *chan) } static int -_channels_close(struct _channels *channels, int64_t cid, _PyChannelState **pchan) +_channels_close(_channels *channels, int64_t cid, _PyChannelState **pchan) { int res = -1; PyThread_acquire_lock(channels->mutex, WAIT_LOCK); @@ -724,7 +724,7 @@ _channels_close(struct _channels *channels, int64_t cid, _PyChannelState **pchan *pchan = NULL; } - struct _channelref *ref = _channelref_find(channels->head, cid, NULL); + _channelref *ref = _channelref_find(channels->head, cid, NULL); if (ref == NULL) { PyErr_Format(ChannelNotFoundError, "channel %d not found", cid); goto done; @@ -751,7 +751,7 @@ _channels_close(struct _channels *channels, int64_t cid, _PyChannelState **pchan } static void -_channels_remove_ref(struct _channels *channels, struct _channelref *ref, struct _channelref *prev, _PyChannelState **pchan) +_channels_remove_ref(_channels *channels, _channelref *ref, _channelref *prev, _PyChannelState **pchan) { if (ref == channels->head) { channels->head = ref->next; @@ -768,7 +768,7 @@ _channels_remove_ref(struct _channels *channels, struct _channelref *ref, struct } static int -_channels_remove(struct _channels *channels, int64_t id, _PyChannelState **pchan) +_channels_remove(_channels *channels, int64_t id, _PyChannelState **pchan) { int res = -1; PyThread_acquire_lock(channels->mutex, WAIT_LOCK); @@ -777,8 +777,8 @@ _channels_remove(struct _channels *channels, int64_t id, _PyChannelState **pchan *pchan = NULL; } - struct _channelref *prev = NULL; - struct _channelref *ref = _channelref_find(channels->head, id, &prev); + _channelref *prev = NULL; + _channelref *ref = _channelref_find(channels->head, id, &prev); if (ref == NULL) { PyErr_Format(ChannelNotFoundError, "channel %d not found", id); goto done; @@ -793,12 +793,12 @@ _channels_remove(struct _channels *channels, int64_t id, _PyChannelState **pchan } static int -_channels_add_id_object(struct _channels *channels, int64_t id) +_channels_add_id_object(_channels *channels, int64_t id) { int res = -1; PyThread_acquire_lock(channels->mutex, WAIT_LOCK); - struct _channelref *ref = _channelref_find(channels->head, id, NULL); + _channelref *ref = _channelref_find(channels->head, id, NULL); if (ref == NULL) { PyErr_Format(ChannelNotFoundError, "channel %d not found", id); goto done; @@ -812,12 +812,12 @@ _channels_add_id_object(struct _channels *channels, int64_t id) } static void -_channels_drop_id_object(struct _channels *channels, int64_t id) +_channels_drop_id_object(_channels *channels, int64_t id) { PyThread_acquire_lock(channels->mutex, WAIT_LOCK); - struct _channelref *prev = NULL; - struct _channelref *ref = _channelref_find(channels->head, id, &prev); + _channelref *prev = NULL; + _channelref *ref = _channelref_find(channels->head, id, &prev); if (ref == NULL) { // Already destroyed. goto done; @@ -838,7 +838,7 @@ _channels_drop_id_object(struct _channels *channels, int64_t id) } int64_t * -_channels_list_all(struct _channels *channels, int64_t *count) +_channels_list_all(_channels *channels, int64_t *count) { int64_t *cids = NULL; PyThread_acquire_lock(channels->mutex, WAIT_LOCK); @@ -846,7 +846,7 @@ _channels_list_all(struct _channels *channels, int64_t *count) if (ids == NULL) { goto done; } - struct _channelref *ref = channels->head; + _channelref *ref = channels->head; for (int64_t i=0; ref != NULL; ref = ref->next, i++) { ids[i] = ref->id; } @@ -861,7 +861,7 @@ _channels_list_all(struct _channels *channels, int64_t *count) /* "high"-level channel-related functions */ static int64_t -_channel_create(struct _channels *channels) +_channel_create(_channels *channels) { _PyChannelState *chan = _channel_new(); if (chan == NULL) { @@ -876,7 +876,7 @@ _channel_create(struct _channels *channels) } static int -_channel_destroy(struct _channels *channels, int64_t id) +_channel_destroy(_channels *channels, int64_t id) { _PyChannelState *chan = NULL; if (_channels_remove(channels, id, &chan) != 0) { @@ -889,7 +889,7 @@ _channel_destroy(struct _channels *channels, int64_t id) } static int -_channel_send(struct _channels *channels, int64_t id, PyObject *obj) +_channel_send(_channels *channels, int64_t id, PyObject *obj) { PyInterpreterState *interp = _get_current(); if (interp == NULL) { @@ -928,7 +928,7 @@ _channel_send(struct _channels *channels, int64_t id, PyObject *obj) } static PyObject * -_channel_recv(struct _channels *channels, int64_t id) +_channel_recv(_channels *channels, int64_t id) { PyInterpreterState *interp = _get_current(); if (interp == NULL) { @@ -962,7 +962,7 @@ _channel_recv(struct _channels *channels, int64_t id) } static int -_channel_drop(struct _channels *channels, int64_t id, int send, int recv) +_channel_drop(_channels *channels, int64_t id, int send, int recv) { PyInterpreterState *interp = _get_current(); if (interp == NULL) { @@ -984,7 +984,7 @@ _channel_drop(struct _channels *channels, int64_t id, int send, int recv) } static int -_channel_close(struct _channels *channels, int64_t id) +_channel_close(_channels *channels, int64_t id) { return _channels_close(channels, id, NULL); } @@ -1000,11 +1000,11 @@ typedef struct channelid { PyObject_HEAD int64_t id; int end; - struct _channels *channels; + _channels *channels; } channelid; static channelid * -newchannelid(PyTypeObject *cls, int64_t cid, int end, struct _channels *channels, int force) +newchannelid(PyTypeObject *cls, int64_t cid, int end, _channels *channels, int force) { channelid *self = PyObject_New(channelid, cls); if (self == NULL) { @@ -1027,7 +1027,7 @@ newchannelid(PyTypeObject *cls, int64_t cid, int end, struct _channels *channels return self; } -static struct _channels * _global_channels(void); +static _channels * _global_channels(void); static PyObject * channelid_new(PyTypeObject *cls, PyObject *args, PyObject *kwds) @@ -1075,7 +1075,7 @@ static void channelid_dealloc(PyObject *v) { int64_t cid = ((channelid *)v)->id; - struct _channels *channels = ((channelid *)v)->channels; + _channels *channels = ((channelid *)v)->channels; Py_TYPE(v)->tp_free(v); _channels_drop_id_object(channels, cid); @@ -1364,8 +1364,8 @@ _ensure_not_running(PyInterpreterState *interp) static int _run_script(PyInterpreterState *interp, const char *codestr, - struct _shareditem *shared, Py_ssize_t num_shared, - struct _shared_exception **exc) + _shareditem *shared, Py_ssize_t num_shared, + _shared_exception **exc) { assert(num_shared >= 0); PyObject *main_mod = PyMapping_GetItemString(interp->modules, "__main__"); @@ -1382,7 +1382,7 @@ _run_script(PyInterpreterState *interp, const char *codestr, // Apply the cross-interpreter data. if (shared != NULL) { for (Py_ssize_t i=0; i < num_shared; i++) { - struct _shareditem *item = &shared[i]; + _shareditem *item = &shared[i]; if (_shareditem_apply(item, ns) != 0) { Py_DECREF(ns); goto error; @@ -1417,7 +1417,7 @@ _run_script_in_interpreter(PyInterpreterState *interp, const char *codestr, } Py_ssize_t num_shared = -1; - struct _shareditem *shared = _get_shared_ns(shareables, &num_shared); + _shareditem *shared = _get_shared_ns(shareables, &num_shared); if (shared == NULL && PyErr_Occurred()) { return -1; } @@ -1427,7 +1427,7 @@ _run_script_in_interpreter(PyInterpreterState *interp, const char *codestr, PyThreadState *save_tstate = PyThreadState_Swap(tstate); // Run the script. - struct _shared_exception *exc = NULL; + _shared_exception *exc = NULL; int result = _run_script(interp, codestr, shared, num_shared, &exc); // Switch back. @@ -1460,7 +1460,7 @@ _run_script_in_interpreter(PyInterpreterState *interp, const char *codestr, the data that we need to share between interpreters, so it cannot hold PyObject values. */ static struct globals { - struct _channels channels; + _channels channels; } _globals = {0}; static int @@ -1472,7 +1472,7 @@ _init_globals(void) return 0; } -static struct _channels * +static _channels * _global_channels(void) { return &_globals.channels; } From 9bb05cfb62fca1281c3af71fcbab17de7fd3b0d7 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 26 Jan 2018 21:06:00 +0000 Subject: [PATCH 66/78] _shared_exception -> _sharedexception --- Modules/_xxsubinterpretersmodule.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 21b0bcb8ed42e9..6f86df95bb83a4 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -130,14 +130,14 @@ _shareditem_apply(_shareditem *item, PyObject *ns) // simulate, a la traceback.TracebackException), and even chain, a copy // of the exception in the calling interpreter. -typedef struct _shared_exception { +typedef struct _sharedexception { char *msg; -} _shared_exception; +} _sharedexception; -static _shared_exception * +static _sharedexception * _get_shared_exception(void) { - _shared_exception *err = PyMem_NEW(_shared_exception, 1); + _sharedexception *err = PyMem_NEW(_sharedexception, 1); if (err == NULL) { return NULL; } @@ -184,7 +184,7 @@ interp_exceptions_init(PyObject *ns) } static void -_apply_shared_exception(_shared_exception *exc) +_apply_shared_exception(_sharedexception *exc) { PyErr_SetString(RunFailedError, exc->msg); } @@ -1365,7 +1365,7 @@ _ensure_not_running(PyInterpreterState *interp) static int _run_script(PyInterpreterState *interp, const char *codestr, _shareditem *shared, Py_ssize_t num_shared, - _shared_exception **exc) + _sharedexception **exc) { assert(num_shared >= 0); PyObject *main_mod = PyMapping_GetItemString(interp->modules, "__main__"); @@ -1427,7 +1427,7 @@ _run_script_in_interpreter(PyInterpreterState *interp, const char *codestr, PyThreadState *save_tstate = PyThreadState_Swap(tstate); // Run the script. - _shared_exception *exc = NULL; + _sharedexception *exc = NULL; int result = _run_script(interp, codestr, shared, num_shared, &exc); // Switch back. From 633a3d39060ecd8ec0e071fab4905257923bbda5 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 26 Jan 2018 21:08:03 +0000 Subject: [PATCH 67/78] Free memory in the error case. --- Modules/_xxsubinterpretersmodule.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 6f86df95bb83a4..209e9af8ccce45 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -332,6 +332,7 @@ _channel_new(void) } chan->mutex = PyThread_allocate_lock(); if (chan->mutex == NULL) { + PyMem_Free(chan); PyErr_SetString(ChannelError, "can't initialize mutex for new channel"); return NULL; } From 74e7241ea6975e29b2be39b699524b3696bc4d03 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 26 Jan 2018 21:10:30 +0000 Subject: [PATCH 68/78] Whitelist instead of blacklist for richcompare ops. --- Modules/_xxsubinterpretersmodule.c | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 209e9af8ccce45..1fd5ca363d1e9d 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -1164,11 +1164,7 @@ channelid_hash(PyObject *self) static PyObject * channelid_richcompare(PyObject *self, PyObject *other, int op) { - switch(op) { - case Py_LT: - case Py_LE: - case Py_GT: - case Py_GE: + if (op != Py_EQ && op != Py_NE) { Py_RETURN_NOTIMPLEMENTED; } From bb8619af9155e19bb3a3cf1889d0929ec2d9160a Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 26 Jan 2018 21:12:29 +0000 Subject: [PATCH 69/78] Decref in the error case. --- Modules/_xxsubinterpretersmodule.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 1fd5ca363d1e9d..0f5a49cf710cfb 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -1573,10 +1573,12 @@ interp_list_all(PyObject *self) while (interp != NULL) { id = _get_id(interp); if (id == NULL) { + Py_DECREF(ids); return NULL; } // insert at front of list if (PyList_Insert(ids, 0, id) < 0) { + Py_DECREF(ids); return NULL; } From d7855734948c8e2af20359dca382ebdca29393d2 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 26 Jan 2018 21:29:55 +0000 Subject: [PATCH 70/78] Add a Misc/NEWS entry. --- .../next/Tests/2018-01-26-21-29-09.bpo-32604.7iazNx.rst | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 Misc/NEWS.d/next/Tests/2018-01-26-21-29-09.bpo-32604.7iazNx.rst diff --git a/Misc/NEWS.d/next/Tests/2018-01-26-21-29-09.bpo-32604.7iazNx.rst b/Misc/NEWS.d/next/Tests/2018-01-26-21-29-09.bpo-32604.7iazNx.rst new file mode 100644 index 00000000000000..f5472f9fe23f8d --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2018-01-26-21-29-09.bpo-32604.7iazNx.rst @@ -0,0 +1,4 @@ +Add a new "_xxsubinterpreters" extension module that exposes the existing +subinterpreter C-API and a new cross-interpreter data sharing mechanism. The +module is primarily intended for more thorough testing of the existing +subinterpreter support. From f856035e450c4ce1e2ea3aedb7be86bdbec843bd Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Sat, 27 Jan 2018 00:11:45 +0000 Subject: [PATCH 71/78] Do not try to print a bytes object. --- Lib/test/test__xxsubinterpreters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test__xxsubinterpreters.py b/Lib/test/test__xxsubinterpreters.py index 144b90d42fc96f..2b170443a3b638 100644 --- a/Lib/test/test__xxsubinterpreters.py +++ b/Lib/test/test__xxsubinterpreters.py @@ -864,7 +864,7 @@ def test_drop_multiple_users(self): import _xxsubinterpreters as _interpreters obj = _interpreters.channel_recv({int(cid)}) _interpreters.channel_drop_interpreter({int(cid)}) - print(obj) + print(repr(obj)) """)) interpreters.run_string(id1, dedent(f""" _interpreters.channel_drop_interpreter({int(cid)}) From ba945715b241629e0045cb19a798dd653594e313 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Sat, 27 Jan 2018 00:35:38 +0000 Subject: [PATCH 72/78] Drop some outdated comments. --- Modules/_xxsubinterpretersmodule.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 0f5a49cf710cfb..e796d995548d51 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -1086,7 +1086,6 @@ static PyObject * channelid_repr(PyObject *self) { PyTypeObject *type = Py_TYPE(self); - // XXX Use the qualname? const char *name = _PyType_Name(type); channelid *cid = (channelid *)self; @@ -1299,8 +1298,12 @@ static PyTypeObject ChannelIDtype = { 0, /* tp_dictoffset */ 0, /* tp_init */ 0, /* tp_alloc */ + // Note that we do not set tp_new to channelid_new. Instead we + // set it to NULL, meaning it cannot be instantiated from Python + // code. We do this because there is a strong relationship between + // channel IDs and the channel lifecycle, so this limitation avoids + // related complications. NULL, /* tp_new */ - //(newfunc)channelid_new, /* tp_new */ }; /* interpreter-specific functions *******************************************/ From 90650014e1412fec176e7e467fdbedd9554e2a86 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Sat, 27 Jan 2018 00:53:48 +0000 Subject: [PATCH 73/78] Wrap long lines. --- Modules/_xxsubinterpretersmodule.c | 71 +++++++++++++++++++----------- 1 file changed, 45 insertions(+), 26 deletions(-) diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index e796d995548d51..908f8bc570a5eb 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -22,24 +22,29 @@ _coerce_id(PyObject *id) id = PyNumber_Long(id); if (id == NULL) { if (PyErr_ExceptionMatches(PyExc_TypeError)) { - PyErr_SetString(PyExc_TypeError, "'id' must be a non-negative int"); + PyErr_SetString(PyExc_TypeError, + "'id' must be a non-negative int"); } else { - PyErr_SetString(PyExc_ValueError, "'id' must be a non-negative int"); + PyErr_SetString(PyExc_ValueError, + "'id' must be a non-negative int"); } return -1; } long long cid = PyLong_AsLongLong(id); if (cid == -1 && PyErr_Occurred() != NULL) { - PyErr_SetString(PyExc_ValueError, "'id' must be a non-negative int"); + PyErr_SetString(PyExc_ValueError, + "'id' must be a non-negative int"); return -1; } if (cid < 0) { - PyErr_SetString(PyExc_ValueError, "'id' must be a non-negative int"); + PyErr_SetString(PyExc_ValueError, + "'id' must be a non-negative int"); return -1; } if (cid > INT64_MAX) { - PyErr_SetString(PyExc_ValueError, "'id' too large (must be 64-bit int)"); + PyErr_SetString(PyExc_ValueError, + "'id' too large (must be 64-bit int)"); return -1; } return cid; @@ -212,8 +217,8 @@ channel_exceptions_init(PyObject *ns) } // An operation tried to use a channel that doesn't exist. - ChannelNotFoundError = PyErr_NewException("_xxsubinterpreters.ChannelNotFoundError", - ChannelError, NULL); + ChannelNotFoundError = PyErr_NewException( + "_xxsubinterpreters.ChannelNotFoundError", ChannelError, NULL); if (ChannelNotFoundError == NULL) { return -1; } @@ -222,8 +227,8 @@ channel_exceptions_init(PyObject *ns) } // An operation tried to use a closed channel. - ChannelClosedError = PyErr_NewException("_xxsubinterpreters.ChannelClosedError", - ChannelError, NULL); + ChannelClosedError = PyErr_NewException( + "_xxsubinterpreters.ChannelClosedError", ChannelError, NULL); if (ChannelClosedError == NULL) { return -1; } @@ -232,8 +237,8 @@ channel_exceptions_init(PyObject *ns) } // An operation tried to pop from an empty channel. - ChannelEmptyError = PyErr_NewException("_xxsubinterpreters.ChannelEmptyError", - ChannelError, NULL); + ChannelEmptyError = PyErr_NewException( + "_xxsubinterpreters.ChannelEmptyError", ChannelError, NULL); if (ChannelEmptyError == NULL) { return -1; } @@ -333,7 +338,8 @@ _channel_new(void) chan->mutex = PyThread_allocate_lock(); if (chan->mutex == NULL) { PyMem_Free(chan); - PyErr_SetString(ChannelError, "can't initialize mutex for new channel"); + PyErr_SetString(ChannelError, + "can't initialize mutex for new channel"); return NULL; } @@ -352,7 +358,8 @@ _channel_new(void) } static _channelend * -_channel_add_end(_PyChannelState *chan, _channelend *prev, int64_t interp, int send) +_channel_add_end(_PyChannelState *chan, _channelend *prev, int64_t interp, + int send) { _channelend *end = _channelend_new(interp); if (end == NULL) { @@ -388,7 +395,8 @@ _channel_associate_end(_PyChannelState *chan, int64_t interp, int send) } _channelend *prev; - _channelend *end = _channelend_find(send ? chan->send : chan->recv, interp, &prev); + _channelend *end = _channelend_find(send ? chan->send : chan->recv, + interp, &prev); if (end != NULL) { if (!end->open) { PyErr_SetString(ChannelClosedError, "channel already closed"); @@ -494,7 +502,8 @@ _channel_close_all(_PyChannelState *chan) } static int -_channel_add(_PyChannelState *chan, int64_t interp, _PyCrossInterpreterData *data) +_channel_add(_PyChannelState *chan, int64_t interp, + _PyCrossInterpreterData *data) { int res = -1; @@ -632,7 +641,8 @@ _channels_init(_channels *channels) channels->mutex = PyThread_allocate_lock(); if (channels->mutex == NULL) { PyMem_Free(channels); - PyErr_SetString(ChannelError, "can't initialize mutex for channel management"); + PyErr_SetString(ChannelError, + "can't initialize mutex for channel management"); return -1; } } @@ -752,7 +762,8 @@ _channels_close(_channels *channels, int64_t cid, _PyChannelState **pchan) } static void -_channels_remove_ref(_channels *channels, _channelref *ref, _channelref *prev, _PyChannelState **pchan) +_channels_remove_ref(_channels *channels, _channelref *ref, _channelref *prev, + _PyChannelState **pchan) { if (ref == channels->head) { channels->head = ref->next; @@ -906,7 +917,7 @@ _channel_send(_channels *channels, int64_t id, PyObject *obj) // Past this point we are responsible for releasing the mutex. // Convert the object to cross-interpreter data. - _PyCrossInterpreterData *data = PyMem_Malloc(sizeof(_PyCrossInterpreterData)); + _PyCrossInterpreterData *data = PyMem_NEW(_PyCrossInterpreterData, 1); if (data == NULL) { PyThread_release_lock(mutex); return -1; @@ -1005,7 +1016,8 @@ typedef struct channelid { } channelid; static channelid * -newchannelid(PyTypeObject *cls, int64_t cid, int end, _channels *channels, int force) +newchannelid(PyTypeObject *cls, int64_t cid, int end, _channels *channels, + int force) { channelid *self = PyObject_New(channelid, cls); if (self == NULL) { @@ -1038,7 +1050,8 @@ channelid_new(PyTypeObject *cls, PyObject *args, PyObject *kwds) int send = -1; int recv = -1; int force = 0; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|$ppp:ChannelID.__init__", kwlist, + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "O|$ppp:ChannelID.__init__", kwlist, &id, &send, &recv, &force)) return NULL; @@ -1056,7 +1069,8 @@ channelid_new(PyTypeObject *cls, PyObject *args, PyObject *kwds) // Handle "send" and "recv". if (send == 0 && recv == 0) { - PyErr_SetString(PyExc_ValueError, "'send' and 'recv' cannot both be False"); + PyErr_SetString(PyExc_ValueError, + "'send' and 'recv' cannot both be False"); return NULL; } int end = 0; @@ -1205,7 +1219,8 @@ static PyObject * _channelid_from_xid(_PyCrossInterpreterData *data) { struct _channelid_xid *xid = (struct _channelid_xid *)data->data; - return (PyObject *)newchannelid(&ChannelIDtype, xid->id, xid->end, _global_channels(), 0); + return (PyObject *)newchannelid(&ChannelIDtype, xid->id, xid->end, + _global_channels(), 0); } static int @@ -1231,7 +1246,8 @@ channelid_end(PyObject *self, void *end) int force = 1; channelid *cid = (channelid *)self; if (end != NULL) { - return (PyObject *)newchannelid(Py_TYPE(self), cid->id, *(int *)end, cid->channels, force); + return (PyObject *)newchannelid(Py_TYPE(self), cid->id, *(int *)end, + cid->channels, force); } if (cid->end == CHANNEL_SEND) { @@ -1736,7 +1752,8 @@ channel_create(PyObject *self) if (cid < 0) { return NULL; } - PyObject *id = (PyObject *)newchannelid(&ChannelIDtype, cid, 0, &_globals.channels, 0); + PyObject *id = (PyObject *)newchannelid(&ChannelIDtype, cid, 0, + &_globals.channels, 0); if (id == NULL) { if (_channel_destroy(&_globals.channels, cid) != 0) { // XXX issue a warning? @@ -1793,7 +1810,8 @@ channel_list_all(PyObject *self) return NULL; } for (int64_t i=0; i < count; cids++, i++) { - PyObject *id = (PyObject *)newchannelid(&ChannelIDtype, *cids, 0, &_globals.channels, 0); + PyObject *id = (PyObject *)newchannelid(&ChannelIDtype, *cids, 0, + &_globals.channels, 0); if (id == NULL) { Py_DECREF(ids); ids = NULL; @@ -1886,7 +1904,8 @@ channel_drop_interpreter(PyObject *self, PyObject *args, PyObject *kwds) PyObject *id; int send = -1; int recv = -1; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|$pp:channel_drop_interpreter", kwlist, + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "O|$pp:channel_drop_interpreter", kwlist, &id, &send, &recv)) return NULL; From 2ebc68c970cb904731912b6665ba7311e9fdb178 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Sat, 27 Jan 2018 01:56:24 +0000 Subject: [PATCH 74/78] Fix some compiler warnings. --- Modules/_xxsubinterpretersmodule.c | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 908f8bc570a5eb..0ddb16f0fa04f2 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -1185,12 +1185,15 @@ channelid_richcompare(PyObject *self, PyObject *other, int op) Py_RETURN_NOTIMPLEMENTED; } - int64_t cid = ((channelid *)self)->id; - int64_t othercid; + channelid *cid = (channelid *)self; + int equal; if (PyObject_TypeCheck(other, &ChannelIDtype)) { - othercid = ((channelid *)other)->id; - if (((channelid *)other)->end != ((channelid *)self)->end) { - Py_RETURN_FALSE; + channelid *othercid = (channelid *)other; + if (cid->end != othercid->end) { + equal = 0; + } + else { + equal = (cid->id == othercid->id); } } else { @@ -1199,15 +1202,23 @@ channelid_richcompare(PyObject *self, PyObject *other, int op) PyErr_Clear(); Py_RETURN_NOTIMPLEMENTED; } - othercid = PyLong_AsLongLong(other); + int64_t othercid = PyLong_AsLongLong(other); + // XXX decref other here? if (othercid == -1 && PyErr_Occurred() != NULL) { return NULL; } if (othercid < 0 || othercid > INT64_MAX) { - Py_RETURN_FALSE; + equal = 0; + } + else { + equal = (cid->id == othercid); } } - Py_RETURN_RICHCOMPARE(othercid != cid, 0, op); + + if ((op == Py_EQ && equal) || (op == Py_NE && !equal)) { + Py_RETURN_TRUE; + } + Py_RETURN_FALSE; } struct _channelid_xid { @@ -1477,7 +1488,7 @@ _run_script_in_interpreter(PyInterpreterState *interp, const char *codestr, hold PyObject values. */ static struct globals { _channels channels; -} _globals = {0}; +} _globals = {{0}}; static int _init_globals(void) From 48291affab362c6156f8cfc5125897042ba2cf85 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Sat, 27 Jan 2018 20:12:54 +0000 Subject: [PATCH 75/78] Always use PyMem_NEW(). --- Modules/_xxsubinterpretersmodule.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 0ddb16f0fa04f2..57aaf73d132cb0 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -260,7 +260,7 @@ typedef struct _channelend { static _channelend * _channelend_new(int64_t interp) { - _channelend *end = PyMem_Malloc(sizeof(_channelend)); + _channelend *end = PyMem_NEW(_channelend, 1); if (end == NULL) { return NULL; } @@ -331,7 +331,7 @@ typedef struct _channel { static _PyChannelState * _channel_new(void) { - _PyChannelState *chan = PyMem_Malloc(sizeof(_PyChannelState)); + _PyChannelState *chan = PyMem_NEW(_PyChannelState, 1); if (chan == NULL) { return NULL; } @@ -512,7 +512,7 @@ _channel_add(_PyChannelState *chan, int64_t interp, goto done; } - _channelitem *item = PyMem_Malloc(sizeof(_channelitem)); + _channelitem *item = PyMem_NEW(_channelitem, 1); if (item == NULL) { goto done; } @@ -598,7 +598,7 @@ typedef struct _channelref { static _channelref * _channelref_new(int64_t id, _PyChannelState *chan) { - _channelref *ref = PyMem_Malloc(sizeof(_channelref)); + _channelref *ref = PyMem_NEW(_channelref, 1); if (ref == NULL) { return NULL; } @@ -854,7 +854,7 @@ _channels_list_all(_channels *channels, int64_t *count) { int64_t *cids = NULL; PyThread_acquire_lock(channels->mutex, WAIT_LOCK); - int64_t *ids = PyMem_Malloc(sizeof(int64_t) * channels->numopen); + int64_t *ids = PyMem_NEW(int64_t, channels->numopen); if (ids == NULL) { goto done; } @@ -1237,7 +1237,7 @@ _channelid_from_xid(_PyCrossInterpreterData *data) static int _channelid_shared(PyObject *obj, _PyCrossInterpreterData *data) { - struct _channelid_xid *xid = PyMem_Malloc(sizeof(struct _channelid_xid)); + struct _channelid_xid *xid = PyMem_NEW(struct _channelid_xid, 1); if (xid == NULL) { return -1; } From 39df95ee5c276988bba02655d8d5ded23fa1c9b9 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Sat, 27 Jan 2018 21:07:02 +0000 Subject: [PATCH 76/78] Do not try to allocate too much space. --- Modules/_xxsubinterpretersmodule.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 57aaf73d132cb0..ed79a13d4a4006 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -854,7 +854,12 @@ _channels_list_all(_channels *channels, int64_t *count) { int64_t *cids = NULL; PyThread_acquire_lock(channels->mutex, WAIT_LOCK); - int64_t *ids = PyMem_NEW(int64_t, channels->numopen); + int64_t numopen = channels->numopen; + if (numopen >= PY_SSIZE_T_MAX) { + PyErr_SetString(PyExc_RuntimeError, "too many channels open"); + goto done; + } + int64_t *ids = PyMem_NEW(int64_t, (Py_ssize_t)(channels->numopen)); if (ids == NULL) { goto done; } From b07e92803f380c9ab4486fce4f3f3657d7d9ce6d Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Sat, 27 Jan 2018 19:16:21 +0000 Subject: [PATCH 77/78] Add Windows build support. --- PCbuild/_xxsubinterpreters.vcxproj | 82 ++++++++++++++++++++++ PCbuild/_xxsubinterpreters.vcxproj.filters | 13 ++++ PCbuild/pcbuild.proj | 2 +- PCbuild/pcbuild.sln | 18 +++++ PCbuild/readme.txt | 1 + 5 files changed, 115 insertions(+), 1 deletion(-) create mode 100644 PCbuild/_xxsubinterpreters.vcxproj create mode 100644 PCbuild/_xxsubinterpreters.vcxproj.filters diff --git a/PCbuild/_xxsubinterpreters.vcxproj b/PCbuild/_xxsubinterpreters.vcxproj new file mode 100644 index 00000000000000..e71f1261c49dfb --- /dev/null +++ b/PCbuild/_xxsubinterpreters.vcxproj @@ -0,0 +1,82 @@ + + + + + Debug + Win32 + + + Debug + x64 + + + PGInstrument + Win32 + + + PGInstrument + x64 + + + PGUpdate + Win32 + + + PGUpdate + x64 + + + Release + Win32 + + + Release + x64 + + + + {XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX} + _xxsubinterpreters + Win32Proj + + + + + DynamicLibrary + NotSet + + + + .pyd + + + + + + + + + + <_ProjectFileVersion>10.0.30319.1 + + + + Py_BUILD_CORE;%(PreprocessorDefinitions) + + + + + + + + + + + {cf7ac3d1-e2df-41d2-bea6-1e2556cdea26} + false + + + + + + diff --git a/PCbuild/_xxsubinterpreters.vcxproj.filters b/PCbuild/_xxsubinterpreters.vcxproj.filters new file mode 100644 index 00000000000000..5bcc0fdeb6a60d --- /dev/null +++ b/PCbuild/_xxsubinterpreters.vcxproj.filters @@ -0,0 +1,13 @@ + + + + + {YYYYYYYY-YYYY-YYYY-YYYY-YYYYYYYYYYYY} + + + + + Source Files + + + diff --git a/PCbuild/pcbuild.proj b/PCbuild/pcbuild.proj index 5e341959bd3a2d..f8ad2c51b9c8a8 100644 --- a/PCbuild/pcbuild.proj +++ b/PCbuild/pcbuild.proj @@ -59,7 +59,7 @@ - + diff --git a/PCbuild/pcbuild.sln b/PCbuild/pcbuild.sln index 0443610331e79b..cc045316b23e53 100644 --- a/PCbuild/pcbuild.sln +++ b/PCbuild/pcbuild.sln @@ -95,6 +95,8 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "liblzma", "liblzma.vcxproj" EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_distutils_findvs", "_distutils_findvs.vcxproj", "{41ADEDF9-11D8-474E-B4D7-BB82332C878E}" EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_xxsubinterpreters", "_xxsubinterpreters.vcxproj", "{XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Win32 = Debug|Win32 @@ -711,6 +713,22 @@ Global {41ADEDF9-11D8-474E-B4D7-BB82332C878E}.Release|Win32.Build.0 = Release|Win32 {41ADEDF9-11D8-474E-B4D7-BB82332C878E}.Release|x64.ActiveCfg = Release|x64 {41ADEDF9-11D8-474E-B4D7-BB82332C878E}.Release|x64.Build.0 = Release|x64 + {XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX}.Debug|Win32.ActiveCfg = Debug|Win32 + {XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX}.Debug|Win32.Build.0 = Debug|Win32 + {XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX}.Debug|x64.ActiveCfg = Debug|x64 + {XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX}.Debug|x64.Build.0 = Debug|x64 + {XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 + {XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 + {XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 + {XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX}.PGInstrument|x64.Build.0 = PGInstrument|x64 + {XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 + {XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 + {XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 + {XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX}.PGUpdate|x64.Build.0 = PGUpdate|x64 + {XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX}.Release|Win32.ActiveCfg = Release|Win32 + {XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX}.Release|Win32.Build.0 = Release|Win32 + {XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX}.Release|x64.ActiveCfg = Release|x64 + {XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX}.Release|x64.Build.0 = Release|x64 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/PCbuild/readme.txt b/PCbuild/readme.txt index d28435b4c4d7d9..d258098e74f9ff 100644 --- a/PCbuild/readme.txt +++ b/PCbuild/readme.txt @@ -145,6 +145,7 @@ _testconsole _testimportmultiple _testmultiphase _tkinter +_xxsubinterpreters pyexpat select unicodedata From 1bfed0f4ed5836cfc77b152cf0c0658b7f2127e5 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 30 Jan 2018 00:51:25 +0000 Subject: [PATCH 78/78] Drop Windows build support (for now). --- PCbuild/_xxsubinterpreters.vcxproj | 82 ---------------------- PCbuild/_xxsubinterpreters.vcxproj.filters | 13 ---- PCbuild/pcbuild.proj | 2 +- PCbuild/pcbuild.sln | 18 ----- PCbuild/readme.txt | 1 - 5 files changed, 1 insertion(+), 115 deletions(-) delete mode 100644 PCbuild/_xxsubinterpreters.vcxproj delete mode 100644 PCbuild/_xxsubinterpreters.vcxproj.filters diff --git a/PCbuild/_xxsubinterpreters.vcxproj b/PCbuild/_xxsubinterpreters.vcxproj deleted file mode 100644 index e71f1261c49dfb..00000000000000 --- a/PCbuild/_xxsubinterpreters.vcxproj +++ /dev/null @@ -1,82 +0,0 @@ - - - - - Debug - Win32 - - - Debug - x64 - - - PGInstrument - Win32 - - - PGInstrument - x64 - - - PGUpdate - Win32 - - - PGUpdate - x64 - - - Release - Win32 - - - Release - x64 - - - - {XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX} - _xxsubinterpreters - Win32Proj - - - - - DynamicLibrary - NotSet - - - - .pyd - - - - - - - - - - <_ProjectFileVersion>10.0.30319.1 - - - - Py_BUILD_CORE;%(PreprocessorDefinitions) - - - - - - - - - - - {cf7ac3d1-e2df-41d2-bea6-1e2556cdea26} - false - - - - - - diff --git a/PCbuild/_xxsubinterpreters.vcxproj.filters b/PCbuild/_xxsubinterpreters.vcxproj.filters deleted file mode 100644 index 5bcc0fdeb6a60d..00000000000000 --- a/PCbuild/_xxsubinterpreters.vcxproj.filters +++ /dev/null @@ -1,13 +0,0 @@ - - - - - {YYYYYYYY-YYYY-YYYY-YYYY-YYYYYYYYYYYY} - - - - - Source Files - - - diff --git a/PCbuild/pcbuild.proj b/PCbuild/pcbuild.proj index f8ad2c51b9c8a8..5e341959bd3a2d 100644 --- a/PCbuild/pcbuild.proj +++ b/PCbuild/pcbuild.proj @@ -59,7 +59,7 @@ - + diff --git a/PCbuild/pcbuild.sln b/PCbuild/pcbuild.sln index cc045316b23e53..0443610331e79b 100644 --- a/PCbuild/pcbuild.sln +++ b/PCbuild/pcbuild.sln @@ -95,8 +95,6 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "liblzma", "liblzma.vcxproj" EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_distutils_findvs", "_distutils_findvs.vcxproj", "{41ADEDF9-11D8-474E-B4D7-BB82332C878E}" EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_xxsubinterpreters", "_xxsubinterpreters.vcxproj", "{XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX}" -EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Win32 = Debug|Win32 @@ -713,22 +711,6 @@ Global {41ADEDF9-11D8-474E-B4D7-BB82332C878E}.Release|Win32.Build.0 = Release|Win32 {41ADEDF9-11D8-474E-B4D7-BB82332C878E}.Release|x64.ActiveCfg = Release|x64 {41ADEDF9-11D8-474E-B4D7-BB82332C878E}.Release|x64.Build.0 = Release|x64 - {XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX}.Debug|Win32.ActiveCfg = Debug|Win32 - {XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX}.Debug|Win32.Build.0 = Debug|Win32 - {XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX}.Debug|x64.ActiveCfg = Debug|x64 - {XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX}.Debug|x64.Build.0 = Debug|x64 - {XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 - {XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 - {XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 - {XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX}.PGInstrument|x64.Build.0 = PGInstrument|x64 - {XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 - {XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 - {XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 - {XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX}.PGUpdate|x64.Build.0 = PGUpdate|x64 - {XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX}.Release|Win32.ActiveCfg = Release|Win32 - {XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX}.Release|Win32.Build.0 = Release|Win32 - {XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX}.Release|x64.ActiveCfg = Release|x64 - {XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX}.Release|x64.Build.0 = Release|x64 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/PCbuild/readme.txt b/PCbuild/readme.txt index d258098e74f9ff..d28435b4c4d7d9 100644 --- a/PCbuild/readme.txt +++ b/PCbuild/readme.txt @@ -145,7 +145,6 @@ _testconsole _testimportmultiple _testmultiphase _tkinter -_xxsubinterpreters pyexpat select unicodedata