From c3a669edb6d84ebd725fb02f6f96acf96156cc4f Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 20 Sep 2023 11:13:06 -0600 Subject: [PATCH 01/12] Update the Interpreter.run() docstring. --- Lib/test/support/interpreters.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/Lib/test/support/interpreters.py b/Lib/test/support/interpreters.py index d2beba31e80283..2f2551d765be4f 100644 --- a/Lib/test/support/interpreters.py +++ b/Lib/test/support/interpreters.py @@ -94,7 +94,20 @@ def close(self): def run(self, src_str, /, *, channels=None): """Run the given source code in the interpreter. - This blocks the current Python thread until done. + This is essentially the same as calling the builtin "exec" + with this interpreter, using the __dict__ of its __main__ + module as both globals and locals. + + There is no return value. + + If the code raises an unhandled exception then a RunFailedError + is raised, which summarizes the unhandled exception. The actual + exception is discarded because objects cannot be shared between + interpreters. + + This blocks the current Python thread until done. During + that time, the previous interpreter is allowed to run + in other threads. """ _interpreters.run_string(self._id, src_str, channels) From 5cf8df62b2b87c20b198b300d8e3e48a77b2330e Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 20 Sep 2023 11:41:28 -0600 Subject: [PATCH 02/12] run_string() -> exec(). --- Lib/test/support/interpreters.py | 3 ++- Lib/test/test__xxsubinterpreters.py | 1 + Modules/_xxsubinterpretersmodule.c | 19 ++++++++++--------- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/Lib/test/support/interpreters.py b/Lib/test/support/interpreters.py index 2f2551d765be4f..2f6e90340a6421 100644 --- a/Lib/test/support/interpreters.py +++ b/Lib/test/support/interpreters.py @@ -91,6 +91,7 @@ def close(self): """ return _interpreters.destroy(self._id) + # XXX Rename "run" to "exec"? def run(self, src_str, /, *, channels=None): """Run the given source code in the interpreter. @@ -109,7 +110,7 @@ def run(self, src_str, /, *, channels=None): that time, the previous interpreter is allowed to run in other threads. """ - _interpreters.run_string(self._id, src_str, channels) + _interpreters.exec(self._id, src_str, channels) def create_channel(): diff --git a/Lib/test/test__xxsubinterpreters.py b/Lib/test/test__xxsubinterpreters.py index ac2280eb7090e9..5fbc5335752175 100644 --- a/Lib/test/test__xxsubinterpreters.py +++ b/Lib/test/test__xxsubinterpreters.py @@ -14,6 +14,7 @@ interpreters = import_helper.import_module('_xxsubinterpreters') +interpreters.run_string = interpreters.exec ################################## diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index e1c7d4ab2fd78f..678807e0c96ebc 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -667,13 +667,13 @@ Return the ID of main interpreter."); static PyObject * -interp_run_string(PyObject *self, PyObject *args, PyObject *kwds) +interp_exec(PyObject *self, PyObject *args, PyObject *kwds) { static char *kwlist[] = {"id", "script", "shared", NULL}; PyObject *id, *code; PyObject *shared = NULL; if (!PyArg_ParseTupleAndKeywords(args, kwds, - "OU|O:run_string", kwlist, + "OU|O:" MODULE_NAME ".exec", kwlist, &id, &code, &shared)) { return NULL; } @@ -697,18 +697,19 @@ interp_run_string(PyObject *self, PyObject *args, PyObject *kwds) } // Run the code in the interpreter. - if (_run_script_in_interpreter(self, interp, codestr, shared) != 0) { + if (_run_in_interpreter(self, interp, codestr, shared) != 0) { return NULL; } Py_RETURN_NONE; } -PyDoc_STRVAR(run_string_doc, -"run_string(id, script, shared)\n\ +PyDoc_STRVAR(exec_doc, +"exec(id, script, shared)\n\ \n\ Execute the provided string in the identified interpreter.\n\ -\n\ -See PyRun_SimpleStrings."); +This is equivalent to running the builtin exec() under the target\n\ +interpreter, using the __dict__ of its __main__ module as both\n\ +globals and locals."); static PyObject * @@ -775,8 +776,8 @@ static PyMethodDef module_functions[] = { {"is_running", _PyCFunction_CAST(interp_is_running), METH_VARARGS | METH_KEYWORDS, is_running_doc}, - {"run_string", _PyCFunction_CAST(interp_run_string), - METH_VARARGS | METH_KEYWORDS, run_string_doc}, + {"exec", _PyCFunction_CAST(interp_exec), + METH_VARARGS | METH_KEYWORDS, exec_doc}, {"is_shareable", _PyCFunction_CAST(object_is_shareable), METH_VARARGS | METH_KEYWORDS, is_shareable_doc}, From 7a39e9f320e15a856b774d55bda534ddd29a8764 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 20 Sep 2023 12:09:58 -0600 Subject: [PATCH 03/12] Generalize interp_exec(). --- Modules/_xxsubinterpretersmodule.c | 65 ++++++++++++++++++++++-------- 1 file changed, 48 insertions(+), 17 deletions(-) diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 678807e0c96ebc..7044ca2dab96f6 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -10,6 +10,7 @@ #include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() #include "pycore_pystate.h" // _PyInterpreterState_SetRunningMain() #include "interpreteridobject.h" +#include "marshal.h" // PyMarshal_ReadObjectFromString() #define MODULE_NAME "_xxsubinterpreters" @@ -332,6 +333,35 @@ _sharedexception_apply(_sharedexception *exc, PyObject *wrapperclass) } +/* Python code **************************************************************/ + +#define RUN_TEXT 1 + +static const char * +get_code_str(PyObject *arg, int *flags_p) +{ + if (PyUnicode_Check(arg)) { + Py_ssize_t size; + const char *codestr = PyUnicode_AsUTF8AndSize(arg, &size); + if (codestr == NULL) { + return NULL; + } + if (strlen(codestr) != (size_t)size) { + PyErr_SetString(PyExc_ValueError, + "source code string cannot contain null bytes"); + return NULL; + } + *flags_p = RUN_TEXT; + return codestr; + } + else { + PyErr_SetString(PyExc_TypeError, + "unsupported code arg"); + return NULL; + } +} + + /* interpreter-specific code ************************************************/ static int @@ -360,7 +390,7 @@ exceptions_init(PyObject *mod) static int _run_script(PyInterpreterState *interp, const char *codestr, - _sharedns *shared, _sharedexception *sharedexc) + _sharedns *shared, _sharedexception *sharedexc, int flags) { if (_PyInterpreterState_SetRunningMain(interp) < 0) { // We skip going through the shared exception. @@ -387,8 +417,14 @@ _run_script(PyInterpreterState *interp, const char *codestr, } } - // Run the string (see PyRun_SimpleStringFlags). - PyObject *result = PyRun_StringFlags(codestr, Py_file_input, ns, ns, NULL); + // Run the script/code/etc. + PyObject *result = NULL; + if (flags & RUN_TEXT) { + result = PyRun_StringFlags(codestr, Py_file_input, ns, ns, NULL); + } + else { + Py_FatalError("unsupported codestr"); + } Py_DECREF(ns); if (result == NULL) { goto error; @@ -417,8 +453,8 @@ _run_script(PyInterpreterState *interp, const char *codestr, } static int -_run_script_in_interpreter(PyObject *mod, PyInterpreterState *interp, - const char *codestr, PyObject *shareables) +_run_in_interpreter(PyObject *mod, PyInterpreterState *interp, + const char *codestr, PyObject *shareables, int flags) { module_state *state = get_module_state(mod); @@ -456,7 +492,7 @@ _run_script_in_interpreter(PyObject *mod, PyInterpreterState *interp, // Run the script. _sharedexception exc = {NULL, NULL}; - int result = _run_script(interp, codestr, shared, &exc); + int result = _run_script(interp, codestr, shared, &exc, flags); // Switch back. if (save_tstate != NULL) { @@ -669,12 +705,12 @@ Return the ID of main interpreter."); static PyObject * interp_exec(PyObject *self, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"id", "script", "shared", NULL}; - PyObject *id, *code; + static char *kwlist[] = {"id", "code", "shared", NULL}; + PyObject *id, *code_arg; PyObject *shared = NULL; if (!PyArg_ParseTupleAndKeywords(args, kwds, "OU|O:" MODULE_NAME ".exec", kwlist, - &id, &code, &shared)) { + &id, &code_arg, &shared)) { return NULL; } @@ -685,19 +721,14 @@ interp_exec(PyObject *self, PyObject *args, PyObject *kwds) } // Extract code. - Py_ssize_t size; - const char *codestr = PyUnicode_AsUTF8AndSize(code, &size); + int flags = 0; + const char *codestr = get_code_str(code_arg, &flags); if (codestr == NULL) { return NULL; } - if (strlen(codestr) != (size_t)size) { - PyErr_SetString(PyExc_ValueError, - "source code string cannot contain null bytes"); - return NULL; - } // Run the code in the interpreter. - if (_run_in_interpreter(self, interp, codestr, shared) != 0) { + if (_run_in_interpreter(self, interp, codestr, shared, flags) != 0) { return NULL; } Py_RETURN_NONE; From a25c7c763ba98ad7ee207b24d668d748c3809e15 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 20 Sep 2023 13:31:15 -0600 Subject: [PATCH 04/12] Support passing basic functions to Interpreter.run(). --- Modules/_xxsubinterpretersmodule.c | 127 ++++++++++++++++++++++++----- 1 file changed, 108 insertions(+), 19 deletions(-) diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 7044ca2dab96f6..4a89b19102a408 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -335,30 +335,96 @@ _sharedexception_apply(_sharedexception *exc, PyObject *wrapperclass) /* Python code **************************************************************/ +static int +validate_code_object(PyCodeObject *code) +{ + if (code->co_argcount > 0 + || code->co_posonlyargcount > 0 + || code->co_kwonlyargcount > 0) + { + PyErr_SetString(PyExc_ValueError, "arguments not supported"); + return -1; + } + if (code->co_ncellvars > 0) { + PyErr_SetString(PyExc_ValueError, "closures not supported"); + return -1; + } + // We trust that no code objects under co_consts have unbound cell vars. + + if (code->co_executors != NULL + || code->_co_instrumentation_version > 0) + { + PyErr_SetString(PyExc_ValueError, "only basic functions are supported"); + return -1; + } + if (code->_co_monitoring != NULL) { + PyErr_SetString(PyExc_ValueError, "only basic functions are supported"); + return -1; + } + if (code->co_extra != NULL) { + PyErr_SetString(PyExc_ValueError, "only basic functions are supported"); + return -1; + } + + return 0; +} + #define RUN_TEXT 1 +#define RUN_CODE 2 static const char * -get_code_str(PyObject *arg, int *flags_p) +get_code_str(PyObject *arg, Py_ssize_t *len_p, PyObject **bytes_p, int *flags_p) { + const char *codestr = NULL; + Py_ssize_t len = -1; + PyObject *bytes_obj = NULL; + int flags = 0; + if (PyUnicode_Check(arg)) { - Py_ssize_t size; - const char *codestr = PyUnicode_AsUTF8AndSize(arg, &size); + // XXX Validate that it parses? + codestr = PyUnicode_AsUTF8AndSize(arg, &len); if (codestr == NULL) { return NULL; } - if (strlen(codestr) != (size_t)size) { + if (strlen(codestr) != (size_t)len) { PyErr_SetString(PyExc_ValueError, "source code string cannot contain null bytes"); return NULL; } - *flags_p = RUN_TEXT; - return codestr; + flags = RUN_TEXT; } else { - PyErr_SetString(PyExc_TypeError, - "unsupported code arg"); - return NULL; + PyObject *code = arg; + if (PyFunction_Check(arg)) { + if (PyFunction_GetClosure(arg)) { + PyErr_SetString(PyExc_ValueError, "closures not supported"); + return NULL; + } + code = PyFunction_GetCode(arg); + } + else if (!PyCode_Check(arg)) { + PyErr_SetString(PyExc_TypeError, "unsupported type"); + return NULL; + } + flags = RUN_CODE; + + if (validate_code_object((PyCodeObject *)code) < 0) { + return NULL; + } + + // Serialize the code object. + bytes_obj = PyMarshal_WriteObjectToString(code, Py_MARSHAL_VERSION); + if (bytes_obj == NULL) { + return NULL; + } + codestr = PyBytes_AS_STRING(bytes_obj); + len = PyBytes_GET_SIZE(bytes_obj); } + + *flags_p = flags; + *bytes_p = bytes_obj; + *len_p = len; + return codestr; } @@ -389,7 +455,8 @@ exceptions_init(PyObject *mod) } static int -_run_script(PyInterpreterState *interp, const char *codestr, +_run_script(PyInterpreterState *interp, + const char *codestr, Py_ssize_t codestrlen, _sharedns *shared, _sharedexception *sharedexc, int flags) { if (_PyInterpreterState_SetRunningMain(interp) < 0) { @@ -422,8 +489,14 @@ _run_script(PyInterpreterState *interp, const char *codestr, if (flags & RUN_TEXT) { result = PyRun_StringFlags(codestr, Py_file_input, ns, ns, NULL); } + else if (flags & RUN_CODE) { + PyObject *code = PyMarshal_ReadObjectFromString(codestr, codestrlen); + if (code != NULL) { + result = PyEval_EvalCode(code, ns, ns); + } + } else { - Py_FatalError("unsupported codestr"); + Py_UNREACHABLE(); } Py_DECREF(ns); if (result == NULL) { @@ -454,7 +527,8 @@ _run_script(PyInterpreterState *interp, const char *codestr, static int _run_in_interpreter(PyObject *mod, PyInterpreterState *interp, - const char *codestr, PyObject *shareables, int flags) + const char *codestr, Py_ssize_t codestrlen, + PyObject *shareables, int flags) { module_state *state = get_module_state(mod); @@ -492,7 +566,7 @@ _run_in_interpreter(PyObject *mod, PyInterpreterState *interp, // Run the script. _sharedexception exc = {NULL, NULL}; - int result = _run_script(interp, codestr, shared, &exc, flags); + int result = _run_script(interp, codestr, codestrlen, shared, &exc, flags); // Switch back. if (save_tstate != NULL) { @@ -709,7 +783,7 @@ interp_exec(PyObject *self, PyObject *args, PyObject *kwds) PyObject *id, *code_arg; PyObject *shared = NULL; if (!PyArg_ParseTupleAndKeywords(args, kwds, - "OU|O:" MODULE_NAME ".exec", kwlist, + "OO|O:" MODULE_NAME ".exec", kwlist, &id, &code_arg, &shared)) { return NULL; } @@ -721,26 +795,41 @@ interp_exec(PyObject *self, PyObject *args, PyObject *kwds) } // Extract code. + Py_ssize_t codestrlen = -1; + PyObject *bytes_obj = NULL; int flags = 0; - const char *codestr = get_code_str(code_arg, &flags); + const char *codestr = get_code_str(code_arg, + &codestrlen, &bytes_obj, &flags); if (codestr == NULL) { return NULL; } // Run the code in the interpreter. - if (_run_in_interpreter(self, interp, codestr, shared, flags) != 0) { + int res = _run_in_interpreter(self, interp, codestr, codestrlen, + shared, flags); + Py_XDECREF(bytes_obj); + if (res != 0) { return NULL; } Py_RETURN_NONE; } PyDoc_STRVAR(exec_doc, -"exec(id, script, shared)\n\ +"exec(id, code, shared)\n\ \n\ -Execute the provided string in the identified interpreter.\n\ +Execute the provided code in the identified interpreter.\n\ This is equivalent to running the builtin exec() under the target\n\ interpreter, using the __dict__ of its __main__ module as both\n\ -globals and locals."); +globals and locals.\n\ +\n\ +\"code\" may be a string containing the text of a Python script.\n\ +\n\ +Functions (and code objects) are also supported, with some restrictions.\n\ +The code/function must not take any arguments or be a closure\n\ +(i.e. have cell vars).\n\ +\n\ +If a function is provided, its code object is used and all its state\n\ +is ignored, including its __globals__ dict."); static PyObject * From 5b3f4e8e4cb4acf79a4dab4584a5e0dca3fab771 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 2 Oct 2023 18:28:59 -0600 Subject: [PATCH 05/12] Fix validate_code_object(). --- Modules/_xxsubinterpretersmodule.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 4a89b19102a408..09aaa9db0a2cdc 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -340,7 +340,8 @@ validate_code_object(PyCodeObject *code) { if (code->co_argcount > 0 || code->co_posonlyargcount > 0 - || code->co_kwonlyargcount > 0) + || code->co_kwonlyargcount > 0 + || code->co_flags & (CO_VARARGS | CO_VARKEYWORDS)) { PyErr_SetString(PyExc_ValueError, "arguments not supported"); return -1; From eece6f97d2ed452e070619c47c05db6917a5ccdd Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 2 Oct 2023 18:29:08 -0600 Subject: [PATCH 06/12] Add tests. --- Lib/test/test__xxsubinterpreters.py | 106 ++++++++++++++++++++++++++++ 1 file changed, 106 insertions(+) diff --git a/Lib/test/test__xxsubinterpreters.py b/Lib/test/test__xxsubinterpreters.py index 5fbc5335752175..8dbb48457770f6 100644 --- a/Lib/test/test__xxsubinterpreters.py +++ b/Lib/test/test__xxsubinterpreters.py @@ -15,6 +15,7 @@ interpreters = import_helper.import_module('_xxsubinterpreters') interpreters.run_string = interpreters.exec +interpreters.run_func = interpreters.exec ################################## @@ -926,5 +927,110 @@ def f(): self.assertEqual(retcode, 0) +class RunFuncTests(TestBase): + + def setUp(self): + super().setUp() + self.id = interpreters.create() + + def test_success(self): + r, w = os.pipe() + def script(): + global w + import contextlib + with open(w, 'w', encoding="utf-8") as spipe: + with contextlib.redirect_stdout(spipe): + print('it worked!', end='') + interpreters.run_func(self.id, script, shared=dict(w=w)) + + with open(r, encoding="utf-8") as outfile: + out = outfile.read() + + self.assertEqual(out, 'it worked!') + + def test_in_thread(self): + r, w = os.pipe() + def script(): + global w + import contextlib + with open(w, 'w', encoding="utf-8") as spipe: + with contextlib.redirect_stdout(spipe): + print('it worked!', end='') + def f(): + interpreters.run_func(self.id, script, shared=dict(w=w)) + t = threading.Thread(target=f) + t.start() + t.join() + + with open(r, encoding="utf-8") as outfile: + out = outfile.read() + + self.assertEqual(out, 'it worked!') + + def test_code_object(self): + r, w = os.pipe() + + def script(): + global w + import contextlib + with open(w, 'w', encoding="utf-8") as spipe: + with contextlib.redirect_stdout(spipe): + print('it worked!', end='') + code = script.__code__ + interpreters.run_func(self.id, code, shared=dict(w=w)) + + with open(r, encoding="utf-8") as outfile: + out = outfile.read() + + self.assertEqual(out, 'it worked!') + + def test_closure(self): + spam = True + def script(): + assert spam + + with self.assertRaises(ValueError): + interpreters.run_func(self.id, script) + + # XXX This hasn't been fixed yet. + @unittest.expectedFailure + def test_return_value(self): + def script(): + return 'spam' + with self.assertRaises(ValueError): + interpreters.run_func(self.id, script) + + def test_args(self): + with self.subTest('args'): + def script(a, b=0): + assert a == b + with self.assertRaises(ValueError): + interpreters.run_func(self.id, script) + + with self.subTest('*args'): + def script(*args): + assert not args + with self.assertRaises(ValueError): + interpreters.run_func(self.id, script) + + with self.subTest('**kwargs'): + def script(**kwargs): + assert not kwargs + with self.assertRaises(ValueError): + interpreters.run_func(self.id, script) + + with self.subTest('kwonly'): + def script(*, spam=True): + assert spam + with self.assertRaises(ValueError): + interpreters.run_func(self.id, script) + + with self.subTest('posonly'): + def script(spam, /): + assert spam + with self.assertRaises(ValueError): + interpreters.run_func(self.id, script) + + if __name__ == '__main__': unittest.main() From ed70ffc67a8a5befd1301910ccd4e27ce20f9037 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 4 Oct 2023 11:45:21 -0600 Subject: [PATCH 07/12] Factor out _interp_exec(). --- Modules/_xxsubinterpretersmodule.c | 41 +++++++++++++++++++----------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 09aaa9db0a2cdc..7d5e1ad1bd8ab1 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -777,22 +777,14 @@ PyDoc_STRVAR(get_main_doc, Return the ID of main interpreter."); -static PyObject * -interp_exec(PyObject *self, PyObject *args, PyObject *kwds) +static int +_interp_exec(PyObject *self, + PyObject *id_arg, PyObject *code_arg, PyObject *shared_arg) { - static char *kwlist[] = {"id", "code", "shared", NULL}; - PyObject *id, *code_arg; - PyObject *shared = NULL; - if (!PyArg_ParseTupleAndKeywords(args, kwds, - "OO|O:" MODULE_NAME ".exec", kwlist, - &id, &code_arg, &shared)) { - return NULL; - } - // Look up the interpreter. - PyInterpreterState *interp = PyInterpreterID_LookUp(id); + PyInterpreterState *interp = PyInterpreterID_LookUp(id_arg); if (interp == NULL) { - return NULL; + return -1; } // Extract code. @@ -802,14 +794,33 @@ interp_exec(PyObject *self, PyObject *args, PyObject *kwds) const char *codestr = get_code_str(code_arg, &codestrlen, &bytes_obj, &flags); if (codestr == NULL) { - return NULL; + return -1; } // Run the code in the interpreter. int res = _run_in_interpreter(self, interp, codestr, codestrlen, - shared, flags); + shared_arg, flags); Py_XDECREF(bytes_obj); if (res != 0) { + return -1; + } + + return 0; +} + +static PyObject * +interp_exec(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"id", "code", "shared", NULL}; + PyObject *id, *code; + PyObject *shared = NULL; + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "OO|O:" MODULE_NAME ".exec", kwlist, + &id, &code, &shared)) { + return NULL; + } + + if (_interp_exec(self, id, code, shared) < 0) { return NULL; } Py_RETURN_NONE; From 0c443ad0575798c31d75bd6a3ff5bb4a95bf848c Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 4 Oct 2023 11:46:21 -0600 Subject: [PATCH 08/12] Fix the docstring. --- Modules/_xxsubinterpretersmodule.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 7d5e1ad1bd8ab1..95b21bd86dcce3 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -827,7 +827,7 @@ interp_exec(PyObject *self, PyObject *args, PyObject *kwds) } PyDoc_STRVAR(exec_doc, -"exec(id, code, shared)\n\ +"exec(id, code, shared=None)\n\ \n\ Execute the provided code in the identified interpreter.\n\ This is equivalent to running the builtin exec() under the target\n\ @@ -838,7 +838,7 @@ globals and locals.\n\ \n\ Functions (and code objects) are also supported, with some restrictions.\n\ The code/function must not take any arguments or be a closure\n\ -(i.e. have cell vars).\n\ +(i.e. have cell vars). Methods and other callables are not supported.\n\ \n\ If a function is provided, its code object is used and all its state\n\ is ignored, including its __globals__ dict."); From b8e32fefb862c243d8051b5088ed425686940a4b Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 4 Oct 2023 16:18:54 -0600 Subject: [PATCH 09/12] Clean up arg parsing. --- Modules/_xxsubinterpretersmodule.c | 151 +++++++++++++++++++++++------ 1 file changed, 119 insertions(+), 32 deletions(-) diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 95b21bd86dcce3..cc3b9585e00768 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -335,39 +335,48 @@ _sharedexception_apply(_sharedexception *exc, PyObject *wrapperclass) /* Python code **************************************************************/ -static int -validate_code_object(PyCodeObject *code) +static const char * +check_code_str(PyUnicodeObject *text) +{ + assert(text != NULL); + if (PyUnicode_GET_LENGTH(text) == 0) { + return "too short"; + } + + // XXX Verify that it parses? + + return NULL; +} + +static const char * +check_code_object(PyCodeObject *code) { + assert(code != NULL); if (code->co_argcount > 0 || code->co_posonlyargcount > 0 || code->co_kwonlyargcount > 0 || code->co_flags & (CO_VARARGS | CO_VARKEYWORDS)) { - PyErr_SetString(PyExc_ValueError, "arguments not supported"); - return -1; + return "arguments not supported"; } if (code->co_ncellvars > 0) { - PyErr_SetString(PyExc_ValueError, "closures not supported"); - return -1; + return "closures not supported"; } // We trust that no code objects under co_consts have unbound cell vars. if (code->co_executors != NULL || code->_co_instrumentation_version > 0) { - PyErr_SetString(PyExc_ValueError, "only basic functions are supported"); - return -1; + return "only basic functions are supported"; } if (code->_co_monitoring != NULL) { - PyErr_SetString(PyExc_ValueError, "only basic functions are supported"); - return -1; + return "only basic functions are supported"; } if (code->co_extra != NULL) { - PyErr_SetString(PyExc_ValueError, "only basic functions are supported"); - return -1; + return "only basic functions are supported"; } - return 0; + return NULL; } #define RUN_TEXT 1 @@ -382,7 +391,8 @@ get_code_str(PyObject *arg, Py_ssize_t *len_p, PyObject **bytes_p, int *flags_p) int flags = 0; if (PyUnicode_Check(arg)) { - // XXX Validate that it parses? + assert(PyUnicode_CheckExact(arg) + && (check_code_str((PyUnicodeObject *)arg) == NULL)); codestr = PyUnicode_AsUTF8AndSize(arg, &len); if (codestr == NULL) { return NULL; @@ -395,26 +405,12 @@ get_code_str(PyObject *arg, Py_ssize_t *len_p, PyObject **bytes_p, int *flags_p) flags = RUN_TEXT; } else { - PyObject *code = arg; - if (PyFunction_Check(arg)) { - if (PyFunction_GetClosure(arg)) { - PyErr_SetString(PyExc_ValueError, "closures not supported"); - return NULL; - } - code = PyFunction_GetCode(arg); - } - else if (!PyCode_Check(arg)) { - PyErr_SetString(PyExc_TypeError, "unsupported type"); - return NULL; - } + assert(PyCode_Check(arg) + && (check_code_object((PyCodeObject *)arg) == NULL)); flags = RUN_CODE; - if (validate_code_object((PyCodeObject *)code) < 0) { - return NULL; - } - // Serialize the code object. - bytes_obj = PyMarshal_WriteObjectToString(code, Py_MARSHAL_VERSION); + bytes_obj = PyMarshal_WriteObjectToString(arg, Py_MARSHAL_VERSION); if (bytes_obj == NULL) { return NULL; } @@ -777,6 +773,82 @@ PyDoc_STRVAR(get_main_doc, Return the ID of main interpreter."); +static PyUnicodeObject * +convert_script_arg(PyObject *arg, const char *fname, const char *displayname, + const char *expected) +{ + PyUnicodeObject *str = NULL; + if (PyUnicode_CheckExact(arg)) { + str = (PyUnicodeObject *)Py_NewRef(arg); + } + else if (PyUnicode_Check(arg)) { + // XXX str = PyUnicode_FromObject(arg); + str = (PyUnicodeObject *)Py_NewRef(arg); + } + else { + _PyArg_BadArgument(fname, displayname, expected, arg); + return NULL; + } + + const char *err = check_code_str(str); + if (err != NULL) { + Py_DECREF(str); + PyErr_Format(PyExc_ValueError, + "%.200s(): bad script text (%s)", fname, err); + return NULL; + } + + return str; +} + +static PyCodeObject * +convert_code_arg(PyObject *arg, const char *fname, const char *displayname, + const char *expected) +{ + const char *kind = NULL; + PyCodeObject *code = NULL; + if (PyFunction_Check(arg)) { + if (PyFunction_GetClosure(arg) != NULL) { + PyErr_Format(PyExc_ValueError, + "%.200s(): closures not supported", fname); + return NULL; + } + code = (PyCodeObject *)PyFunction_GetCode(arg); + if (code == NULL) { + if (PyErr_Occurred()) { + // This chains. + PyErr_Format(PyExc_ValueError, + "%.200s(): bad func", fname); + } + else { + PyErr_Format(PyExc_ValueError, + "%.200s(): func.__code__ missing", fname); + } + return NULL; + } + Py_INCREF(code); + kind = "func"; + } + else if (PyCode_Check(arg)) { + code = (PyCodeObject *)Py_NewRef(arg); + kind = "code object"; + } + else { + _PyArg_BadArgument(fname, displayname, expected, arg); + return NULL; + } + + const char *err = check_code_object(code); + if (err != NULL) { + Py_DECREF(code); + PyErr_Format(PyExc_ValueError, + "%.200s(): bad %s (%s)", fname, kind, err); + return NULL; + } + + return code; +} + static int _interp_exec(PyObject *self, PyObject *id_arg, PyObject *code_arg, PyObject *shared_arg) @@ -820,7 +892,22 @@ interp_exec(PyObject *self, PyObject *args, PyObject *kwds) return NULL; } - if (_interp_exec(self, id, code, shared) < 0) { + const char *expected = "a string, a function, or a code object"; + if (PyUnicode_Check(code)) { + code = (PyObject *)convert_script_arg(code, MODULE_NAME ".exec", + "argument 2", expected); + } + else { + code = (PyObject *)convert_code_arg(code, MODULE_NAME ".exec", + "argument 2", expected); + } + if (code == NULL) { + return NULL; + } + + int res = _interp_exec(self, id, code, shared); + Py_DECREF(code); + if (res < 0) { return NULL; } Py_RETURN_NONE; From 7e8320c903a6eb36c7c094b5a6a59953dfd14743 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 4 Oct 2023 16:25:49 -0600 Subject: [PATCH 10/12] Restore _interpreters.run_string(). --- Lib/test/test__xxsubinterpreters.py | 1 - Modules/_xxsubinterpretersmodule.c | 33 +++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/Lib/test/test__xxsubinterpreters.py b/Lib/test/test__xxsubinterpreters.py index 8dbb48457770f6..c76986b50b62c9 100644 --- a/Lib/test/test__xxsubinterpreters.py +++ b/Lib/test/test__xxsubinterpreters.py @@ -14,7 +14,6 @@ interpreters = import_helper.import_module('_xxsubinterpreters') -interpreters.run_string = interpreters.exec interpreters.run_func = interpreters.exec diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index cc3b9585e00768..80f33d76802f8e 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -930,6 +930,37 @@ The code/function must not take any arguments or be a closure\n\ If a function is provided, its code object is used and all its state\n\ is ignored, including its __globals__ dict."); +static PyObject * +interp_run_string(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"id", "script", "shared", NULL}; + PyObject *id, *script; + PyObject *shared = NULL; + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "OU|O:" MODULE_NAME ".run_string", kwlist, + &id, &script, &shared)) { + return NULL; + } + + script = (PyObject *)convert_script_arg(script, MODULE_NAME ".exec", + "argument 2", "a string"); + if (script == NULL) { + return NULL; + } + + if (_interp_exec(self, id, (PyObject *)script, shared) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + +PyDoc_STRVAR(run_string_doc, +"run_string(id, script, shared=None)\n\ +\n\ +Execute the provided string in the identified interpreter.\n\ +\n\ +(See " MODULE_NAME ".exec()."); + static PyObject * object_is_shareable(PyObject *self, PyObject *args, PyObject *kwds) @@ -997,6 +1028,8 @@ static PyMethodDef module_functions[] = { METH_VARARGS | METH_KEYWORDS, is_running_doc}, {"exec", _PyCFunction_CAST(interp_exec), METH_VARARGS | METH_KEYWORDS, exec_doc}, + {"run_string", _PyCFunction_CAST(interp_run_string), + METH_VARARGS | METH_KEYWORDS, run_string_doc}, {"is_shareable", _PyCFunction_CAST(object_is_shareable), METH_VARARGS | METH_KEYWORDS, is_shareable_doc}, From f5589e00680bec8a3a54d517a4d04ddf07e26ac0 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 4 Oct 2023 16:26:26 -0600 Subject: [PATCH 11/12] Add _interpreters.run_func(). --- Lib/test/test__xxsubinterpreters.py | 1 - Modules/_xxsubinterpretersmodule.c | 36 +++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/Lib/test/test__xxsubinterpreters.py b/Lib/test/test__xxsubinterpreters.py index c76986b50b62c9..e3c917aa2eb19d 100644 --- a/Lib/test/test__xxsubinterpreters.py +++ b/Lib/test/test__xxsubinterpreters.py @@ -14,7 +14,6 @@ interpreters = import_helper.import_module('_xxsubinterpreters') -interpreters.run_func = interpreters.exec ################################## diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 80f33d76802f8e..b1b25d85623d08 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -961,6 +961,40 @@ Execute the provided string in the identified interpreter.\n\ \n\ (See " MODULE_NAME ".exec()."); +static PyObject * +interp_run_func(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"id", "func", "shared", NULL}; + PyObject *id, *func; + PyObject *shared = NULL; + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "OO|O:" MODULE_NAME ".run_func", kwlist, + &id, &func, &shared)) { + return NULL; + } + + PyCodeObject *code = convert_code_arg(func, MODULE_NAME ".exec", + "argument 2", + "a function or a code object"); + if (code == NULL) { + return NULL; + } + + if (_interp_exec(self, id, (PyObject *)code, shared) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + +PyDoc_STRVAR(run_func_doc, +"run_func(id, func, shared=None)\n\ +\n\ +Execute the body of the provided function in the identified interpreter.\n\ +Code objects are also supported. In both cases, closures and args\n\ +are not supported. Methods and other callables are not supported either.\n\ +\n\ +(See " MODULE_NAME ".exec()."); + static PyObject * object_is_shareable(PyObject *self, PyObject *args, PyObject *kwds) @@ -1030,6 +1064,8 @@ static PyMethodDef module_functions[] = { METH_VARARGS | METH_KEYWORDS, exec_doc}, {"run_string", _PyCFunction_CAST(interp_run_string), METH_VARARGS | METH_KEYWORDS, run_string_doc}, + {"run_func", _PyCFunction_CAST(interp_run_func), + METH_VARARGS | METH_KEYWORDS, run_func_doc}, {"is_shareable", _PyCFunction_CAST(object_is_shareable), METH_VARARGS | METH_KEYWORDS, is_shareable_doc}, From e12d3a234b7448847c7163f18a7488f4a1030d0d Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 5 Oct 2023 17:06:21 -0600 Subject: [PATCH 12/12] Fix ref leaks. --- Modules/_xxsubinterpretersmodule.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 85a9dc3eabf081..12c98ea61fb7ac 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -531,6 +531,7 @@ _run_script(PyInterpreterState *interp, PyObject *code = PyMarshal_ReadObjectFromString(codestr, codestrlen); if (code != NULL) { result = PyEval_EvalCode(code, ns, ns); + Py_DECREF(code); } } else { @@ -977,7 +978,9 @@ interp_run_string(PyObject *self, PyObject *args, PyObject *kwds) return NULL; } - if (_interp_exec(self, id, (PyObject *)script, shared) < 0) { + int res = _interp_exec(self, id, (PyObject *)script, shared); + Py_DECREF(script); + if (res < 0) { return NULL; } Py_RETURN_NONE; @@ -1009,7 +1012,9 @@ interp_run_func(PyObject *self, PyObject *args, PyObject *kwds) return NULL; } - if (_interp_exec(self, id, (PyObject *)code, shared) < 0) { + int res = _interp_exec(self, id, (PyObject *)code, shared); + Py_DECREF(code); + if (res < 0) { return NULL; } Py_RETURN_NONE;