diff --git a/Doc/c-api/cell.rst b/Doc/c-api/cell.rst index ac4ef5adc5cc20..69e2723f2d2184 100644 --- a/Doc/c-api/cell.rst +++ b/Doc/c-api/cell.rst @@ -39,13 +39,14 @@ Cell objects are not likely to be useful elsewhere. .. c:function:: PyObject* PyCell_Get(PyObject *cell) - Return the contents of the cell *cell*. + Return a new reference to the contents of the cell *cell*. .. c:function:: PyObject* PyCell_GET(PyObject *cell) - Return the contents of the cell *cell*, but without checking that *cell* is - non-``NULL`` and a cell object. + Borrow a reference to the contents of the cell *cell*. No reference counts are + adjusted, and no checks are made for safety; *cell* must be non-``NULL`` and must + be a cell object. .. c:function:: int PyCell_Set(PyObject *cell, PyObject *value) @@ -58,6 +59,6 @@ Cell objects are not likely to be useful elsewhere. .. c:function:: void PyCell_SET(PyObject *cell, PyObject *value) - Sets the value of the cell object *cell* to *value*. No reference counts are + Sets the value of the cell object *cell* to *value*. No reference counts are adjusted, and no checks are made for safety; *cell* must be non-``NULL`` and must be a cell object. diff --git a/Doc/c-api/reflection.rst b/Doc/c-api/reflection.rst index 64ce4d1d0c34df..9998b038abc123 100644 --- a/Doc/c-api/reflection.rst +++ b/Doc/c-api/reflection.rst @@ -10,12 +10,21 @@ Reflection Return a dictionary of the builtins in the current execution frame, or the interpreter of the thread state if no frame is currently executing. +.. c:function:: PyObject* PyLocals_Get(void) + + Return a dictionary of the local variables in the current execution frame, + or ``NULL`` if no frame is currently executing. + + Equivalent to calling the Python level ``locals()`` builtin. + +.. TODO: cover the rest of the PEP 558 API here .. c:function:: PyObject* PyEval_GetLocals(void) Return a dictionary of the local variables in the current execution frame, or ``NULL`` if no frame is currently executing. + TODO: Clarify just how this relates to PyLocals_Get(). .. c:function:: PyObject* PyEval_GetGlobals(void) diff --git a/Doc/data/stable_abi.dat b/Doc/data/stable_abi.dat index e373e2314a6517..b7fd1bce20641b 100644 --- a/Doc/data/stable_abi.dat +++ b/Doc/data/stable_abi.dat @@ -332,6 +332,10 @@ function,PyList_SetSlice,3.2, function,PyList_Size,3.2, function,PyList_Sort,3.2, var,PyList_Type,3.2, +function,PyLocals_Get,3.11, +function,PyLocals_GetCopy,3.11, +function,PyLocals_Kind,3.11, +function,PyLocals_GetView,3.11, type,PyLongObject,3.2, var,PyLongRangeIter_Type,3.2, function,PyLong_AsDouble,3.2, diff --git a/Include/ceval.h b/Include/ceval.h index 0f687666e2bccf..ab7c64b80afdc2 100644 --- a/Include/ceval.h +++ b/Include/ceval.h @@ -33,6 +33,36 @@ PyAPI_FUNC(PyObject *) PyEval_GetGlobals(void); PyAPI_FUNC(PyObject *) PyEval_GetLocals(void); PyAPI_FUNC(PyFrameObject *) PyEval_GetFrame(void); +// TODO: Update PyEval_GetLocals() documentation as described in +// https://discuss.python.org/t/pep-558-defined-semantics-for-locals/2936/11 + +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030B0000 +/* Access the frame locals mapping in an implementation independent way */ + +/* PyLocals_Get() is equivalent to the Python locals() builtin. + * It returns a read/write reference or a shallow copy depending on the scope + * of the active frame. + */ +PyAPI_FUNC(PyObject *) PyLocals_Get(void); + +/* PyLocals_GetCopy() returns a fresh shallow copy of the active local namespace */ +PyAPI_FUNC(PyObject *) PyLocals_GetCopy(void); + +/* PyLocals_GetView() returns a read-only proxy for the active local namespace */ +PyAPI_FUNC(PyObject *) PyLocals_GetView(void); + +/* PyLocals_GetKind()reports the behaviour of PyLocals_Get() in the active scope */ +typedef enum { + PyLocals_UNDEFINED = -1, // Indicates error (e.g. no thread state defined) + PyLocals_DIRECT_REFERENCE = 0, + PyLocals_SHALLOW_COPY = 1, + _PyLocals_ENSURE_32BIT_ENUM = 2147483647 +} PyLocals_Kind; + +PyAPI_FUNC(PyLocals_Kind) PyLocals_GetKind(void); +#endif + + PyAPI_FUNC(int) Py_AddPendingCall(int (*func)(void *), void *arg); PyAPI_FUNC(int) Py_MakePendingCalls(void); diff --git a/Include/cpython/frameobject.h b/Include/cpython/frameobject.h index 2bf458cab35451..799f81e6443a0a 100644 --- a/Include/cpython/frameobject.h +++ b/Include/cpython/frameobject.h @@ -24,6 +24,7 @@ struct _frame { struct _frame *f_back; /* previous frame, or NULL */ PyObject **f_valuestack; /* points after the last local */ PyObject *f_trace; /* Trace function */ + PyObject *f_fast_refs; /* Name -> index-or-cell lookup for fast locals */ /* Borrowed reference to a generator, or NULL */ PyObject *f_gen; int f_stackdepth; /* Depth of value stack */ @@ -64,13 +65,39 @@ _PyFrame_New_NoTrack(PyThreadState *, PyFrameConstructor *, PyObject *, PyObject /* The rest of the interface is specific for frame objects */ -/* Conversions between "fast locals" and locals in dictionary */ - -PyAPI_FUNC(void) PyFrame_LocalsToFast(PyFrameObject *, int); - +/* Legacy conversions between "fast locals" and locals in dictionary */ PyAPI_FUNC(int) PyFrame_FastToLocalsWithError(PyFrameObject *f); PyAPI_FUNC(void) PyFrame_FastToLocals(PyFrameObject *); +/* This always raises RuntimeError now (use the PyLocals_* API instead) */ +PyAPI_FUNC(void) PyFrame_LocalsToFast(PyFrameObject *, int); + +/* Frame object memory management */ +PyAPI_FUNC(int) PyFrame_ClearFreeList(void); PyAPI_FUNC(void) _PyFrame_DebugMallocStats(FILE *out); PyAPI_FUNC(PyFrameObject *) PyFrame_GetBack(PyFrameObject *frame); + +/* Fast locals proxy allows for reliable write-through from trace functions */ +// TODO: Perhaps this should be hidden, and API users told to query for +// PyFrame_GetLocalsKind() instead. Having this available +// seems like a nice way to let folks write some useful debug assertions, +// though. +PyAPI_DATA(PyTypeObject) _PyFastLocalsProxy_Type; +#define _PyFastLocalsProxy_CheckExact(op) Py_IS_TYPE(op, &_PyFastLocalsProxy_Type) + + +// Underlying implementation API supporting the stable PyLocals_*() APIs +// TODO: Add specific test cases for these (as any PyLocals_* tests won't cover +// checking the status of a frame other than the currently active one) +PyAPI_FUNC(PyObject *) PyFrame_GetLocals(PyFrameObject *); +PyAPI_FUNC(PyObject *) PyFrame_GetLocalsCopy(PyFrameObject *); +PyAPI_FUNC(PyObject *) PyFrame_GetLocalsView(PyFrameObject *); +PyAPI_FUNC(PyLocals_Kind) PyFrame_GetLocalsKind(PyFrameObject *); + +// Underlying API supporting PyEval_GetLocals() +PyAPI_FUNC(PyObject *) _PyFrame_BorrowLocals(PyFrameObject *); + +#ifdef __cplusplus +} +#endif diff --git a/Include/descrobject.h b/Include/descrobject.h index 703bc8fd6df213..9a27f2dd27a11a 100644 --- a/Include/descrobject.h +++ b/Include/descrobject.h @@ -96,7 +96,14 @@ PyAPI_FUNC(PyObject *) PyDescr_NewWrapper(PyTypeObject *, PyAPI_FUNC(int) PyDescr_IsData(PyObject *); #endif +/* PyDictProxy should really have its own header/impl pair, but keeping + * it here for now... */ + PyAPI_FUNC(PyObject *) PyDictProxy_New(PyObject *); +#ifdef Py_BUILD_CORE +PyAPI_DATA(PyTypeObject) PyDictProxy_Type; +#endif + PyAPI_FUNC(PyObject *) PyWrapper_New(PyObject *, PyObject *); diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index bc469763670d4e..3a2fc398cc0b2c 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -164,19 +164,19 @@ extern Py_ssize_t _Py_QuickenedCount; #define CO_FAST_CELL 0x40 #define CO_FAST_FREE 0x80 -typedef unsigned char _PyLocals_Kind; +typedef unsigned char _PyLocal_VarKind; -static inline _PyLocals_Kind -_PyLocals_GetKind(PyObject *kinds, int i) +static inline _PyLocal_VarKind +_PyLocal_GetVarKind(PyObject *kinds, int i) { assert(PyBytes_Check(kinds)); assert(0 <= i && i < PyBytes_GET_SIZE(kinds)); char *ptr = PyBytes_AS_STRING(kinds); - return (_PyLocals_Kind)(ptr[i]); + return (_PyLocal_VarKind)(ptr[i]); } static inline void -_PyLocals_SetKind(PyObject *kinds, int i, _PyLocals_Kind kind) +_PyLocal_SetVarKind(PyObject *kinds, int i, _PyLocal_VarKind kind) { assert(PyBytes_Check(kinds)); assert(0 <= i && i < PyBytes_GET_SIZE(kinds)); diff --git a/Include/internal/pycore_frameobject.h b/Include/internal/pycore_frameobject.h new file mode 100644 index 00000000000000..6725c290a8b746 --- /dev/null +++ b/Include/internal/pycore_frameobject.h @@ -0,0 +1,17 @@ +#ifndef Py_INTERNAL_FRAMEOBJECT_H +#define Py_INTERNAL_FRAMEOBJECT_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +struct _frame; +typedef struct _frame PyFrameObject; + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_FRAMEOBJECT_H */ diff --git a/Lib/test/test_frame.py b/Lib/test/test_frame.py index a715e725a7e45b..f41b24ddb6aebb 100644 --- a/Lib/test/test_frame.py +++ b/Lib/test/test_frame.py @@ -5,6 +5,7 @@ import weakref from test import support +from test.support import import_helper class ClearTest(unittest.TestCase): @@ -183,8 +184,8 @@ def test_clear_locals(self): def test_locals_clear_locals(self): # Test f_locals before and after clear() (to exercise caching) f, outer, inner = self.make_frames() - outer.f_locals - inner.f_locals + self.assertNotEqual(outer.f_locals, {}) + self.assertNotEqual(inner.f_locals, {}) outer.clear() inner.clear() self.assertEqual(outer.f_locals, {}) @@ -195,6 +196,364 @@ def test_f_lineno_del_segfault(self): with self.assertRaises(AttributeError): del f.f_lineno +class FastLocalsProxyTest(unittest.TestCase): + + def check_proxy_contents(self, proxy, expected_contents): + # These checks should never implicitly resync the frame proxy's cache, + # even if the proxy is referenced as a local variable in the frame + # However, the first executed check may trigger the initial lazy sync + self.assertEqual(len(proxy), len(expected_contents)) + self.assertCountEqual(proxy.items(), expected_contents.items()) + + def test_dict_query_operations(self): + # Check retrieval of individual keys via the proxy + proxy = sys._getframe().f_locals + self.assertIs(proxy["self"], self) + self.assertIs(proxy.get("self"), self) + self.assertIs(proxy.get("no-such-key"), None) + self.assertIs(proxy.get("no-such-key", Ellipsis), Ellipsis) + + # Proxy value cache is lazily refreshed on the first operation that cares + # about the full contents of the mapping (such as querying the length) + expected_proxy_contents = {"self": self, "proxy": proxy} + expected_proxy_contents["expected_proxy_contents"] = expected_proxy_contents + self.check_proxy_contents(proxy, expected_proxy_contents) + + # Ensuring copying the proxy produces a plain dict instance + dict_copy = proxy.copy() + self.assertIsInstance(dict_copy, dict) + self.check_proxy_contents(dict_copy, expected_proxy_contents) + + # The proxy automatically updates its cache for O(n) operations like copying, + # but won't pick up new local variables until it is resync'ed with the frame + # or that particular key is accessed or queried + self.check_proxy_contents(proxy, dict_copy) + self.assertIn("dict_copy", proxy) # Implicitly updates cache for this key + expected_proxy_contents["dict_copy"] = dict_copy + self.check_proxy_contents(proxy, expected_proxy_contents) + + # Check forward iteration (order is abitrary, so only check overall contents) + # Note: len() and the items() method are covered by "check_proxy_contents" + self.assertCountEqual(proxy, expected_proxy_contents) + self.assertCountEqual(proxy.keys(), expected_proxy_contents.keys()) + self.assertCountEqual(proxy.values(), expected_proxy_contents.values()) + # Check reversed iteration (order should be reverse of forward iteration) + self.assertEqual(list(reversed(proxy)), list(reversed(list(proxy)))) + + # Check dict union operations (these implicitly refresh the value cache) + extra_contents = dict(a=1, b=2) + expected_proxy_contents["extra_contents"] = extra_contents + self.check_proxy_contents(proxy | proxy, expected_proxy_contents) + self.assertIsInstance(proxy | proxy, dict) + self.check_proxy_contents(proxy | extra_contents, expected_proxy_contents | extra_contents) + self.check_proxy_contents(extra_contents | proxy, expected_proxy_contents | extra_contents) + + def test_dict_mutation_operations(self): + # Check mutation of local variables via proxy + proxy = sys._getframe().f_locals + if not len(proxy): # Trigger the initial implicit cache update + # This code block never actually runs + unbound_local = None + self.assertNotIn("unbound_local", proxy) + proxy["unbound_local"] = "set via proxy" + self.assertEqual(unbound_local, "set via proxy") + del proxy["unbound_local"] + with self.assertRaises(UnboundLocalError): + unbound_local + # Check mutation of cell variables via proxy + cell_variable = None + proxy["cell_variable"] = "set via proxy" + self.assertEqual(cell_variable, "set via proxy") + def inner(): + return cell_variable + self.assertEqual(inner(), "set via proxy") + del proxy["cell_variable"] + with self.assertRaises(UnboundLocalError): + cell_variable + with self.assertRaises(NameError): + inner() + # Check storage of additional variables in the frame value cache via proxy + proxy["extra_variable"] = "added via proxy" + self.assertEqual(proxy["extra_variable"], "added via proxy") + with self.assertRaises(NameError): + extra_variable + del proxy["extra_variable"] + self.assertNotIn("extra_variable", proxy) + + # Check pop() on all 3 kinds of variable + unbound_local = "set directly" + self.assertEqual(proxy.pop("unbound_local"), "set directly") + self.assertIs(proxy.pop("unbound_local", None), None) + with self.assertRaises(KeyError): + proxy.pop("unbound_local") + cell_variable = "set directly" + self.assertEqual(proxy.pop("cell_variable"), "set directly") + self.assertIs(proxy.pop("cell_variable", None), None) + with self.assertRaises(KeyError): + proxy.pop("cell_variable") + proxy["extra_variable"] = "added via proxy" + self.assertEqual(proxy.pop("extra_variable"), "added via proxy") + self.assertIs(proxy.pop("extra_variable", None), None) + with self.assertRaises(KeyError): + proxy.pop("extra_variable") + + # Check setdefault() on all 3 kinds of variable + expected_value = "set via setdefault()" + self.assertEqual(proxy.setdefault("unbound_local", expected_value), expected_value) + self.assertEqual(proxy.setdefault("unbound_local", "ignored"), expected_value) + del unbound_local + self.assertEqual(proxy.setdefault("unbound_local"), None) + self.assertIs(unbound_local, None) + self.assertEqual(proxy.setdefault("cell_variable", expected_value), expected_value) + self.assertEqual(proxy.setdefault("cell_variable", "ignored"), expected_value) + del cell_variable + self.assertEqual(proxy.setdefault("cell_variable"), None) + self.assertIs(cell_variable, None) + self.assertEqual(proxy.setdefault("extra_variable", expected_value), expected_value) + self.assertEqual(proxy.setdefault("extra_variable", "ignored"), expected_value) + del proxy["extra_variable"] + self.assertEqual(proxy.setdefault("extra_variable"), None) + self.assertIs(cell_variable, None) + + + # Check updating all 3 kinds of variable via update() + updated_keys = { + "unbound_local": "set via proxy.update()", + "cell_variable": "set via proxy.update()", + "extra_variable": "set via proxy.update()", + } + proxy.update(updated_keys) + self.assertEqual(unbound_local, "set via proxy.update()") + self.assertEqual(cell_variable, "set via proxy.update()") + self.assertEqual(proxy["extra_variable"], "set via proxy.update()") + + # Check updating all 3 kinds of variable via an in-place dict union + updated_keys = { + "unbound_local": "set via proxy |=", + "cell_variable": "set via proxy |=", + "extra_variable": "set via proxy |=", + } + proxy |= updated_keys + self.assertEqual(unbound_local, "set via proxy |=") + self.assertEqual(cell_variable, "set via proxy |=") + self.assertEqual(proxy["extra_variable"], "set via proxy |=") + + # Check clearing all variables via the proxy + # Use a nested generator to allow the test case reference to be + # restored even after the frame variables are cleared + def clear_frame_via_proxy(test_case_arg): + inner_proxy = sys._getframe().f_locals + inner_proxy["extra_variable"] = "added via inner_proxy" + test_case_arg.assertEqual(inner_proxy, { + "inner_proxy": inner_proxy, + "cell_variable": cell_variable, + "test_case_arg": test_case_arg, + "extra_variable": "added via inner_proxy", + }) + inner_proxy.clear() + test_case = yield None + with test_case.assertRaises(UnboundLocalError): + inner_proxy + with test_case.assertRaises(UnboundLocalError): + test_case_arg + with test_case.assertRaises(NameError): + cell_variable + inner_proxy = sys._getframe().f_locals + test_case.assertNotIn("extra_variable", inner_proxy) + # Clearing the inner frame even clears the cell in the outer frame + clear_iter = clear_frame_via_proxy(self) + next(clear_iter) + with self.assertRaises(UnboundLocalError): + cell_variable + # Run the final checks in the inner frame + try: + clear_iter.send(self) + self.fail("Inner proxy clearing iterator didn't stop") + except StopIteration: + pass + + def test_popitem(self): + # Check popitem() in a controlled inner frame + # This is a separate test case so it can be skipped if the test case + # detects that something is injecting extra keys into the frame state + if len(sys._getframe().f_locals) != 1: + self.skipTest("Locals other than 'self' detected, test case will be unreliable") + + # With no local variables, trying to pop one should fail + def popitem_exception(): + return sys._getframe().f_locals.popitem() + with self.assertRaises(KeyError): + popitem_exception() + + # With exactly one local variable, it should be popped + def popitem_local(arg="only proxy entry"): + return sys._getframe().f_locals.popitem(), list(sys._getframe().f_locals) + popped_item, remaining_vars = popitem_local() + self.assertEqual(popped_item, ("arg", "only proxy entry")) + self.assertEqual(remaining_vars, []) + + # With exactly one cell variable, it should be popped + cell_variable = initial_cell_ref = "only proxy entry" + def popitem_cell(): + return cell_variable, sys._getframe().f_locals.popitem(), list(sys._getframe().f_locals) + cell_ref, popped_item, remaining_vars = popitem_cell() + self.assertEqual(popped_item, ("cell_variable", "only proxy entry")) + self.assertEqual(remaining_vars, []) + self.assertIs(cell_ref, initial_cell_ref) + with self.assertRaises(UnboundLocalError): + cell_variable + + # With exactly one extra variable, it should be popped + def popitem_extra(): + sys._getframe().f_locals["extra_variable"] = "only proxy entry" + return sys._getframe().f_locals.popitem(), list(sys._getframe().f_locals) + popped_item, remaining_vars = popitem_extra() + self.assertEqual(popped_item, ("extra_variable", "only proxy entry")) + self.assertEqual(remaining_vars, []) + + def test_sync_frame_cache(self): + proxy = sys._getframe().f_locals + self.assertEqual(len(proxy), 2) # Trigger the initial implicit cache update + new_variable = None + # No implicit value cache refresh + self.assertNotIn("new_variable", set(proxy)) + # But an explicit refresh adds the new key + proxy.sync_frame_cache() + self.assertIn("new_variable", set(proxy)) + + @support.cpython_only + def test_proxy_sizeof(self): + # Proxy should only be storing a frame reference and the flag that + # indicates whether or not the proxy has refreshed the value cache + proxy = sys._getframe().f_locals + expected_size = support.calcobjsize("Pi") + support.check_sizeof(self, proxy, expected_size) + + def test_active_frame_c_apis(self): + # Use ctypes to access the C APIs under test + ctypes = import_helper.import_module('ctypes') + Py_IncRef = ctypes.pythonapi.Py_IncRef + PyEval_GetLocals = ctypes.pythonapi.PyEval_GetLocals + PyLocals_Get = ctypes.pythonapi.PyLocals_Get + PyLocals_GetKind = ctypes.pythonapi.PyLocals_GetKind + PyLocals_GetCopy = ctypes.pythonapi.PyLocals_GetCopy + PyLocals_GetView = ctypes.pythonapi.PyLocals_GetView + for capi_func in (Py_IncRef,): + capi_func.argtypes = (ctypes.py_object,) + for capi_func in (PyEval_GetLocals, + PyLocals_Get, PyLocals_GetCopy, PyLocals_GetView): + capi_func.restype = ctypes.py_object + + # PyEval_GetLocals() always accesses the running frame, + # so Py_IncRef has to be called inline (no helper function) + + # This test covers the retrieval APIs, the behavioural tests are covered + # elsewhere using the `frame.f_locals` attribute and the locals() builtin + + # Test retrieval API behaviour in an optimised scope + c_locals_cache = PyEval_GetLocals() + Py_IncRef(c_locals_cache) # Make the borrowed reference a real one + Py_IncRef(c_locals_cache) # Account for next check's borrowed reference + self.assertIs(PyEval_GetLocals(), c_locals_cache) + self.assertEqual(PyLocals_GetKind(), 1) # PyLocals_SHALLOW_COPY + locals_get = PyLocals_Get() + self.assertIsInstance(locals_get, dict) + self.assertIsNot(locals_get, c_locals_cache) + locals_copy = PyLocals_GetCopy() + self.assertIsInstance(locals_copy, dict) + self.assertIsNot(locals_copy, c_locals_cache) + locals_view = PyLocals_GetView() + self.assertIsInstance(locals_view, types.MappingProxyType) + + # Test API behaviour in an unoptimised scope + class ExecFrame: + c_locals_cache = PyEval_GetLocals() + Py_IncRef(c_locals_cache) # Make the borrowed reference a real one + Py_IncRef(c_locals_cache) # Account for next check's borrowed reference + self.assertIs(PyEval_GetLocals(), c_locals_cache) + self.assertEqual(PyLocals_GetKind(), 0) # PyLocals_DIRECT_REFERENCE + locals_get = PyLocals_Get() + self.assertIs(locals_get, c_locals_cache) + locals_copy = PyLocals_GetCopy() + self.assertIsInstance(locals_copy, dict) + self.assertIsNot(locals_copy, c_locals_cache) + locals_view = PyLocals_GetView() + self.assertIsInstance(locals_view, types.MappingProxyType) + + def test_arbitrary_frame_c_apis(self): + # Use ctypes to access the C APIs under test + ctypes = import_helper.import_module('ctypes') + Py_IncRef = ctypes.pythonapi.Py_IncRef + _PyFrame_BorrowLocals = ctypes.pythonapi._PyFrame_BorrowLocals + PyFrame_GetLocals = ctypes.pythonapi.PyFrame_GetLocals + PyFrame_GetLocalsKind = ctypes.pythonapi.PyFrame_GetLocalsKind + PyFrame_GetLocalsCopy = ctypes.pythonapi.PyFrame_GetLocalsCopy + PyFrame_GetLocalsView = ctypes.pythonapi.PyFrame_GetLocalsView + for capi_func in (Py_IncRef, _PyFrame_BorrowLocals, + PyFrame_GetLocals, PyFrame_GetLocalsKind, + PyFrame_GetLocalsCopy, PyFrame_GetLocalsView): + capi_func.argtypes = (ctypes.py_object,) + for capi_func in (_PyFrame_BorrowLocals, PyFrame_GetLocals, + PyFrame_GetLocalsCopy, PyFrame_GetLocalsView): + capi_func.restype = ctypes.py_object + + def get_c_locals(frame): + c_locals = _PyFrame_BorrowLocals(frame) + Py_IncRef(c_locals) # Make the borrowed reference a real one + return c_locals + + # This test covers the retrieval APIs, the behavioural tests are covered + # elsewhere using the `frame.f_locals` attribute and the locals() builtin + + # Test querying an optimised frame from an unoptimised scope + func_frame = sys._getframe() + cls_frame = None + def set_cls_frame(f): + nonlocal cls_frame + cls_frame = f + class ExecFrame: + c_locals_cache = get_c_locals(func_frame) + self.assertIs(get_c_locals(func_frame), c_locals_cache) + self.assertEqual(PyFrame_GetLocalsKind(func_frame), 1) # PyLocals_SHALLOW_COPY + locals_get = PyFrame_GetLocals(func_frame) + self.assertIsInstance(locals_get, dict) + self.assertIsNot(locals_get, c_locals_cache) + locals_copy = PyFrame_GetLocalsCopy(func_frame) + self.assertIsInstance(locals_copy, dict) + self.assertIsNot(locals_copy, c_locals_cache) + locals_view = PyFrame_GetLocalsView(func_frame) + self.assertIsInstance(locals_view, types.MappingProxyType) + + # Keep the class frame alive for the functions below to access + set_cls_frame(sys._getframe()) + + # Test querying an unoptimised frame from an optimised scope + c_locals_cache = get_c_locals(cls_frame) + self.assertIs(get_c_locals(cls_frame), c_locals_cache) + self.assertEqual(PyFrame_GetLocalsKind(cls_frame), 0) # PyLocals_DIRECT_REFERENCE + locals_get = PyFrame_GetLocals(cls_frame) + self.assertIs(locals_get, c_locals_cache) + locals_copy = PyFrame_GetLocalsCopy(cls_frame) + self.assertIsInstance(locals_copy, dict) + self.assertIsNot(locals_copy, c_locals_cache) + locals_view = PyFrame_GetLocalsView(cls_frame) + self.assertIsInstance(locals_view, types.MappingProxyType) + + def test_locals_to_fast_error(self): + # Use ctypes to access the C APIs under test + ctypes = import_helper.import_module('ctypes') + Py_IncRef = ctypes.pythonapi.Py_IncRef + PyFrame_LocalsToFast = ctypes.pythonapi.PyFrame_LocalsToFast + PyFrame_LocalsToFast.argtypes = (ctypes.py_object, ctypes.c_int) + frame = sys._getframe() + # Ensure the error message recommends the replacement API + replacement_api = r'PyObject_GetAttrString\(frame, "f_locals"\)' + with self.assertRaisesRegex(RuntimeError, replacement_api): + PyFrame_LocalsToFast(frame, 0) + # Ensure the error is still raised when the "clear" parameter is set + with self.assertRaisesRegex(RuntimeError, replacement_api): + PyFrame_LocalsToFast(frame, 1) + class ReprTest(unittest.TestCase): """ diff --git a/Lib/test/test_scope.py b/Lib/test/test_scope.py index 29d60ffe53f036..53548febd7df35 100644 --- a/Lib/test/test_scope.py +++ b/Lib/test/test_scope.py @@ -808,5 +808,81 @@ def dig(self): self.assertIsNone(ref()) + +LOCALS_SEMANTICS_TEST_CODE = """\ +global_ns = globals() +known_var = "original" +local_ns_1 = locals() +local_ns_1["known_var"] = "set_via_locals" +local_ns_1["unknown_var"] = "set_via_locals" +local_ns_2 = locals() +""" + + +class TestLocalsSemantics(unittest.TestCase): + # This is a new set of test cases added as part of the implementation + # of PEP 558 to cover the expected behaviour of locals() + # The expected behaviour of frame.f_locals is covered in test_sys_settrace + + def test_locals_update_semantics_at_module_scope(self): + # At module scope, globals() and locals() are the same namespace + global_ns = dict() + exec(LOCALS_SEMANTICS_TEST_CODE, global_ns) + self.assertIs(global_ns["global_ns"], global_ns) + self.assertIs(global_ns["local_ns_1"], global_ns) + self.assertIs(global_ns["local_ns_2"], global_ns) + self.assertEqual(global_ns["known_var"], "set_via_locals") + self.assertEqual(global_ns["unknown_var"], "set_via_locals") + + def test_locals_update_semantics_at_class_scope(self): + # At class scope, globals() and locals() are different namespaces + global_ns = dict() + local_ns = dict() + exec(LOCALS_SEMANTICS_TEST_CODE, global_ns, local_ns) + self.assertIs(local_ns["global_ns"], global_ns) + self.assertIs(local_ns["local_ns_1"], local_ns) + self.assertIs(local_ns["local_ns_2"], local_ns) + self.assertEqual(local_ns["known_var"], "set_via_locals") + self.assertEqual(local_ns["unknown_var"], "set_via_locals") + + def test_locals_snapshot_semantics_at_function_scope(self): + def function_local_semantics(): + global_ns = globals() + known_var = "original" + to_be_deleted = "not_yet_deleted" + local_ns1 = locals() + local_ns1["known_var"] = "set_via_ns1" + local_ns1["unknown_var"] = "set_via_ns1" + del to_be_deleted + local_ns2 = locals() + local_ns2["known_var"] = "set_via_ns2" + local_ns2["unknown_var"] = "set_via_ns2" + return dict(ns1=local_ns1, ns2=local_ns2) + + global_ns = globals() + self.assertIsInstance(global_ns, dict) + local_namespaces = function_local_semantics() + # Check internal consistency of each snapshot + for name, local_ns in local_namespaces.items(): + with self.subTest(namespace=name): + self.assertIsInstance(local_ns, dict) + self.assertIs(local_ns["global_ns"], global_ns) + expected_text = "set_via_" + name + self.assertEqual(local_ns["known_var"], expected_text) + self.assertEqual(local_ns["unknown_var"], expected_text) + # Check independence of the snapshots + local_ns1 = local_namespaces["ns1"] + local_ns2 = local_namespaces["ns2"] + self.assertIsNot(local_ns1, local_ns2) + # Check that not yet set local variables are excluded from snapshot + self.assertNotIn("local_ns1", local_ns1) + self.assertNotIn("local_ns2", local_ns1) + self.assertIs(local_ns2["local_ns1"], local_ns1) + self.assertNotIn("local_ns2", local_ns2) + # Check that deleted variables are excluded from snapshot + self.assertEqual(local_ns1["to_be_deleted"], "not_yet_deleted") + self.assertNotIn("to_be_deleted", local_ns2) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index 8b6631879fb336..c6ac2184a68d44 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -5,6 +5,7 @@ import sys import difflib import gc +import textwrap from functools import wraps import asyncio @@ -2072,6 +2073,188 @@ def gen(): next(gen()) output.append(5) +class FrameLocalsTestCase(unittest.TestCase): + def setUp(self): + self.addCleanup(sys.settrace, sys.gettrace()) + sys.settrace(None) + + def test_closures_are_not_implicitly_reset_to_previous_state(self): + # See https://bugs.python.org/issue30744 for details + i_from_generator = [] + x_from_generator = [] + x_from_nested_ref = [] + x_from_nested_locals = [] + def outer(): + x = 0 + + def update_nested_refs(): + x_from_nested_ref.append(x) + x_from_nested_locals.append(locals()["x"]) + + yield update_nested_refs + for i in range(1, 10): + i_from_generator.append(i) + x += 1 + yield x + + incrementing_generator = outer() + update_nested_refs = next(incrementing_generator) + + def tracefunc(frame, event, arg): + x_from_generator.append(next(incrementing_generator)) + return tracefunc + + sys.settrace(tracefunc) + try: + update_nested_refs() + update_nested_refs() + finally: + sys.settrace(None) + self.assertEqual(x_from_generator, i_from_generator) + self.assertEqual(x_from_nested_ref, [2, 6]) + self.assertEqual(x_from_nested_locals, [3, 7]) + + def test_locals_writethrough_proxy(self): + # To support interactive debuggers, trace functions are expected to + # be able to reliably modify function locals. However, this should + # NOT enable writebacks via locals() at function scope. + # + # This is a simple scenario to test the core behaviour of the + # writethrough proxy + local_var = "original" + # Check regular locals() snapshot + locals_snapshot = locals() + self.assertEqual(locals_snapshot["local_var"], "original") + locals_snapshot["local_var"] = "modified" + self.assertEqual(local_var, "original") + # Check writethrough proxy on frame + locals_proxy = sys._getframe().f_locals + self.assertEqual(locals_proxy["local_var"], "original") + locals_proxy["local_var"] = "modified" + self.assertEqual(local_var, "modified") + # Check handling of closures + def nested_scope(): + nonlocal local_var + return local_var, locals(), sys._getframe().f_locals + closure_var, inner_snapshot, inner_proxy = nested_scope() + self.assertEqual(closure_var, "modified") + self.assertEqual(inner_snapshot["local_var"], "modified") + self.assertEqual(inner_proxy["local_var"], "modified") + inner_snapshot["local_var"] = "modified_again" + self.assertEqual(local_var, "modified") + self.assertEqual(inner_snapshot["local_var"], "modified_again") + self.assertEqual(inner_proxy["local_var"], "modified") + inner_proxy["local_var"] = "modified_yet_again" + self.assertEqual(local_var, "modified_yet_again") + self.assertEqual(inner_snapshot["local_var"], "modified_again") + self.assertEqual(inner_proxy["local_var"], "modified_yet_again") + + def test_locals_writeback_complex_scenario(self): + # Further locals writeback testing using a more complex scenario + # involving multiple scopes of different kinds + # + # Note: the sample values have numbers in them so mixing up variable + # names in the checks can't accidentally make the test pass - + # you'd have to get both the name *and* expected number wrong + self.maxDiff = None + code = textwrap.dedent(""" + locals()['a_global'] = 'created1' # We expect this to be retained + another_global = 'original2' # Trace func will modify this + + class C: + locals()['an_attr'] = 'created3' # We expect this to be retained + another_attr = 'original4' # Trace func will modify this + a_class_attribute = C.an_attr + another_class_attribute = C.another_attr + del C + + def outer(): + a_nonlocal = 'original5' # We expect this to be retained + another_nonlocal = 'original6' # Trace func will modify this + def inner(): + nonlocal a_nonlocal, another_nonlocal + a_local = 'original7' # We expect this to be retained + another_local = 'original8' # Trace func will modify this + ns = locals() + ns['a_local'] = 'modified7' # We expect this to be retained + ns['a_nonlocal'] = 'modified5' # We expect this to be retained + ns['a_new_local'] = 'created9' # We expect this to be retained + return a_local, another_local, ns + outer_local = 'original10' # Trace func will modify this + # Trigger any updates from the inner function & trace function + inner_result = inner() + outer_result = a_nonlocal, another_nonlocal, outer_local, locals() + return outer_result, inner_result + outer_result, inner_result = outer() + a_nonlocal, another_nonlocal, outer_local, outer_ns = outer_result + a_nonlocal_via_ns = outer_ns['a_nonlocal'] + another_nonlocal_via_ns = outer_ns['another_nonlocal'] + outer_local_via_ns = outer_ns['outer_local'] + a_local, another_local, inner_ns = inner_result + a_local_via_ns = inner_ns['a_local'] + a_nonlocal_via_inner_ns = inner_ns['a_nonlocal'] + print(a_nonlocal_via_inner_ns) + another_nonlocal_via_inner_ns = inner_ns['another_nonlocal'] + another_local_via_ns = inner_ns['another_local'] + a_new_local_via_ns = inner_ns['a_new_local'] + del outer, outer_result, outer_ns, inner_result, inner_ns + """ + ) + def tracefunc(frame, event, arg): + ns = frame.f_locals + co_name = frame.f_code.co_name + if event == "return": + # We leave most state manipulation to the very end + if co_name == "C": + # Modify class attributes + ns["another_attr"] = "modified4" + elif co_name == "inner": + # Modify variables in outer scope + ns["another_nonlocal"] = "modified6" + outer_ns = frame.f_back.f_locals + outer_ns["outer_local"] = "modified10" + elif co_name == "": + # Modify globals + ns["another_global"] = "modified2" + elif event == "line" and co_name == "inner": + # This is the one item we can't leave to the end, as the + # return tuple is already built by the time the return event + # fires, and we want to manipulate one of the entries in that + # So instead, we just mutate it on every line trace event + ns["another_local"] = "modified8" + print(event) + return tracefunc + actual_ns = {} + sys.settrace(tracefunc) + try: + exec(code, actual_ns) + finally: + sys.settrace(None) + for k in list(actual_ns.keys()): + if k.startswith("_"): + del actual_ns[k] + expected_ns = { + "a_global": "created1", + "another_global": "modified2", + "a_class_attribute": "created3", + "another_class_attribute": "modified4", + "a_nonlocal": "original5", + "a_nonlocal_via_ns": "original5", + "a_nonlocal_via_inner_ns": "modified5", + "another_nonlocal": "modified6", + "another_nonlocal_via_ns": "modified6", + "another_nonlocal_via_inner_ns": "original6", + "a_local": "original7", + "a_local_via_ns": "modified7", + "another_local": "modified8", + "another_local_via_ns": "modified8", + "a_new_local_via_ns": "created9", + "outer_local": "modified10", + "outer_local_via_ns": "modified10", + } + # Expected is first so any error diff is easier to read + self.assertEqual(expected_ns, actual_ns) + if __name__ == "__main__": unittest.main() diff --git a/Misc/stable_abi.txt b/Misc/stable_abi.txt index f104f84e451da1..b858d50961070e 100644 --- a/Misc/stable_abi.txt +++ b/Misc/stable_abi.txt @@ -2133,5 +2133,17 @@ function PyGC_IsEnabled added 3.10 -# (Detailed comments aren't really needed for further entries: from here on -# we can use version control logs.) +# While detailed comments aren't technically needed for further entries +# (we can use version control logs for this file), they're still helpful +# when the full changelog isn't readily available + +# PEP 558: New locals() access functions + +function PyLocals_Get + added 3.11 +function PyLocals_GetCopy + added 3.11 +function PyLocals_GetKind + added 3.11 +function PyLocals_GetView + added 3.11 diff --git a/Objects/clinic/frameobject.c.h b/Objects/clinic/frameobject.c.h new file mode 100644 index 00000000000000..43b3a587914887 --- /dev/null +++ b/Objects/clinic/frameobject.c.h @@ -0,0 +1,29 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +static PyObject * +fastlocalsproxy_new_impl(PyTypeObject *type, PyObject *frame); + +static PyObject * +fastlocalsproxy_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"frame", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "fastlocalsproxy", 0}; + PyObject *argsbuf[1]; + PyObject * const *fastargs; + Py_ssize_t nargs = PyTuple_GET_SIZE(args); + PyObject *frame; + + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 1, 0, argsbuf); + if (!fastargs) { + goto exit; + } + frame = fastargs[0]; + return_value = fastlocalsproxy_new_impl(type, frame); + +exit: + return return_value; +} +/*[clinic end generated code: output=34617a00e21738f3 input=a9049054013a1b77]*/ diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 3dc9fd787f3859..c0206043278109 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -156,12 +156,12 @@ validate_and_copy_tuple(PyObject *tup) // This is also used in compile.c. void -_Py_set_localsplus_info(int offset, PyObject *name, _PyLocals_Kind kind, +_Py_set_localsplus_info(int offset, PyObject *name, _PyLocal_VarKind kind, PyObject *names, PyObject *kinds) { Py_INCREF(name); PyTuple_SET_ITEM(names, offset, name); - _PyLocals_SetKind(kinds, offset, kind); + _PyLocal_SetVarKind(kinds, offset, kind); } static void @@ -175,7 +175,7 @@ get_localsplus_counts(PyObject *names, PyObject *kinds, int nfreevars = 0; Py_ssize_t nlocalsplus = PyTuple_GET_SIZE(names); for (int i = 0; i < nlocalsplus; i++) { - _PyLocals_Kind kind = _PyLocals_GetKind(kinds, i); + _PyLocal_VarKind kind = _PyLocal_GetVarKind(kinds, i); if (kind & CO_FAST_LOCAL) { nlocals += 1; if (kind & CO_FAST_CELL) { @@ -205,7 +205,7 @@ get_localsplus_counts(PyObject *names, PyObject *kinds, } static PyObject * -get_localsplus_names(PyCodeObject *co, _PyLocals_Kind kind, int num) +get_localsplus_names(PyCodeObject *co, _PyLocal_VarKind kind, int num) { PyObject *names = PyTuple_New(num); if (names == NULL) { @@ -213,7 +213,7 @@ get_localsplus_names(PyCodeObject *co, _PyLocals_Kind kind, int num) } int index = 0; for (int offset = 0; offset < co->co_nlocalsplus; offset++) { - _PyLocals_Kind k = _PyLocals_GetKind(co->co_localspluskinds, offset); + _PyLocal_VarKind k = _PyLocal_GetVarKind(co->co_localspluskinds, offset); if ((k & kind) == 0) { continue; } @@ -458,8 +458,8 @@ PyCode_NewWithPosOnlyArgs(int argcount, int posonlyargcount, int kwonlyargcount, // Merge the localsplus indices. nlocalsplus -= 1; offset -= 1; - _PyLocals_Kind kind = _PyLocals_GetKind(localspluskinds, argoffset); - _PyLocals_SetKind(localspluskinds, argoffset, kind | CO_FAST_CELL); + _PyLocal_VarKind kind = _PyLocal_GetVarKind(localspluskinds, argoffset); + _PyLocal_SetVarKind(localspluskinds, argoffset, kind | CO_FAST_CELL); continue; } _Py_set_localsplus_info(offset, name, CO_FAST_CELL, diff --git a/Objects/descrobject.c b/Objects/descrobject.c index 0565992bdb79f7..61faa74cc0247b 100644 --- a/Objects/descrobject.c +++ b/Objects/descrobject.c @@ -1249,6 +1249,7 @@ PyDictProxy_New(PyObject *mapping) } + /* --- Wrapper object for "slot" methods --- */ /* This has no reason to be in this file except that adding new files is a @@ -1863,8 +1864,9 @@ PyTypeObject PyDictProxy_Type = { PyObject_GenericGetAttr, /* tp_getattro */ 0, /* tp_setattro */ 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | - Py_TPFLAGS_MAPPING, /* tp_flags */ + Py_TPFLAGS_DEFAULT | + Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_MAPPING, /* tp_flags */ 0, /* tp_doc */ mappingproxy_traverse, /* tp_traverse */ 0, /* tp_clear */ diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 813ec561463ba5..3c2231834551c5 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -13,6 +13,8 @@ #define OFF(x) offsetof(PyFrameObject, x) +static PyObject *_PyFastLocalsProxy_New(PyObject *frame); + static PyMemberDef frame_memberlist[] = { {"f_back", T_OBJECT, OFF(f_back), READONLY}, {"f_trace_lines", T_BOOL, OFF(f_trace_lines), 0}, @@ -27,17 +29,122 @@ get_frame_state(void) return &interp->frame; } +static PyObject * +_frame_get_locals_mapping(PyFrameObject *f) +{ + PyObject *locals = _PyFrame_Specials(f)[FRAME_SPECIALS_LOCALS_OFFSET]; + if (locals == NULL) { + locals = _PyFrame_Specials(f)[FRAME_SPECIALS_LOCALS_OFFSET] = PyDict_New(); + } + return locals; +} static PyObject * -frame_getlocals(PyFrameObject *f, void *closure) +_frame_get_updated_locals(PyFrameObject *f) { if (PyFrame_FastToLocalsWithError(f) < 0) return NULL; PyObject *locals = _PyFrame_Specials(f)[FRAME_SPECIALS_LOCALS_OFFSET]; - Py_INCREF(locals); + assert(locals != NULL); + // Internal borrowed reference, caller increfs for external sharing return locals; } +PyObject * +_PyFrame_BorrowLocals(PyFrameObject *f) +{ + // This frame API supports the PyEval_GetLocals() stable API, which has + // historically returned a borrowed reference (so this does the same) + // It also ensures the cache is up to date + return _frame_get_updated_locals(f); +} + +PyObject * +PyFrame_GetLocals(PyFrameObject *f) +{ + // This frame API implements the Python level locals() builtin + // and supports the PyLocals_Get() stable API + PyObject *updated_locals = _frame_get_updated_locals(f); + if (updated_locals == NULL) { + return NULL; + } + PyCodeObject *co = _PyFrame_GetCode(f); + + assert(co); + if (co->co_flags & CO_OPTIMIZED) { + // PEP 558: Make a copy of optimised scopes to avoid quirky side effects + updated_locals = PyDict_Copy(updated_locals); + } else { + // Share a direct locals reference for class and module scopes + Py_INCREF(updated_locals); + } + + return updated_locals; +} + +PyLocals_Kind +PyFrame_GetLocalsKind(PyFrameObject *f) +{ + // This frame API supports the stable PyLocals_GetKind() API + PyCodeObject *co = _PyFrame_GetCode(f); + assert(co); + if (co->co_flags & CO_OPTIMIZED) { + return PyLocals_SHALLOW_COPY; + } + return PyLocals_DIRECT_REFERENCE; +} + +PyObject * +PyFrame_GetLocalsCopy(PyFrameObject *f) +{ + // This frame API supports the stable PyLocals_GetCopy() API + PyObject *updated_locals = _frame_get_updated_locals(f); + if (updated_locals == NULL) { + return NULL; + } + return PyDict_Copy(updated_locals); +} + +static PyObject * +frame_getlocals(PyFrameObject *f, void *Py_UNUSED(ignored)) +{ + // This API implements the Python level frame.f_locals descriptor + PyObject *f_locals_attr = NULL; + PyCodeObject *co = _PyFrame_GetCode(f); + + assert(co); + if (co->co_flags & CO_OPTIMIZED) { + /* PEP 558: If this is an optimized frame, ensure f_locals at the Python + * layer is a new fastlocalsproxy instance, while f_locals at the C + * layer still refers to the underlying shared namespace mapping. + * + * To minimise runtime overhead when the frame value cache isn't used, + * each new proxy instance postpones refreshing the cache until the + * first operation that assumes the value cache is up to date. + */ + f_locals_attr = _PyFastLocalsProxy_New((PyObject *) f); + } else { + // Share a direct locals reference for class and module scopes + f_locals_attr = _frame_get_locals_mapping(f); + if (f_locals_attr == NULL) { + return NULL; + } + Py_INCREF(f_locals_attr); + } + return f_locals_attr; +} + +PyObject * +PyFrame_GetLocalsView(PyFrameObject *f) +{ + // This frame API supports the stable PyLocals_GetView() API + PyObject *rw_locals = frame_getlocals(f, NULL); + if (rw_locals == NULL) { + return NULL; + } + return PyDictProxy_New(rw_locals); +} + int PyFrame_GetLineNumber(PyFrameObject *f) { @@ -665,6 +772,7 @@ frame_traverse(PyFrameObject *f, visitproc visit, void *arg) { Py_VISIT(f->f_back); Py_VISIT(f->f_trace); + Py_VISIT(f->f_fast_refs); /* locals */ PyObject **localsplus = f->f_localsptr; @@ -691,9 +799,16 @@ frame_tp_clear(PyFrameObject *f) Py_CLEAR(f->f_trace); PyCodeObject *co = _PyFrame_GetCode(f); - /* locals */ + /* fast locals */ for (int i = 0; i < co->co_nlocalsplus; i++) { Py_CLEAR(f->f_localsptr[i]); + if (f->f_fast_refs != NULL) { + // Keep a record of the names defined on the code object, + // but don't keep any cells alive and discard the slot locations + PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i); + assert(name); + PyDict_SetItem(f->f_fast_refs, name, Py_None); + } } /* stack */ @@ -888,6 +1003,7 @@ _PyFrame_New_NoTrack(PyThreadState *tstate, PyFrameConstructor *con, PyObject *l specials[FRAME_SPECIALS_GLOBALS_OFFSET] = Py_NewRef(con->fc_globals); specials[FRAME_SPECIALS_LOCALS_OFFSET] = Py_XNewRef(locals); f->f_trace = NULL; + f->f_fast_refs = NULL; f->f_stackdepth = 0; f->f_trace_lines = 1; f->f_trace_opcodes = 0; @@ -949,21 +1065,20 @@ PyFrame_FastToLocalsWithError(PyFrameObject *f) PyErr_BadInternalCall(); return -1; } - locals = _PyFrame_Specials(f)[FRAME_SPECIALS_LOCALS_OFFSET]; + locals = _frame_get_locals_mapping(f); if (locals == NULL) { - locals = _PyFrame_Specials(f)[FRAME_SPECIALS_LOCALS_OFFSET] = PyDict_New(); - if (locals == NULL) - return -1; + return -1; } co = _PyFrame_GetCode(f); + assert(co); fast = f->f_localsptr; for (int i = 0; i < co->co_nlocalsplus; i++) { - _PyLocals_Kind kind = _PyLocals_GetKind(co->co_localspluskinds, i); + _PyLocal_VarKind kind = _PyLocal_GetVarKind(co->co_localspluskinds, i); /* If the namespace is unoptimized, then one of the following cases applies: 1. It does not contain free variables, because it - uses import * or is a top-level namespace. + is a top-level namespace. 2. It is a class namespace. We don't want to accidentally copy free variables into the locals dict used by the class. @@ -994,8 +1109,8 @@ PyFrame_FastToLocalsWithError(PyFrameObject *f) } // (likely) Otherwise it it is an arg (kind & CO_FAST_LOCAL), // with the initial value set by _PyEval_MakeFrameVector()... - // (unlikely) ...or it was set to some initial value by - // an earlier call to PyFrame_LocalsToFast(). + // (unlikely) ...or it was set to some initial value via + // the frame locals proxy } } } @@ -1018,6 +1133,7 @@ PyFrame_FastToLocalsWithError(PyFrameObject *f) } } } + return 0; } @@ -1036,70 +1152,100 @@ PyFrame_FastToLocals(PyFrameObject *f) void PyFrame_LocalsToFast(PyFrameObject *f, int clear) { - /* Merge locals into fast locals */ - PyObject *locals; - PyObject **fast; - PyObject *error_type, *error_value, *error_traceback; + PyErr_SetString( + PyExc_RuntimeError, + "PyFrame_LocalsToFast is no longer supported. " + "Use PyObject_GetAttrString(frame, \"f_locals\") " + "to obtain a write-through mapping proxy instead." + ); +} + +static int +set_fast_ref(PyObject *fast_refs, PyObject *key, PyObject *value) +{ + // NOTE: Steals the "value" reference, so borrowed values need an INCREF + assert(PyUnicode_Check(key)); + int status = PyDict_SetItem(fast_refs, key, value); + Py_DECREF(value); + return status; +} + +static PyObject * +_PyFrame_BuildFastRefs(PyFrameObject *f) +{ + /* Construct a combined mapping from local variable names to indices + * in the fast locals array, and from nonlocal variable names directly + * to the corresponding cell objects + */ + PyObject **fast_locals; PyCodeObject *co; - if (f == NULL || f->f_state == FRAME_CLEARED) { - return; + PyObject *fast_refs; + + + if (f == NULL) { + PyErr_BadInternalCall(); + return NULL; } - locals = _PyFrame_Specials(f)[FRAME_SPECIALS_LOCALS_OFFSET]; - if (locals == NULL) - return; - fast = f->f_localsptr; co = _PyFrame_GetCode(f); + assert(co); + fast_locals = f->f_localsptr; - PyErr_Fetch(&error_type, &error_value, &error_traceback); - for (int i = 0; i < co->co_nlocalsplus; i++) { - _PyLocals_Kind kind = _PyLocals_GetKind(co->co_localspluskinds, i); + if (fast_locals == NULL || !(co->co_flags & CO_OPTIMIZED)) { + PyErr_SetString(PyExc_SystemError, + "attempted to build fast refs lookup table for non-optimized scope"); + return NULL; + } - /* Same test as in PyFrame_FastToLocals() above. */ - if (kind & CO_FAST_FREE && !(co->co_flags & CO_OPTIMIZED)) { - continue; - } + fast_refs = PyDict_New(); + if (fast_refs == NULL) { + return NULL; + } + + for (int i = 0; i < co->co_nlocalsplus; i++) { + _PyLocal_VarKind kind = _PyLocal_GetVarKind(co->co_localspluskinds, i); PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i); - PyObject *value = PyObject_GetItem(locals, name); - /* We only care about NULLs if clear is true. */ - if (value == NULL) { - PyErr_Clear(); - if (!clear) { - continue; - } - } - PyObject *oldvalue = fast[i]; - PyObject *cell = NULL; - if (kind == CO_FAST_FREE) { - // The cell was set by _PyEval_MakeFrameVector() from - // the function's closure. - assert(oldvalue != NULL && PyCell_Check(oldvalue)); - cell = oldvalue; - } - else if (kind & CO_FAST_CELL && oldvalue != NULL) { - /* Same test as in PyFrame_FastToLocals() above. */ - if (PyCell_Check(oldvalue) && + assert(name); + PyObject *target = NULL; + if (f->f_state == FRAME_CLEARED) { + // Frame is cleared, map all names defined on the frame to None + target = Py_None; + Py_INCREF(target); + } else if (kind & CO_FAST_FREE) { + // Reference to closure cell, save it as the proxy target + target = fast_locals[i]; + assert(target != NULL && PyCell_Check(target)); + Py_INCREF(target); + } else if (kind & CO_FAST_CELL) { + // Closure cell referenced from nested scopes + // Save it as the proxy target if the cell already exists, + // otherwise save the index and fix it up later on access + target = fast_locals[i]; + if (target != NULL && PyCell_Check(target) && _PyFrame_OpAlreadyRan(f, MAKE_CELL, i)) { - // (likely) MAKE_CELL must have executed already. - cell = oldvalue; + // MAKE_CELL built the cell, so use it as the proxy target + Py_INCREF(target); + } else { + // MAKE_CELL hasn't run yet, so just store the lookup index + // The proxy will check the kind on access, and switch over + // to using the cell once MAKE_CELL creates it + target = PyLong_FromSsize_t(i); } - // (unlikely) Otherwise, it must have been set to some - // initial value by an earlier call to PyFrame_LocalsToFast(). + } else if (kind & CO_FAST_LOCAL) { + // Ordinary fast local variable. Save index as the proxy target + target = PyLong_FromSsize_t(i); + } else { + PyErr_SetString(PyExc_SystemError, + "unknown local variable kind while building fast refs lookup table"); } - if (cell != NULL) { - oldvalue = PyCell_GET(cell); - if (value != oldvalue) { - Py_XDECREF(oldvalue); - Py_XINCREF(value); - PyCell_SET(cell, value); - } + if (target == NULL) { + Py_DECREF(fast_refs); + return NULL; } - else if (value != oldvalue) { - Py_XINCREF(value); - Py_XSETREF(fast[i], value); + if (set_fast_ref(fast_refs, name, target) != 0) { + return NULL; } - Py_XDECREF(value); } - PyErr_Restore(error_type, error_value, error_traceback); + return fast_refs; } /* Clear out the free list */ @@ -1174,3 +1320,1023 @@ _PyEval_BuiltinsFromGlobals(PyThreadState *tstate, PyObject *globals) return _PyEval_GetBuiltins(tstate); } + +/* _PyFastLocalsProxy_Type + * + * Mapping object that provides name-based access to the fast locals on a frame + */ +/*[clinic input] +class fastlocalsproxy "fastlocalsproxyobject *" "&_PyFastLocalsProxy_Type" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=a2dd0ae6e1642243]*/ + + +typedef struct { + PyObject_HEAD + PyFrameObject *frame; + int frame_cache_updated; /* Assume cache is out of date if this is not set */ +} fastlocalsproxyobject; + +static PyObject * +fastlocalsproxy_sizeof(fastlocalsproxyobject *flp, PyObject *Py_UNUSED(ignored)) +{ + Py_ssize_t res; + res = sizeof(fastlocalsproxyobject); + return PyLong_FromSsize_t(res); +} + +static PyObject * +fastlocalsproxy_get_updated_value_cache(fastlocalsproxyobject *flp) +{ + // Retrieve the locals value cache from an optimised frame, + // ensuring it is up to date with the current state of the frame + assert(flp); + assert(flp->frame); + flp->frame_cache_updated = 1; + return _frame_get_updated_locals(flp->frame); +} + +static PyObject * +fastlocalsproxy_get_value_cache(fastlocalsproxyobject *flp) +{ + // Retrieve the locals value cache from an optimised frame. + // If this proxy hasn't previously updated the locals value cache, + // assume the cache may be out of date and update it first + assert(flp); + if (flp->frame_cache_updated) { + assert(flp->frame); + return _frame_get_locals_mapping(flp->frame); + } + return fastlocalsproxy_get_updated_value_cache(flp); +} + + +static int +fastlocalsproxy_init_fast_refs(fastlocalsproxyobject *flp) +{ + // Build fast ref mapping if it hasn't been built yet + assert(flp); + assert(flp->frame); + if (flp->frame->f_fast_refs != NULL) { + return 0; + } + PyObject *fast_refs = _PyFrame_BuildFastRefs(flp->frame); + if (fast_refs == NULL) { + return -1; + } + flp->frame->f_fast_refs = fast_refs; + return 0; +} + +static Py_ssize_t +fastlocalsproxy_len(fastlocalsproxyobject *flp) +{ + // Extra keys may have been added, and some variables may not have been + // bound yet, so use the value cache on the frame rather than the + // keys in the fast locals reverse lookup mapping + // As long as it has been updated at least once, assume value cache + // is up to date (as actually checking is O(n)) + PyObject *locals = fastlocalsproxy_get_value_cache(flp); + return PyObject_Size(locals); +} + +static int +fastlocalsproxy_set_value_cache_entry(fastlocalsproxyobject *flp, PyObject *key, PyObject *value) +{ + PyFrameObject *f = flp->frame; + if (f->f_state == FRAME_CLEARED) { + // Don't touch the locals cache on already cleared frames + return 0; + } + PyObject *locals = _frame_get_locals_mapping(f); + if (locals == NULL) { + return -1; + } + if (value == NULL) { + // Ensure key is absent from cache (deleting if necessary) + if (PyDict_Contains(locals, key)) { + return PyObject_DelItem(locals, key); + } + return 0; + } + // Set cached value for the given key + return PyObject_SetItem(locals, key, value); +} + +static PyObject * +fastlocalsproxy_getitem(fastlocalsproxyobject *flp, PyObject *key) +{ + assert(flp); + if (fastlocalsproxy_init_fast_refs(flp) != 0) { + return NULL; + } + PyObject *fast_ref = PyDict_GetItem(flp->frame->f_fast_refs, key); + if (fast_ref == NULL) { + // No such local variable, delegate the request to the f_locals mapping + // Used by pdb (at least) to access __return__ and __exception__ values + PyObject *locals = _frame_get_locals_mapping(flp->frame); + if (locals == NULL) { + return NULL; + } + return PyObject_GetItem(locals, key); + } + // If the frame has been cleared, disallow access to locals and cell variables + PyFrameObject *f = flp->frame; + if (f->f_state == FRAME_CLEARED) { + PyErr_Format(PyExc_RuntimeError, + "Fast locals proxy attempted to read from cleared frame (%R)", f); + return NULL; + } + // Key is a valid Python variable for the frame, so retrieve the value + PyObject *value = NULL; + if (PyCell_Check(fast_ref)) { + // Closure cells are accessed directly via the fast refs mapping + value = PyCell_GET(fast_ref); + } else { + // Fast ref is a Python int mapping into the fast locals array + assert(PyLong_CheckExact(fast_ref)); + Py_ssize_t offset = PyLong_AsSsize_t(fast_ref); + if (offset < 0) { + return NULL; + } + PyCodeObject *co = _PyFrame_GetCode(f); + assert(co); + Py_ssize_t max_offset = co->co_nlocalsplus - 1; + if (offset > max_offset) { + PyErr_Format(PyExc_SystemError, + "Fast locals ref (%zd) exceeds array bound (%zd)", + offset, max_offset); + return NULL; + } + PyObject **fast_locals = f->f_localsptr; + value = fast_locals[offset]; + // Check if MAKE_CELL has been called since the proxy was created + _PyLocal_VarKind kind = _PyLocal_GetVarKind(co->co_localspluskinds, offset); + if (kind & CO_FAST_CELL) { + // Value hadn't been converted to a cell yet when the proxy was created + // Update the proxy if MAKE_CELL has run since the last access, + // otherwise continue treating it as a regular local variable + PyObject *target = value; + if (target != NULL && PyCell_Check(target) && + _PyFrame_OpAlreadyRan(f, MAKE_CELL, offset)) { + // MAKE_CELL has built the cell, so use it as the proxy target + Py_INCREF(target); + if (set_fast_ref(flp->frame->f_fast_refs, key, target) != 0) { + return NULL; + } + value = PyCell_GET(target); + } + } + } + + // Local variable, or cell variable that either hasn't been converted yet + // or was only just converted since the last cache sync + // Ensure the value cache is up to date if the frame is still live + if (!PyErr_Occurred()) { + if (fastlocalsproxy_set_value_cache_entry(flp, key, value) != 0) { + return NULL; + } + } + if (value == NULL && !PyErr_Occurred()) { + // Report KeyError if the variable hasn't been bound to a value yet + // (akin to getting an UnboundLocalError in running code) + PyErr_SetObject(PyExc_KeyError, key); + } else { + Py_XINCREF(value); + } + return value; +} + + +static int +fastlocalsproxy_write_to_frame(fastlocalsproxyobject *flp, PyObject *key, PyObject *value) +{ + assert(flp); + if (fastlocalsproxy_init_fast_refs(flp) != 0) { + return -1; + } + PyObject *fast_ref = PyDict_GetItem(flp->frame->f_fast_refs, key); + if (fast_ref == NULL) { + // No such local variable, delegate the request to the f_locals mapping + // Used by pdb (at least) to store __return__ and __exception__ values + return fastlocalsproxy_set_value_cache_entry(flp, key, value); + } + // If the frame has been cleared, disallow access to locals and cell variables + PyFrameObject *f = flp->frame; + if (f->f_state == FRAME_CLEARED) { + PyErr_Format(PyExc_RuntimeError, + "Fast locals proxy attempted to write to cleared frame (%R)", f); + return -1; + } + // Key is a valid Python variable for the frame, so update that reference + if (PyCell_Check(fast_ref)) { + // Closure cells are accessed directly via the fast refs mapping + int result = PyCell_Set(fast_ref, value); + if (result == 0) { + // Ensure the value cache is up to date if the frame is still live + result = fastlocalsproxy_set_value_cache_entry(flp, key, value); + } + return result; + } + // Fast ref is a Python int mapping into the fast locals array + assert(PyLong_CheckExact(fast_ref)); + Py_ssize_t offset = PyLong_AsSsize_t(fast_ref); + if (offset < 0) { + return -1; + } + PyCodeObject *co = _PyFrame_GetCode(f); + assert(co); + Py_ssize_t max_offset = co->co_nlocalsplus - 1; + if (offset > max_offset) { + PyErr_Format(PyExc_SystemError, + "Fast locals ref (%zd) exceeds array bound (%zd)", + offset, max_offset); + return -1; + } + PyObject **fast_locals = f->f_localsptr; + // Check if MAKE_CELL has been called since the proxy was created + _PyLocal_VarKind kind = _PyLocal_GetVarKind(co->co_localspluskinds, offset); + if (kind & CO_FAST_CELL) { + // Value hadn't been converted to a cell yet when the proxy was created + // Update the proxy if MAKE_CELL has run since the last access, + // otherwise continue treating it as a regular local variable + PyObject *target = fast_locals[offset]; + if (target != NULL && PyCell_Check(target) && + _PyFrame_OpAlreadyRan(f, MAKE_CELL, offset)) { + // MAKE_CELL has built the cell, so use it as the proxy target + Py_INCREF(target); + if (set_fast_ref(flp->frame->f_fast_refs, key, target) != 0) { + return -1; + } + int result = PyCell_Set(target, value); + if (result == 0) { + // Ensure the value cache is up to date if the frame is still live + result = fastlocalsproxy_set_value_cache_entry(flp, key, value); + } + return result; + } + } + + // Local variable, or future cell variable that hasn't been converted yet + Py_XINCREF(value); + Py_XSETREF(fast_locals[offset], value); + // Ensure the value cache is up to date if the frame is still live + return fastlocalsproxy_set_value_cache_entry(flp, key, value); +} + +static int +fastlocalsproxy_setitem(fastlocalsproxyobject *flp, PyObject *key, PyObject *value) +{ + return fastlocalsproxy_write_to_frame(flp, key, value); +} + +static int +fastlocalsproxy_delitem(fastlocalsproxyobject *flp, PyObject *key) +{ + return fastlocalsproxy_write_to_frame(flp, key, NULL); +} + +static PyMappingMethods fastlocalsproxy_as_mapping = { + (lenfunc)fastlocalsproxy_len, /* mp_length */ + (binaryfunc)fastlocalsproxy_getitem, /* mp_subscript */ + (objobjargproc)fastlocalsproxy_setitem, /* mp_ass_subscript */ +}; + +static int mutablemapping_update_arg(PyObject*, PyObject*); + +static PyObject * +fastlocalsproxy_or(PyObject *left, PyObject *right) +{ + // Binary union operations are delegated to the frame value cache + // Ensure it is up to date, as the union operation is already O(m+n) + int repeated_operand = (left == right); + + if (_PyFastLocalsProxy_CheckExact(left)) { + left = fastlocalsproxy_get_updated_value_cache((fastlocalsproxyobject *)left); + } + if (_PyFastLocalsProxy_CheckExact(right)) { + if (repeated_operand) { + right = left; + } else { + right = fastlocalsproxy_get_updated_value_cache((fastlocalsproxyobject *)right); + } + } + return PyNumber_Or(left, right); +} + +static PyObject * +fastlocalsproxy_ior(PyObject *self, PyObject *other) +{ + // In-place union operations are equivalent to an update() method call + if (mutablemapping_update_arg(self, other) < 0) { + return NULL; + } + Py_INCREF(self); + return self; +} + +/* tp_as_number */ +static PyNumberMethods fastlocalsproxy_as_number = { + .nb_or = fastlocalsproxy_or, + .nb_inplace_or = fastlocalsproxy_ior, +}; + + +static int +fastlocalsproxy_contains(fastlocalsproxyobject *flp, PyObject *key) +{ + // This runs a full key lookup so it will return false if the name + // hasn't been bound yet, but still runs in O(1) time without needing + // to rely on the f_locals cache already being up to date + PyObject *value = fastlocalsproxy_getitem(flp, key); + int found = (value != NULL); + if (!found && PyErr_ExceptionMatches(PyExc_KeyError)) { + PyErr_Clear(); + } + Py_XDECREF(value); + return found; +} + +static PySequenceMethods fastlocalsproxy_as_sequence = { + 0, /* sq_length */ + 0, /* sq_concat */ + 0, /* sq_repeat */ + 0, /* sq_item */ + 0, /* sq_slice */ + 0, /* sq_ass_item */ + 0, /* sq_ass_slice */ + (objobjproc)fastlocalsproxy_contains, /* sq_contains */ + 0, /* sq_inplace_concat */ + 0, /* sq_inplace_repeat */ +}; + +static PyObject * +fastlocalsproxy_get(fastlocalsproxyobject *flp, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *key = NULL; + PyObject *failobj = Py_None; + + if (!_PyArg_UnpackStack(args, nargs, "get", 1, 2, + &key, &failobj)) + { + return NULL; + } + + PyObject *value = fastlocalsproxy_getitem(flp, key); + if (value == NULL && PyErr_ExceptionMatches(PyExc_KeyError)) { + PyErr_Clear(); + value = failobj; + Py_INCREF(value); + } + return value; +} + +static PyObject * +fastlocalsproxy_keys(fastlocalsproxyobject *flp, PyObject *Py_UNUSED(ignored)) +{ + // Extra keys may have been added, and some variables may not have been + // bound yet, so use the value cache on the frame rather than the + // keys in the fast locals reverse lookup mapping + // As long as it has been updated at least once, assume value cache + // is up to date (as actually checking is O(n)) + PyObject *locals = fastlocalsproxy_get_value_cache(flp); + return PyDict_Keys(locals); +} + +static PyObject * +fastlocalsproxy_values(fastlocalsproxyobject *flp, PyObject *Py_UNUSED(ignored)) +{ + // Need values, so use the value cache on the frame + // As long as it has been updated at least once, assume value cache + // is up to date (as actually checking is O(n)) + PyObject *locals = fastlocalsproxy_get_value_cache(flp); + return PyDict_Values(locals); +} + +static PyObject * +fastlocalsproxy_items(fastlocalsproxyobject *flp, PyObject *Py_UNUSED(ignored)) +{ + // Need values, so use the value cache on the frame + // As long as it has been updated at least once, assume value cache + // is up to date (as actually checking is O(n)) + PyObject *locals = fastlocalsproxy_get_value_cache(flp); + return PyDict_Items(locals); +} + +static PyObject * +fastlocalsproxy_copy(fastlocalsproxyobject *flp, PyObject *Py_UNUSED(ignored)) +{ + // Need values, so use the value cache on the frame + // Ensure it is up to date, as copying the entire mapping is already O(n) + PyObject *locals = fastlocalsproxy_get_updated_value_cache(flp); + return PyDict_Copy(locals); +} + +static PyObject * +fastlocalsproxy_reversed(fastlocalsproxyobject *flp, PyObject *Py_UNUSED(ignored)) +{ + _Py_IDENTIFIER(__reversed__); + // Extra keys may have been added, and some variables may not have been + // bound yet, so use the value cache on the frame rather than the + // keys in the fast locals reverse lookup mapping + // As long as it has been updated at least once, assume value cache + // is up to date (as actually checking is O(n)) + PyObject *locals = fastlocalsproxy_get_value_cache(flp); + return _PyObject_CallMethodIdNoArgs(locals, &PyId___reversed__); +} + +static PyObject * +fastlocalsproxy_getiter(fastlocalsproxyobject *flp) +{ + // Extra keys may have been added, and some variables may not have been + // bound yet, so use the value cache on the frame rather than the + // keys in the fast locals reverse lookup mapping + // As long as it has been updated at least once, assume value cache + // is up to date (as actually checking is O(n)) + PyObject *locals = fastlocalsproxy_get_value_cache(flp); + return PyObject_GetIter(locals); +} + +static PyObject * +fastlocalsproxy_richcompare(fastlocalsproxyobject *flp, PyObject *w, int op) +{ + // Need values, so use the value cache on the frame + // As long as it has been updated at least once, assume value cache + // is up to date, as even though the worst case comparison is O(n) to + // determine equality, there are O(1) shortcuts for inequality checks + // (i.e. different sizes) + PyObject *locals = fastlocalsproxy_get_value_cache(flp); + return PyObject_RichCompare(locals, w, op); +} + +/* setdefault() */ + +PyDoc_STRVAR(fastlocalsproxy_setdefault__doc__, +"flp.setdefault(k[, d=None]) -> v, Bind key to given default if key\n\ + is not already bound to a value.\n\n\ + Return the value for key if key is already bound, else default."); + + +static PyObject * +_fastlocalsproxy_setdefault_impl(fastlocalsproxyobject *flp, PyObject *key, PyObject *failobj) +{ + assert(flp); + if (fastlocalsproxy_init_fast_refs(flp) != 0) { + return NULL; + } + + PyObject *value = fastlocalsproxy_getitem(flp, key); + if (value == NULL && PyErr_ExceptionMatches(PyExc_KeyError)) { + // Given key is currently unbound, so bind it to the specified default + // and return that object + PyErr_Clear(); + value = failobj; + if (fastlocalsproxy_setitem(flp, key, value)) { + return NULL; + } + Py_INCREF(value); + } + return value; +} + +static PyObject * +fastlocalsproxy_setdefault(PyObject *flp, PyObject *args, PyObject *kwargs) +{ + static char *kwlist[] = {"key", "default", 0}; + PyObject *key, *failobj = Py_None; + + /* borrowed */ + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|O:setdefault", kwlist, + &key, &failobj)) { + return NULL; + } + + return _fastlocalsproxy_setdefault_impl((fastlocalsproxyobject *)flp, key, failobj); +} + +/* pop() */ + +PyDoc_STRVAR(fastlocalsproxy_pop__doc__, +"flp.pop(k[,d]) -> v, unbind specified variable and return the corresponding\n\ + value. If key is not found, d is returned if given, otherwise KeyError\n\ + is raised."); + +static PyObject * +_fastlocalsproxy_pop_impl(fastlocalsproxyobject *flp, PyObject *key, PyObject *failobj) +{ + // TODO: Similar to the odict implementation, the fast locals proxy + // could benefit from an internal API that accepts already calculated + // hashes, rather than recalculating the hash multiple times for the + // same key in a single operation (see _odict_popkey_hash) + assert(flp); + if (fastlocalsproxy_init_fast_refs(flp) != 0) { + return NULL; + } + + // Just implement naive lookup through the object based C API for now + PyObject *value = fastlocalsproxy_getitem(flp, key); + if (value != NULL) { + if (fastlocalsproxy_delitem(flp, key) != 0) { + Py_CLEAR(value); + } + } else if (failobj != NULL && PyErr_ExceptionMatches(PyExc_KeyError)) { + PyErr_Clear(); + value = failobj; + Py_INCREF(value); + } + + return value; +} + +static PyObject * +fastlocalsproxy_pop(PyObject *flp, PyObject *args, PyObject *kwargs) +{ + static char *kwlist[] = {"key", "default", 0}; + PyObject *key, *failobj = NULL; + + /* borrowed */ + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|O:pop", kwlist, + &key, &failobj)) { + return NULL; + } + + return _fastlocalsproxy_pop_impl((fastlocalsproxyobject *)flp, key, failobj); +} + +/* popitem() */ + +PyDoc_STRVAR(fastlocalsproxy_popitem__doc__, +"flp.popitem() -> (k, v), unbind and return some (key, value) pair as a\n\ + 2-tuple; but raise KeyError if underlying frame has no bound values."); + +static PyObject * +fastlocalsproxy_popitem(fastlocalsproxyobject *flp, PyObject *Py_UNUSED(ignored)) +{ + _Py_IDENTIFIER(popitem); + // Need values, so use the value cache on the frame + // As long as it has been updated at least once, assume value cache + // is up to date (as actually checking is O(n)) + PyObject *locals = fastlocalsproxy_get_value_cache(flp); + if (locals == NULL) { + return NULL; + } + PyObject *result = _PyObject_CallMethodIdNoArgs(locals, &PyId_popitem); + if (result != NULL) { + // We popped a key from the cache, so ensure it is also cleared on the frame + assert(PyTuple_CheckExact(result)); + assert(PyTuple_GET_SIZE(result) == 2); + PyObject *key = PyTuple_GET_ITEM(result, 0); + if (fastlocalsproxy_delitem(flp, key)) { + Py_DECREF(result); + return NULL; + } + } + + return result; +} + +/* update() */ + +/* MutableMapping.update() does not have a docstring. */ +PyDoc_STRVAR(fastlocalsproxy_update__doc__, ""); + +/* forward */ +static PyObject * mutablemapping_update(PyObject *, PyObject *, PyObject *); + +#define fastlocalsproxy_update mutablemapping_update + +/* clear() */ + +PyDoc_STRVAR(fastlocalsproxy_clear__doc__, + "flp.clear() -> None. Unbind all variables in frame."); + +static PyObject * +fastlocalsproxy_clear(register PyObject *flp, PyObject *Py_UNUSED(ignored)) +{ + /* Merge fast locals into f->f_locals */ + PyFrameObject *f; + PyObject *locals; + + assert(flp); + f = ((fastlocalsproxyobject *)flp)->frame; + if (f == NULL) { + PyErr_BadInternalCall(); + return NULL; + } + + // Clear any local and still referenced cell variables + // Nothing to do in this step if the frame itself has already been cleared + if (f->f_state != FRAME_CLEARED) { + PyCodeObject *co = _PyFrame_GetCode(f); + PyObject **fast = f->f_localsptr; + // Fast locals proxies only get created for optimised frames + assert(co); + assert(co->co_flags & CO_OPTIMIZED); + assert(fast); + for (int i = 0; i < co->co_nlocalsplus; i++) { + _PyLocal_VarKind kind = _PyLocal_GetVarKind(co->co_localspluskinds, i); + + PyObject *value = fast[i]; + if (kind & CO_FAST_FREE) { + // The cell was set by _PyEval_MakeFrameVector() from + // the function's closure. + assert(value != NULL && PyCell_Check(value)); + if (PyCell_Set(value, NULL)) { + return NULL; + } + } + else if (kind & CO_FAST_CELL) { + // If the cell has already been created, unbind its reference, + // otherwise clear its initial value (if any) + if (value != NULL) { + if (PyCell_Check(value) && + _PyFrame_OpAlreadyRan(f, MAKE_CELL, i)) { + // (likely) MAKE_CELL must have executed already. + if (PyCell_Set(value, NULL)) { + return NULL; + } + } else { + // Clear the initial value + Py_CLEAR(fast[i]); + } + } + } else if (value != NULL) { + // Clear local variable reference + Py_CLEAR(fast[i]); + } + } + } + + // Finally, clear the frame value cache (including any extra variables) + locals = _frame_get_locals_mapping(f); + if (locals == NULL) { + return NULL; + } + PyDict_Clear(locals); + + Py_RETURN_NONE; +} + + +PyDoc_STRVAR(fastlocalsproxy_sync_frame_cache__doc__, + "flp.sync_frame_cache() -> None. Ensure f_locals snapshot is in sync with underlying frame."); + +static PyObject * +fastlocalsproxy_sync_frame_cache(register PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + fastlocalsproxyobject *flp = (fastlocalsproxyobject *)self; + if (PyFrame_FastToLocalsWithError(flp->frame) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + + +static PyMethodDef fastlocalsproxy_methods[] = { + {"get", (PyCFunction)(void(*)(void))fastlocalsproxy_get, METH_FASTCALL, + PyDoc_STR("P.get(k[,d]) -> P[k] if k in P, else d." + " d defaults to None.")}, + {"keys", (PyCFunction)fastlocalsproxy_keys, METH_NOARGS, + PyDoc_STR("P.keys() -> virtual set of proxy's bound keys")}, + {"values", (PyCFunction)fastlocalsproxy_values, METH_NOARGS, + PyDoc_STR("P.values() -> virtual multiset of proxy's bound values")}, + {"items", (PyCFunction)fastlocalsproxy_items, METH_NOARGS, + PyDoc_STR("P.items() -> virtual set of P's (key, value) pairs, as 2-tuples")}, + {"copy", (PyCFunction)fastlocalsproxy_copy, METH_NOARGS, + PyDoc_STR("P.copy() -> a shallow copy of P as a regular dict")}, + {"__class_getitem__", (PyCFunction)Py_GenericAlias, METH_O|METH_CLASS, + PyDoc_STR("See PEP 585")}, + {"__reversed__", (PyCFunction)fastlocalsproxy_reversed, METH_NOARGS, + PyDoc_STR("P.__reversed__() -> reverse iterator over P's keys")}, + {"__sizeof__", (PyCFunction)fastlocalsproxy_sizeof, METH_NOARGS, + PyDoc_STR("P.__sizeof__: size of P in memory, in bytes (excludes frame)")}, + // PEP 558 TODO: Convert METH_VARARGS/METH_KEYWORDS methods to METH_FASTCALL + {"setdefault", (PyCFunction)(void(*)(void))fastlocalsproxy_setdefault, + METH_VARARGS | METH_KEYWORDS, fastlocalsproxy_setdefault__doc__}, + {"pop", (PyCFunction)(void(*)(void))fastlocalsproxy_pop, + METH_VARARGS | METH_KEYWORDS, fastlocalsproxy_pop__doc__}, + {"popitem", (PyCFunction)fastlocalsproxy_popitem, + METH_NOARGS, fastlocalsproxy_popitem__doc__}, + {"update", (PyCFunction)(void(*)(void))fastlocalsproxy_update, + METH_VARARGS | METH_KEYWORDS, fastlocalsproxy_update__doc__}, + {"clear", (PyCFunction)fastlocalsproxy_clear, + METH_NOARGS, fastlocalsproxy_clear__doc__}, + {"sync_frame_cache", (PyCFunction)fastlocalsproxy_sync_frame_cache, + METH_NOARGS, fastlocalsproxy_sync_frame_cache__doc__}, + {NULL, NULL} /* sentinel */ +}; + +static void +fastlocalsproxy_dealloc(fastlocalsproxyobject *flp) +{ + if (_PyObject_GC_IS_TRACKED(flp)) + _PyObject_GC_UNTRACK(flp); + + Py_TRASHCAN_SAFE_BEGIN(flp) + + Py_CLEAR(flp->frame); + PyObject_GC_Del(flp); + + Py_TRASHCAN_SAFE_END(flp) +} + +static PyObject * +fastlocalsproxy_repr(fastlocalsproxyobject *flp) +{ + return PyUnicode_FromFormat("fastlocalsproxy(%R)", flp->frame); +} + +static PyObject * +fastlocalsproxy_str(fastlocalsproxyobject *flp) +{ + // Need values, so use the value cache on the frame + // Ensure it is up to date, as rendering the entire mapping is already O(n) + PyObject *locals = fastlocalsproxy_get_updated_value_cache(flp); + return PyObject_Str(locals); +} + +static int +fastlocalsproxy_traverse(PyObject *self, visitproc visit, void *arg) +{ + fastlocalsproxyobject *flp = (fastlocalsproxyobject *)self; + Py_VISIT(flp->frame); + return 0; +} + +static int +fastlocalsproxy_check_frame(PyObject *maybe_frame) +{ + /* This is an internal-only API, so getting bad arguments means something + * already went wrong elsewhere in the interpreter code. + */ + if (!PyFrame_Check(maybe_frame)) { + PyErr_Format(PyExc_SystemError, + "fastlocalsproxy() argument must be a frame, not %s", + Py_TYPE(maybe_frame)->tp_name); + return -1; + } + + PyFrameObject *frame = (PyFrameObject *) maybe_frame; + if (!(_PyFrame_GetCode(frame)->co_flags & CO_OPTIMIZED)) { + PyErr_SetString(PyExc_SystemError, + "fastlocalsproxy() argument must be a frame using fast locals"); + return -1; + } + return 0; +} + + +static PyObject * +_PyFastLocalsProxy_New(PyObject *frame) +{ + fastlocalsproxyobject *flp; + + if (fastlocalsproxy_check_frame(frame) == -1) { + return NULL; + } + + flp = PyObject_GC_New(fastlocalsproxyobject, &_PyFastLocalsProxy_Type); + if (flp == NULL) + return NULL; + flp->frame = (PyFrameObject *) frame; + Py_INCREF(flp->frame); + flp->frame_cache_updated = 0; + _PyObject_GC_TRACK(flp); + return (PyObject *)flp; +} + +/*[clinic input] +@classmethod +fastlocalsproxy.__new__ as fastlocalsproxy_new + + frame: object + +[clinic start generated code]*/ + +static PyObject * +fastlocalsproxy_new_impl(PyTypeObject *type, PyObject *frame) +/*[clinic end generated code: output=058c588af86f1525 input=049f74502e02fc63]*/ +{ + return _PyFastLocalsProxy_New(frame); +} + +#include "clinic/frameobject.c.h" + +PyTypeObject _PyFastLocalsProxy_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "fastlocalsproxy", /* tp_name */ + sizeof(fastlocalsproxyobject), /* tp_basicsize */ + 0, /* tp_itemsize */ + (destructor)fastlocalsproxy_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_reserved */ + (reprfunc)fastlocalsproxy_repr, /* tp_repr */ + &fastlocalsproxy_as_number, /* tp_as_number */ + &fastlocalsproxy_as_sequence, /* tp_as_sequence */ + &fastlocalsproxy_as_mapping, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + (reprfunc)fastlocalsproxy_str, /* tp_str */ + PyObject_GenericGetAttr, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | + Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_MAPPING, /* tp_flags */ + 0, /* tp_doc */ + (traverseproc)fastlocalsproxy_traverse, /* tp_traverse */ + 0, /* tp_clear */ + (richcmpfunc)fastlocalsproxy_richcompare, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + (getiterfunc)fastlocalsproxy_getiter, /* tp_iter */ + 0, /* tp_iternext */ + fastlocalsproxy_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + fastlocalsproxy_new, /* tp_new */ + 0, /* tp_free */ +}; + + +//========================================================================== +// The rest of this file is currently DUPLICATED CODE from odictobject.c +// +// PEP 558 TODO: move the duplicated code to Objects/mutablemapping.c and +// expose it to the linker as a private API +// +//========================================================================== + +static int +mutablemapping_add_pairs(PyObject *self, PyObject *pairs) +{ + PyObject *pair, *iterator, *unexpected; + int res = 0; + + iterator = PyObject_GetIter(pairs); + if (iterator == NULL) + return -1; + PyErr_Clear(); + + while ((pair = PyIter_Next(iterator)) != NULL) { + /* could be more efficient (see UNPACK_SEQUENCE in ceval.c) */ + PyObject *key = NULL, *value = NULL; + PyObject *pair_iterator = PyObject_GetIter(pair); + if (pair_iterator == NULL) + goto Done; + + key = PyIter_Next(pair_iterator); + if (key == NULL) { + if (!PyErr_Occurred()) + PyErr_SetString(PyExc_ValueError, + "need more than 0 values to unpack"); + goto Done; + } + + value = PyIter_Next(pair_iterator); + if (value == NULL) { + if (!PyErr_Occurred()) + PyErr_SetString(PyExc_ValueError, + "need more than 1 value to unpack"); + goto Done; + } + + unexpected = PyIter_Next(pair_iterator); + if (unexpected != NULL) { + Py_DECREF(unexpected); + PyErr_SetString(PyExc_ValueError, + "too many values to unpack (expected 2)"); + goto Done; + } + else if (PyErr_Occurred()) + goto Done; + + res = PyObject_SetItem(self, key, value); + +Done: + Py_DECREF(pair); + Py_XDECREF(pair_iterator); + Py_XDECREF(key); + Py_XDECREF(value); + if (PyErr_Occurred()) + break; + } + Py_DECREF(iterator); + + if (res < 0 || PyErr_Occurred() != NULL) + return -1; + else + return 0; +} + +static int +mutablemapping_update_arg(PyObject *self, PyObject *arg) +{ + int res = 0; + if (PyDict_CheckExact(arg)) { + PyObject *items = PyDict_Items(arg); + if (items == NULL) { + return -1; + } + res = mutablemapping_add_pairs(self, items); + Py_DECREF(items); + return res; + } + _Py_IDENTIFIER(keys); + PyObject *func; + if (_PyObject_LookupAttrId(arg, &PyId_keys, &func) < 0) { + return -1; + } + if (func != NULL) { + PyObject *keys = _PyObject_CallNoArg(func); + Py_DECREF(func); + if (keys == NULL) { + return -1; + } + PyObject *iterator = PyObject_GetIter(keys); + Py_DECREF(keys); + if (iterator == NULL) { + return -1; + } + PyObject *key; + while (res == 0 && (key = PyIter_Next(iterator))) { + PyObject *value = PyObject_GetItem(arg, key); + if (value != NULL) { + res = PyObject_SetItem(self, key, value); + Py_DECREF(value); + } + else { + res = -1; + } + Py_DECREF(key); + } + Py_DECREF(iterator); + if (res != 0 || PyErr_Occurred()) { + return -1; + } + return 0; + } + _Py_IDENTIFIER(items); + if (_PyObject_LookupAttrId(arg, &PyId_items, &func) < 0) { + return -1; + } + if (func != NULL) { + PyObject *items = _PyObject_CallNoArg(func); + Py_DECREF(func); + if (items == NULL) { + return -1; + } + res = mutablemapping_add_pairs(self, items); + Py_DECREF(items); + return res; + } + res = mutablemapping_add_pairs(self, arg); + return res; +} + +static PyObject * +mutablemapping_update(PyObject *self, PyObject *args, PyObject *kwargs) +{ + int res; + /* first handle args, if any */ + assert(args == NULL || PyTuple_Check(args)); + Py_ssize_t len = (args != NULL) ? PyTuple_GET_SIZE(args) : 0; + if (len > 1) { + const char *msg = "update() takes at most 1 positional argument (%zd given)"; + PyErr_Format(PyExc_TypeError, msg, len); + return NULL; + } + + if (len) { + PyObject *other = PyTuple_GET_ITEM(args, 0); /* borrowed reference */ + assert(other != NULL); + Py_INCREF(other); + res = mutablemapping_update_arg(self, other); + Py_DECREF(other); + if (res < 0) { + return NULL; + } + } + + /* now handle kwargs */ + assert(kwargs == NULL || PyDict_Check(kwargs)); + if (kwargs != NULL && PyDict_GET_SIZE(kwargs)) { + PyObject *items = PyDict_Items(kwargs); + if (items == NULL) + return NULL; + res = mutablemapping_add_pairs(self, items); + Py_DECREF(items); + if (res == -1) + return NULL; + } + + Py_RETURN_NONE; +} diff --git a/Objects/object.c b/Objects/object.c index 446c974f8e614b..86a4010277c8af 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -1814,6 +1814,7 @@ _PyTypes_Init(void) INIT_TYPE(PyDictIterValue_Type); INIT_TYPE(PyDictKeys_Type); INIT_TYPE(PyDictProxy_Type); + INIT_TYPE(_PyFastLocalsProxy_Type); INIT_TYPE(PyDictRevIterItem_Type); INIT_TYPE(PyDictRevIterKey_Type); INIT_TYPE(PyDictRevIterValue_Type); diff --git a/Objects/typeobject.c b/Objects/typeobject.c index badd7064fe3f43..6013a179a15577 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -8888,7 +8888,7 @@ super_init_without_args(PyFrameObject *f, PyCodeObject *co, PyObject *firstarg = f->f_localsptr[0]; // The first argument might be a cell. - if (firstarg != NULL && (_PyLocals_GetKind(co->co_localspluskinds, 0) & CO_FAST_CELL)) { + if (firstarg != NULL && (_PyLocal_GetVarKind(co->co_localspluskinds, 0) & CO_FAST_CELL)) { // "firstarg" is a cell here unless (very unlikely) super() // was called from the C-API before the first MAKE_CELL op. if (f->f_lasti >= 0) { @@ -8907,7 +8907,7 @@ super_init_without_args(PyFrameObject *f, PyCodeObject *co, PyTypeObject *type = NULL; int i = co->co_nlocals + co->co_nplaincellvars; for (; i < co->co_nlocalsplus; i++) { - assert((_PyLocals_GetKind(co->co_localspluskinds, i) & CO_FAST_FREE) != 0); + assert((_PyLocal_GetVarKind(co->co_localspluskinds, i) & CO_FAST_FREE) != 0); PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i); assert(PyUnicode_Check(name)); if (_PyUnicode_EqualToASCIIId(name, &PyId___class__)) { diff --git a/PC/python3dll.c b/PC/python3dll.c index 0ebb56efaecb2c..ac6d857d56a171 100755 --- a/PC/python3dll.c +++ b/PC/python3dll.c @@ -313,6 +313,10 @@ EXPORT_FUNC(PyList_SetItem) EXPORT_FUNC(PyList_SetSlice) EXPORT_FUNC(PyList_Size) EXPORT_FUNC(PyList_Sort) +EXPORT_FUNC(PyLocals_Get) +EXPORT_FUNC(PyLocals_GetCopy) +EXPORT_FUNC(PyLocals_GetKind) +EXPORT_FUNC(PyLocals_GetView) EXPORT_FUNC(PyLong_AsDouble) EXPORT_FUNC(PyLong_AsLong) EXPORT_FUNC(PyLong_AsLongAndOverflow) diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index bfe21ad6d0c4c7..49ff54e87d0efe 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -923,9 +923,15 @@ builtin_eval_impl(PyObject *module, PyObject *source, PyObject *globals, PyObject *locals) /*[clinic end generated code: output=0a0824aa70093116 input=11ee718a8640e527]*/ { - PyObject *result, *source_copy; + PyObject *result, *source_copy = NULL, *locals_ref = NULL; const char *str; + // PEP 558: "locals_ref" is a quick fix to adapt this function to + // PyLocals_Get() returning a new reference, whereas PyEval_GetLocals() + // returned a borrowed reference. The proper fix will be to factor out + // a common set up function shared between eval() and exec() that means + // the main function body can always just call Py_DECREF(locals) at the end. + if (locals != Py_None && !PyMapping_Check(locals)) { PyErr_SetString(PyExc_TypeError, "locals must be a mapping"); return NULL; @@ -939,18 +945,22 @@ builtin_eval_impl(PyObject *module, PyObject *source, PyObject *globals, if (globals == Py_None) { globals = PyEval_GetGlobals(); if (locals == Py_None) { - locals = PyEval_GetLocals(); + locals = PyLocals_Get(); if (locals == NULL) return NULL; + // This function owns the locals ref, need to decref on exit + locals_ref = locals; } } - else if (locals == Py_None) + else if (locals == Py_None) { locals = globals; + } if (globals == NULL || locals == NULL) { PyErr_SetString(PyExc_TypeError, "eval must be given globals and locals " "when called without a frame"); + Py_XDECREF(locals_ref); return NULL; } @@ -965,22 +975,28 @@ builtin_eval_impl(PyObject *module, PyObject *source, PyObject *globals, if (PyCode_Check(source)) { if (PySys_Audit("exec", "O", source) < 0) { + Py_XDECREF(locals_ref); return NULL; } if (PyCode_GetNumFree((PyCodeObject *)source) > 0) { PyErr_SetString(PyExc_TypeError, "code object passed to eval() may not contain free variables"); + Py_XDECREF(locals_ref); return NULL; } - return PyEval_EvalCode(source, globals, locals); + result = PyEval_EvalCode(source, globals, locals); + Py_XDECREF(locals_ref); + return result; } PyCompilerFlags cf = _PyCompilerFlags_INIT; cf.cf_flags = PyCF_SOURCE_IS_UTF8; str = _Py_SourceAsString(source, "eval", "string, bytes or code", &cf, &source_copy); - if (str == NULL) + if (str == NULL) { + Py_XDECREF(locals_ref); return NULL; + } while (*str == ' ' || *str == '\t') str++; @@ -988,6 +1004,7 @@ builtin_eval_impl(PyObject *module, PyObject *source, PyObject *globals, (void)PyEval_MergeCompilerFlags(&cf); result = PyRun_StringFlags(str, Py_eval_input, globals, locals, &cf); Py_XDECREF(source_copy); + Py_XDECREF(locals_ref); return result; } @@ -1013,33 +1030,44 @@ builtin_exec_impl(PyObject *module, PyObject *source, PyObject *globals, PyObject *locals) /*[clinic end generated code: output=3c90efc6ab68ef5d input=01ca3e1c01692829]*/ { - PyObject *v; + PyObject *v, *locals_ref = NULL; + + // (ncoghlan) There is an annoying level of gratuitous differences between + // the exec setup code and the eval setup code, when ideally they would be + // sharing a common helper function at least as far as the call to + // PyCode_Check... if (globals == Py_None) { globals = PyEval_GetGlobals(); if (locals == Py_None) { - locals = PyEval_GetLocals(); + locals = PyLocals_Get(); if (locals == NULL) return NULL; + // This function owns the locals ref, need to decref on exit + locals_ref = locals; } if (!globals || !locals) { PyErr_SetString(PyExc_SystemError, "globals and locals cannot be NULL"); + Py_XDECREF(locals_ref); return NULL; } } - else if (locals == Py_None) + else if (locals == Py_None) { locals = globals; + } if (!PyDict_Check(globals)) { PyErr_Format(PyExc_TypeError, "exec() globals must be a dict, not %.100s", Py_TYPE(globals)->tp_name); + Py_XDECREF(locals_ref); return NULL; } if (!PyMapping_Check(locals)) { PyErr_Format(PyExc_TypeError, "locals must be a mapping or None, not %.100s", Py_TYPE(locals)->tp_name); + Py_XDECREF(locals_ref); return NULL; } int r = _PyDict_ContainsId(globals, &PyId___builtins__); @@ -1053,6 +1081,7 @@ builtin_exec_impl(PyObject *module, PyObject *source, PyObject *globals, if (PyCode_Check(source)) { if (PySys_Audit("exec", "O", source) < 0) { + Py_XDECREF(locals_ref); return NULL; } @@ -1060,6 +1089,7 @@ builtin_exec_impl(PyObject *module, PyObject *source, PyObject *globals, PyErr_SetString(PyExc_TypeError, "code object passed to exec() may not " "contain free variables"); + Py_XDECREF(locals_ref); return NULL; } v = PyEval_EvalCode(source, globals, locals); @@ -1072,8 +1102,10 @@ builtin_exec_impl(PyObject *module, PyObject *source, PyObject *globals, str = _Py_SourceAsString(source, "exec", "string, bytes or code", &cf, &source_copy); - if (str == NULL) + if (str == NULL) { + Py_XDECREF(locals_ref); return NULL; + } if (PyEval_MergeCompilerFlags(&cf)) v = PyRun_StringFlags(str, Py_file_input, globals, locals, &cf); @@ -1081,6 +1113,7 @@ builtin_exec_impl(PyObject *module, PyObject *source, PyObject *globals, v = PyRun_String(str, Py_file_input, globals, locals); Py_XDECREF(source_copy); } + Py_XDECREF(locals_ref); if (v == NULL) return NULL; Py_DECREF(v); @@ -1694,20 +1727,14 @@ locals as builtin_locals Return a dictionary containing the current scope's local variables. -NOTE: Whether or not updates to this dictionary will affect name lookups in -the local scope and vice-versa is *implementation dependent* and not -covered by any backwards compatibility guarantees. +TODO: Update the docstring with the gist of PEP 558 semantics. [clinic start generated code]*/ static PyObject * builtin_locals_impl(PyObject *module) -/*[clinic end generated code: output=b46c94015ce11448 input=7874018d478d5c4b]*/ +/*[clinic end generated code: output=b46c94015ce11448 input=9869b08c278df34f]*/ { - PyObject *d; - - d = PyEval_GetLocals(); - Py_XINCREF(d); - return d; + return PyLocals_Get(); } @@ -2391,8 +2418,7 @@ builtin_vars(PyObject *self, PyObject *args) if (!PyArg_UnpackTuple(args, "vars", 0, 1, &v)) return NULL; if (v == NULL) { - d = PyEval_GetLocals(); - Py_XINCREF(d); + d = PyLocals_Get(); } else { if (_PyObject_LookupAttrId(v, &PyId___dict__, &d) == 0) { diff --git a/Python/ceval.c b/Python/ceval.c index 90112aa3f9476e..35457176e3ad1d 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -3003,7 +3003,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int throwflag) case TARGET(MAKE_CELL): { // "initial" is probably NULL but not if it's an arg (or set - // via PyFrame_LocalsToFast() before MAKE_CELL has run). + // via a frame locals proxy before MAKE_CELL has run). PyObject *initial = GETLOCAL(oparg); PyObject *cell = PyCell_New(initial); if (cell == NULL) { @@ -3591,10 +3591,12 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int throwflag) case TARGET(IMPORT_STAR): { PyObject *from = POP(), *locals; int err; - if (PyFrame_FastToLocalsWithError(f) < 0) { - Py_DECREF(from); - goto error; - } + /* PEP 558 TODO: + * Report an error here for CO_OPTIMIZED frames + * The 3.x compiler treats wildcard imports as an error inside + * functions, but they can still happen with independently + * constructed opcode sequences + */ locals = LOCALS(); if (locals == NULL) { @@ -3604,7 +3606,6 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int throwflag) goto error; } err = import_all_from(tstate, locals, from); - PyFrame_LocalsToFast(f, 0); Py_DECREF(from); if (err != 0) goto error; @@ -5786,16 +5787,62 @@ PyEval_GetLocals(void) return NULL; } - if (PyFrame_FastToLocalsWithError(current_frame) < 0) { + return _PyFrame_BorrowLocals(current_frame); +} + +PyObject * +PyLocals_Get(void) +{ + PyThreadState *tstate = _PyThreadState_GET(); + PyFrameObject *current_frame = tstate->frame; + if (current_frame == NULL) { + _PyErr_SetString(tstate, PyExc_SystemError, "frame does not exist"); return NULL; } - PyObject *locals = current_frame->f_valuestack[ - FRAME_SPECIALS_LOCALS_OFFSET-FRAME_SPECIALS_SIZE]; - assert(locals != NULL); - return locals; + return PyFrame_GetLocals(current_frame); +} + +PyLocals_Kind +PyLocals_GetKind(void) +{ + PyThreadState *tstate = _PyThreadState_GET(); + PyFrameObject *current_frame = tstate->frame; + if (current_frame == NULL) { + _PyErr_SetString(tstate, PyExc_SystemError, "frame does not exist"); + return PyLocals_UNDEFINED; + } + + return PyFrame_GetLocalsKind(current_frame); +} + +PyObject * +PyLocals_GetCopy(void) +{ + PyThreadState *tstate = _PyThreadState_GET(); + PyFrameObject *current_frame = tstate->frame; + if (current_frame == NULL) { + _PyErr_SetString(tstate, PyExc_SystemError, "frame does not exist"); + return NULL; + } + + return PyFrame_GetLocalsCopy(current_frame); +} + +PyObject * +PyLocals_GetView(void) +{ + PyThreadState *tstate = _PyThreadState_GET(); + PyFrameObject *current_frame = tstate->frame; + if (current_frame == NULL) { + _PyErr_SetString(tstate, PyExc_SystemError, "frame does not exist"); + return NULL; + } + + return PyFrame_GetLocalsView(current_frame); } + PyObject * PyEval_GetGlobals(void) { diff --git a/Python/clinic/bltinmodule.c.h b/Python/clinic/bltinmodule.c.h index 1fade994f40e30..dd962a678801f5 100644 --- a/Python/clinic/bltinmodule.c.h +++ b/Python/clinic/bltinmodule.c.h @@ -592,9 +592,7 @@ PyDoc_STRVAR(builtin_locals__doc__, "\n" "Return a dictionary containing the current scope\'s local variables.\n" "\n" -"NOTE: Whether or not updates to this dictionary will affect name lookups in\n" -"the local scope and vice-versa is *implementation dependent* and not\n" -"covered by any backwards compatibility guarantees."); +"TODO: Update the docstring with the gist of PEP 558 semantics."); #define BUILTIN_LOCALS_METHODDEF \ {"locals", (PyCFunction)builtin_locals, METH_NOARGS, builtin_locals__doc__}, @@ -951,4 +949,4 @@ builtin_issubclass(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=77ace832b3fb38e0 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=57e6c861ff19ed53 input=a9049054013a1b77]*/ diff --git a/Python/compile.c b/Python/compile.c index 3a20f6b57eb367..a13ad7121225c2 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -7385,7 +7385,7 @@ compute_localsplus_info(struct compiler *c, int nlocalsplus, assert(offset >= 0); assert(offset < nlocalsplus); // For now we do not distinguish arg kinds. - _PyLocals_Kind kind = CO_FAST_LOCAL; + _PyLocal_VarKind kind = CO_FAST_LOCAL; if (PyDict_GetItem(c->u->u_cellvars, k) != NULL) { kind |= CO_FAST_CELL; } diff --git a/Python/sysmodule.c b/Python/sysmodule.c index f809e2fda56044..df7ea7baa7b128 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -950,11 +950,8 @@ static PyObject * call_trampoline(PyThreadState *tstate, PyObject* callback, PyFrameObject *frame, int what, PyObject *arg) { - if (PyFrame_FastToLocalsWithError(frame) < 0) { - return NULL; - } - PyObject *stack[3]; + stack[0] = (PyObject *)frame; stack[1] = whatstrings[what]; stack[2] = (arg != NULL) ? arg : Py_None; @@ -962,7 +959,6 @@ call_trampoline(PyThreadState *tstate, PyObject* callback, /* call the Python-level function */ PyObject *result = _PyObject_FastCallTstate(tstate, callback, stack, 3); - PyFrame_LocalsToFast(frame, 1); if (result == NULL) { PyTraceBack_Here(frame); }