Pull in main

erlend-aasland · erlend-aasland · commit 858399155f23 · 2023-02-23T14:40:52.000+01:00
diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst
@@ -818,7 +818,7 @@ Process-wide parameters
    .. deprecated:: 3.11
 
 
-.. c:function:: w_char* Py_GetPythonHome()
+.. c:function:: wchar_t* Py_GetPythonHome()
 
    Return the default "home", that is, the value set by a previous call to
    :c:func:`Py_SetPythonHome`, or the value of the :envvar:`PYTHONHOME`
diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst
@@ -86,6 +86,14 @@ The :mod:`functools` module defines the following functions:
    The cached value can be cleared by deleting the attribute.  This
    allows the *cached_property* method to run again.
 
+   The *cached_property* does not prevent a possible race condition in
+   multi-threaded usage. The getter function could run more than once on the
+   same instance, with the latest run setting the cached value. If the cached
+   property is idempotent or otherwise not harmful to run more than once on an
+   instance, this is fine. If synchronization is needed, implement the necessary
+   locking inside the decorated getter function or around the cached property
+   access.
+
    Note, this decorator interferes with the operation of :pep:`412`
    key-sharing dictionaries.  This means that instance dictionaries
    can take more space than usual.
@@ -110,6 +118,14 @@ The :mod:`functools` module defines the following functions:
            def stdev(self):
                return statistics.stdev(self._data)
 
+
+   .. versionchanged:: 3.12
+      Prior to Python 3.12, ``cached_property`` included an undocumented lock to
+      ensure that in multi-threaded usage the getter function was guaranteed to
+      run only once per instance. However, the lock was per-property, not
+      per-instance, which could result in unacceptably high lock contention. In
+      Python 3.12+ this locking is removed.
+
    .. versionadded:: 3.8
 
 
diff --git a/Doc/library/struct.rst b/Doc/library/struct.rst
@@ -371,7 +371,7 @@ ordering::
     >>> from struct import *
     >>> pack(">bhl", 1, 2, 3)
     b'\x01\x00\x02\x00\x00\x00\x03'
-    >>> unpack('>bhl', b'\x01\x00\x02\x00\x00\x00\x03'
+    >>> unpack('>bhl', b'\x01\x00\x02\x00\x00\x00\x03')
     (1, 2, 3)
     >>> calcsize('>bhl')
     7
diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst
@@ -761,6 +761,15 @@ Changes in the Python API
   around process-global resources, which are best managed from the main interpreter.
   (Contributed by Dong-hee Na in :gh:`99127`.)
 
+* The undocumented locking behavior of :func:`~functools.cached_property`
+  is removed, because it locked across all instances of the class, leading to high
+  lock contention. This means that a cached property getter function could now run
+  more than once for a single instance, if two threads race. For most simple
+  cached properties (e.g. those that are idempotent and simply calculate a value
+  based on other attributes of the instance) this will be fine.  If
+  synchronization is needed, implement locking within the cached property getter
+  function or around multi-threaded access points.
+
 
 Build Changes
 =============
@@ -861,6 +870,19 @@ New Features
   get a frame variable by its name.
   (Contributed by Victor Stinner in :gh:`91248`.)
 
+* Add :c:func:`PyErr_GetRaisedException` and :c:func:`PyErr_SetRaisedException`
+  for saving and restoring the current exception.
+  These functions return and accept a single exception object,
+  rather than the triple arguments of the now-deprecated
+  :c:func:`PyErr_Fetch` and :c:func:`PyErr_Restore`.
+  This is less error prone and a bit more efficient.
+  (Contributed by Mark Shannon in :gh:`101578`.)
+
+* Add :c:func:`PyException_GetArgs` and :c:func:`PyException_SetArgs`
+  as convenience functions for retrieving and modifying
+  the :attr:`~BaseException.args` passed to the exception's constructor.
+  (Contributed by Mark Shannon in :gh:`101578`.)
+
 Porting to Python 3.12
 ----------------------
 
@@ -984,6 +1006,11 @@ Deprecated
   (Contributed in :gh:`47146` by Petr Viktorin, based on
   earlier work by Alexander Belopolsky and Matthias Braun.)
 
+* :c:func:`PyErr_Fetch` and :c:func:`PyErr_Restore` are deprecated.
+  Use :c:func:`PyErr_GetRaisedException` and
+  :c:func:`PyErr_SetRaisedException` instead.
+  (Contributed by Mark Shannon in :gh:`101578`.)
+
 
 Removed
 -------
diff --git a/Include/cpython/genobject.h b/Include/cpython/genobject.h
@@ -13,8 +13,6 @@ extern "C" {
    and coroutine objects. */
 #define _PyGenObject_HEAD(prefix)                                           \
     PyObject_HEAD                                                           \
-    /* The code object backing the generator */                             \
-    PyCodeObject *prefix##_code;                                            \
     /* List of weak reference. */                                           \
     PyObject *prefix##_weakreflist;                                         \
     /* Name of the generator. */                                            \
@@ -28,7 +26,7 @@ extern "C" {
     char prefix##_running_async;                                            \
     /* The frame */                                                         \
     int8_t prefix##_frame_state;                                            \
-    PyObject *prefix##_iframe[1];
+    PyObject *prefix##_iframe[1];                                           \
 
 typedef struct {
     /* The gi_ prefix is intended to remind of generator-iterator. */
@@ -46,6 +44,7 @@ PyAPI_FUNC(PyObject *) PyGen_NewWithQualName(PyFrameObject *,
 PyAPI_FUNC(int) _PyGen_SetStopIterationValue(PyObject *);
 PyAPI_FUNC(int) _PyGen_FetchStopIterationValue(PyObject **);
 PyAPI_FUNC(void) _PyGen_Finalize(PyObject *self);
+PyAPI_FUNC(PyCodeObject *) PyGen_GetCode(PyGenObject *gen);
 
 
 /* --- PyCoroObject ------------------------------------------------------- */
diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h
@@ -209,7 +209,7 @@ _PyFrame_GetFrameObject(_PyInterpreterFrame *frame)
  * frames like the ones in generators and coroutines.
  */
 void
-_PyFrame_Clear(_PyInterpreterFrame * frame);
+_PyFrame_ClearExceptCode(_PyInterpreterFrame * frame);
 
 int
 _PyFrame_Traverse(_PyInterpreterFrame *frame, visitproc visit, void *arg);
diff --git a/Lib/functools.py b/Lib/functools.py
@@ -959,15 +959,12 @@ def __isabstractmethod__(self):
 ### cached_property() - computed once per instance, cached as attribute
 ################################################################################
 
-_NOT_FOUND = object()
-
 
 class cached_property:
     def __init__(self, func):
         self.func = func
         self.attrname = None
         self.__doc__ = func.__doc__
-        self.lock = RLock()
 
     def __set_name__(self, owner, name):
         if self.attrname is None:
@@ -992,21 +989,15 @@ def __get__(self, instance, owner=None):
                 f"instance to cache {self.attrname!r} property."
             )
             raise TypeError(msg) from None
-        val = cache.get(self.attrname, _NOT_FOUND)
-        if val is _NOT_FOUND:
-            with self.lock:
-                # check if another thread filled cache while we awaited lock
-                val = cache.get(self.attrname, _NOT_FOUND)
-                if val is _NOT_FOUND:
-                    val = self.func(instance)
-                    try:
-                        cache[self.attrname] = val
-                    except TypeError:
-                        msg = (
-                            f"The '__dict__' attribute on {type(instance).__name__!r} instance "
-                            f"does not support item assignment for caching {self.attrname!r} property."
-                        )
-                        raise TypeError(msg) from None
+        val = self.func(instance)
+        try:
+            cache[self.attrname] = val
+        except TypeError:
+            msg = (
+                f"The '__dict__' attribute on {type(instance).__name__!r} instance "
+                f"does not support item assignment for caching {self.attrname!r} property."
+            )
+            raise TypeError(msg) from None
         return val
 
     __class_getitem__ = classmethod(GenericAlias)
diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py
@@ -1213,6 +1213,11 @@ def test_pendingcalls_non_threaded(self):
         self.pendingcalls_submit(l, n)
         self.pendingcalls_wait(l, n)
 
+    def test_gen_get_code(self):
+       def genf(): yield
+       gen = genf()
+       self.assertEqual(_testcapi.gen_get_code(gen), gen.gi_code)
+
 
 class SubinterpreterTest(unittest.TestCase):
 
diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py
@@ -2931,21 +2931,6 @@ def get_cost(self):
     cached_cost = py_functools.cached_property(get_cost)
 
 
-class CachedCostItemWait:
-
-    def __init__(self, event):
-        self._cost = 1
-        self.lock = py_functools.RLock()
-        self.event = event
-
-    @py_functools.cached_property
-    def cost(self):
-        self.event.wait(1)
-        with self.lock:
-            self._cost += 1
-        return self._cost
-
-
 class CachedCostItemWithSlots:
     __slots__ = ('_cost')
 
@@ -2970,27 +2955,6 @@ def test_cached_attribute_name_differs_from_func_name(self):
         self.assertEqual(item.get_cost(), 4)
         self.assertEqual(item.cached_cost, 3)
 
-    @threading_helper.requires_working_threading()
-    def test_threaded(self):
-        go = threading.Event()
-        item = CachedCostItemWait(go)
-
-        num_threads = 3
-
-        orig_si = sys.getswitchinterval()
-        sys.setswitchinterval(1e-6)
-        try:
-            threads = [
-                threading.Thread(target=lambda: item.cost)
-                for k in range(num_threads)
-            ]
-            with threading_helper.start_threads(threads):
-                go.set()
-        finally:
-            sys.setswitchinterval(orig_si)
-
-        self.assertEqual(item.cost, 2)
-
     def test_object_with_slots(self):
         item = CachedCostItemWithSlots()
         with self.assertRaisesRegex(
diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py
@@ -1460,7 +1460,7 @@ def bar(cls):
             check(bar, size('PP'))
         # generator
         def get_gen(): yield 1
-        check(get_gen(), size('P2P4P4c7P2ic??2P'))
+        check(get_gen(), size('PP4P4c7P2ic??2P'))
         # iterator
         check(iter('abc'), size('lP'))
         # callable-iterator
diff --git a/Misc/NEWS.d/next/C API/2023-02-06-16-14-30.gh-issue-101578.PW5fA9.rst b/Misc/NEWS.d/next/C API/2023-02-06-16-14-30.gh-issue-101578.PW5fA9.rst
@@ -1,13 +1,10 @@
-Add new C-API functions for saving and restoring the current exception:
-``PyErr_GetRaisedException`` and ``PyErr_SetRaisedException``.
-These functions take and return a single exception rather than
-the triple of  ``PyErr_Fetch`` and ``PyErr_Restore``.
+Add :c:func:`PyErr_GetRaisedException` and :c:func:`PyErr_SetRaisedException`
+for saving and restoring the current exception.
+These functions return and accept a single exception object,
+rather than the triple arguments of the now-deprecated
+:c:func:`PyErr_Fetch` and :c:func:`PyErr_Restore`.
 This is less error prone and a bit more efficient.
 
-The three arguments forms of saving and restoring the
-current exception: ``PyErr_Fetch`` and ``PyErr_Restore``
-are deprecated.
-
-Also add ``PyException_GetArgs`` and ``PyException_SetArgs``
-as convenience functions to help dealing with
-exceptions in the C API.
+Add :c:func:`PyException_GetArgs` and :c:func:`PyException_SetArgs`
+as convenience functions for retrieving and modifying
+the :attr:`~BaseException.args` passed to the exception's constructor.
diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-03-10-21-48-05.bpo-46978.f5QFfw.rst b/Misc/NEWS.d/next/Core and Builtins/2022-03-10-21-48-05.bpo-46978.f5QFfw.rst
diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-01-04-12-49-33.gh-issue-100719.uRPccL.rst b/Misc/NEWS.d/next/Core and Builtins/2023-01-04-12-49-33.gh-issue-100719.uRPccL.rst
@@ -0,0 +1,3 @@
+Remove gi_code field from generator (and coroutine and async generator)
+objects as it is redundant. The frame already includes a reference to the
+code object.
diff --git a/Misc/NEWS.d/next/Library/2023-02-13-12-55-48.gh-issue-87634.q-SBhJ.rst b/Misc/NEWS.d/next/Library/2023-02-13-12-55-48.gh-issue-87634.q-SBhJ.rst
@@ -0,0 +1 @@
+Remove locking behavior from :func:`functools.cached_property`.
diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c
@@ -3076,6 +3076,16 @@ eval_get_func_desc(PyObject *self, PyObject *func)
     return PyUnicode_FromString(PyEval_GetFuncDesc(func));
 }
 
+static PyObject *
+gen_get_code(PyObject *self, PyObject *gen)
+{
+    if (!PyGen_Check(gen)) {
+        PyErr_SetString(PyExc_TypeError, "argument must be a generator object");
+        return NULL;
+    }
+    return (PyObject *)PyGen_GetCode((PyGenObject *)gen);
+}
+
 static PyObject *
 eval_eval_code_ex(PyObject *mod, PyObject *pos_args)
 {
@@ -3657,6 +3667,7 @@ static PyMethodDef TestMethods[] = {
     {"frame_getvarstring", test_frame_getvarstring, METH_VARARGS, NULL},
     {"eval_get_func_name", eval_get_func_name, METH_O, NULL},
     {"eval_get_func_desc", eval_get_func_desc, METH_O, NULL},
+    {"gen_get_code", gen_get_code, METH_O, NULL},
     {"get_feature_macros", get_feature_macros, METH_NOARGS, NULL},
     {"test_code_api", test_code_api, METH_NOARGS, NULL},
     {"settrace_to_record", settrace_to_record, METH_O, NULL},
diff --git a/Objects/genobject.c b/Objects/genobject.c
diff --git a/Objects/typeobject.c b/Objects/typeobject.c
diff --git a/Python/ceval.c b/Python/ceval.c
diff --git a/Python/frame.c b/Python/frame.c

Original file line number	Diff line number	Diff line change
`@@ -371,7 +371,7 @@ ordering::`
`371`	`371`	`>>> from struct import *`
`372`	`372`	`>>> pack(">bhl", 1, 2, 3)`
`373`	`373`	`b'\x01\x00\x02\x00\x00\x00\x03'`
`374`		`- >>> unpack('>bhl', b'\x01\x00\x02\x00\x00\x00\x03'`
	`374`	`+ >>> unpack('>bhl', b'\x01\x00\x02\x00\x00\x00\x03')`
`375`	`375`	`(1, 2, 3)`
`376`	`376`	`>>> calcsize('>bhl')`
`377`	`377`	`7`