From 926232384a836372fa1e0a05c373d302b702f633 Mon Sep 17 00:00:00 2001 From: "d.grigonis" Date: Mon, 30 Sep 2024 17:37:52 +0300 Subject: [PATCH 01/15] fix --- Lib/functools.py | 191 ++++++++++++++------------ Lib/inspect.py | 34 +---- Lib/test/test_functools.py | 16 ++- Lib/test/test_inspect/test_inspect.py | 15 +- Modules/_functoolsmodule.c | 124 +++++++++-------- 5 files changed, 185 insertions(+), 195 deletions(-) diff --git a/Lib/functools.py b/Lib/functools.py index 83b8895794e7c0..956573aa0dde6b 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -14,12 +14,12 @@ 'partial', 'partialmethod', 'singledispatch', 'singledispatchmethod', 'cached_property', 'Placeholder'] -from abc import get_cache_token +from abc import abstractmethod, get_cache_token from collections import namedtuple # import types, weakref # Deferred to single_dispatch() from operator import itemgetter from reprlib import recursive_repr -from types import MethodType +from types import FunctionType, MethodType from _thread import RLock # Avoid importing types, so we can speedup import time @@ -316,49 +316,6 @@ def _partial_prepare_merger(args): merger = itemgetter(*order) if phcount else None return phcount, merger -def _partial_new(cls, func, /, *args, **keywords): - if issubclass(cls, partial): - base_cls = partial - if not callable(func): - raise TypeError("the first argument must be callable") - else: - base_cls = partialmethod - # func could be a descriptor like classmethod which isn't callable - if not callable(func) and not hasattr(func, "__get__"): - raise TypeError(f"the first argument {func!r} must be a callable " - "or a descriptor") - if args and args[-1] is Placeholder: - raise TypeError("trailing Placeholders are not allowed") - if isinstance(func, base_cls): - pto_phcount = func._phcount - tot_args = func.args - if args: - tot_args += args - if pto_phcount: - # merge args with args of `func` which is `partial` - nargs = len(args) - if nargs < pto_phcount: - tot_args += (Placeholder,) * (pto_phcount - nargs) - tot_args = func._merger(tot_args) - if nargs > pto_phcount: - tot_args += args[pto_phcount:] - phcount, merger = _partial_prepare_merger(tot_args) - else: # works for both pto_phcount == 0 and != 0 - phcount, merger = pto_phcount, func._merger - keywords = {**func.keywords, **keywords} - func = func.func - else: - tot_args = args - phcount, merger = _partial_prepare_merger(tot_args) - - self = object.__new__(cls) - self.func = func - self.args = tot_args - self.keywords = keywords - self._phcount = phcount - self._merger = merger - return self - def _partial_repr(self): cls = type(self) module = cls.__module__ @@ -377,7 +334,49 @@ class partial: __slots__ = ("func", "args", "keywords", "_phcount", "_merger", "__dict__", "__weakref__") - __new__ = _partial_new + def __new__(cls, func, /, *args, **keywords): + if not callable(func): + raise TypeError("the first argument must be callable") + if args and args[-1] is Placeholder: + # Trim trailing placeholders + j = len(args) - 1 + if not j: + args = () + else: + while (j := j - 1) >= 0: + if args[j] is not Placeholder: + break + args = args[:j + 1] + if isinstance(func, partial): + pto_phcount = func._phcount + tot_args = func.args + if args: + tot_args += args + if pto_phcount: + # merge args with args of `func` which is `partial` + nargs = len(args) + if nargs < pto_phcount: + tot_args += (Placeholder,) * (pto_phcount - nargs) + tot_args = func._merger(tot_args) + if nargs > pto_phcount: + tot_args += args[pto_phcount:] + phcount, merger = _partial_prepare_merger(tot_args) + else: # works for both pto_phcount == 0 and != 0 + phcount, merger = pto_phcount, func._merger + keywords = {**func.keywords, **keywords} + func = func.func + else: + tot_args = args + phcount, merger = _partial_prepare_merger(tot_args) + + self = object.__new__(cls) + self.func = func + self.args = tot_args + self.keywords = keywords + self._phcount = phcount + self._merger = merger + return self + __repr__ = recursive_repr()(_partial_repr) def __call__(self, /, *args, **keywords): @@ -416,7 +415,7 @@ def __setstate__(self, state): raise TypeError("invalid partial state") if args and args[-1] is Placeholder: - raise TypeError("trailing Placeholders are not allowed") + raise TypeError("unexpected trailing Placeholders") phcount, merger = _partial_prepare_merger(args) args = tuple(args) # just in case it's a subclass @@ -439,6 +438,7 @@ def __setstate__(self, state): except ImportError: pass + # Descriptor version class partialmethod: """Method descriptor with partial application of the given arguments @@ -447,50 +447,65 @@ class partialmethod: Supports wrapping existing descriptors and handles non-descriptor callables as instance methods. """ - __new__ = _partial_new + + __slots__ = ("func", "args", "keywords", "wrapper", + "__isabstractmethod__", "__dict__", "__weakref__") __repr__ = _partial_repr - def _make_unbound_method(self): - def _method(cls_or_self, /, *args, **keywords): - phcount = self._phcount - if phcount: - try: - pto_args = self._merger(self.args + args) - args = args[phcount:] - except IndexError: - raise TypeError("missing positional arguments " - "in 'partialmethod' call; expected " - f"at least {phcount}, got {len(args)}") - else: - pto_args = self.args - keywords = {**self.keywords, **keywords} - return self.func(cls_or_self, *pto_args, *args, **keywords) - _method.__isabstractmethod__ = self.__isabstractmethod__ - _method.__partialmethod__ = self - return _method + def __init__(self, func, /, *args, **keywords): + if isinstance(func, partialmethod): + # Subclass optimization + temp = partial(lambda: None, *func.args, **func.keywords) + temp = partial(temp, *args, **keywords) + func = func.func + args = temp.args + keywords = temp.keywords + self.func = func + self.args = args + self.keywords = keywords + self.__isabstractmethod__ = getattr(func, "__isabstractmethod__", False) + + # 5 cases + rewrap = None + if isinstance(func, staticmethod): + self.wrapper = partial(func.__wrapped__, *args, **keywords) + rewrap = staticmethod + elif isinstance(func, classmethod): + self.wrapper = partial(func.__wrapped__, Placeholder, *args, **keywords) + rewrap = classmethod + elif isinstance(func, (FunctionType, partial)): + # instance method + self.wrapper = partial(func, Placeholder, *args, **keywords) + elif getattr(func, '__get__', None) is None: + if not callable(func): + raise TypeError(f"the first argument {func!r} must be a callable " + "or a descriptor") + # callable object without __get__ + # treat this like an instance method + self.wrapper = partial(func, Placeholder, *args, **keywords) + else: + # Unknown descriptor + self.wrapper = None - def __get__(self, obj, cls=None): - get = getattr(self.func, "__get__", None) - result = None - if get is not None: - new_func = get(obj, cls) - if new_func is not self.func: - # Assume __get__ returning something new indicates the - # creation of an appropriate callable - result = partial(new_func, *self.args, **self.keywords) - try: - result.__self__ = new_func.__self__ - except AttributeError: - pass - if result is None: - # If the underlying descriptor didn't do anything, treat this - # like an instance method - result = self._make_unbound_method().__get__(obj, cls) - return result + # Adjust for abstract and rewrap if needed + if self.wrapper is not None: + if self.__isabstractmethod__: + self.wrapper = abstractmethod(self.wrapper) + if rewrap is not None: + self.wrapper = rewrap(self.wrapper) - @property - def __isabstractmethod__(self): - return getattr(self.func, "__isabstractmethod__", False) + def __get__(self, obj, cls=None): + if self.wrapper is not None: + return self.wrapper.__get__(obj, cls) + else: + # Unknown descriptor + new_func = getattr(self.func, '__get__')(obj, cls) + result = partial(new_func, *self.args, **self.keywords) + try: + result.__self__ = new_func.__self__ + except AttributeError: + pass + return result __class_getitem__ = classmethod(GenericAlias) @@ -506,8 +521,6 @@ def _unwrap_partialmethod(func): prev = None while func is not prev: prev = func - while isinstance(getattr(func, "__partialmethod__", None), partialmethod): - func = func.__partialmethod__ while isinstance(func, partialmethod): func = getattr(func, 'func') func = _unwrap_partial(func) diff --git a/Lib/inspect.py b/Lib/inspect.py index 2b25300fcb2509..c0323764ff2c9b 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -2436,40 +2436,8 @@ def _signature_from_callable(obj, *, 'attribute'.format(o_sig)) return sig - try: - partialmethod = obj.__partialmethod__ - except AttributeError: - pass - else: - if isinstance(partialmethod, functools.partialmethod): - # Unbound partialmethod (see functools.partialmethod) - # This means, that we need to calculate the signature - # as if it's a regular partial object, but taking into - # account that the first positional argument - # (usually `self`, or `cls`) will not be passed - # automatically (as for boundmethods) - - wrapped_sig = _get_signature_of(partialmethod.func) - - sig = _signature_get_partial(wrapped_sig, partialmethod, (None,)) - first_wrapped_param = tuple(wrapped_sig.parameters.values())[0] - if first_wrapped_param.kind is Parameter.VAR_POSITIONAL: - # First argument of the wrapped callable is `*args`, as in - # `partialmethod(lambda *args)`. - return sig - else: - sig_params = tuple(sig.parameters.values()) - assert (not sig_params or - first_wrapped_param is not sig_params[0]) - # If there were placeholders set, - # first param is transformed to positional only - if partialmethod.args.count(functools.Placeholder): - first_wrapped_param = first_wrapped_param.replace( - kind=Parameter.POSITIONAL_ONLY) - new_params = (first_wrapped_param,) + sig_params - return sig.replace(parameters=new_params) - if isinstance(obj, functools.partial): + print('HERE?', obj) wrapped_sig = _get_signature_of(obj.func) return _signature_get_partial(wrapped_sig, obj) diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index bdaa9a7ec4f020..1d426e71d8e24e 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -211,11 +211,16 @@ def foo(bar): p2.new_attr = 'spam' self.assertEqual(p2.new_attr, 'spam') - def test_placeholders_trailing_raise(self): + def test_placeholders_trailing_trim(self): PH = self.module.Placeholder - for args in [(PH,), (0, PH), (0, PH, 1, PH, PH, PH)]: - with self.assertRaises(TypeError): - self.partial(capture, *args) + for args, call_args, expected_args in [ + [(PH,), (), ()], + [(0, PH), (), (0,)], + [(0, PH, 1, PH, PH, PH), (2,), (0, 2, 1)] + ]: + actual_args, actual_kwds = self.partial(capture, *args)(*call_args) + self.assertEqual(actual_args, expected_args) + self.assertEqual(actual_kwds, {}) def test_placeholders(self): PH = self.module.Placeholder @@ -370,7 +375,7 @@ def test_setstate(self): # Trailing Placeholder error f = self.partial(signature) - msg_regex = re.escape("trailing Placeholders are not allowed") + msg_regex = re.escape("unexpected trailing Placeholders") with self.assertRaisesRegex(TypeError, f'^{msg_regex}$') as cm: f.__setstate__((capture, (1, PH), dict(a=10), dict(attr=[]))) @@ -713,6 +718,7 @@ class PartialMethodSubclass(functools.partialmethod): p = functools.partialmethod(min, 2) p2 = PartialMethodSubclass(p, 1) self.assertIs(p2.func, min) + print(p2.__get__(0)()) self.assertEqual(p2.__get__(0)(), 0) # `partialmethod` subclass input to `partialmethod` subclass p = PartialMethodSubclass(min, 2) diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index aeee504fb8b555..e1a3829b19a469 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -3606,14 +3606,6 @@ def foo(a=0, b=1, /, c=2, d=3): def test_signature_on_partialmethod(self): from functools import partialmethod - class Spam: - def test(): - pass - ham = partialmethod(test) - - with self.assertRaisesRegex(ValueError, "has incorrect arguments"): - inspect.signature(Spam.ham) - class Spam: def test(it, a, b, *, c) -> 'spam': pass @@ -3651,14 +3643,9 @@ def test(self: 'anno', x): g = partialmethod(test, 1) self.assertEqual(self.signature(Spam.g, eval_str=False), - ((('self', ..., 'anno', 'positional_or_keyword'),), + ((('self', ..., 'anno', 'positional_only'),), ...)) - def test_signature_on_fake_partialmethod(self): - def foo(a): pass - foo.__partialmethod__ = 'spam' - self.assertEqual(str(inspect.signature(foo)), '(a)') - def test_signature_on_decorated(self): def decorator(func): @functools.wraps(func) diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index 31cf7bcc09782c..ccfb22055911ba 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -184,14 +184,6 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) if (state == NULL) { return NULL; } - phold = state->placeholder; - - /* Placeholder restrictions */ - if (new_nargs && PyTuple_GET_ITEM(args, new_nargs) == phold) { - PyErr_SetString(PyExc_TypeError, - "trailing Placeholders are not allowed"); - return NULL; - } /* check wrapped function / object */ pto_args = pto_kw = NULL; @@ -217,66 +209,91 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) if (pto == NULL) return NULL; + phold = state->placeholder; pto->fn = Py_NewRef(func); pto->placeholder = phold; - new_args = PyTuple_GetSlice(args, 1, new_nargs + 1); - if (new_args == NULL) { - Py_DECREF(pto); - return NULL; + /* Get new_args with trailing Placeholders trimmed */ + while (new_nargs > 0 && PyTuple_GET_ITEM(args, new_nargs) == phold) { + new_nargs--; } - /* Count placeholders */ + /* process args */ Py_ssize_t phcount = 0; - for (Py_ssize_t i = 0; i < new_nargs - 1; i++) { - if (PyTuple_GET_ITEM(new_args, i) == phold) { - phcount++; + if (new_nargs == 0) { + if (pto_args == NULL) { + pto->args = PyTuple_New(0); + pto->phcount = 0; + } + else { + pto->args = pto_args; + pto->phcount = pto_phcount; + Py_INCREF(pto_args); + assert(PyTuple_Check(pto->args)); } } - /* merge args with args of `func` which is `partial` */ - if (pto_phcount > 0 && new_nargs > 0) { - Py_ssize_t npargs = PyTuple_GET_SIZE(pto_args); - Py_ssize_t tot_nargs = npargs; - if (new_nargs > pto_phcount) { - tot_nargs += new_nargs - pto_phcount; + else { + /* Count placeholders */ + for (Py_ssize_t i = 0; i < new_nargs - 1; i++) { + if (PyTuple_GET_ITEM(args, i + 1) == phold) { + phcount++; + } } - PyObject *item; - PyObject *tot_args = PyTuple_New(tot_nargs); - for (Py_ssize_t i = 0, j = 0; i < tot_nargs; i++) { - if (i < npargs) { - item = PyTuple_GET_ITEM(pto_args, i); - if (j < new_nargs && item == phold) { - item = PyTuple_GET_ITEM(new_args, j); - j++; - pto_phcount--; + if (pto_args == NULL) { + new_args = PyTuple_GetSlice(args, 1, new_nargs + 1); + if (new_args == NULL) { + Py_DECREF(pto); + return NULL; + } + pto->args = new_args; + pto->phcount = phcount; + } + else { + /* merge args with args of `func` which is `partial` */ + Py_ssize_t npargs = PyTuple_GET_SIZE(pto_args); + Py_ssize_t tot_nargs = npargs; + if (new_nargs > pto_phcount) { + tot_nargs += new_nargs - pto_phcount; + } + PyObject *tot_args = PyTuple_New(tot_nargs); + PyObject *item; + if (pto_phcount > 0) { + for (Py_ssize_t i = 0, j = 0; i < tot_nargs; ++i) { + if (i < npargs) { + item = PyTuple_GET_ITEM(pto_args, i); + if (j < new_nargs && item == phold) { + item = PyTuple_GET_ITEM(args, j + 1); + j++; + pto_phcount--; + } + } + else { + item = PyTuple_GET_ITEM(args, j + 1); + j++; + } + Py_INCREF(item); + PyTuple_SET_ITEM(tot_args, i, item); } } else { - item = PyTuple_GET_ITEM(new_args, j); - j++; + for (Py_ssize_t i = 0; i < npargs; ++i) { + item = PyTuple_GET_ITEM(pto_args, i); + Py_INCREF(item); + PyTuple_SET_ITEM(tot_args, i, item); + } + for (Py_ssize_t i = 0; i < new_nargs; ++i) { + item = PyTuple_GET_ITEM(args, i + 1); + Py_INCREF(item); + PyTuple_SET_ITEM(tot_args, npargs + i, item); + } } - Py_INCREF(item); - PyTuple_SET_ITEM(tot_args, i, item); + pto->args = tot_args; + pto->phcount = pto_phcount + phcount; + assert(PyTuple_Check(pto->args)); } - pto->args = tot_args; - pto->phcount = pto_phcount + phcount; - Py_DECREF(new_args); - } - else if (pto_args == NULL) { - pto->args = new_args; - pto->phcount = phcount; - } - else { - pto->args = PySequence_Concat(pto_args, new_args); - pto->phcount = pto_phcount + phcount; - Py_DECREF(new_args); - if (pto->args == NULL) { - Py_DECREF(pto); - return NULL; - } - assert(PyTuple_Check(pto->args)); } + /* process keywords */ if (pto_kw == NULL || PyDict_GET_SIZE(pto_kw) == 0) { if (kw == NULL) { pto->kw = PyDict_New(); @@ -686,8 +703,7 @@ partial_setstate(partialobject *pto, PyObject *state) Py_ssize_t nargs = PyTuple_GET_SIZE(fnargs); if (nargs && PyTuple_GET_ITEM(fnargs, nargs - 1) == pto->placeholder) { - PyErr_SetString(PyExc_TypeError, - "trailing Placeholders are not allowed"); + PyErr_SetString(PyExc_TypeError, "unexpected trailing Placeholders"); return NULL; } /* Count placeholders */ From 3c4edd82e34e7edecee56914666779d9be6517cf Mon Sep 17 00:00:00 2001 From: "d.grigonis" Date: Mon, 30 Sep 2024 18:31:00 +0300 Subject: [PATCH 02/15] test fixes --- Lib/inspect.py | 1 - Lib/test/test_functools.py | 1 - Lib/test/test_inspect/test_inspect.py | 10 ++++++++++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/Lib/inspect.py b/Lib/inspect.py index c0323764ff2c9b..dbb3b85c140590 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -2437,7 +2437,6 @@ def _signature_from_callable(obj, *, return sig if isinstance(obj, functools.partial): - print('HERE?', obj) wrapped_sig = _get_signature_of(obj.func) return _signature_get_partial(wrapped_sig, obj) diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index 1d426e71d8e24e..ed399f92613f25 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -718,7 +718,6 @@ class PartialMethodSubclass(functools.partialmethod): p = functools.partialmethod(min, 2) p2 = PartialMethodSubclass(p, 1) self.assertIs(p2.func, min) - print(p2.__get__(0)()) self.assertEqual(p2.__get__(0)(), 0) # `partialmethod` subclass input to `partialmethod` subclass p = PartialMethodSubclass(min, 2) diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index e1a3829b19a469..8f58166857f59e 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -3606,6 +3606,16 @@ def foo(a=0, b=1, /, c=2, d=3): def test_signature_on_partialmethod(self): from functools import partialmethod + class Spam: + def test(): + pass + ham = partialmethod(test) + + self.assertEqual(self.signature(Spam.ham, eval_str=False), + ((), Ellipsis)) + with self.assertRaisesRegex(ValueError, "invalid method signature"): + inspect.signature(Spam().ham) + class Spam: def test(it, a, b, *, c) -> 'spam': pass From f43b69ebf87ee2345a59cec3028dc851bdf64f02 Mon Sep 17 00:00:00 2001 From: "d.grigonis" Date: Mon, 30 Sep 2024 18:39:47 +0300 Subject: [PATCH 03/15] minor edits --- Lib/functools.py | 1 - Lib/test/test_inspect/test_inspect.py | 6 +++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/Lib/functools.py b/Lib/functools.py index 6adc8e635b8ff3..6e29167e766207 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -435,7 +435,6 @@ def __setstate__(self, state): except ImportError: pass - # Descriptor version class partialmethod: """Method descriptor with partial application of the given arguments diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index 73338afee8cef8..233aecbdab0ea2 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -3664,9 +3664,9 @@ def test_signature_on_partialmethod(self): from functools import partialmethod class Spam: - def test(): - pass - ham = partialmethod(test) + def test(): + pass + ham = partialmethod(test) self.assertEqual(self.signature(Spam.ham, eval_str=False), ((), Ellipsis)) From 90dc0fd85096062eacb60c4adbb5a29c4087165b Mon Sep 17 00:00:00 2001 From: "d.grigonis" Date: Tue, 1 Oct 2024 00:35:59 +0300 Subject: [PATCH 04/15] trailing placeholder restriction removed --- Lib/functools.py | 73 +++++++++++++-------------- Lib/test/test_functools.py | 30 +++++------ Lib/test/test_inspect/test_inspect.py | 2 +- Modules/_functoolsmodule.c | 20 +++----- 4 files changed, 59 insertions(+), 66 deletions(-) diff --git a/Lib/functools.py b/Lib/functools.py index 6e29167e766207..30edaa3b224d59 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -334,16 +334,6 @@ class partial: def __new__(cls, func, /, *args, **keywords): if not callable(func): raise TypeError("the first argument must be callable") - if args and args[-1] is Placeholder: - # Trim trailing placeholders - j = len(args) - 1 - if not j: - args = () - else: - while (j := j - 1) >= 0: - if args[j] is not Placeholder: - break - args = args[:j + 1] if isinstance(func, partial): pto_phcount = func._phcount tot_args = func.args @@ -411,8 +401,8 @@ def __setstate__(self, state): (namespace is not None and not isinstance(namespace, dict))): raise TypeError("invalid partial state") - if args and args[-1] is Placeholder: - raise TypeError("unexpected trailing Placeholders") + # if args and args[-1] is Placeholder: + # raise TypeError("unexpected trailing Placeholders") phcount, merger = _partial_prepare_merger(args) args = tuple(args) # just in case it's a subclass @@ -435,6 +425,7 @@ def __setstate__(self, state): except ImportError: pass + # Descriptor version class partialmethod: """Method descriptor with partial application of the given arguments @@ -446,50 +437,49 @@ class partialmethod: __slots__ = ("func", "args", "keywords", "wrapper", "__isabstractmethod__", "__dict__", "__weakref__") + __repr__ = _partial_repr + __class_getitem__ = classmethod(GenericAlias) def __init__(self, func, /, *args, **keywords): if isinstance(func, partialmethod): # Subclass optimization temp = partial(lambda: None, *func.args, **func.keywords) temp = partial(temp, *args, **keywords) + isabstract = func.__isabstractmethod__ func = func.func args = temp.args keywords = temp.keywords + else: + isabstract = getattr(func, '__isabstractmethod__', False) self.func = func self.args = args self.keywords = keywords - self.__isabstractmethod__ = getattr(func, "__isabstractmethod__", False) + self.__isabstractmethod__ = isabstract # 5 cases - rewrap = None if isinstance(func, staticmethod): - self.wrapper = partial(func.__wrapped__, *args, **keywords) - rewrap = staticmethod + wrapper = partial(func.__wrapped__, *args, **keywords) + self.wrapper = _rewrap_func(wrapper, isabstract, staticmethod) elif isinstance(func, classmethod): - self.wrapper = partial(func.__wrapped__, Placeholder, *args, **keywords) - rewrap = classmethod + wrapper = _partial_unbound(func.__wrapped__, args, keywords) + self.wrapper = _rewrap_func(wrapper, isabstract, classmethod) elif isinstance(func, (FunctionType, partial)): # instance method - self.wrapper = partial(func, Placeholder, *args, **keywords) + wrapper = _partial_unbound(func, args, keywords) + self.wrapper = _rewrap_func(wrapper, isabstract) elif getattr(func, '__get__', None) is None: - if not callable(func): - raise TypeError(f"the first argument {func!r} must be a callable " - "or a descriptor") # callable object without __get__ # treat this like an instance method - self.wrapper = partial(func, Placeholder, *args, **keywords) + if not callable(func): + raise TypeError(f'the first argument {func!r} must be a callable ' + 'or a descriptor') + wrapper = _partial_unbound(func, args, keywords) + self.wrapper = _rewrap_func(wrapper, isabstract) else: # Unknown descriptor self.wrapper = None - # Adjust for abstract and rewrap if needed - if self.wrapper is not None: - if self.__isabstractmethod__: - self.wrapper = abstractmethod(self.wrapper) - if rewrap is not None: - self.wrapper = rewrap(self.wrapper) - def __get__(self, obj, cls=None): if self.wrapper is not None: return self.wrapper.__get__(obj, cls) @@ -503,25 +493,32 @@ def __get__(self, obj, cls=None): pass return result - __class_getitem__ = classmethod(GenericAlias) - # Helper functions +def _partial_unbound(func, args, keywords): + if not args: + return partial(func, **keywords) + return partial(func, Placeholder, *args, **keywords) + +def _rewrap_func(func, isabstract, decorator=None): + if isabstract: + func = abstractmethod(func) + if decorator is not None: + func = decorator(func) + return func + def _unwrap_partial(func): while isinstance(func, partial): func = func.func return func def _unwrap_partialmethod(func): - prev = None - while func is not prev: - prev = func - while isinstance(func, partialmethod): - func = getattr(func, 'func') - func = _unwrap_partial(func) + while isinstance(func, (partial, partialmethod)): + func = func.func return func + ################################################################################ ### LRU Cache function decorator ################################################################################ diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index ed399f92613f25..a098891224360c 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -211,16 +211,16 @@ def foo(bar): p2.new_attr = 'spam' self.assertEqual(p2.new_attr, 'spam') - def test_placeholders_trailing_trim(self): - PH = self.module.Placeholder - for args, call_args, expected_args in [ - [(PH,), (), ()], - [(0, PH), (), (0,)], - [(0, PH, 1, PH, PH, PH), (2,), (0, 2, 1)] - ]: - actual_args, actual_kwds = self.partial(capture, *args)(*call_args) - self.assertEqual(actual_args, expected_args) - self.assertEqual(actual_kwds, {}) + # def test_placeholders_trailing_trim(self): + # PH = self.module.Placeholder + # for args, call_args, expected_args in [ + # [(PH,), (), ()], + # [(0, PH), (), (0,)], + # [(0, PH, 1, PH, PH, PH), (2,), (0, 2, 1)] + # ]: + # actual_args, actual_kwds = self.partial(capture, *args)(*call_args) + # self.assertEqual(actual_args, expected_args) + # self.assertEqual(actual_kwds, {}) def test_placeholders(self): PH = self.module.Placeholder @@ -373,11 +373,11 @@ def test_setstate(self): f() self.assertEqual(f(2), ((2, 1), dict(a=10))) - # Trailing Placeholder error - f = self.partial(signature) - msg_regex = re.escape("unexpected trailing Placeholders") - with self.assertRaisesRegex(TypeError, f'^{msg_regex}$') as cm: - f.__setstate__((capture, (1, PH), dict(a=10), dict(attr=[]))) + # # Trailing Placeholder error + # f = self.partial(signature) + # msg_regex = re.escape("unexpected trailing Placeholders") + # with self.assertRaisesRegex(TypeError, f'^{msg_regex}$') as cm: + # f.__setstate__((capture, (1, PH), dict(a=10), dict(attr=[]))) def test_setstate_errors(self): f = self.partial(signature) diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index 233aecbdab0ea2..21ba48eab7ac57 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -3671,7 +3671,7 @@ def test(): self.assertEqual(self.signature(Spam.ham, eval_str=False), ((), Ellipsis)) with self.assertRaisesRegex(ValueError, "invalid method signature"): - inspect.signature(Spam().ham) + inspect.signature(Spam().ham) class Spam: def test(it, a, b, *, c) -> 'spam': diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index ccfb22055911ba..22bc2b12f7e8a9 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -213,13 +213,7 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) pto->fn = Py_NewRef(func); pto->placeholder = phold; - /* Get new_args with trailing Placeholders trimmed */ - while (new_nargs > 0 && PyTuple_GET_ITEM(args, new_nargs) == phold) { - new_nargs--; - } - /* process args */ - Py_ssize_t phcount = 0; if (new_nargs == 0) { if (pto_args == NULL) { pto->args = PyTuple_New(0); @@ -234,7 +228,8 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) } else { /* Count placeholders */ - for (Py_ssize_t i = 0; i < new_nargs - 1; i++) { + Py_ssize_t phcount = 0; + for (Py_ssize_t i = 0; i < new_nargs; i++) { if (PyTuple_GET_ITEM(args, i + 1) == phold) { phcount++; } @@ -414,6 +409,12 @@ partial_vectorcall(partialobject *pto, PyObject *const *args, pto_args, pto_nargs, NULL); } + /* Fast path if all Placeholders */ + if (pto_nargs == pto_phcount) { + return _PyObject_VectorcallTstate(tstate, pto->fn, + args, nargs, kwnames); + } + /* Fast path using PY_VECTORCALL_ARGUMENTS_OFFSET to prepend a single * positional argument */ if (pto_nargs == 1 && (nargsf & PY_VECTORCALL_ARGUMENTS_OFFSET)) { @@ -701,11 +702,6 @@ partial_setstate(partialobject *pto, PyObject *state) return NULL; } - Py_ssize_t nargs = PyTuple_GET_SIZE(fnargs); - if (nargs && PyTuple_GET_ITEM(fnargs, nargs - 1) == pto->placeholder) { - PyErr_SetString(PyExc_TypeError, "unexpected trailing Placeholders"); - return NULL; - } /* Count placeholders */ Py_ssize_t phcount = 0; for (Py_ssize_t i = 0; i < nargs - 1; i++) { From 10ee9725bce91150af0ff1a45b99eaba6c421758 Mon Sep 17 00:00:00 2001 From: "d.grigonis" Date: Tue, 1 Oct 2024 00:39:13 +0300 Subject: [PATCH 05/15] rm commented code --- Lib/functools.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/Lib/functools.py b/Lib/functools.py index 30edaa3b224d59..4a90a90ec26ce7 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -401,8 +401,6 @@ def __setstate__(self, state): (namespace is not None and not isinstance(namespace, dict))): raise TypeError("invalid partial state") - # if args and args[-1] is Placeholder: - # raise TypeError("unexpected trailing Placeholders") phcount, merger = _partial_prepare_merger(args) args = tuple(args) # just in case it's a subclass From b76ffeefa62f7cc365f4eb9d48a88905a207646f Mon Sep 17 00:00:00 2001 From: "d.grigonis" Date: Tue, 1 Oct 2024 00:51:01 +0300 Subject: [PATCH 06/15] put back accidental removal --- Modules/_functoolsmodule.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index 22bc2b12f7e8a9..99e171ec7313dd 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -703,6 +703,7 @@ partial_setstate(partialobject *pto, PyObject *state) } /* Count placeholders */ + Py_ssize_t nargs = PyTuple_GET_SIZE(fnargs); Py_ssize_t phcount = 0; for (Py_ssize_t i = 0; i < nargs - 1; i++) { if (PyTuple_GET_ITEM(fnargs, i) == pto->placeholder) { From 8fa0ec50e50dfd373e32d66ba474b06a61256972 Mon Sep 17 00:00:00 2001 From: "d.grigonis" Date: Tue, 1 Oct 2024 02:42:26 +0300 Subject: [PATCH 07/15] test_trailing_placeholders and bug fix --- Lib/functools.py | 13 +++++++++++-- Lib/test/test_functools.py | 26 ++++++++++---------------- 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/Lib/functools.py b/Lib/functools.py index 4a90a90ec26ce7..f7099a273f736c 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -310,7 +310,15 @@ def _partial_prepare_merger(args): else: order.append(i) phcount = j - nargs - merger = itemgetter(*order) if phcount else None + if phcount: + if nargs == 1: + i = order[0] + def merger(all_args): + return (all_args[i],) + else: + merger = itemgetter(*order) + else: + merger = None return phcount, merger def _partial_repr(self): @@ -403,7 +411,8 @@ def __setstate__(self, state): phcount, merger = _partial_prepare_merger(args) - args = tuple(args) # just in case it's a subclass + if type(args) is not tuple: + args = tuple(args) # just in case it's a subclass if kwds is None: kwds = {} elif type(kwds) is not dict: # XXX does it need to be *exactly* dict? diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index a098891224360c..71112b889541f7 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -211,16 +211,16 @@ def foo(bar): p2.new_attr = 'spam' self.assertEqual(p2.new_attr, 'spam') - # def test_placeholders_trailing_trim(self): - # PH = self.module.Placeholder - # for args, call_args, expected_args in [ - # [(PH,), (), ()], - # [(0, PH), (), (0,)], - # [(0, PH, 1, PH, PH, PH), (2,), (0, 2, 1)] - # ]: - # actual_args, actual_kwds = self.partial(capture, *args)(*call_args) - # self.assertEqual(actual_args, expected_args) - # self.assertEqual(actual_kwds, {}) + def test_trailing_placeholders(self): + PH = self.module.Placeholder + for args, call_args, expected_args in [ + [(PH,), (1,), (1,)], + [(0, PH), (1,), (0, 1)], + [(0, PH, 2, PH, PH), (1, 3, 4), (0, 1, 2, 3, 4)] + ]: + actual_args, actual_kwds = self.partial(capture, *args)(*call_args) + self.assertEqual(actual_args, expected_args) + self.assertEqual(actual_kwds, {}) def test_placeholders(self): PH = self.module.Placeholder @@ -373,12 +373,6 @@ def test_setstate(self): f() self.assertEqual(f(2), ((2, 1), dict(a=10))) - # # Trailing Placeholder error - # f = self.partial(signature) - # msg_regex = re.escape("unexpected trailing Placeholders") - # with self.assertRaisesRegex(TypeError, f'^{msg_regex}$') as cm: - # f.__setstate__((capture, (1, PH), dict(a=10), dict(attr=[]))) - def test_setstate_errors(self): f = self.partial(signature) self.assertRaises(TypeError, f.__setstate__, (capture, (), {})) From f323dbdf0f787aa3dd781ffb5dc18ceb19efa8e2 Mon Sep 17 00:00:00 2001 From: "d.grigonis" Date: Wed, 2 Oct 2024 11:32:33 +0300 Subject: [PATCH 08/15] full backwards compatibility --- Lib/functools.py | 117 +++++++++++++++++++++++++++-------------------- 1 file changed, 68 insertions(+), 49 deletions(-) diff --git a/Lib/functools.py b/Lib/functools.py index f7099a273f736c..06905fcaf694f6 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -432,6 +432,9 @@ def __setstate__(self, state): except ImportError: pass +_NULL = object() +_UNKNOWN_DESCRIPTOR = object() +_STD_METHOD_TYPES = (staticmethod, classmethod, FunctionType, partial) # Descriptor version class partialmethod: @@ -443,86 +446,102 @@ class partialmethod: """ __slots__ = ("func", "args", "keywords", "wrapper", - "__isabstractmethod__", "__dict__", "__weakref__") + "__dict__", "__weakref__") __repr__ = _partial_repr - __class_getitem__ = classmethod(GenericAlias) def __init__(self, func, /, *args, **keywords): + if not callable(func) and getattr(func, '__get__', None) is None: + raise TypeError(f'the first argument {func!r} must be a callable ' + 'or a descriptor') + if isinstance(func, partialmethod): # Subclass optimization temp = partial(lambda: None, *func.args, **func.keywords) temp = partial(temp, *args, **keywords) - isabstract = func.__isabstractmethod__ func = func.func args = temp.args keywords = temp.keywords - else: - isabstract = getattr(func, '__isabstractmethod__', False) + self.func = func self.args = args self.keywords = keywords - self.__isabstractmethod__ = isabstract - # 5 cases + if (isinstance(func, _STD_METHOD_TYPES) or + getattr(func, '__get__', None) is None): + self.method = None + else: + # Unknown descriptor + self.method = _UNKNOWN_DESCRIPTOR + + def _set_func_attrs(self, func): + func.__partialmethod__ = self + if self.__isabstractmethod__: + func = abstractmethod(func) + return func + + def _make_method(self): + args = self.args + func = self.func + + # 4 cases if isinstance(func, staticmethod): - wrapper = partial(func.__wrapped__, *args, **keywords) - self.wrapper = _rewrap_func(wrapper, isabstract, staticmethod) + func = partial(func.__wrapped__, *args, **self.keywords) + self._set_func_attrs(func) + return staticmethod(func) elif isinstance(func, classmethod): - wrapper = _partial_unbound(func.__wrapped__, args, keywords) - self.wrapper = _rewrap_func(wrapper, isabstract, classmethod) - elif isinstance(func, (FunctionType, partial)): - # instance method - wrapper = _partial_unbound(func, args, keywords) - self.wrapper = _rewrap_func(wrapper, isabstract) - elif getattr(func, '__get__', None) is None: - # callable object without __get__ - # treat this like an instance method - if not callable(func): - raise TypeError(f'the first argument {func!r} must be a callable ' - 'or a descriptor') - wrapper = _partial_unbound(func, args, keywords) - self.wrapper = _rewrap_func(wrapper, isabstract) + ph_args = (Placeholder,) if args else () + func = partial(func.__wrapped__, *ph_args, *args, **self.keywords) + self._set_func_attrs(func) + return classmethod(func) else: - # Unknown descriptor - self.wrapper = None + # instance method. 2 cases: + # a) FunctionType | partial + # b) callable object without __get__ + ph_args = (Placeholder,) if args else () + func = partial(func, *ph_args, *args, **self.keywords) + self._set_func_attrs(func) + return func def __get__(self, obj, cls=None): - if self.wrapper is not None: - return self.wrapper.__get__(obj, cls) - else: - # Unknown descriptor - new_func = getattr(self.func, '__get__')(obj, cls) + method = self.method + if method is _UNKNOWN_DESCRIPTOR: + # Unknown descriptor == unknown binding + # Need to get callable at runtime and apply partial on top + new_func = self.func.__get__(obj, cls) result = partial(new_func, *self.args, **self.keywords) - try: - result.__self__ = new_func.__self__ - except AttributeError: - pass + self._set_func_attrs(func) + __self__ = getattr(new_func, '__self__', _NULL) + if __self__ is not _NULL: + result.__self__ = __self__ return result + if method is None: + # Cache method + self.method = method = self._make_method() + return method.__get__(obj, cls) + @property + def __isabstractmethod__(self): + return getattr(self.func, '__isabstractmethod__', False) -# Helper functions + __class_getitem__ = classmethod(GenericAlias) -def _partial_unbound(func, args, keywords): - if not args: - return partial(func, **keywords) - return partial(func, Placeholder, *args, **keywords) - -def _rewrap_func(func, isabstract, decorator=None): - if isabstract: - func = abstractmethod(func) - if decorator is not None: - func = decorator(func) - return func +# Helper functions def _unwrap_partial(func): - while isinstance(func, partial): + if isinstance(func, partial): func = func.func return func def _unwrap_partialmethod(func): - while isinstance(func, (partial, partialmethod)): - func = func.func + prev = None + while func is not prev: + prev = func + __partialmethod__ = getattr(func, "__partialmethod__", None) + if isinstance(__partialmethod__, partialmethod): + func = __partialmethod__.func + if isinstance(func, (partial, partialmethod)): + func = func.func return func From d217592c6ec964c3513ab331e79a158cd58efad0 Mon Sep 17 00:00:00 2001 From: "d.grigonis" Date: Wed, 2 Oct 2024 11:53:52 +0300 Subject: [PATCH 09/15] small edits --- Lib/functools.py | 38 ++++++++++++++++++++++---------------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/Lib/functools.py b/Lib/functools.py index 06905fcaf694f6..decb4204d46de5 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -451,10 +451,6 @@ class partialmethod: __repr__ = _partial_repr def __init__(self, func, /, *args, **keywords): - if not callable(func) and getattr(func, '__get__', None) is None: - raise TypeError(f'the first argument {func!r} must be a callable ' - 'or a descriptor') - if isinstance(func, partialmethod): # Subclass optimization temp = partial(lambda: None, *func.args, **func.keywords) @@ -467,8 +463,12 @@ def __init__(self, func, /, *args, **keywords): self.args = args self.keywords = keywords - if (isinstance(func, _STD_METHOD_TYPES) or - getattr(func, '__get__', None) is None): + if isinstance(func, _STD_METHOD_TYPES): + self.method = None + elif getattr(func, '__get__', None) is None: + if not callable(func): + raise TypeError(f'the first argument {func!r} must be a callable ' + 'or a descriptor') self.method = None else: # Unknown descriptor @@ -486,22 +486,26 @@ def _make_method(self): # 4 cases if isinstance(func, staticmethod): - func = partial(func.__wrapped__, *args, **self.keywords) - self._set_func_attrs(func) - return staticmethod(func) + deco = staticmethod + method = partial(func.__wrapped__, *args, **self.keywords) elif isinstance(func, classmethod): + deco = classmethod ph_args = (Placeholder,) if args else () - func = partial(func.__wrapped__, *ph_args, *args, **self.keywords) - self._set_func_attrs(func) - return classmethod(func) + method = partial(func.__wrapped__, *ph_args, *args, **self.keywords) else: # instance method. 2 cases: # a) FunctionType | partial # b) callable object without __get__ + deco = None ph_args = (Placeholder,) if args else () - func = partial(func, *ph_args, *args, **self.keywords) - self._set_func_attrs(func) - return func + method = partial(func, *ph_args, *args, **self.keywords) + + method.__partialmethod__ = self + if self.__isabstractmethod__: + method = abstractmethod(method) + if deco is not None: + method = deco(method) + return method def __get__(self, obj, cls=None): method = self.method @@ -510,7 +514,9 @@ def __get__(self, obj, cls=None): # Need to get callable at runtime and apply partial on top new_func = self.func.__get__(obj, cls) result = partial(new_func, *self.args, **self.keywords) - self._set_func_attrs(func) + result.__partialmethod__ = self + if self.__isabstractmethod__: + result = abstractmethod(result) __self__ = getattr(new_func, '__self__', _NULL) if __self__ is not _NULL: result.__self__ = __self__ From 88422c4cae294f929eef06adbaa85d1105221e4d Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Thu, 17 Oct 2024 00:50:34 +0000 Subject: [PATCH 10/15] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20b?= =?UTF-8?q?lurb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../next/Library/2024-10-17-00-50-32.gh-issue-124652.AK3PDp.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2024-10-17-00-50-32.gh-issue-124652.AK3PDp.rst diff --git a/Misc/NEWS.d/next/Library/2024-10-17-00-50-32.gh-issue-124652.AK3PDp.rst b/Misc/NEWS.d/next/Library/2024-10-17-00-50-32.gh-issue-124652.AK3PDp.rst new file mode 100644 index 00000000000000..260c0dcc8b314b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-17-00-50-32.gh-issue-124652.AK3PDp.rst @@ -0,0 +1,2 @@ +:func:`functools.partial` now allows trailing placeholders, which are converted to positional-only arguments. +:func:`functools.partialmethod` is simplified by making use of new :func:`functools.partial` placeholder functionality. From 27493b3c3791fc52cfc2976a5ae21059a3816286 Mon Sep 17 00:00:00 2001 From: "d.grigonis" Date: Sun, 27 Oct 2024 04:07:33 +0200 Subject: [PATCH 11/15] restore previous _unwrap_partial --- Lib/functools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/functools.py b/Lib/functools.py index decb4204d46de5..b48125251c187e 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -535,7 +535,7 @@ def __isabstractmethod__(self): # Helper functions def _unwrap_partial(func): - if isinstance(func, partial): + while isinstance(func, partial): func = func.func return func From 7b247274f7a3d3286f9482e859d62ccbbc12d56e Mon Sep 17 00:00:00 2001 From: "d.grigonis" Date: Sat, 4 Jan 2025 15:45:34 +0200 Subject: [PATCH 12/15] is tuple rollback --- Lib/functools.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Lib/functools.py b/Lib/functools.py index b48125251c187e..62f9349abbf62c 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -411,8 +411,7 @@ def __setstate__(self, state): phcount, merger = _partial_prepare_merger(args) - if type(args) is not tuple: - args = tuple(args) # just in case it's a subclass + args = tuple(args) # just in case it's a subclass if kwds is None: kwds = {} elif type(kwds) is not dict: # XXX does it need to be *exactly* dict? From 4f334599dd9ee19d3e47f6530bd7d367b63a047e Mon Sep 17 00:00:00 2001 From: "d.grigonis" Date: Wed, 8 Jan 2025 23:44:05 +0200 Subject: [PATCH 13/15] leading trailing placeholder lift factored out --- Lib/functools.py | 14 +- Lib/test/test_functools.py | 19 +-- Lib/test/test_inspect/test_inspect.py | 5 + ...-10-17-00-50-32.gh-issue-124652.AK3PDp.rst | 1 - Modules/_functoolsmodule.c | 136 ++++++++---------- 5 files changed, 82 insertions(+), 93 deletions(-) diff --git a/Lib/functools.py b/Lib/functools.py index 62f9349abbf62c..bb3e3e0a78e3b9 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -310,15 +310,7 @@ def _partial_prepare_merger(args): else: order.append(i) phcount = j - nargs - if phcount: - if nargs == 1: - i = order[0] - def merger(all_args): - return (all_args[i],) - else: - merger = itemgetter(*order) - else: - merger = None + merger = itemgetter(*order) if phcount else None return phcount, merger def _partial_repr(self): @@ -342,6 +334,8 @@ class partial: def __new__(cls, func, /, *args, **keywords): if not callable(func): raise TypeError("the first argument must be callable") + if args and args[-1] is Placeholder: + raise TypeError("trailing Placeholders are not allowed") if isinstance(func, partial): pto_phcount = func._phcount tot_args = func.args @@ -409,6 +403,8 @@ def __setstate__(self, state): (namespace is not None and not isinstance(namespace, dict))): raise TypeError("invalid partial state") + if args and args[-1] is Placeholder: + raise TypeError("trailing Placeholders are not allowed") phcount, merger = _partial_prepare_merger(args) args = tuple(args) # just in case it's a subclass diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index 71112b889541f7..bdaa9a7ec4f020 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -211,16 +211,11 @@ def foo(bar): p2.new_attr = 'spam' self.assertEqual(p2.new_attr, 'spam') - def test_trailing_placeholders(self): + def test_placeholders_trailing_raise(self): PH = self.module.Placeholder - for args, call_args, expected_args in [ - [(PH,), (1,), (1,)], - [(0, PH), (1,), (0, 1)], - [(0, PH, 2, PH, PH), (1, 3, 4), (0, 1, 2, 3, 4)] - ]: - actual_args, actual_kwds = self.partial(capture, *args)(*call_args) - self.assertEqual(actual_args, expected_args) - self.assertEqual(actual_kwds, {}) + for args in [(PH,), (0, PH), (0, PH, 1, PH, PH, PH)]: + with self.assertRaises(TypeError): + self.partial(capture, *args) def test_placeholders(self): PH = self.module.Placeholder @@ -373,6 +368,12 @@ def test_setstate(self): f() self.assertEqual(f(2), ((2, 1), dict(a=10))) + # Trailing Placeholder error + f = self.partial(signature) + msg_regex = re.escape("trailing Placeholders are not allowed") + with self.assertRaisesRegex(TypeError, f'^{msg_regex}$') as cm: + f.__setstate__((capture, (1, PH), dict(a=10), dict(attr=[]))) + def test_setstate_errors(self): f = self.partial(signature) self.assertRaises(TypeError, f.__setstate__, (capture, (), {})) diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index 6f61e00b28b53b..894d7b10f8cbf8 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -3714,6 +3714,11 @@ def test(self: 'anno', x): ((('self', ..., 'anno', 'positional_only'),), ...)) + def test_signature_on_fake_partialmethod(self): + def foo(a): pass + foo.__partialmethod__ = 'spam' + self.assertEqual(str(inspect.signature(foo)), '(a)') + def test_signature_on_decorated(self): def decorator(func): @functools.wraps(func) diff --git a/Misc/NEWS.d/next/Library/2024-10-17-00-50-32.gh-issue-124652.AK3PDp.rst b/Misc/NEWS.d/next/Library/2024-10-17-00-50-32.gh-issue-124652.AK3PDp.rst index 260c0dcc8b314b..43eb2e90285ba7 100644 --- a/Misc/NEWS.d/next/Library/2024-10-17-00-50-32.gh-issue-124652.AK3PDp.rst +++ b/Misc/NEWS.d/next/Library/2024-10-17-00-50-32.gh-issue-124652.AK3PDp.rst @@ -1,2 +1 @@ -:func:`functools.partial` now allows trailing placeholders, which are converted to positional-only arguments. :func:`functools.partialmethod` is simplified by making use of new :func:`functools.partial` placeholder functionality. diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index 9ec90ceb19324a..339e2dd6b00d7b 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -187,6 +187,14 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) if (state == NULL) { return NULL; } + phold = state->placeholder; + + /* Placeholder restrictions */ + if (new_nargs && PyTuple_GET_ITEM(args, new_nargs) == phold) { + PyErr_SetString(PyExc_TypeError, + "trailing Placeholders are not allowed"); + return NULL; + } /* check wrapped function / object */ pto_args = pto_kw = NULL; @@ -212,86 +220,66 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) if (pto == NULL) return NULL; - phold = state->placeholder; pto->fn = Py_NewRef(func); pto->placeholder = phold; - /* process args */ - if (new_nargs == 0) { - if (pto_args == NULL) { - pto->args = PyTuple_New(0); - pto->phcount = 0; - } - else { - pto->args = pto_args; - pto->phcount = pto_phcount; - Py_INCREF(pto_args); - assert(PyTuple_Check(pto->args)); - } + new_args = PyTuple_GetSlice(args, 1, new_nargs + 1); + if (new_args == NULL) { + Py_DECREF(pto); + return NULL; } - else { - /* Count placeholders */ - Py_ssize_t phcount = 0; - for (Py_ssize_t i = 0; i < new_nargs; i++) { - if (PyTuple_GET_ITEM(args, i + 1) == phold) { - phcount++; - } + + /* Count placeholders */ + Py_ssize_t phcount = 0; + for (Py_ssize_t i = 0; i < new_nargs - 1; i++) { + if (PyTuple_GET_ITEM(new_args, i) == phold) { + phcount++; } - if (pto_args == NULL) { - new_args = PyTuple_GetSlice(args, 1, new_nargs + 1); - if (new_args == NULL) { - Py_DECREF(pto); - return NULL; - } - pto->args = new_args; - pto->phcount = phcount; + } + /* merge args with args of `func` which is `partial` */ + if (pto_phcount > 0 && new_nargs > 0) { + Py_ssize_t npargs = PyTuple_GET_SIZE(pto_args); + Py_ssize_t tot_nargs = npargs; + if (new_nargs > pto_phcount) { + tot_nargs += new_nargs - pto_phcount; } - else { - /* merge args with args of `func` which is `partial` */ - Py_ssize_t npargs = PyTuple_GET_SIZE(pto_args); - Py_ssize_t tot_nargs = npargs; - if (new_nargs > pto_phcount) { - tot_nargs += new_nargs - pto_phcount; - } - PyObject *tot_args = PyTuple_New(tot_nargs); - PyObject *item; - if (pto_phcount > 0) { - for (Py_ssize_t i = 0, j = 0; i < tot_nargs; ++i) { - if (i < npargs) { - item = PyTuple_GET_ITEM(pto_args, i); - if (j < new_nargs && item == phold) { - item = PyTuple_GET_ITEM(args, j + 1); - j++; - pto_phcount--; - } - } - else { - item = PyTuple_GET_ITEM(args, j + 1); - j++; - } - Py_INCREF(item); - PyTuple_SET_ITEM(tot_args, i, item); + PyObject *item; + PyObject *tot_args = PyTuple_New(tot_nargs); + for (Py_ssize_t i = 0, j = 0; i < tot_nargs; i++) { + if (i < npargs) { + item = PyTuple_GET_ITEM(pto_args, i); + if (j < new_nargs && item == phold) { + item = PyTuple_GET_ITEM(new_args, j); + j++; + pto_phcount--; } } else { - for (Py_ssize_t i = 0; i < npargs; ++i) { - item = PyTuple_GET_ITEM(pto_args, i); - Py_INCREF(item); - PyTuple_SET_ITEM(tot_args, i, item); - } - for (Py_ssize_t i = 0; i < new_nargs; ++i) { - item = PyTuple_GET_ITEM(args, i + 1); - Py_INCREF(item); - PyTuple_SET_ITEM(tot_args, npargs + i, item); - } + item = PyTuple_GET_ITEM(new_args, j); + j++; } - pto->args = tot_args; - pto->phcount = pto_phcount + phcount; - assert(PyTuple_Check(pto->args)); + Py_INCREF(item); + PyTuple_SET_ITEM(tot_args, i, item); } + pto->args = tot_args; + pto->phcount = pto_phcount + phcount; + Py_DECREF(new_args); + } + else if (pto_args == NULL) { + pto->args = new_args; + pto->phcount = phcount; + } + else { + pto->args = PySequence_Concat(pto_args, new_args); + pto->phcount = pto_phcount + phcount; + Py_DECREF(new_args); + if (pto->args == NULL) { + Py_DECREF(pto); + return NULL; + } + assert(PyTuple_Check(pto->args)); } - /* process keywords */ if (pto_kw == NULL || PyDict_GET_SIZE(pto_kw) == 0) { if (kw == NULL) { pto->kw = PyDict_New(); @@ -414,12 +402,6 @@ partial_vectorcall(PyObject *self, PyObject *const *args, pto_args, pto_nargs, NULL); } - /* Fast path if all Placeholders */ - if (pto_nargs == pto_phcount) { - return _PyObject_VectorcallTstate(tstate, pto->fn, - args, nargs, kwnames); - } - /* Fast path using PY_VECTORCALL_ARGUMENTS_OFFSET to prepend a single * positional argument */ if (pto_nargs == 1 && (nargsf & PY_VECTORCALL_ARGUMENTS_OFFSET)) { @@ -711,8 +693,14 @@ partial_setstate(PyObject *self, PyObject *state) return NULL; } - /* Count placeholders */ Py_ssize_t nargs = PyTuple_GET_SIZE(fnargs); + if (nargs && PyTuple_GET_ITEM(fnargs, nargs - 1) == pto->placeholder) { + PyErr_SetString(PyExc_TypeError, + "trailing Placeholders are not allowed"); + return NULL; + } + + /* Count placeholders */ Py_ssize_t phcount = 0; for (Py_ssize_t i = 0; i < nargs - 1; i++) { if (PyTuple_GET_ITEM(fnargs, i) == pto->placeholder) { From c3df6e00595a71d0f458f9098fff4c44d86b3279 Mon Sep 17 00:00:00 2001 From: "d.grigonis" Date: Wed, 8 Jan 2025 23:45:31 +0200 Subject: [PATCH 14/15] remove extra blank line --- Modules/_functoolsmodule.c | 1 - 1 file changed, 1 deletion(-) diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index 339e2dd6b00d7b..802b1cf792c555 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -699,7 +699,6 @@ partial_setstate(PyObject *self, PyObject *state) "trailing Placeholders are not allowed"); return NULL; } - /* Count placeholders */ Py_ssize_t phcount = 0; for (Py_ssize_t i = 0; i < nargs - 1; i++) { From fca3d7df08c31781950cafcf17c0b12a2fbfdaa4 Mon Sep 17 00:00:00 2001 From: "d.grigonis" Date: Wed, 8 Jan 2025 23:46:49 +0200 Subject: [PATCH 15/15] rollback unnecessary changes --- Lib/functools.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/functools.py b/Lib/functools.py index bb3e3e0a78e3b9..0ee436a03394f5 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -407,7 +407,7 @@ def __setstate__(self, state): raise TypeError("trailing Placeholders are not allowed") phcount, merger = _partial_prepare_merger(args) - args = tuple(args) # just in case it's a subclass + args = tuple(args) # just in case it's a subclass if kwds is None: kwds = {} elif type(kwds) is not dict: # XXX does it need to be *exactly* dict? @@ -523,7 +523,7 @@ def __get__(self, obj, cls=None): @property def __isabstractmethod__(self): - return getattr(self.func, '__isabstractmethod__', False) + return getattr(self.func, "__isabstractmethod__", False) __class_getitem__ = classmethod(GenericAlias)