From bb7713bbe24506a4642cc1cecae24f5abfe64a8a Mon Sep 17 00:00:00 2001 From: Yurii Karabas <1998uriyyo@gmail.com> Date: Thu, 1 Jul 2021 12:02:10 +0300 Subject: [PATCH 01/13] bpo-44490: Add __parameters__ to types.Union --- Include/genericaliasobject.h | 2 + Lib/test/test_types.py | 5 ++ .../2021-07-01-11-59-34.bpo-44490.xY80VR.rst | 2 + Objects/genericaliasobject.c | 50 +++++++++++------- Objects/unionobject.c | 52 +++++++++++++++++++ 5 files changed, 92 insertions(+), 19 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2021-07-01-11-59-34.bpo-44490.xY80VR.rst diff --git a/Include/genericaliasobject.h b/Include/genericaliasobject.h index cf002976b27cd7..6df56fbf41df79 100644 --- a/Include/genericaliasobject.h +++ b/Include/genericaliasobject.h @@ -5,6 +5,8 @@ extern "C" { #endif +PyAPI_FUNC(PyObject *) _Py_apply_parameters(PyObject *, PyObject *, PyObject *, PyObject *); +PyAPI_FUNC(PyObject *) _Py_make_parameters(PyObject *); PyAPI_FUNC(PyObject *) Py_GenericAlias(PyObject *, PyObject *); PyAPI_DATA(PyTypeObject) Py_GenericAliasType; diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py index ae7b17bd590e61..3f938c3a81f98f 100644 --- a/Lib/test/test_types.py +++ b/Lib/test/test_types.py @@ -666,6 +666,11 @@ def test_or_type_operator_with_TypeVar(self): assert TV | str == typing.Union[TV, str] assert str | TV == typing.Union[str, TV] + def test_union_parameter_chaining(self): + T = typing.TypeVar("T") + + assert (float | list[T])[int] == float | list[int] + def test_or_type_operator_with_forward(self): T = typing.TypeVar('T') ForwardAfter = T | 'Forward' diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-07-01-11-59-34.bpo-44490.xY80VR.rst b/Misc/NEWS.d/next/Core and Builtins/2021-07-01-11-59-34.bpo-44490.xY80VR.rst new file mode 100644 index 00000000000000..1f8f3e90505d14 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2021-07-01-11-59-34.bpo-44490.xY80VR.rst @@ -0,0 +1,2 @@ +Add ``__parameters__`` attribute to ``types.Union``. Add ``__getitem__`` +operator to ``types.Union``. Patch provided by Yurii Karabas. diff --git a/Objects/genericaliasobject.c b/Objects/genericaliasobject.c index 756a7ce474aee9..d64c323a51edb2 100644 --- a/Objects/genericaliasobject.c +++ b/Objects/genericaliasobject.c @@ -198,8 +198,8 @@ tuple_add(PyObject *self, Py_ssize_t len, PyObject *item) return 0; } -static PyObject * -make_parameters(PyObject *args) +PyObject * +_Py_make_parameters(PyObject *args) { Py_ssize_t nargs = PyTuple_GET_SIZE(args); Py_ssize_t len = nargs; @@ -294,18 +294,10 @@ subs_tvars(PyObject *obj, PyObject *params, PyObject **argitems) return obj; } -static PyObject * -ga_getitem(PyObject *self, PyObject *item) +PyObject * +_Py_apply_parameters(PyObject *self, PyObject *args, PyObject *parameters, PyObject *item) { - gaobject *alias = (gaobject *)self; - // do a lookup for __parameters__ so it gets populated (if not already) - if (alias->parameters == NULL) { - alias->parameters = make_parameters(alias->args); - if (alias->parameters == NULL) { - return NULL; - } - } - Py_ssize_t nparams = PyTuple_GET_SIZE(alias->parameters); + Py_ssize_t nparams = PyTuple_GET_SIZE(parameters); if (nparams == 0) { return PyErr_Format(PyExc_TypeError, "There are no type variables left in %R", @@ -320,32 +312,32 @@ ga_getitem(PyObject *self, PyObject *item) nitems > nparams ? "many" : "few", self); } - /* Replace all type variables (specified by alias->parameters) + /* Replace all type variables (specified by parameters) with corresponding values specified by argitems. t = list[T]; t[int] -> newargs = [int] t = dict[str, T]; t[int] -> newargs = [str, int] t = dict[T, list[S]]; t[str, int] -> newargs = [str, list[int]] */ - Py_ssize_t nargs = PyTuple_GET_SIZE(alias->args); + Py_ssize_t nargs = PyTuple_GET_SIZE(args); PyObject *newargs = PyTuple_New(nargs); if (newargs == NULL) { return NULL; } for (Py_ssize_t iarg = 0; iarg < nargs; iarg++) { - PyObject *arg = PyTuple_GET_ITEM(alias->args, iarg); + PyObject *arg = PyTuple_GET_ITEM(args, iarg); int typevar = is_typevar(arg); if (typevar < 0) { Py_DECREF(newargs); return NULL; } if (typevar) { - Py_ssize_t iparam = tuple_index(alias->parameters, nparams, arg); + Py_ssize_t iparam = tuple_index(parameters, nparams, arg); assert(iparam >= 0); arg = argitems[iparam]; Py_INCREF(arg); } else { - arg = subs_tvars(arg, alias->parameters, argitems); + arg = subs_tvars(arg, parameters, argitems); if (arg == NULL) { Py_DECREF(newargs); return NULL; @@ -354,6 +346,26 @@ ga_getitem(PyObject *self, PyObject *item) PyTuple_SET_ITEM(newargs, iarg, arg); } + return newargs; +} + +static PyObject * +ga_getitem(PyObject *self, PyObject *item) +{ + gaobject *alias = (gaobject *)self; + // do a lookup for __parameters__ so it gets populated (if not already) + if (alias->parameters == NULL) { + alias->parameters = _Py_make_parameters(alias->args); + if (alias->parameters == NULL) { + return NULL; + } + } + + PyObject *newargs = _Py_apply_parameters(self, alias->args, alias->parameters, item); + if (newargs == NULL) { + return NULL; + } + PyObject *res = Py_GenericAlias(alias->origin, newargs); Py_DECREF(newargs); @@ -550,7 +562,7 @@ ga_parameters(PyObject *self, void *unused) { gaobject *alias = (gaobject *)self; if (alias->parameters == NULL) { - alias->parameters = make_parameters(alias->args); + alias->parameters = _Py_make_parameters(alias->args); if (alias->parameters == NULL) { return NULL; } diff --git a/Objects/unionobject.c b/Objects/unionobject.c index cc7181d2475caf..81c613c150e0c4 100644 --- a/Objects/unionobject.c +++ b/Objects/unionobject.c @@ -8,6 +8,7 @@ typedef struct { PyObject_HEAD PyObject *args; + PyObject *parameters; } unionobject; static void @@ -18,6 +19,7 @@ unionobject_dealloc(PyObject *self) _PyObject_GC_UNTRACK(self); Py_XDECREF(alias->args); + Py_XDECREF(alias->parameters); Py_TYPE(self)->tp_free(self); } @@ -435,6 +437,53 @@ static PyMethodDef union_methods[] = { {"__subclasscheck__", union_subclasscheck, METH_O}, {0}}; + +static PyObject * +union_getitem(PyObject *self, PyObject *item) +{ + unionobject *alias = (unionobject *)self; + // do a lookup for __parameters__ so it gets populated (if not already) + if (alias->parameters == NULL) { + alias->parameters = _Py_make_parameters(alias->args); + if (alias->parameters == NULL) { + return NULL; + } + } + + PyObject *newargs = _Py_apply_parameters(self, alias->args, alias->parameters, item); + if (newargs == NULL) { + return NULL; + } + + PyObject *res = _Py_Union(newargs); + + Py_DECREF(newargs); + return res; +} + +static PyMappingMethods union_as_mapping = { + .mp_subscript = union_getitem, +}; + +static PyObject * +union_parameters(PyObject *self, void *unused) +{ + unionobject *alias = (unionobject *)self; + if (alias->parameters == NULL) { + alias->parameters = _Py_make_parameters(alias->args); + if (alias->parameters == NULL) { + return NULL; + } + } + Py_INCREF(alias->parameters); + return alias->parameters; +} + +static PyGetSetDef union_properties[] = { + {"__parameters__", union_parameters, (setter)NULL, "Type variables in the types.Union.", NULL}, + {0} +}; + static PyNumberMethods union_as_number = { .nb_or = _Py_union_type_or, // Add __or__ function }; @@ -456,8 +505,10 @@ PyTypeObject _Py_UnionType = { .tp_members = union_members, .tp_methods = union_methods, .tp_richcompare = union_richcompare, + .tp_as_mapping = &union_as_mapping, .tp_as_number = &union_as_number, .tp_repr = union_repr, + .tp_getset = union_properties, }; PyObject * @@ -489,6 +540,7 @@ _Py_Union(PyObject *args) return NULL; } + result->parameters = NULL; result->args = dedup_and_flatten_args(args); if (result->args == NULL) { PyObject_GC_Del(result); From 4d1a60b3f37cbb3a89774a5804a3f02fffafb459 Mon Sep 17 00:00:00 2001 From: Yurii Karabas <1998uriyyo@gmail.com> Date: Thu, 1 Jul 2021 12:14:46 +0300 Subject: [PATCH 02/13] Add Py_LIMITED_API to genericaliasobject --- Include/genericaliasobject.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Include/genericaliasobject.h b/Include/genericaliasobject.h index 6df56fbf41df79..7d52e8a8e57b2f 100644 --- a/Include/genericaliasobject.h +++ b/Include/genericaliasobject.h @@ -5,8 +5,11 @@ extern "C" { #endif +#ifndef Py_LIMITED_API PyAPI_FUNC(PyObject *) _Py_apply_parameters(PyObject *, PyObject *, PyObject *, PyObject *); PyAPI_FUNC(PyObject *) _Py_make_parameters(PyObject *); +#endif + PyAPI_FUNC(PyObject *) Py_GenericAlias(PyObject *, PyObject *); PyAPI_DATA(PyTypeObject) Py_GenericAliasType; From 71b306ef6cabc59a4ad2e72a5a29b0872423e4b3 Mon Sep 17 00:00:00 2001 From: Yurii Karabas <1998uriyyo@gmail.com> Date: Thu, 1 Jul 2021 18:21:57 +0300 Subject: [PATCH 03/13] Update Lib/test/test_types.py Co-authored-by: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> --- Lib/test/test_types.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py index 3f938c3a81f98f..edc4d1b0203b5f 100644 --- a/Lib/test/test_types.py +++ b/Lib/test/test_types.py @@ -669,7 +669,7 @@ def test_or_type_operator_with_TypeVar(self): def test_union_parameter_chaining(self): T = typing.TypeVar("T") - assert (float | list[T])[int] == float | list[int] + self.assertEqual((float | list[T])[int], float | list[int]) def test_or_type_operator_with_forward(self): T = typing.TypeVar('T') From 09d441cf0e6187bba2d1e712671b768e19ca98e5 Mon Sep 17 00:00:00 2001 From: Yurii Karabas <1998uriyyo@gmail.com> Date: Thu, 1 Jul 2021 18:22:08 +0300 Subject: [PATCH 04/13] Update Misc/NEWS.d/next/Core and Builtins/2021-07-01-11-59-34.bpo-44490.xY80VR.rst Co-authored-by: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> --- .../Core and Builtins/2021-07-01-11-59-34.bpo-44490.xY80VR.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-07-01-11-59-34.bpo-44490.xY80VR.rst b/Misc/NEWS.d/next/Core and Builtins/2021-07-01-11-59-34.bpo-44490.xY80VR.rst index 1f8f3e90505d14..4912bca837bb10 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2021-07-01-11-59-34.bpo-44490.xY80VR.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2021-07-01-11-59-34.bpo-44490.xY80VR.rst @@ -1,2 +1,2 @@ -Add ``__parameters__`` attribute to ``types.Union``. Add ``__getitem__`` +Add ``__parameters__`` attribute and ``__getitem__`` operator to ``types.Union``. Patch provided by Yurii Karabas. From 8033e53ea44a77b2fe9c8ff52f82c6ef7cce754c Mon Sep 17 00:00:00 2001 From: Yurii Karabas <1998uriyyo@gmail.com> Date: Thu, 1 Jul 2021 18:22:25 +0300 Subject: [PATCH 05/13] Update Objects/unionobject.c Co-authored-by: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> --- Objects/unionobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/unionobject.c b/Objects/unionobject.c index 81c613c150e0c4..7c3d436a4c2eee 100644 --- a/Objects/unionobject.c +++ b/Objects/unionobject.c @@ -466,7 +466,7 @@ static PyMappingMethods union_as_mapping = { }; static PyObject * -union_parameters(PyObject *self, void *unused) +union_parameters(PyObject *self, void *Py_UNUSED(unused)) { unionobject *alias = (unionobject *)self; if (alias->parameters == NULL) { From 59f54afa5ede83df2e00b679ffd406b507b6ef3f Mon Sep 17 00:00:00 2001 From: Yurii Karabas <1998uriyyo@gmail.com> Date: Thu, 1 Jul 2021 18:32:58 +0300 Subject: [PATCH 06/13] Update Include/genericaliasobject.h Co-authored-by: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> --- Include/genericaliasobject.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/genericaliasobject.h b/Include/genericaliasobject.h index 7d52e8a8e57b2f..4ce9244bb4ce79 100644 --- a/Include/genericaliasobject.h +++ b/Include/genericaliasobject.h @@ -6,7 +6,7 @@ extern "C" { #endif #ifndef Py_LIMITED_API -PyAPI_FUNC(PyObject *) _Py_apply_parameters(PyObject *, PyObject *, PyObject *, PyObject *); +PyAPI_FUNC(PyObject *) _Py_subs_parameters(PyObject *, PyObject *, PyObject *, PyObject *); PyAPI_FUNC(PyObject *) _Py_make_parameters(PyObject *); #endif From 2072f7e73a22ff29718a3238b228a1b2988ed314 Mon Sep 17 00:00:00 2001 From: Yurii Karabas <1998uriyyo@gmail.com> Date: Thu, 1 Jul 2021 18:34:34 +0300 Subject: [PATCH 07/13] Update genericaliasobject.c --- Objects/genericaliasobject.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Objects/genericaliasobject.c b/Objects/genericaliasobject.c index d64c323a51edb2..30e11ab7b4da87 100644 --- a/Objects/genericaliasobject.c +++ b/Objects/genericaliasobject.c @@ -295,7 +295,7 @@ subs_tvars(PyObject *obj, PyObject *params, PyObject **argitems) } PyObject * -_Py_apply_parameters(PyObject *self, PyObject *args, PyObject *parameters, PyObject *item) +_Py_subs_parameters(PyObject *self, PyObject *args, PyObject *parameters, PyObject *item) { Py_ssize_t nparams = PyTuple_GET_SIZE(parameters); if (nparams == 0) { @@ -361,7 +361,7 @@ ga_getitem(PyObject *self, PyObject *item) } } - PyObject *newargs = _Py_apply_parameters(self, alias->args, alias->parameters, item); + PyObject *newargs = _Py_subs_parameters(self, alias->args, alias->parameters, item); if (newargs == NULL) { return NULL; } From cea63d4ac51938e2a250e622a6ce07a68a86af3c Mon Sep 17 00:00:00 2001 From: Yurii Karabas <1998uriyyo@gmail.com> Date: Thu, 1 Jul 2021 18:35:21 +0300 Subject: [PATCH 08/13] Update unionobject.c --- Objects/unionobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/unionobject.c b/Objects/unionobject.c index 7c3d436a4c2eee..2c2221c7f9b84f 100644 --- a/Objects/unionobject.c +++ b/Objects/unionobject.c @@ -450,7 +450,7 @@ union_getitem(PyObject *self, PyObject *item) } } - PyObject *newargs = _Py_apply_parameters(self, alias->args, alias->parameters, item); + PyObject *newargs = _Py_subs_parameters(self, alias->args, alias->parameters, item); if (newargs == NULL) { return NULL; } From 27980f1c896b39955da20f327f68af083371004c Mon Sep 17 00:00:00 2001 From: Yurii Karabas <1998uriyyo@gmail.com> Date: Thu, 1 Jul 2021 19:09:22 +0300 Subject: [PATCH 09/13] Update Lib/test/test_types.py Co-authored-by: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> --- Lib/test/test_types.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py index edc4d1b0203b5f..070692fa0c6eda 100644 --- a/Lib/test/test_types.py +++ b/Lib/test/test_types.py @@ -670,7 +670,8 @@ def test_union_parameter_chaining(self): T = typing.TypeVar("T") self.assertEqual((float | list[T])[int], float | list[int]) - +self.assertEqual(list[int | list[T]].__parameters__, (T,)) +self.assertEqual(list[int | list[T]][str], list[int | list[str]]) def test_or_type_operator_with_forward(self): T = typing.TypeVar('T') ForwardAfter = T | 'Forward' From afbb03868e6ecc8ab963ff736a7d47d82c8ad6da Mon Sep 17 00:00:00 2001 From: Yurii Karabas <1998uriyyo@gmail.com> Date: Thu, 1 Jul 2021 19:12:04 +0300 Subject: [PATCH 10/13] Update test_types.py --- Lib/test/test_types.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py index 070692fa0c6eda..22ae20d8c4efc3 100644 --- a/Lib/test/test_types.py +++ b/Lib/test/test_types.py @@ -670,8 +670,9 @@ def test_union_parameter_chaining(self): T = typing.TypeVar("T") self.assertEqual((float | list[T])[int], float | list[int]) -self.assertEqual(list[int | list[T]].__parameters__, (T,)) -self.assertEqual(list[int | list[T]][str], list[int | list[str]]) + self.assertEqual(list[int | list[T]].__parameters__, (T,)) + self.assertEqual(list[int | list[T]][str], list[int | list[str]]) + def test_or_type_operator_with_forward(self): T = typing.TypeVar('T') ForwardAfter = T | 'Forward' From 8cfa980308abec90a29f54cc17218335d079f882 Mon Sep 17 00:00:00 2001 From: Yurii Karabas <1998uriyyo@gmail.com> Date: Sat, 3 Jul 2021 17:27:58 +0300 Subject: [PATCH 11/13] Add parameters to union_traverse --- Objects/unionobject.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Objects/unionobject.c b/Objects/unionobject.c index 2c2221c7f9b84f..9fc0620a28f380 100644 --- a/Objects/unionobject.c +++ b/Objects/unionobject.c @@ -28,6 +28,7 @@ union_traverse(PyObject *self, visitproc visit, void *arg) { unionobject *alias = (unionobject *)self; Py_VISIT(alias->args); + Py_VISIT(alias->parameters); return 0; } From 1a1ac841b4f3ad0b5b1ca09afd6d753e327cf5ad Mon Sep 17 00:00:00 2001 From: Yurii Karabas <1998uriyyo@gmail.com> Date: Mon, 5 Jul 2021 20:00:19 +0300 Subject: [PATCH 12/13] Update Objects/genericaliasobject.c Co-authored-by: Guido van Rossum --- Objects/genericaliasobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/genericaliasobject.c b/Objects/genericaliasobject.c index 30e11ab7b4da87..87b04851e08da1 100644 --- a/Objects/genericaliasobject.c +++ b/Objects/genericaliasobject.c @@ -353,7 +353,7 @@ static PyObject * ga_getitem(PyObject *self, PyObject *item) { gaobject *alias = (gaobject *)self; - // do a lookup for __parameters__ so it gets populated (if not already) + // Populate __parameters__ if needed. if (alias->parameters == NULL) { alias->parameters = _Py_make_parameters(alias->args); if (alias->parameters == NULL) { From 7799a159b616e9d90754bfa013f6ac27a284e4f6 Mon Sep 17 00:00:00 2001 From: Yurii Karabas <1998uriyyo@gmail.com> Date: Mon, 5 Jul 2021 20:09:43 +0300 Subject: [PATCH 13/13] Add more tests to cover types.Union --- Lib/test/test_types.py | 3 +++ Objects/unionobject.c | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py index 22ae20d8c4efc3..7f7ce86ff08ef3 100644 --- a/Lib/test/test_types.py +++ b/Lib/test/test_types.py @@ -668,10 +668,13 @@ def test_or_type_operator_with_TypeVar(self): def test_union_parameter_chaining(self): T = typing.TypeVar("T") + S = typing.TypeVar("S") self.assertEqual((float | list[T])[int], float | list[int]) self.assertEqual(list[int | list[T]].__parameters__, (T,)) self.assertEqual(list[int | list[T]][str], list[int | list[str]]) + self.assertEqual((list[T] | list[S]).__parameters__, (T, S)) + self.assertEqual((list[T] | list[S])[int, T], list[int] | list[T]) def test_or_type_operator_with_forward(self): T = typing.TypeVar('T') diff --git a/Objects/unionobject.c b/Objects/unionobject.c index 9fc0620a28f380..9d714fb073711a 100644 --- a/Objects/unionobject.c +++ b/Objects/unionobject.c @@ -443,7 +443,7 @@ static PyObject * union_getitem(PyObject *self, PyObject *item) { unionobject *alias = (unionobject *)self; - // do a lookup for __parameters__ so it gets populated (if not already) + // Populate __parameters__ if needed. if (alias->parameters == NULL) { alias->parameters = _Py_make_parameters(alias->args); if (alias->parameters == NULL) {