From 5932a3224b65d11d0671e5b34a4a3c11efeccb5d Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 24 Mar 2018 19:36:27 +0000 Subject: [PATCH 1/5] Treat type variables and special typing forms as immputable by copy and pickle --- Lib/test/test_typing.py | 19 +++++++++++++++---- Lib/typing.py | 23 +++++++++++++++++++++-- 2 files changed, 36 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index f56caa13a29594..87914bb20fd8a6 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -1068,9 +1068,7 @@ class C(B[int]): for proto in range(pickle.HIGHEST_PROTOCOL + 1): z = pickle.dumps(s, proto) x = pickle.loads(z) - self.assertEqual(repr(s), repr(x)) # TODO: fix this - # see also comment in test_copy_and_deepcopy - # the issue is typing/#512 + self.assertEqual(s, x) def test_copy_and_deepcopy(self): T = TypeVar('T') @@ -1082,7 +1080,20 @@ class Node(Generic[T]): ... Union['T', int], List['T'], typing.Mapping['T', int]] for t in things + [Any]: self.assertEqual(t, copy(t)) - self.assertEqual(repr(t), repr(deepcopy(t))) # Use repr() because of TypeVars + self.assertEqual(t, deepcopy(t)) + + def test_immutability_by_copy_and_pickle(self): + # Special forms like Union, Any, etc., generic aliases to containers like List, + # Mapping, etc., and type variabcles are considered immutable by copy and pickle. + global TP, TPB, TPV # for pickle + TP = TypeVar('TP') + TPB = TypeVar('TPB', bound=int) + TPV = TypeVar('TPV', bytes, str) + for X in [TP, List, typing.Mapping, ClassVar, typing.Iterable, + Union, Any, Tuple, Callable]: + self.assertIs(copy(X), X) + self.assertIs(deepcopy(X), X) + self.assertIs(pickle.loads(pickle.dumps(X)), X) def test_copy_generic_instances(self): T = TypeVar('T') diff --git a/Lib/typing.py b/Lib/typing.py index 56126cfc02470e..703f60ff3c7423 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -328,7 +328,10 @@ def __hash__(self): def __repr__(self): return 'typing.' + self._name - def __copy__(self): + def __reduce__(self): + return self._name + + def __deepcopy__(self, memo): return self # Special forms are immutable. def __call__(self, *args, **kwds): @@ -496,6 +499,10 @@ def __repr__(self): return f'ForwardRef({self.__forward_arg__!r})' +def _find_name(mod, name): + return getattr(sys.modules[mod], name) + + class TypeVar(_Final, _root=True): """Type variable. @@ -539,7 +546,7 @@ def longest(x: A, y: A) -> A: """ __slots__ = ('__name__', '__bound__', '__constraints__', - '__covariant__', '__contravariant__') + '__covariant__', '__contravariant__', '_def_mod') def __init__(self, name, *constraints, bound=None, covariant=False, contravariant=False): @@ -558,6 +565,7 @@ def __init__(self, name, *constraints, bound=None, self.__bound__ = _type_check(bound, "Bound must be a type.") else: self.__bound__ = None + self._def_mod = sys._getframe(1).f_globals['__name__'] # for pickling def __getstate__(self): return {'name': self.__name__, @@ -582,6 +590,12 @@ def __repr__(self): prefix = '~' return prefix + self.__name__ + def __reduce__(self): + return (_find_name, (self._def_mod, self.__name__)) + + def __deepcopy__(self, memo): + return self + # Special typing constructs Union, Optional, Generic, Callable and Tuple # use three special attributes for internal bookkeeping of generic types: @@ -724,6 +738,11 @@ def __subclasscheck__(self, cls): raise TypeError("Subscripted generics cannot be used with" " class and instance checks") + def __reduce__(self): + if self._special: + return self._name + return super().__reduce__() + class _VariadicGenericAlias(_GenericAlias, _root=True): """Same as _GenericAlias above but for variadic aliases. Currently, From 277b49048f251c658a0b3f2df8d4ef7198645790 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 24 Mar 2018 19:55:09 +0000 Subject: [PATCH 2/5] Add NEWS entry --- .../next/Library/2018-03-24-19-54-48.bpo-32873.cHyoAm.rst | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2018-03-24-19-54-48.bpo-32873.cHyoAm.rst diff --git a/Misc/NEWS.d/next/Library/2018-03-24-19-54-48.bpo-32873.cHyoAm.rst b/Misc/NEWS.d/next/Library/2018-03-24-19-54-48.bpo-32873.cHyoAm.rst new file mode 100644 index 00000000000000..99f8088cf13878 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-24-19-54-48.bpo-32873.cHyoAm.rst @@ -0,0 +1,3 @@ +Treat type variables and special typing forms as immutable by copy and +pickle. This fixes several minor issues and inconsistencies, and improves +backwards compatibility with Python 3.6. From e9157394030b4cdb07e4cda0bb57b5f52f120c90 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 24 Mar 2018 20:08:29 +0000 Subject: [PATCH 3/5] Add more samples for pickling subscripted generics (which is now possible) --- Lib/test/test_typing.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 87914bb20fd8a6..104ce8e2561608 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -1057,13 +1057,15 @@ class C(B[int]): self.assertEqual(x.foo, 42) self.assertEqual(x.bar, 'abc') self.assertEqual(x.__dict__, {'foo': 42, 'bar': 'abc'}) - samples = [Any, Union, Tuple, Callable, ClassVar] + samples = [Any, Union, Tuple, Callable, ClassVar, + Union[int, str], ClassVar[List], Tuple[int, ...], Callable[[str], bytes]] for s in samples: for proto in range(pickle.HIGHEST_PROTOCOL + 1): z = pickle.dumps(s, proto) x = pickle.loads(z) self.assertEqual(s, x) - more_samples = [List, typing.Iterable, typing.Type] + more_samples = [List, typing.Iterable, typing.Type, List[int], + typing.Type[typing.Mapping]] for s in more_samples: for proto in range(pickle.HIGHEST_PROTOCOL + 1): z = pickle.dumps(s, proto) From 44321a45161210dafa1c3d9415ff1b5ca9d75111 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Mon, 26 Mar 2018 22:33:56 +0100 Subject: [PATCH 4/5] Addres review comments --- Lib/test/test_typing.py | 9 ++++++++- Lib/typing.py | 21 +++++++++++++-------- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 104ce8e2561608..a9a88dd22a19f4 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -1091,11 +1091,18 @@ def test_immutability_by_copy_and_pickle(self): TP = TypeVar('TP') TPB = TypeVar('TPB', bound=int) TPV = TypeVar('TPV', bytes, str) - for X in [TP, List, typing.Mapping, ClassVar, typing.Iterable, + for X in [TP, TPB, TPV, List, typing.Mapping, ClassVar, typing.Iterable, Union, Any, Tuple, Callable]: self.assertIs(copy(X), X) self.assertIs(deepcopy(X), X) self.assertIs(pickle.loads(pickle.dumps(X)), X) + # Check that local type variables are copyable. + TL = TypeVar('TL') + TLB = TypeVar('TLB', bound=int) + TLV = TypeVar('TLV', bytes, str) + for X in [TP, TPB, TPV]: + self.assertIs(copy(X), X) + self.assertIs(deepcopy(X), X) def test_copy_generic_instances(self): T = TypeVar('T') diff --git a/Lib/typing.py b/Lib/typing.py index 703f60ff3c7423..510574c413e300 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -285,8 +285,17 @@ def __init_subclass__(self, *args, **kwds): if '_root' not in kwds: raise TypeError("Cannot subclass special typing classes") +class _Immutable: + """Mixin to indicate that object should not be copied.""" -class _SpecialForm(_Final, _root=True): + def __copy__(self): + return self + + def __deepcopy__(self, memo): + return self + + +class _SpecialForm(_Final, _Immutable, _root=True): """Internal indicator of special typing constructs. See _doc instance attribute for specific docs. """ @@ -331,9 +340,6 @@ def __repr__(self): def __reduce__(self): return self._name - def __deepcopy__(self, memo): - return self # Special forms are immutable. - def __call__(self, *args, **kwds): raise TypeError(f"Cannot instantiate {self!r}") @@ -503,7 +509,7 @@ def _find_name(mod, name): return getattr(sys.modules[mod], name) -class TypeVar(_Final, _root=True): +class TypeVar(_Final, _Immutable, _root=True): """Type variable. Usage:: @@ -543,6 +549,8 @@ def longest(x: A, y: A) -> A: T.__covariant__ == False T.__contravariant__ = False A.__constraints__ == (str, bytes) + + Note that only type variables defined in global scope can be pickled. """ __slots__ = ('__name__', '__bound__', '__constraints__', @@ -593,9 +601,6 @@ def __repr__(self): def __reduce__(self): return (_find_name, (self._def_mod, self.__name__)) - def __deepcopy__(self, memo): - return self - # Special typing constructs Union, Optional, Generic, Callable and Tuple # use three special attributes for internal bookkeeping of generic types: From 68333359cb17c8d1e8ededfd83624aa79c184d8b Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Mon, 26 Mar 2018 22:36:49 +0100 Subject: [PATCH 5/5] Fix a typo in test --- Lib/test/test_typing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index a9a88dd22a19f4..09e39fec45e428 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -1100,7 +1100,7 @@ def test_immutability_by_copy_and_pickle(self): TL = TypeVar('TL') TLB = TypeVar('TLB', bound=int) TLV = TypeVar('TLV', bytes, str) - for X in [TP, TPB, TPV]: + for X in [TL, TLB, TLV]: self.assertIs(copy(X), X) self.assertIs(deepcopy(X), X)