Skip to content

bpo-22005: Fixed unpickling instances of datetime classes pickled by Python 2. #794

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions Doc/library/pickle.rst
Original file line number Diff line number Diff line change
Expand Up @@ -243,6 +243,9 @@ process more convenient:
*errors* tell pickle how to decode 8-bit string instances pickled by Python
2; these default to 'ASCII' and 'strict', respectively. The *encoding* can
be 'bytes' to read these 8-bit string instances as bytes objects.
Using ``encoding='latin1'`` is required for unpickling NumPy arrays and
instances of :class:`~datetime.datetime`, :class:`~datetime.date` and
:class:`~datetime.time` pickled by Python 2.

.. function:: loads(bytes_object, \*, fix_imports=True, encoding="ASCII", errors="strict")

Expand All @@ -260,6 +263,9 @@ process more convenient:
*errors* tell pickle how to decode 8-bit string instances pickled by Python
2; these default to 'ASCII' and 'strict', respectively. The *encoding* can
be 'bytes' to read these 8-bit string instances as bytes objects.
Using ``encoding='latin1'`` is required for unpickling NumPy arrays and
instances of :class:`~datetime.datetime`, :class:`~datetime.date` and
:class:`~datetime.time` pickled by Python 2.


The :mod:`pickle` module defines three exceptions:
Expand Down
38 changes: 34 additions & 4 deletions Lib/datetime.py
Original file line number Diff line number Diff line change
Expand Up @@ -808,9 +808,19 @@ def __new__(cls, year, month=None, day=None):

year, month, day (required, base 1)
"""
if month is None and isinstance(year, bytes) and len(year) == 4 and \
1 <= year[2] <= 12:
if (month is None and
isinstance(year, (bytes, str)) and len(year) == 4 and
1 <= ord(year[2:3]) <= 12):
# Pickle support
if isinstance(year, str):
try:
year = year.encode('latin1')
except UnicodeEncodeError:
# More informative error message.
raise ValueError(
"Failed to encode latin1 string when unpickling "
"a date object. "
"pickle.load(data, encoding='latin1') is assumed.")
self = object.__new__(cls)
self.__setstate(year)
self._hashcode = -1
Expand Down Expand Up @@ -1184,8 +1194,18 @@ def __new__(cls, hour=0, minute=0, second=0, microsecond=0, tzinfo=None, *, fold
tzinfo (default to None)
fold (keyword only, default to zero)
"""
if isinstance(hour, bytes) and len(hour) == 6 and hour[0]&0x7F < 24:
if (isinstance(hour, (bytes, str)) and len(hour) == 6 and
ord(hour[0:1])&0x7F < 24):
# Pickle support
if isinstance(hour, str):
try:
hour = hour.encode('latin1')
except UnicodeEncodeError:
# More informative error message.
raise ValueError(
"Failed to encode latin1 string when unpickling "
"a time object. "
"pickle.load(data, encoding='latin1') is assumed.")
self = object.__new__(cls)
self.__setstate(hour, minute or None)
self._hashcode = -1
Expand Down Expand Up @@ -1496,8 +1516,18 @@ class datetime(date):

def __new__(cls, year, month=None, day=None, hour=0, minute=0, second=0,
microsecond=0, tzinfo=None, *, fold=0):
if isinstance(year, bytes) and len(year) == 10 and 1 <= year[2]&0x7F <= 12:
if (isinstance(year, (bytes, str)) and len(year) == 10 and
1 <= ord(year[2:3])&0x7F <= 12):
# Pickle support
if isinstance(year, str):
try:
year = bytes(year, 'latin1')
except UnicodeEncodeError:
# More informative error message.
raise ValueError(
"Failed to encode latin1 string when unpickling "
"a datetime object. "
"pickle.load(data, encoding='latin1') is assumed.")
self = object.__new__(cls)
self.__setstate(year, month)
self._hashcode = -1
Expand Down
116 changes: 116 additions & 0 deletions Lib/test/datetimetester.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@
import _strptime
#

pickle_loads = {pickle.loads, pickle._loads}

pickle_choices = [(pickle, pickle, proto)
for proto in range(pickle.HIGHEST_PROTOCOL + 1)]
Expand Down Expand Up @@ -1434,6 +1435,19 @@ def test_pickling(self):
self.assertEqual(orig, derived)
self.assertEqual(orig.__reduce__(), orig.__reduce_ex__(2))

def test_compat_unpickle(self):
tests = [
b"cdatetime\ndate\n(S'\\x07\\xdf\\x0b\\x1b'\ntR.",
b'cdatetime\ndate\n(U\x04\x07\xdf\x0b\x1btR.',
b'\x80\x02cdatetime\ndate\nU\x04\x07\xdf\x0b\x1b\x85R.',
]
args = 2015, 11, 27
expected = self.theclass(*args)
for data in tests:
for loads in pickle_loads:
derived = loads(data, encoding='latin1')
self.assertEqual(derived, expected)

def test_compare(self):
t1 = self.theclass(2, 3, 4)
t2 = self.theclass(2, 3, 4)
Expand Down Expand Up @@ -2098,6 +2112,24 @@ def test_pickling_subclass_datetime(self):
derived = unpickler.loads(green)
self.assertEqual(orig, derived)

def test_compat_unpickle(self):
tests = [
b'cdatetime\ndatetime\n('
b"S'\\x07\\xdf\\x0b\\x1b\\x14;\\x01\\x00\\x10\\x00'\ntR.",

b'cdatetime\ndatetime\n('
b'U\n\x07\xdf\x0b\x1b\x14;\x01\x00\x10\x00tR.',

b'\x80\x02cdatetime\ndatetime\n'
b'U\n\x07\xdf\x0b\x1b\x14;\x01\x00\x10\x00\x85R.',
]
args = 2015, 11, 27, 20, 59, 1, 64**2
expected = self.theclass(*args)
for data in tests:
for loads in pickle_loads:
derived = loads(data, encoding='latin1')
self.assertEqual(derived, expected)

def test_more_compare(self):
# The test_compare() inherited from TestDate covers the error cases.
# We just want to test lexicographic ordering on the members datetime
Expand Down Expand Up @@ -3069,6 +3101,19 @@ def test_pickling_subclass_time(self):
derived = unpickler.loads(green)
self.assertEqual(orig, derived)

def test_compat_unpickle(self):
tests = [
b"cdatetime\ntime\n(S'\\x14;\\x10\\x00\\x10\\x00'\ntR.",
b'cdatetime\ntime\n(U\x06\x14;\x10\x00\x10\x00tR.',
b'\x80\x02cdatetime\ntime\nU\x06\x14;\x10\x00\x10\x00\x85R.',
]
args = 20, 59, 16, 64**2
expected = self.theclass(*args)
for data in tests:
for loads in pickle_loads:
derived = loads(data, encoding='latin1')
self.assertEqual(derived, expected)

def test_bool(self):
# time is always True.
cls = self.theclass
Expand Down Expand Up @@ -3441,6 +3486,40 @@ def test_pickling(self):
self.assertEqual(derived.tzname(), 'cookie')
self.assertEqual(orig.__reduce__(), orig.__reduce_ex__(2))

def test_compat_unpickle(self):
tests = [
b"cdatetime\ntime\n(S'\\x05\\x06\\x07\\x01\\xe2@'\n"
b"ctest.datetimetester\nPicklableFixedOffset\n(tR"
b"(dS'_FixedOffset__offset'\ncdatetime\ntimedelta\n"
b"(I-1\nI68400\nI0\ntRs"
b"S'_FixedOffset__dstoffset'\nNs"
b"S'_FixedOffset__name'\nS'cookie'\nsbtR.",

b'cdatetime\ntime\n(U\x06\x05\x06\x07\x01\xe2@'
b'ctest.datetimetester\nPicklableFixedOffset\n)R'
b'}(U\x14_FixedOffset__offsetcdatetime\ntimedelta\n'
b'(J\xff\xff\xff\xffJ0\x0b\x01\x00K\x00tR'
b'U\x17_FixedOffset__dstoffsetN'
b'U\x12_FixedOffset__nameU\x06cookieubtR.',

b'\x80\x02cdatetime\ntime\nU\x06\x05\x06\x07\x01\xe2@'
b'ctest.datetimetester\nPicklableFixedOffset\n)R'
b'}(U\x14_FixedOffset__offsetcdatetime\ntimedelta\n'
b'J\xff\xff\xff\xffJ0\x0b\x01\x00K\x00\x87R'
b'U\x17_FixedOffset__dstoffsetN'
b'U\x12_FixedOffset__nameU\x06cookieub\x86R.',
]

tinfo = PicklableFixedOffset(-300, 'cookie')
expected = self.theclass(5, 6, 7, 123456, tzinfo=tinfo)
for data in tests:
for loads in pickle_loads:
derived = loads(data, encoding='latin1')
self.assertEqual(derived, expected, repr(data))
self.assertIsInstance(derived.tzinfo, PicklableFixedOffset)
self.assertEqual(derived.utcoffset(), timedelta(minutes=-300))
self.assertEqual(derived.tzname(), 'cookie')

def test_more_bool(self):
# time is always True.
cls = self.theclass
Expand Down Expand Up @@ -3789,6 +3868,43 @@ def test_pickling(self):
self.assertEqual(derived.tzname(), 'cookie')
self.assertEqual(orig.__reduce__(), orig.__reduce_ex__(2))

def test_compat_unpickle(self):
tests = [
b'cdatetime\ndatetime\n'
b"(S'\\x07\\xdf\\x0b\\x1b\\x14;\\x01\\x01\\xe2@'\n"
b'ctest.datetimetester\nPicklableFixedOffset\n(tR'
b"(dS'_FixedOffset__offset'\ncdatetime\ntimedelta\n"
b'(I-1\nI68400\nI0\ntRs'
b"S'_FixedOffset__dstoffset'\nNs"
b"S'_FixedOffset__name'\nS'cookie'\nsbtR.",

b'cdatetime\ndatetime\n'
b'(U\n\x07\xdf\x0b\x1b\x14;\x01\x01\xe2@'
b'ctest.datetimetester\nPicklableFixedOffset\n)R'
b'}(U\x14_FixedOffset__offsetcdatetime\ntimedelta\n'
b'(J\xff\xff\xff\xffJ0\x0b\x01\x00K\x00tR'
b'U\x17_FixedOffset__dstoffsetN'
b'U\x12_FixedOffset__nameU\x06cookieubtR.',

b'\x80\x02cdatetime\ndatetime\n'
b'U\n\x07\xdf\x0b\x1b\x14;\x01\x01\xe2@'
b'ctest.datetimetester\nPicklableFixedOffset\n)R'
b'}(U\x14_FixedOffset__offsetcdatetime\ntimedelta\n'
b'J\xff\xff\xff\xffJ0\x0b\x01\x00K\x00\x87R'
b'U\x17_FixedOffset__dstoffsetN'
b'U\x12_FixedOffset__nameU\x06cookieub\x86R.',
]
args = 2015, 11, 27, 20, 59, 1, 123456
tinfo = PicklableFixedOffset(-300, 'cookie')
expected = self.theclass(*args, **{'tzinfo': tinfo})
for data in tests:
for loads in pickle_loads:
derived = loads(data, encoding='latin1')
self.assertEqual(derived, expected)
self.assertIsInstance(derived.tzinfo, PicklableFixedOffset)
self.assertEqual(derived.utcoffset(), timedelta(minutes=-300))
self.assertEqual(derived.tzname(), 'cookie')

def test_extreme_hashes(self):
# If an attempt is made to hash these via subtracting the offset
# then hashing a datetime object, OverflowError results. The
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
Implemented unpickling instances of :class:`~datetime.datetime`,
:class:`~datetime.date` and :class:`~datetime.time` pickled by Python 2.
``encoding='latin1'`` should be used for successful decoding.
Loading