bpo-22005: Fixed unpickling instances of datetime classes pickled by Python 2. (GH-11017)
encoding='latin1' should be used for successful decoding.
diff --git a/Lib/datetime.py b/Lib/datetime.py
index 292919f..4780b6d 100644
--- a/Lib/datetime.py
+++ b/Lib/datetime.py
@@ -808,9 +808,19 @@
year, month, day (required, base 1)
"""
- if month is None and isinstance(year, bytes) and len(year) == 4 and \
- 1 <= year[2] <= 12:
+ if (month is None and
+ isinstance(year, (bytes, str)) and len(year) == 4 and
+ 1 <= ord(year[2:3]) <= 12):
# Pickle support
+ if isinstance(year, str):
+ try:
+ year = year.encode('latin1')
+ except UnicodeEncodeError:
+ # More informative error message.
+ raise ValueError(
+ "Failed to encode latin1 string when unpickling "
+ "a date object. "
+ "pickle.load(data, encoding='latin1') is assumed.")
self = object.__new__(cls)
self.__setstate(year)
self._hashcode = -1
@@ -1184,8 +1194,18 @@
tzinfo (default to None)
fold (keyword only, default to zero)
"""
- if isinstance(hour, bytes) and len(hour) == 6 and hour[0]&0x7F < 24:
+ if (isinstance(hour, (bytes, str)) and len(hour) == 6 and
+ ord(hour[0:1])&0x7F < 24):
# Pickle support
+ if isinstance(hour, str):
+ try:
+ hour = hour.encode('latin1')
+ except UnicodeEncodeError:
+ # More informative error message.
+ raise ValueError(
+ "Failed to encode latin1 string when unpickling "
+ "a time object. "
+ "pickle.load(data, encoding='latin1') is assumed.")
self = object.__new__(cls)
self.__setstate(hour, minute or None)
self._hashcode = -1
@@ -1496,8 +1516,18 @@
def __new__(cls, year, month=None, day=None, hour=0, minute=0, second=0,
microsecond=0, tzinfo=None, *, fold=0):
- if isinstance(year, bytes) and len(year) == 10 and 1 <= year[2]&0x7F <= 12:
+ if (isinstance(year, (bytes, str)) and len(year) == 10 and
+ 1 <= ord(year[2:3])&0x7F <= 12):
# Pickle support
+ if isinstance(year, str):
+ try:
+ year = bytes(year, 'latin1')
+ except UnicodeEncodeError:
+ # More informative error message.
+ raise ValueError(
+ "Failed to encode latin1 string when unpickling "
+ "a datetime object. "
+ "pickle.load(data, encoding='latin1') is assumed.")
self = object.__new__(cls)
self.__setstate(year, month)
self._hashcode = -1
diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py
index 78b123f..2f838c4 100644
--- a/Lib/test/datetimetester.py
+++ b/Lib/test/datetimetester.py
@@ -38,6 +38,7 @@
import _strptime
#
+pickle_loads = {pickle.loads, pickle._loads}
pickle_choices = [(pickle, pickle, proto)
for proto in range(pickle.HIGHEST_PROTOCOL + 1)]
@@ -1434,6 +1435,19 @@
self.assertEqual(orig, derived)
self.assertEqual(orig.__reduce__(), orig.__reduce_ex__(2))
+ def test_compat_unpickle(self):
+ tests = [
+ b"cdatetime\ndate\n(S'\\x07\\xdf\\x0b\\x1b'\ntR.",
+ b'cdatetime\ndate\n(U\x04\x07\xdf\x0b\x1btR.',
+ b'\x80\x02cdatetime\ndate\nU\x04\x07\xdf\x0b\x1b\x85R.',
+ ]
+ args = 2015, 11, 27
+ expected = self.theclass(*args)
+ for data in tests:
+ for loads in pickle_loads:
+ derived = loads(data, encoding='latin1')
+ self.assertEqual(derived, expected)
+
def test_compare(self):
t1 = self.theclass(2, 3, 4)
t2 = self.theclass(2, 3, 4)
@@ -2098,6 +2112,24 @@
derived = unpickler.loads(green)
self.assertEqual(orig, derived)
+ def test_compat_unpickle(self):
+ tests = [
+ b'cdatetime\ndatetime\n('
+ b"S'\\x07\\xdf\\x0b\\x1b\\x14;\\x01\\x00\\x10\\x00'\ntR.",
+
+ b'cdatetime\ndatetime\n('
+ b'U\n\x07\xdf\x0b\x1b\x14;\x01\x00\x10\x00tR.',
+
+ b'\x80\x02cdatetime\ndatetime\n'
+ b'U\n\x07\xdf\x0b\x1b\x14;\x01\x00\x10\x00\x85R.',
+ ]
+ args = 2015, 11, 27, 20, 59, 1, 64**2
+ expected = self.theclass(*args)
+ for data in tests:
+ for loads in pickle_loads:
+ derived = loads(data, encoding='latin1')
+ self.assertEqual(derived, expected)
+
def test_more_compare(self):
# The test_compare() inherited from TestDate covers the error cases.
# We just want to test lexicographic ordering on the members datetime
@@ -3069,6 +3101,19 @@
derived = unpickler.loads(green)
self.assertEqual(orig, derived)
+ def test_compat_unpickle(self):
+ tests = [
+ b"cdatetime\ntime\n(S'\\x14;\\x10\\x00\\x10\\x00'\ntR.",
+ b'cdatetime\ntime\n(U\x06\x14;\x10\x00\x10\x00tR.',
+ b'\x80\x02cdatetime\ntime\nU\x06\x14;\x10\x00\x10\x00\x85R.',
+ ]
+ args = 20, 59, 16, 64**2
+ expected = self.theclass(*args)
+ for data in tests:
+ for loads in pickle_loads:
+ derived = loads(data, encoding='latin1')
+ self.assertEqual(derived, expected)
+
def test_bool(self):
# time is always True.
cls = self.theclass
@@ -3441,6 +3486,40 @@
self.assertEqual(derived.tzname(), 'cookie')
self.assertEqual(orig.__reduce__(), orig.__reduce_ex__(2))
+ def test_compat_unpickle(self):
+ tests = [
+ b"cdatetime\ntime\n(S'\\x05\\x06\\x07\\x01\\xe2@'\n"
+ b"ctest.datetimetester\nPicklableFixedOffset\n(tR"
+ b"(dS'_FixedOffset__offset'\ncdatetime\ntimedelta\n"
+ b"(I-1\nI68400\nI0\ntRs"
+ b"S'_FixedOffset__dstoffset'\nNs"
+ b"S'_FixedOffset__name'\nS'cookie'\nsbtR.",
+
+ b'cdatetime\ntime\n(U\x06\x05\x06\x07\x01\xe2@'
+ b'ctest.datetimetester\nPicklableFixedOffset\n)R'
+ b'}(U\x14_FixedOffset__offsetcdatetime\ntimedelta\n'
+ b'(J\xff\xff\xff\xffJ0\x0b\x01\x00K\x00tR'
+ b'U\x17_FixedOffset__dstoffsetN'
+ b'U\x12_FixedOffset__nameU\x06cookieubtR.',
+
+ b'\x80\x02cdatetime\ntime\nU\x06\x05\x06\x07\x01\xe2@'
+ b'ctest.datetimetester\nPicklableFixedOffset\n)R'
+ b'}(U\x14_FixedOffset__offsetcdatetime\ntimedelta\n'
+ b'J\xff\xff\xff\xffJ0\x0b\x01\x00K\x00\x87R'
+ b'U\x17_FixedOffset__dstoffsetN'
+ b'U\x12_FixedOffset__nameU\x06cookieub\x86R.',
+ ]
+
+ tinfo = PicklableFixedOffset(-300, 'cookie')
+ expected = self.theclass(5, 6, 7, 123456, tzinfo=tinfo)
+ for data in tests:
+ for loads in pickle_loads:
+ derived = loads(data, encoding='latin1')
+ self.assertEqual(derived, expected, repr(data))
+ self.assertIsInstance(derived.tzinfo, PicklableFixedOffset)
+ self.assertEqual(derived.utcoffset(), timedelta(minutes=-300))
+ self.assertEqual(derived.tzname(), 'cookie')
+
def test_more_bool(self):
# time is always True.
cls = self.theclass
@@ -3789,6 +3868,43 @@
self.assertEqual(derived.tzname(), 'cookie')
self.assertEqual(orig.__reduce__(), orig.__reduce_ex__(2))
+ def test_compat_unpickle(self):
+ tests = [
+ b'cdatetime\ndatetime\n'
+ b"(S'\\x07\\xdf\\x0b\\x1b\\x14;\\x01\\x01\\xe2@'\n"
+ b'ctest.datetimetester\nPicklableFixedOffset\n(tR'
+ b"(dS'_FixedOffset__offset'\ncdatetime\ntimedelta\n"
+ b'(I-1\nI68400\nI0\ntRs'
+ b"S'_FixedOffset__dstoffset'\nNs"
+ b"S'_FixedOffset__name'\nS'cookie'\nsbtR.",
+
+ b'cdatetime\ndatetime\n'
+ b'(U\n\x07\xdf\x0b\x1b\x14;\x01\x01\xe2@'
+ b'ctest.datetimetester\nPicklableFixedOffset\n)R'
+ b'}(U\x14_FixedOffset__offsetcdatetime\ntimedelta\n'
+ b'(J\xff\xff\xff\xffJ0\x0b\x01\x00K\x00tR'
+ b'U\x17_FixedOffset__dstoffsetN'
+ b'U\x12_FixedOffset__nameU\x06cookieubtR.',
+
+ b'\x80\x02cdatetime\ndatetime\n'
+ b'U\n\x07\xdf\x0b\x1b\x14;\x01\x01\xe2@'
+ b'ctest.datetimetester\nPicklableFixedOffset\n)R'
+ b'}(U\x14_FixedOffset__offsetcdatetime\ntimedelta\n'
+ b'J\xff\xff\xff\xffJ0\x0b\x01\x00K\x00\x87R'
+ b'U\x17_FixedOffset__dstoffsetN'
+ b'U\x12_FixedOffset__nameU\x06cookieub\x86R.',
+ ]
+ args = 2015, 11, 27, 20, 59, 1, 123456
+ tinfo = PicklableFixedOffset(-300, 'cookie')
+ expected = self.theclass(*args, **{'tzinfo': tinfo})
+ for data in tests:
+ for loads in pickle_loads:
+ derived = loads(data, encoding='latin1')
+ self.assertEqual(derived, expected)
+ self.assertIsInstance(derived.tzinfo, PicklableFixedOffset)
+ self.assertEqual(derived.utcoffset(), timedelta(minutes=-300))
+ self.assertEqual(derived.tzname(), 'cookie')
+
def test_extreme_hashes(self):
# If an attempt is made to hash these via subtracting the offset
# then hashing a datetime object, OverflowError results. The