revert the addition of _pickle because it was causing havok with 64-bit
diff --git a/Lib/pickle.py b/Lib/pickle.py
index 8f1f9f0..25ffd03 100644
--- a/Lib/pickle.py
+++ b/Lib/pickle.py
@@ -174,7 +174,7 @@
# Pickling machinery
-class _Pickler:
+class Pickler:
def __init__(self, file, protocol=None):
"""This takes a binary file for writing a pickle data stream.
@@ -182,19 +182,21 @@
All protocols now read and write bytes.
The optional protocol argument tells the pickler to use the
- given protocol; supported protocols are 0, 1, 2, 3. The default
- protocol is 3; a backward-incompatible protocol designed for
- Python 3.0.
+ given protocol; supported protocols are 0, 1, 2. The default
+ protocol is 2; it's been supported for many years now.
+
+ Protocol 1 is more efficient than protocol 0; protocol 2 is
+ more efficient than protocol 1.
Specifying a negative protocol version selects the highest
protocol version supported. The higher the protocol used, the
more recent the version of Python needed to read the pickle
produced.
- The file argument must have a write() method that accepts a single
- bytes argument. It can thus be a file object opened for binary
- writing, a io.BytesIO instance, or any other custom object that
- meets this interface.
+ The file parameter must have a write() method that accepts a single
+ string argument. It can thus be an open file object, a StringIO
+ object, or any other custom object that meets this interface.
+
"""
if protocol is None:
protocol = DEFAULT_PROTOCOL
@@ -202,10 +204,7 @@
protocol = HIGHEST_PROTOCOL
elif not 0 <= protocol <= HIGHEST_PROTOCOL:
raise ValueError("pickle protocol must be <= %d" % HIGHEST_PROTOCOL)
- try:
- self.write = file.write
- except AttributeError:
- raise TypeError("file must have a 'write' attribute")
+ self.write = file.write
self.memo = {}
self.proto = int(protocol)
self.bin = protocol >= 1
@@ -271,10 +270,10 @@
return GET + repr(i).encode("ascii") + b'\n'
- def save(self, obj, save_persistent_id=True):
+ def save(self, obj):
# Check for persistent id (defined by a subclass)
pid = self.persistent_id(obj)
- if pid is not None and save_persistent_id:
+ if pid:
self.save_pers(pid)
return
@@ -342,7 +341,7 @@
def save_pers(self, pid):
# Save a persistent id reference
if self.bin:
- self.save(pid, save_persistent_id=False)
+ self.save(pid)
self.write(BINPERSID)
else:
self.write(PERSID + str(pid).encode("ascii") + b'\n')
@@ -351,13 +350,13 @@
listitems=None, dictitems=None, obj=None):
# This API is called by some subclasses
- # Assert that args is a tuple
+ # Assert that args is a tuple or None
if not isinstance(args, tuple):
- raise PicklingError("args from save_reduce() should be a tuple")
+ raise PicklingError("args from reduce() should be a tuple")
# Assert that func is callable
if not hasattr(func, '__call__'):
- raise PicklingError("func from save_reduce() should be callable")
+ raise PicklingError("func from reduce should be callable")
save = self.save
write = self.write
@@ -439,6 +438,31 @@
self.write(obj and TRUE or FALSE)
dispatch[bool] = save_bool
+ def save_int(self, obj, pack=struct.pack):
+ if self.bin:
+ # If the int is small enough to fit in a signed 4-byte 2's-comp
+ # format, we can store it more efficiently than the general
+ # case.
+ # First one- and two-byte unsigned ints:
+ if obj >= 0:
+ if obj <= 0xff:
+ self.write(BININT1 + bytes([obj]))
+ return
+ if obj <= 0xffff:
+ self.write(BININT2 + bytes([obj&0xff, obj>>8]))
+ return
+ # Next check for 4-byte signed ints:
+ high_bits = obj >> 31 # note that Python shift sign-extends
+ if high_bits == 0 or high_bits == -1:
+ # All high bits are copies of bit 2**31, so the value
+ # fits in a 4-byte signed int.
+ self.write(BININT + pack("<i", obj))
+ return
+ # Text pickle, or int too big to fit in signed 4-byte format.
+ self.write(INT + repr(obj).encode("ascii") + b'\n')
+ # XXX save_int is merged into save_long
+ # dispatch[int] = save_int
+
def save_long(self, obj, pack=struct.pack):
if self.bin:
# If the int is small enough to fit in a signed 4-byte 2's-comp
@@ -479,7 +503,7 @@
def save_bytes(self, obj, pack=struct.pack):
if self.proto < 3:
- self.save_reduce(bytes, (list(obj),), obj=obj)
+ self.save_reduce(bytes, (list(obj),))
return
n = len(obj)
if n < 256:
@@ -555,6 +579,12 @@
dispatch[tuple] = save_tuple
+ # save_empty_tuple() isn't used by anything in Python 2.3. However, I
+ # found a Pickler subclass in Zope3 that calls it, so it's not harmless
+ # to remove it.
+ def save_empty_tuple(self, obj):
+ self.write(EMPTY_TUPLE)
+
def save_list(self, obj):
write = self.write
@@ -666,7 +696,7 @@
module = whichmodule(obj, name)
try:
- __import__(module, level=0)
+ __import__(module)
mod = sys.modules[module]
klass = getattr(mod, name)
except (ImportError, KeyError, AttributeError):
@@ -690,19 +720,9 @@
else:
write(EXT4 + pack("<i", code))
return
- # Non-ASCII identifiers are supported only with protocols >= 3.
- if self.proto >= 3:
- write(GLOBAL + bytes(module, "utf-8") + b'\n' +
- bytes(name, "utf-8") + b'\n')
- else:
- try:
- write(GLOBAL + bytes(module, "ascii") + b'\n' +
- bytes(name, "ascii") + b'\n')
- except UnicodeEncodeError:
- raise PicklingError(
- "can't pickle global identifier '%s.%s' using "
- "pickle protocol %i" % (module, name, self.proto))
+ write(GLOBAL + bytes(module, "utf-8") + b'\n' +
+ bytes(name, "utf-8") + b'\n')
self.memoize(obj)
dispatch[FunctionType] = save_global
@@ -761,7 +781,7 @@
# Unpickling machinery
-class _Unpickler:
+class Unpickler:
def __init__(self, file, *, encoding="ASCII", errors="strict"):
"""This takes a binary file for reading a pickle data stream.
@@ -821,9 +841,6 @@
while stack[k] is not mark: k = k-1
return k
- def persistent_load(self, pid):
- raise UnpickingError("unsupported persistent id encountered")
-
dispatch = {}
def load_proto(self):
@@ -833,7 +850,7 @@
dispatch[PROTO[0]] = load_proto
def load_persid(self):
- pid = self.readline()[:-1].decode("ascii")
+ pid = self.readline()[:-1]
self.append(self.persistent_load(pid))
dispatch[PERSID[0]] = load_persid
@@ -862,9 +879,9 @@
val = True
else:
try:
- val = int(data, 0)
+ val = int(data)
except ValueError:
- val = int(data, 0)
+ val = int(data)
self.append(val)
dispatch[INT[0]] = load_int
@@ -916,8 +933,7 @@
break
else:
raise ValueError("insecure string pickle: %r" % orig)
- self.append(codecs.escape_decode(rep)[0]
- .decode(self.encoding, self.errors))
+ self.append(codecs.escape_decode(rep)[0])
dispatch[STRING[0]] = load_string
def load_binstring(self):
@@ -959,7 +975,7 @@
dispatch[TUPLE[0]] = load_tuple
def load_empty_tuple(self):
- self.append(())
+ self.stack.append(())
dispatch[EMPTY_TUPLE[0]] = load_empty_tuple
def load_tuple1(self):
@@ -975,11 +991,11 @@
dispatch[TUPLE3[0]] = load_tuple3
def load_empty_list(self):
- self.append([])
+ self.stack.append([])
dispatch[EMPTY_LIST[0]] = load_empty_list
def load_empty_dictionary(self):
- self.append({})
+ self.stack.append({})
dispatch[EMPTY_DICT[0]] = load_empty_dictionary
def load_list(self):
@@ -1006,13 +1022,13 @@
def _instantiate(self, klass, k):
args = tuple(self.stack[k+1:])
del self.stack[k:]
- instantiated = False
+ instantiated = 0
if (not args and
isinstance(klass, type) and
not hasattr(klass, "__getinitargs__")):
value = _EmptyClass()
value.__class__ = klass
- instantiated = True
+ instantiated = 1
if not instantiated:
try:
value = klass(*args)
@@ -1022,8 +1038,8 @@
self.append(value)
def load_inst(self):
- module = self.readline()[:-1].decode("ascii")
- name = self.readline()[:-1].decode("ascii")
+ module = self.readline()[:-1]
+ name = self.readline()[:-1]
klass = self.find_class(module, name)
self._instantiate(klass, self.marker())
dispatch[INST[0]] = load_inst
@@ -1043,8 +1059,8 @@
dispatch[NEWOBJ[0]] = load_newobj
def load_global(self):
- module = self.readline()[:-1].decode("utf-8")
- name = self.readline()[:-1].decode("utf-8")
+ module = self.readline()[:-1]
+ name = self.readline()[:-1]
klass = self.find_class(module, name)
self.append(klass)
dispatch[GLOBAL[0]] = load_global
@@ -1079,7 +1095,11 @@
def find_class(self, module, name):
# Subclasses may override this
- __import__(module, level=0)
+ if isinstance(module, bytes_types):
+ module = module.decode("utf-8")
+ if isinstance(name, bytes_types):
+ name = name.decode("utf-8")
+ __import__(module)
mod = sys.modules[module]
klass = getattr(mod, name)
return klass
@@ -1111,33 +1131,31 @@
dispatch[DUP[0]] = load_dup
def load_get(self):
- i = int(self.readline()[:-1])
- self.append(self.memo[i])
+ self.append(self.memo[self.readline()[:-1].decode("ascii")])
dispatch[GET[0]] = load_get
def load_binget(self):
- i = self.read(1)[0]
- self.append(self.memo[i])
+ i = ord(self.read(1))
+ self.append(self.memo[repr(i)])
dispatch[BINGET[0]] = load_binget
def load_long_binget(self):
i = mloads(b'i' + self.read(4))
- self.append(self.memo[i])
+ self.append(self.memo[repr(i)])
dispatch[LONG_BINGET[0]] = load_long_binget
def load_put(self):
- i = int(self.readline()[:-1])
- self.memo[i] = self.stack[-1]
+ self.memo[self.readline()[:-1].decode("ascii")] = self.stack[-1]
dispatch[PUT[0]] = load_put
def load_binput(self):
- i = self.read(1)[0]
- self.memo[i] = self.stack[-1]
+ i = ord(self.read(1))
+ self.memo[repr(i)] = self.stack[-1]
dispatch[BINPUT[0]] = load_binput
def load_long_binput(self):
i = mloads(b'i' + self.read(4))
- self.memo[i] = self.stack[-1]
+ self.memo[repr(i)] = self.stack[-1]
dispatch[LONG_BINPUT[0]] = load_long_binput
def load_append(self):
@@ -1303,12 +1321,6 @@
n -= 1 << (nbytes * 8)
return n
-# Use the faster _pickle if possible
-try:
- from _pickle import *
-except ImportError:
- Pickler, Unpickler = _Pickler, _Unpickler
-
# Shorthands
def dump(obj, file, protocol=None):
@@ -1321,14 +1333,14 @@
assert isinstance(res, bytes_types)
return res
-def load(file, *, encoding="ASCII", errors="strict"):
- return Unpickler(file, encoding=encoding, errors=errors).load()
+def load(file):
+ return Unpickler(file).load()
-def loads(s, *, encoding="ASCII", errors="strict"):
+def loads(s):
if isinstance(s, str):
raise TypeError("Can't load pickle from unicode string")
file = io.BytesIO(s)
- return Unpickler(file, encoding=encoding, errors=errors).load()
+ return Unpickler(file).load()
# Doctest
diff --git a/Lib/pickletools.py b/Lib/pickletools.py
index 1dd45da..7bc9c67 100644
--- a/Lib/pickletools.py
+++ b/Lib/pickletools.py
@@ -2079,12 +2079,11 @@
70: t TUPLE (MARK at 49)
71: p PUT 5
74: R REDUCE
- 75: p PUT 6
- 78: V UNICODE 'def'
- 83: p PUT 7
- 86: s SETITEM
- 87: a APPEND
- 88: . STOP
+ 75: V UNICODE 'def'
+ 80: p PUT 6
+ 83: s SETITEM
+ 84: a APPEND
+ 85: . STOP
highest protocol among opcodes = 0
Try again with a "binary" pickle.
@@ -2116,12 +2115,11 @@
49: t TUPLE (MARK at 37)
50: q BINPUT 5
52: R REDUCE
- 53: q BINPUT 6
- 55: X BINUNICODE 'def'
- 63: q BINPUT 7
- 65: s SETITEM
- 66: e APPENDS (MARK at 3)
- 67: . STOP
+ 53: X BINUNICODE 'def'
+ 61: q BINPUT 6
+ 63: s SETITEM
+ 64: e APPENDS (MARK at 3)
+ 65: . STOP
highest protocol among opcodes = 1
Exercise the INST/OBJ/BUILD family.
diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py
index c9ebdb8..783d361 100644
--- a/Lib/test/pickletester.py
+++ b/Lib/test/pickletester.py
@@ -362,7 +362,7 @@
return x
class AbstractPickleTests(unittest.TestCase):
- # Subclass must define self.dumps, self.loads.
+ # Subclass must define self.dumps, self.loads, self.error.
_testdata = create_data()
@@ -463,9 +463,8 @@
self.assertEqual(list(x[0].attr.keys()), [1])
self.assert_(x[0].attr[1] is x)
- def test_get(self):
- self.assertRaises(KeyError, self.loads, b'g0\np0')
- self.assertEquals(self.loads(b'((Kdtp0\nh\x00l.))'), [(100,), (100,)])
+ def test_garyp(self):
+ self.assertRaises(self.error, self.loads, b'garyp')
def test_insecure_strings(self):
# XXX Some of these tests are temporarily disabled
@@ -956,7 +955,7 @@
f = open(TESTFN, "wb")
try:
f.close()
- self.assertRaises(ValueError, pickle.dump, 123, f)
+ self.assertRaises(ValueError, self.module.dump, 123, f)
finally:
os.remove(TESTFN)
@@ -965,24 +964,24 @@
f = open(TESTFN, "wb")
try:
f.close()
- self.assertRaises(ValueError, pickle.dump, 123, f)
+ self.assertRaises(ValueError, self.module.dump, 123, f)
finally:
os.remove(TESTFN)
def test_highest_protocol(self):
# Of course this needs to be changed when HIGHEST_PROTOCOL changes.
- self.assertEqual(pickle.HIGHEST_PROTOCOL, 3)
+ self.assertEqual(self.module.HIGHEST_PROTOCOL, 3)
def test_callapi(self):
from io import BytesIO
f = BytesIO()
# With and without keyword arguments
- pickle.dump(123, f, -1)
- pickle.dump(123, file=f, protocol=-1)
- pickle.dumps(123, -1)
- pickle.dumps(123, protocol=-1)
- pickle.Pickler(f, -1)
- pickle.Pickler(f, protocol=-1)
+ self.module.dump(123, f, -1)
+ self.module.dump(123, file=f, protocol=-1)
+ self.module.dumps(123, -1)
+ self.module.dumps(123, protocol=-1)
+ self.module.Pickler(f, -1)
+ self.module.Pickler(f, protocol=-1)
class AbstractPersistentPicklerTests(unittest.TestCase):
diff --git a/Lib/test/test_pickle.py b/Lib/test/test_pickle.py
index 65a7adc..42fb33e 100644
--- a/Lib/test/test_pickle.py
+++ b/Lib/test/test_pickle.py
@@ -7,42 +7,37 @@
from test.pickletester import AbstractPickleModuleTests
from test.pickletester import AbstractPersistentPicklerTests
-try:
- import _pickle
- has_c_implementation = True
-except ImportError:
- has_c_implementation = False
+class PickleTests(AbstractPickleTests, AbstractPickleModuleTests):
+ module = pickle
+ error = KeyError
-class PickleTests(AbstractPickleModuleTests):
- pass
+ def dumps(self, arg, proto=None):
+ return pickle.dumps(arg, proto)
+ def loads(self, buf):
+ return pickle.loads(buf)
-class PyPicklerTests(AbstractPickleTests):
+class PicklerTests(AbstractPickleTests):
- pickler = pickle._Pickler
- unpickler = pickle._Unpickler
+ error = KeyError
def dumps(self, arg, proto=None):
f = io.BytesIO()
- p = self.pickler(f, proto)
+ p = pickle.Pickler(f, proto)
p.dump(arg)
f.seek(0)
return bytes(f.read())
def loads(self, buf):
f = io.BytesIO(buf)
- u = self.unpickler(f)
+ u = pickle.Unpickler(f)
return u.load()
-
-class PyPersPicklerTests(AbstractPersistentPicklerTests):
-
- pickler = pickle._Pickler
- unpickler = pickle._Unpickler
+class PersPicklerTests(AbstractPersistentPicklerTests):
def dumps(self, arg, proto=None):
- class PersPickler(self.pickler):
+ class PersPickler(pickle.Pickler):
def persistent_id(subself, obj):
return self.persistent_id(obj)
f = io.BytesIO()
@@ -52,29 +47,19 @@
return f.read()
def loads(self, buf):
- class PersUnpickler(self.unpickler):
+ class PersUnpickler(pickle.Unpickler):
def persistent_load(subself, obj):
return self.persistent_load(obj)
f = io.BytesIO(buf)
u = PersUnpickler(f)
return u.load()
-
-if has_c_implementation:
- class CPicklerTests(PyPicklerTests):
- pickler = _pickle.Pickler
- unpickler = _pickle.Unpickler
-
- class CPersPicklerTests(PyPersPicklerTests):
- pickler = _pickle.Pickler
- unpickler = _pickle.Unpickler
-
-
def test_main():
- tests = [PickleTests, PyPicklerTests, PyPersPicklerTests]
- if has_c_implementation:
- tests.extend([CPicklerTests, CPersPicklerTests])
- support.run_unittest(*tests)
+ support.run_unittest(
+ PickleTests,
+ PicklerTests,
+ PersPicklerTests
+ )
support.run_doctest(pickle)
if __name__ == "__main__":
diff --git a/Lib/test/test_pickletools.py b/Lib/test/test_pickletools.py
index 3e701b0..ccb6c9c 100644
--- a/Lib/test/test_pickletools.py
+++ b/Lib/test/test_pickletools.py
@@ -12,6 +12,8 @@
def loads(self, buf):
return pickle.loads(buf)
+ module = pickle
+ error = KeyError
def test_main():
support.run_unittest(OptimizedPickleTests)