Issue #9410: Various optimizations to the pickle module, leading to
speedups up to 4x (depending on the benchmark). Mostly ported from
Unladen Swallow; initial patch by Alexandre Vassalotti.
diff --git a/Lib/pickle.py b/Lib/pickle.py
index 8732508..aca8fd1 100644
--- a/Lib/pickle.py
+++ b/Lib/pickle.py
@@ -1287,12 +1287,6 @@
"""
return int.from_bytes(data, byteorder='little', signed=True)
-# Use the faster _pickle if possible
-try:
- from _pickle import *
-except ImportError:
- Pickler, Unpickler = _Pickler, _Unpickler
-
# Shorthands
def dump(obj, file, protocol=None, *, fix_imports=True):
@@ -1316,6 +1310,12 @@
return Unpickler(file, fix_imports=fix_imports,
encoding=encoding, errors=errors).load()
+# Use the faster _pickle if possible
+try:
+ from _pickle import *
+except ImportError:
+ Pickler, Unpickler = _Pickler, _Unpickler
+
# Doctest
def _test():
import doctest
diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py
index 70e5ad0..33c85dc 100644
--- a/Lib/test/pickletester.py
+++ b/Lib/test/pickletester.py
@@ -1068,6 +1068,15 @@
dumped = self.dumps(set([3]), 2)
self.assertEqual(dumped, DATA6)
+ def test_large_pickles(self):
+ # Test the correctness of internal buffering routines when handling
+ # large data.
+ for proto in protocols:
+ data = (1, b'x' * (256 * 1024))
+ dumped = self.dumps(data, proto)
+ loaded = self.loads(dumped)
+ self.assertEqual(loaded, data)
+
# Test classes for reduce_ex
diff --git a/Lib/test/test_pickle.py b/Lib/test/test_pickle.py
index bb681bf..7f642c8 100644
--- a/Lib/test/test_pickle.py
+++ b/Lib/test/test_pickle.py
@@ -37,6 +37,18 @@
return u.load()
+class InMemoryPickleTests(AbstractPickleTests):
+
+ pickler = pickle._Pickler
+ unpickler = pickle._Unpickler
+
+ def dumps(self, arg, proto=None):
+ return pickle.dumps(arg, proto)
+
+ def loads(self, buf):
+ return pickle.loads(buf)
+
+
class PyPersPicklerTests(AbstractPersistentPicklerTests):
pickler = pickle._Pickler
@@ -95,7 +107,8 @@
tests.extend([CPicklerTests, CPersPicklerTests,
CDumpPickle_LoadPickle, DumpPickle_CLoadPickle,
PyPicklerUnpicklerObjectTests,
- CPicklerUnpicklerObjectTests])
+ CPicklerUnpicklerObjectTests,
+ InMemoryPickleTests])
support.run_unittest(*tests)
support.run_doctest(pickle)