This is my patch:
[ 1180995 ] binary formats for marshalling floats
Adds 2 new type codes for marshal (binary floats and binary complexes), a
new marshal version (2), updates MAGIC and fiddles the de-serializing of
code objects to be less likely to clobber the real reason for failing if
it fails.
diff --git a/Include/marshal.h b/Include/marshal.h
index fc491dd..a9f7320 100644
--- a/Include/marshal.h
+++ b/Include/marshal.h
@@ -7,7 +7,7 @@
extern "C" {
#endif
-#define Py_MARSHAL_VERSION 1
+#define Py_MARSHAL_VERSION 2
PyAPI_FUNC(void) PyMarshal_WriteLongToFile(long, FILE *, int);
PyAPI_FUNC(void) PyMarshal_WriteObjectToFile(PyObject *, FILE *, int);
diff --git a/Lib/test/test_marshal.py b/Lib/test/test_marshal.py
index 9901a3c..b66eef5 100644
--- a/Lib/test/test_marshal.py
+++ b/Lib/test/test_marshal.py
@@ -73,20 +73,34 @@
n /= 123.4567
f = 0.0
- s = marshal.dumps(f)
+ s = marshal.dumps(f, 2)
got = marshal.loads(s)
self.assertEqual(f, got)
+ # and with version <= 1 (floats marshalled differently then)
+ s = marshal.dumps(f, 1)
+ got = marshal.loads(s)
+ self.assertEqual(f, got)
n = sys.maxint * 3.7e-250
while n < small:
for expected in (-n, n):
f = float(expected)
+
s = marshal.dumps(f)
got = marshal.loads(s)
self.assertEqual(f, got)
+
+ s = marshal.dumps(f, 1)
+ got = marshal.loads(s)
+ self.assertEqual(f, got)
+
marshal.dump(f, file(test_support.TESTFN, "wb"))
got = marshal.load(file(test_support.TESTFN, "rb"))
self.assertEqual(f, got)
+
+ marshal.dump(f, file(test_support.TESTFN, "wb"), 1)
+ got = marshal.load(file(test_support.TESTFN, "rb"))
+ self.assertEqual(f, got)
n *= 123.4567
os.unlink(test_support.TESTFN)
diff --git a/Misc/NEWS b/Misc/NEWS
index 50f5a27..67ccb42 100644
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -12,6 +12,9 @@
Core and builtins
-----------------
+- SF patch #1180995: marshal now uses a binary format by default when
+ serializing floats.
+
- SF patch #1181301: on platforms that appear to use IEEE 754 floats,
the routines that promise to produce IEEE 754 binary representations
of floats now simply copy bytes around.
diff --git a/Python/import.c b/Python/import.c
index 0362dbd..c03d4cc 100644
--- a/Python/import.c
+++ b/Python/import.c
@@ -50,8 +50,9 @@
Python 2.4a0: 62041
Python 2.4a3: 62051
Python 2.4b1: 62061
+ Python 2.5a0: 62071
*/
-#define MAGIC (62061 | ((long)'\r'<<16) | ((long)'\n'<<24))
+#define MAGIC (62071 | ((long)'\r'<<16) | ((long)'\n'<<24))
/* Magic word as global; note that _PyImport_Init() can change the
value of this global to accommodate for alterations of how the
diff --git a/Python/marshal.c b/Python/marshal.c
index d0fcac7..2535cb5 100644
--- a/Python/marshal.c
+++ b/Python/marshal.c
@@ -15,28 +15,30 @@
*/
#define MAX_MARSHAL_STACK_DEPTH 5000
-#define TYPE_NULL '0'
-#define TYPE_NONE 'N'
-#define TYPE_FALSE 'F'
-#define TYPE_TRUE 'T'
-#define TYPE_STOPITER 'S'
-#define TYPE_ELLIPSIS '.'
-#define TYPE_INT 'i'
-#define TYPE_INT64 'I'
-#define TYPE_FLOAT 'f'
-#define TYPE_COMPLEX 'x'
-#define TYPE_LONG 'l'
-#define TYPE_STRING 's'
-#define TYPE_INTERNED 't'
-#define TYPE_STRINGREF 'R'
-#define TYPE_TUPLE '('
-#define TYPE_LIST '['
-#define TYPE_DICT '{'
-#define TYPE_CODE 'c'
-#define TYPE_UNICODE 'u'
-#define TYPE_UNKNOWN '?'
-#define TYPE_SET '<'
-#define TYPE_FROZENSET '>'
+#define TYPE_NULL '0'
+#define TYPE_NONE 'N'
+#define TYPE_FALSE 'F'
+#define TYPE_TRUE 'T'
+#define TYPE_STOPITER 'S'
+#define TYPE_ELLIPSIS '.'
+#define TYPE_INT 'i'
+#define TYPE_INT64 'I'
+#define TYPE_FLOAT 'f'
+#define TYPE_BINARY_FLOAT 'g'
+#define TYPE_COMPLEX 'x'
+#define TYPE_BINARY_COMPLEX 'y'
+#define TYPE_LONG 'l'
+#define TYPE_STRING 's'
+#define TYPE_INTERNED 't'
+#define TYPE_STRINGREF 'R'
+#define TYPE_TUPLE '('
+#define TYPE_LIST '['
+#define TYPE_DICT '{'
+#define TYPE_CODE 'c'
+#define TYPE_UNICODE 'u'
+#define TYPE_UNKNOWN '?'
+#define TYPE_SET '<'
+#define TYPE_FROZENSET '>'
typedef struct {
FILE *fp;
@@ -47,6 +49,7 @@
char *ptr;
char *end;
PyObject *strings; /* dict on marshal, list on unmarshal */
+ int version;
} WFILE;
#define w_byte(c, p) if (((p)->fp)) putc((c), (p)->fp); \
@@ -165,32 +168,62 @@
w_short(ob->ob_digit[i], p);
}
else if (PyFloat_Check(v)) {
- char buf[256]; /* Plenty to format any double */
- PyFloat_AsReprString(buf, (PyFloatObject *)v);
- n = strlen(buf);
- w_byte(TYPE_FLOAT, p);
- w_byte(n, p);
- w_string(buf, n, p);
+ if (p->version > 1) {
+ char buf[8];
+ if (_PyFloat_Pack8(PyFloat_AsDouble(v),
+ buf, 1) < 0) {
+ p->error = 1;
+ return;
+ }
+ w_byte(TYPE_BINARY_FLOAT, p);
+ w_string(buf, 8, p);
+ }
+ else {
+ char buf[256]; /* Plenty to format any double */
+ PyFloat_AsReprString(buf, (PyFloatObject *)v);
+ n = strlen(buf);
+ w_byte(TYPE_FLOAT, p);
+ w_byte(n, p);
+ w_string(buf, n, p);
+ }
}
#ifndef WITHOUT_COMPLEX
else if (PyComplex_Check(v)) {
- char buf[256]; /* Plenty to format any double */
- PyFloatObject *temp;
- w_byte(TYPE_COMPLEX, p);
- temp = (PyFloatObject*)PyFloat_FromDouble(
- PyComplex_RealAsDouble(v));
- PyFloat_AsReprString(buf, temp);
- Py_DECREF(temp);
- n = strlen(buf);
- w_byte(n, p);
- w_string(buf, n, p);
- temp = (PyFloatObject*)PyFloat_FromDouble(
- PyComplex_ImagAsDouble(v));
- PyFloat_AsReprString(buf, temp);
- Py_DECREF(temp);
- n = strlen(buf);
- w_byte(n, p);
- w_string(buf, n, p);
+ if (p->version > 1) {
+ char buf[8];
+ if (_PyFloat_Pack8(PyComplex_RealAsDouble(v),
+ buf, 1) < 0) {
+ p->error = 1;
+ return;
+ }
+ w_byte(TYPE_BINARY_COMPLEX, p);
+ w_string(buf, 8, p);
+ if (_PyFloat_Pack8(PyComplex_ImagAsDouble(v),
+ buf, 1) < 0) {
+ p->error = 1;
+ return;
+ }
+ w_string(buf, 8, p);
+ }
+ else {
+ char buf[256]; /* Plenty to format any double */
+ PyFloatObject *temp;
+ w_byte(TYPE_COMPLEX, p);
+ temp = (PyFloatObject*)PyFloat_FromDouble(
+ PyComplex_RealAsDouble(v));
+ PyFloat_AsReprString(buf, temp);
+ Py_DECREF(temp);
+ n = strlen(buf);
+ w_byte(n, p);
+ w_string(buf, n, p);
+ temp = (PyFloatObject*)PyFloat_FromDouble(
+ PyComplex_ImagAsDouble(v));
+ PyFloat_AsReprString(buf, temp);
+ Py_DECREF(temp);
+ n = strlen(buf);
+ w_byte(n, p);
+ w_string(buf, n, p);
+ }
}
#endif
else if (PyString_Check(v)) {
@@ -335,6 +368,7 @@
wf.error = 0;
wf.depth = 0;
wf.strings = NULL;
+ wf.version = version;
w_long(x, &wf);
}
@@ -346,6 +380,7 @@
wf.error = 0;
wf.depth = 0;
wf.strings = (version > 0) ? PyDict_New() : NULL;
+ wf.version = version;
w_object(x, &wf);
Py_XDECREF(wf.strings);
}
@@ -519,6 +554,22 @@
return PyFloat_FromDouble(dx);
}
+ case TYPE_BINARY_FLOAT:
+ {
+ char buf[8];
+ double x;
+ if (r_string(buf, 8, p) != 8) {
+ PyErr_SetString(PyExc_EOFError,
+ "EOF read where object expected");
+ return NULL;
+ }
+ x = _PyFloat_Unpack8(buf, 1);
+ if (x == -1.0 && PyErr_Occurred()) {
+ return NULL;
+ }
+ return PyFloat_FromDouble(x);
+ }
+
#ifndef WITHOUT_COMPLEX
case TYPE_COMPLEX:
{
@@ -546,6 +597,31 @@
PyFPE_END_PROTECT(c)
return PyComplex_FromCComplex(c);
}
+
+ case TYPE_BINARY_COMPLEX:
+ {
+ char buf[8];
+ Py_complex c;
+ if (r_string(buf, 8, p) != 8) {
+ PyErr_SetString(PyExc_EOFError,
+ "EOF read where object expected");
+ return NULL;
+ }
+ c.real = _PyFloat_Unpack8(buf, 1);
+ if (c.real == -1.0 && PyErr_Occurred()) {
+ return NULL;
+ }
+ if (r_string(buf, 8, p) != 8) {
+ PyErr_SetString(PyExc_EOFError,
+ "EOF read where object expected");
+ return NULL;
+ }
+ c.imag = _PyFloat_Unpack8(buf, 1);
+ if (c.imag == -1.0 && PyErr_Occurred()) {
+ return NULL;
+ }
+ return PyComplex_FromCComplex(c);
+ }
#endif
case TYPE_INTERNED:
@@ -707,30 +783,63 @@
return NULL;
}
else {
- int argcount = r_long(p);
- int nlocals = r_long(p);
- int stacksize = r_long(p);
- int flags = r_long(p);
- PyObject *code = r_object(p);
- PyObject *consts = r_object(p);
- PyObject *names = r_object(p);
- PyObject *varnames = r_object(p);
- PyObject *freevars = r_object(p);
- PyObject *cellvars = r_object(p);
- PyObject *filename = r_object(p);
- PyObject *name = r_object(p);
- int firstlineno = r_long(p);
- PyObject *lnotab = r_object(p);
+ int argcount;
+ int nlocals;
+ int stacksize;
+ int flags;
+ PyObject *code = NULL;
+ PyObject *consts = NULL;
+ PyObject *names = NULL;
+ PyObject *varnames = NULL;
+ PyObject *freevars = NULL;
+ PyObject *cellvars = NULL;
+ PyObject *filename = NULL;
+ PyObject *name = NULL;
+ int firstlineno;
+ PyObject *lnotab = NULL;
+
+ v = NULL;
- if (!PyErr_Occurred()) {
- v = (PyObject *) PyCode_New(
+ argcount = r_long(p);
+ nlocals = r_long(p);
+ stacksize = r_long(p);
+ flags = r_long(p);
+ code = r_object(p);
+ if (code == NULL)
+ goto code_error;
+ consts = r_object(p);
+ if (consts == NULL)
+ goto code_error;
+ names = r_object(p);
+ if (names == NULL)
+ goto code_error;
+ varnames = r_object(p);
+ if (varnames == NULL)
+ goto code_error;
+ freevars = r_object(p);
+ if (freevars == NULL)
+ goto code_error;
+ cellvars = r_object(p);
+ if (cellvars == NULL)
+ goto code_error;
+ filename = r_object(p);
+ if (filename == NULL)
+ goto code_error;
+ name = r_object(p);
+ if (name == NULL)
+ goto code_error;
+ firstlineno = r_long(p);
+ lnotab = r_object(p);
+ if (lnotab == NULL)
+ goto code_error;
+
+ v = (PyObject *) PyCode_New(
argcount, nlocals, stacksize, flags,
code, consts, names, varnames,
freevars, cellvars, filename, name,
firstlineno, lnotab);
- }
- else
- v = NULL;
+
+ code_error:
Py_XDECREF(code);
Py_XDECREF(consts);
Py_XDECREF(names);
@@ -882,6 +991,7 @@
wf.end = wf.ptr + PyString_Size(wf.str);
wf.error = 0;
wf.depth = 0;
+ wf.version = version;
wf.strings = (version > 0) ? PyDict_New() : NULL;
w_object(x, &wf);
Py_XDECREF(wf.strings);