This is my patch:
[ 1180995 ] binary formats for marshalling floats
Adds 2 new type codes for marshal (binary floats and binary complexes), a
new marshal version (2), updates MAGIC and fiddles the de-serializing of
code objects to be less likely to clobber the real reason for failing if
it fails.
diff --git a/Python/marshal.c b/Python/marshal.c
index d0fcac7..2535cb5 100644
--- a/Python/marshal.c
+++ b/Python/marshal.c
@@ -15,28 +15,30 @@
*/
#define MAX_MARSHAL_STACK_DEPTH 5000
-#define TYPE_NULL '0'
-#define TYPE_NONE 'N'
-#define TYPE_FALSE 'F'
-#define TYPE_TRUE 'T'
-#define TYPE_STOPITER 'S'
-#define TYPE_ELLIPSIS '.'
-#define TYPE_INT 'i'
-#define TYPE_INT64 'I'
-#define TYPE_FLOAT 'f'
-#define TYPE_COMPLEX 'x'
-#define TYPE_LONG 'l'
-#define TYPE_STRING 's'
-#define TYPE_INTERNED 't'
-#define TYPE_STRINGREF 'R'
-#define TYPE_TUPLE '('
-#define TYPE_LIST '['
-#define TYPE_DICT '{'
-#define TYPE_CODE 'c'
-#define TYPE_UNICODE 'u'
-#define TYPE_UNKNOWN '?'
-#define TYPE_SET '<'
-#define TYPE_FROZENSET '>'
+#define TYPE_NULL '0'
+#define TYPE_NONE 'N'
+#define TYPE_FALSE 'F'
+#define TYPE_TRUE 'T'
+#define TYPE_STOPITER 'S'
+#define TYPE_ELLIPSIS '.'
+#define TYPE_INT 'i'
+#define TYPE_INT64 'I'
+#define TYPE_FLOAT 'f'
+#define TYPE_BINARY_FLOAT 'g'
+#define TYPE_COMPLEX 'x'
+#define TYPE_BINARY_COMPLEX 'y'
+#define TYPE_LONG 'l'
+#define TYPE_STRING 's'
+#define TYPE_INTERNED 't'
+#define TYPE_STRINGREF 'R'
+#define TYPE_TUPLE '('
+#define TYPE_LIST '['
+#define TYPE_DICT '{'
+#define TYPE_CODE 'c'
+#define TYPE_UNICODE 'u'
+#define TYPE_UNKNOWN '?'
+#define TYPE_SET '<'
+#define TYPE_FROZENSET '>'
typedef struct {
FILE *fp;
@@ -47,6 +49,7 @@
char *ptr;
char *end;
PyObject *strings; /* dict on marshal, list on unmarshal */
+ int version;
} WFILE;
#define w_byte(c, p) if (((p)->fp)) putc((c), (p)->fp); \
@@ -165,32 +168,62 @@
w_short(ob->ob_digit[i], p);
}
else if (PyFloat_Check(v)) {
- char buf[256]; /* Plenty to format any double */
- PyFloat_AsReprString(buf, (PyFloatObject *)v);
- n = strlen(buf);
- w_byte(TYPE_FLOAT, p);
- w_byte(n, p);
- w_string(buf, n, p);
+ if (p->version > 1) {
+ char buf[8];
+ if (_PyFloat_Pack8(PyFloat_AsDouble(v),
+ buf, 1) < 0) {
+ p->error = 1;
+ return;
+ }
+ w_byte(TYPE_BINARY_FLOAT, p);
+ w_string(buf, 8, p);
+ }
+ else {
+ char buf[256]; /* Plenty to format any double */
+ PyFloat_AsReprString(buf, (PyFloatObject *)v);
+ n = strlen(buf);
+ w_byte(TYPE_FLOAT, p);
+ w_byte(n, p);
+ w_string(buf, n, p);
+ }
}
#ifndef WITHOUT_COMPLEX
else if (PyComplex_Check(v)) {
- char buf[256]; /* Plenty to format any double */
- PyFloatObject *temp;
- w_byte(TYPE_COMPLEX, p);
- temp = (PyFloatObject*)PyFloat_FromDouble(
- PyComplex_RealAsDouble(v));
- PyFloat_AsReprString(buf, temp);
- Py_DECREF(temp);
- n = strlen(buf);
- w_byte(n, p);
- w_string(buf, n, p);
- temp = (PyFloatObject*)PyFloat_FromDouble(
- PyComplex_ImagAsDouble(v));
- PyFloat_AsReprString(buf, temp);
- Py_DECREF(temp);
- n = strlen(buf);
- w_byte(n, p);
- w_string(buf, n, p);
+ if (p->version > 1) {
+ char buf[8];
+ if (_PyFloat_Pack8(PyComplex_RealAsDouble(v),
+ buf, 1) < 0) {
+ p->error = 1;
+ return;
+ }
+ w_byte(TYPE_BINARY_COMPLEX, p);
+ w_string(buf, 8, p);
+ if (_PyFloat_Pack8(PyComplex_ImagAsDouble(v),
+ buf, 1) < 0) {
+ p->error = 1;
+ return;
+ }
+ w_string(buf, 8, p);
+ }
+ else {
+ char buf[256]; /* Plenty to format any double */
+ PyFloatObject *temp;
+ w_byte(TYPE_COMPLEX, p);
+ temp = (PyFloatObject*)PyFloat_FromDouble(
+ PyComplex_RealAsDouble(v));
+ PyFloat_AsReprString(buf, temp);
+ Py_DECREF(temp);
+ n = strlen(buf);
+ w_byte(n, p);
+ w_string(buf, n, p);
+ temp = (PyFloatObject*)PyFloat_FromDouble(
+ PyComplex_ImagAsDouble(v));
+ PyFloat_AsReprString(buf, temp);
+ Py_DECREF(temp);
+ n = strlen(buf);
+ w_byte(n, p);
+ w_string(buf, n, p);
+ }
}
#endif
else if (PyString_Check(v)) {
@@ -335,6 +368,7 @@
wf.error = 0;
wf.depth = 0;
wf.strings = NULL;
+ wf.version = version;
w_long(x, &wf);
}
@@ -346,6 +380,7 @@
wf.error = 0;
wf.depth = 0;
wf.strings = (version > 0) ? PyDict_New() : NULL;
+ wf.version = version;
w_object(x, &wf);
Py_XDECREF(wf.strings);
}
@@ -519,6 +554,22 @@
return PyFloat_FromDouble(dx);
}
+ case TYPE_BINARY_FLOAT:
+ {
+ char buf[8];
+ double x;
+ if (r_string(buf, 8, p) != 8) {
+ PyErr_SetString(PyExc_EOFError,
+ "EOF read where object expected");
+ return NULL;
+ }
+ x = _PyFloat_Unpack8(buf, 1);
+ if (x == -1.0 && PyErr_Occurred()) {
+ return NULL;
+ }
+ return PyFloat_FromDouble(x);
+ }
+
#ifndef WITHOUT_COMPLEX
case TYPE_COMPLEX:
{
@@ -546,6 +597,31 @@
PyFPE_END_PROTECT(c)
return PyComplex_FromCComplex(c);
}
+
+ case TYPE_BINARY_COMPLEX:
+ {
+ char buf[8];
+ Py_complex c;
+ if (r_string(buf, 8, p) != 8) {
+ PyErr_SetString(PyExc_EOFError,
+ "EOF read where object expected");
+ return NULL;
+ }
+ c.real = _PyFloat_Unpack8(buf, 1);
+ if (c.real == -1.0 && PyErr_Occurred()) {
+ return NULL;
+ }
+ if (r_string(buf, 8, p) != 8) {
+ PyErr_SetString(PyExc_EOFError,
+ "EOF read where object expected");
+ return NULL;
+ }
+ c.imag = _PyFloat_Unpack8(buf, 1);
+ if (c.imag == -1.0 && PyErr_Occurred()) {
+ return NULL;
+ }
+ return PyComplex_FromCComplex(c);
+ }
#endif
case TYPE_INTERNED:
@@ -707,30 +783,63 @@
return NULL;
}
else {
- int argcount = r_long(p);
- int nlocals = r_long(p);
- int stacksize = r_long(p);
- int flags = r_long(p);
- PyObject *code = r_object(p);
- PyObject *consts = r_object(p);
- PyObject *names = r_object(p);
- PyObject *varnames = r_object(p);
- PyObject *freevars = r_object(p);
- PyObject *cellvars = r_object(p);
- PyObject *filename = r_object(p);
- PyObject *name = r_object(p);
- int firstlineno = r_long(p);
- PyObject *lnotab = r_object(p);
+ int argcount;
+ int nlocals;
+ int stacksize;
+ int flags;
+ PyObject *code = NULL;
+ PyObject *consts = NULL;
+ PyObject *names = NULL;
+ PyObject *varnames = NULL;
+ PyObject *freevars = NULL;
+ PyObject *cellvars = NULL;
+ PyObject *filename = NULL;
+ PyObject *name = NULL;
+ int firstlineno;
+ PyObject *lnotab = NULL;
+
+ v = NULL;
- if (!PyErr_Occurred()) {
- v = (PyObject *) PyCode_New(
+ argcount = r_long(p);
+ nlocals = r_long(p);
+ stacksize = r_long(p);
+ flags = r_long(p);
+ code = r_object(p);
+ if (code == NULL)
+ goto code_error;
+ consts = r_object(p);
+ if (consts == NULL)
+ goto code_error;
+ names = r_object(p);
+ if (names == NULL)
+ goto code_error;
+ varnames = r_object(p);
+ if (varnames == NULL)
+ goto code_error;
+ freevars = r_object(p);
+ if (freevars == NULL)
+ goto code_error;
+ cellvars = r_object(p);
+ if (cellvars == NULL)
+ goto code_error;
+ filename = r_object(p);
+ if (filename == NULL)
+ goto code_error;
+ name = r_object(p);
+ if (name == NULL)
+ goto code_error;
+ firstlineno = r_long(p);
+ lnotab = r_object(p);
+ if (lnotab == NULL)
+ goto code_error;
+
+ v = (PyObject *) PyCode_New(
argcount, nlocals, stacksize, flags,
code, consts, names, varnames,
freevars, cellvars, filename, name,
firstlineno, lnotab);
- }
- else
- v = NULL;
+
+ code_error:
Py_XDECREF(code);
Py_XDECREF(consts);
Py_XDECREF(names);
@@ -882,6 +991,7 @@
wf.end = wf.ptr + PyString_Size(wf.str);
wf.error = 0;
wf.depth = 0;
+ wf.version = version;
wf.strings = (version > 0) ? PyDict_New() : NULL;
w_object(x, &wf);
Py_XDECREF(wf.strings);