Optimize built-in unicode codecs by avoiding unnecessary copying.
The approach used is similiar to what is currently used in the version
of unicodeobject.c in Python 2.x. The only difference is we use
_PyBytes_Resize instead of _PyString_Resize.
diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c
index e57b60c..fa74631 100644
--- a/Objects/unicodeobject.c
+++ b/Objects/unicodeobject.c
@@ -1873,7 +1873,7 @@
int encodeWhiteSpace,
const char *errors)
{
- PyObject *v, *result;
+ PyObject *v;
/* It might be possible to tighten this worst case */
Py_ssize_t cbAllocated = 5 * size;
int inShift = 0;
@@ -1889,11 +1889,11 @@
if (cbAllocated / 5 != size)
return PyErr_NoMemory();
- v = PyByteArray_FromStringAndSize(NULL, cbAllocated);
+ v = PyBytes_FromStringAndSize(NULL, cbAllocated);
if (v == NULL)
return NULL;
- start = out = PyByteArray_AS_STRING(v);
+ start = out = PyBytes_AS_STRING(v);
for (;i < size; ++i) {
Py_UNICODE ch = s[i];
@@ -1958,10 +1958,9 @@
*out++= B64(charsleft << (6-bitsleft) );
*out++ = '-';
}
-
- result = PyBytes_FromStringAndSize(PyByteArray_AS_STRING(v), out - start);
- Py_DECREF(v);
- return result;
+ if (_PyBytes_Resize(&v, out - start) < 0)
+ return NULL;
+ return v;
}
#undef SPECIAL
@@ -2479,7 +2478,7 @@
const char *errors,
int byteorder)
{
- PyObject *v, *result;
+ PyObject *v;
unsigned char *p;
Py_ssize_t nsize, bytesize;
#ifndef Py_UNICODE_WIDE
@@ -2515,11 +2514,11 @@
bytesize = nsize * 4;
if (bytesize / 4 != nsize)
return PyErr_NoMemory();
- v = PyByteArray_FromStringAndSize(NULL, bytesize);
+ v = PyBytes_FromStringAndSize(NULL, bytesize);
if (v == NULL)
return NULL;
- p = (unsigned char *)PyByteArray_AS_STRING(v);
+ p = (unsigned char *)PyBytes_AS_STRING(v);
if (byteorder == 0)
STORECHAR(0xFEFF);
if (size == 0)
@@ -2556,9 +2555,7 @@
}
done:
- result = PyBytes_FromStringAndSize(PyByteArray_AS_STRING(v), Py_SIZE(v));
- Py_DECREF(v);
- return result;
+ return v;
#undef STORECHAR
}
@@ -2757,7 +2754,7 @@
const char *errors,
int byteorder)
{
- PyObject *v, *result;
+ PyObject *v;
unsigned char *p;
Py_ssize_t nsize, bytesize;
#ifdef Py_UNICODE_WIDE
@@ -2792,11 +2789,11 @@
bytesize = nsize * 2;
if (bytesize / 2 != nsize)
return PyErr_NoMemory();
- v = PyByteArray_FromStringAndSize(NULL, bytesize);
+ v = PyBytes_FromStringAndSize(NULL, bytesize);
if (v == NULL)
return NULL;
- p = (unsigned char *)PyByteArray_AS_STRING(v);
+ p = (unsigned char *)PyBytes_AS_STRING(v);
if (byteorder == 0)
STORECHAR(0xFEFF);
if (size == 0)
@@ -2828,9 +2825,7 @@
}
done:
- result = PyBytes_FromStringAndSize(PyByteArray_AS_STRING(v), Py_SIZE(v));
- Py_DECREF(v);
- return result;
+ return v;
#undef STORECHAR
}
@@ -3120,7 +3115,7 @@
PyObject *PyUnicode_EncodeUnicodeEscape(const Py_UNICODE *s,
Py_ssize_t size)
{
- PyObject *repr, *result;
+ PyObject *repr;
char *p;
#ifdef Py_UNICODE_WIDE
@@ -3147,17 +3142,20 @@
escape.
*/
+ if (size == 0)
+ return PyBytes_FromStringAndSize(NULL, 0);
+
if (size > (PY_SSIZE_T_MAX - 2 - 1) / expandsize)
return PyErr_NoMemory();
- repr = PyByteArray_FromStringAndSize(NULL,
+ repr = PyBytes_FromStringAndSize(NULL,
2
+ expandsize*size
+ 1);
if (repr == NULL)
return NULL;
- p = PyByteArray_AS_STRING(repr);
+ p = PyBytes_AS_STRING(repr);
while (size-- > 0) {
Py_UNICODE ch = *s++;
@@ -3249,13 +3247,13 @@
*p++ = (char) ch;
}
- result = PyBytes_FromStringAndSize(PyByteArray_AS_STRING(repr),
- p - PyByteArray_AS_STRING(repr));
- Py_DECREF(repr);
- return result;
+ assert(p - PyBytes_AS_STRING(repr) > 0);
+ if (_PyBytes_Resize(&repr, p - PyBytes_AS_STRING(repr)) < 0)
+ return NULL;
+ return repr;
}
-PyObject *PyUnicode_AsUnicodeEscapeString(PyObject *unicode)
+PyObject *PyUnicodeAsUnicodeEscapeString(PyObject *unicode)
{
PyObject *s;
if (!PyUnicode_Check(unicode)) {
@@ -3389,7 +3387,7 @@
PyObject *PyUnicode_EncodeRawUnicodeEscape(const Py_UNICODE *s,
Py_ssize_t size)
{
- PyObject *repr, *result;
+ PyObject *repr;
char *p;
char *q;
@@ -3402,13 +3400,13 @@
if (size > PY_SSIZE_T_MAX / expandsize)
return PyErr_NoMemory();
- repr = PyByteArray_FromStringAndSize(NULL, expandsize * size);
+ repr = PyBytes_FromStringAndSize(NULL, expandsize * size);
if (repr == NULL)
return NULL;
if (size == 0)
- goto done;
+ return repr;
- p = q = PyByteArray_AS_STRING(repr);
+ p = q = PyBytes_AS_STRING(repr);
while (size-- > 0) {
Py_UNICODE ch = *s++;
#ifdef Py_UNICODE_WIDE
@@ -3468,10 +3466,10 @@
}
size = p - q;
- done:
- result = PyBytes_FromStringAndSize(PyByteArray_AS_STRING(repr), size);
- Py_DECREF(repr);
- return result;
+ assert(size > 0);
+ if (_PyBytes_Resize(&repr, size) < 0)
+ return NULL;
+ return repr;
}
PyObject *PyUnicode_AsRawUnicodeEscapeString(PyObject *unicode)
@@ -3706,7 +3704,6 @@
const char *reason = (limit == 256) ? "ordinal not in range(256)" : "ordinal not in range(128)";
PyObject *errorHandler = NULL;
PyObject *exc = NULL;
- PyObject *result = NULL;
/* the following variable is used for caching string comparisons
* -1=not initialized, 0=unknown, 1=strict, 2=replace, 3=ignore, 4=xmlcharrefreplace */
int known_errorHandler = -1;
@@ -3715,10 +3712,10 @@
replacements, if we need more, we'll resize */
if (size == 0)
return PyBytes_FromStringAndSize(NULL, 0);
- res = PyByteArray_FromStringAndSize(NULL, size);
+ res = PyBytes_FromStringAndSize(NULL, size);
if (res == NULL)
return NULL;
- str = PyByteArray_AS_STRING(res);
+ str = PyBytes_AS_STRING(res);
ressize = size;
while (p<endp) {
@@ -3768,7 +3765,7 @@
p = collend;
break;
case 4: /* xmlcharrefreplace */
- respos = str - PyByteArray_AS_STRING(res);
+ respos = str - PyBytes_AS_STRING(res);
/* determine replacement size (temporarily (mis)uses p) */
for (p = collstart, repsize = 0; p < collend; ++p) {
if (*p<10)
@@ -3795,9 +3792,9 @@
if (requiredsize > ressize) {
if (requiredsize<2*ressize)
requiredsize = 2*ressize;
- if (PyByteArray_Resize(res, requiredsize))
+ if (_PyBytes_Resize(&res, requiredsize))
goto onError;
- str = PyByteArray_AS_STRING(res) + respos;
+ str = PyBytes_AS_STRING(res) + respos;
ressize = requiredsize;
}
/* generate replacement (temporarily (mis)uses p) */
@@ -3815,17 +3812,17 @@
/* need more space? (at least enough for what we
have+the replacement+the rest of the string, so
we won't have to check space for encodable characters) */
- respos = str - PyByteArray_AS_STRING(res);
+ respos = str - PyBytes_AS_STRING(res);
repsize = PyUnicode_GET_SIZE(repunicode);
requiredsize = respos+repsize+(endp-collend);
if (requiredsize > ressize) {
if (requiredsize<2*ressize)
requiredsize = 2*ressize;
- if (PyByteArray_Resize(res, requiredsize)) {
+ if (_PyBytes_Resize(&res, requiredsize)) {
Py_DECREF(repunicode);
goto onError;
}
- str = PyByteArray_AS_STRING(res) + respos;
+ str = PyBytes_AS_STRING(res) + respos;
ressize = requiredsize;
}
/* check if there is anything unencodable in the replacement
@@ -3845,13 +3842,23 @@
}
}
}
- result = PyBytes_FromStringAndSize(PyByteArray_AS_STRING(res),
- str - PyByteArray_AS_STRING(res));
- onError:
- Py_DECREF(res);
+ /* Resize if we allocated to much */
+ size = str - PyBytes_AS_STRING(res);
+ if (size < ressize) { /* If this falls res will be NULL */
+ assert(size > 0);
+ if (_PyBytes_Resize(&res, size) < 0)
+ goto onError;
+ }
+
Py_XDECREF(errorHandler);
Py_XDECREF(exc);
- return result;
+ return res;
+
+ onError:
+ Py_XDECREF(res);
+ Py_XDECREF(errorHandler);
+ Py_XDECREF(exc);
+ return NULL;
}
PyObject *PyUnicode_EncodeLatin1(const Py_UNICODE *p,
@@ -4104,7 +4111,7 @@
else {
/* Extend string object */
n = PyBytes_Size(*repr);
- if (_PyBytes_Resize(repr, n + mbcssize) < 0)
+ if (_PyBytes_Resize(&repr, n + mbcssize) < 0)
return -1;
}
@@ -4834,7 +4841,8 @@
/* Resize if we allocated to much */
if (respos<PyBytes_GET_SIZE(res))
- _PyBytes_Resize(&res, respos);
+ if (_PyBytes_Resize(&res, respos) < 0)
+ goto onError;
Py_XDECREF(exc);
Py_XDECREF(errorHandler);