Fix and deprecated the unicode_internal codec
unicode_internal codec uses Py_UNICODE instead of the real internal
representation (PEP 393: Py_UCS1, Py_UCS2 or Py_UCS4) for backward
compatibility.
diff --git a/Doc/library/codecs.rst b/Doc/library/codecs.rst
index 4523c7f..a9fae95 100644
--- a/Doc/library/codecs.rst
+++ b/Doc/library/codecs.rst
@@ -1173,6 +1173,8 @@
| unicode_internal | | Return the internal |
| | | representation of the |
| | | operand |
+| | | |
+| | | .. deprecated:: 3.3 |
+--------------------+---------+---------------------------+
The following codecs provide bytes-to-bytes mappings.
diff --git a/Doc/whatsnew/3.3.rst b/Doc/whatsnew/3.3.rst
index 911d8d9..7f4517f 100644
--- a/Doc/whatsnew/3.3.rst
+++ b/Doc/whatsnew/3.3.rst
@@ -250,6 +250,8 @@
(:issue:`12100`)
+The ``unicode_internal`` codec has been deprecated.
+
crypt
-----
diff --git a/Modules/_codecsmodule.c b/Modules/_codecsmodule.c
index 727cf5e..93cb1b7 100644
--- a/Modules/_codecsmodule.c
+++ b/Modules/_codecsmodule.c
@@ -675,18 +675,30 @@
PyObject *obj;
const char *errors = NULL;
const char *data;
- Py_ssize_t size;
+ Py_ssize_t len, size;
if (!PyArg_ParseTuple(args, "O|z:unicode_internal_encode",
&obj, &errors))
return NULL;
if (PyUnicode_Check(obj)) {
+ Py_UNICODE *u;
+
if (PyUnicode_READY(obj) < 0)
return NULL;
- data = PyUnicode_AS_DATA(obj);
- size = PyUnicode_GET_DATA_SIZE(obj);
- return codec_tuple(PyBytes_FromStringAndSize(data, size),
+
+ if (PyErr_WarnEx(PyExc_DeprecationWarning,
+ "unicode_internal codecs has been deprecated",
+ 1))
+ return NULL;
+
+ u = PyUnicode_AsUnicodeAndSize(obj, &len);
+ if (u == NULL)
+ return NULL;
+ if (len > PY_SSIZE_T_MAX / sizeof(Py_UNICODE))
+ return PyErr_NoMemory();
+ size = len * sizeof(Py_UNICODE);
+ return codec_tuple(PyBytes_FromStringAndSize((const char*)u, size),
PyUnicode_GET_LENGTH(obj));
}
else {
diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c
index 61534b4..3f580b5 100644
--- a/Objects/unicodeobject.c
+++ b/Objects/unicodeobject.c
@@ -6237,6 +6237,11 @@
PyObject *errorHandler = NULL;
PyObject *exc = NULL;
+ if (PyErr_WarnEx(PyExc_DeprecationWarning,
+ "unicode_internal codecs has been deprecated",
+ 1))
+ return NULL;
+
/* XXX overflow detection missing */
v = PyUnicode_New((size+Py_UNICODE_SIZE-1)/ Py_UNICODE_SIZE, 127);
if (v == NULL)
@@ -6270,15 +6275,26 @@
errors, &errorHandler,
"unicode_internal", reason,
&starts, &end, &startinpos, &endinpos, &exc, &s,
- &v, &outpos)) {
+ &v, &outpos))
goto onError;
+ continue;
+ }
+
+ s += Py_UNICODE_SIZE;
+#ifndef Py_UNICODE_WIDE
+ if (ch >= 0xD800 && ch <= 0xDBFF && s < end)
+ {
+ Py_UCS4 ch2 = *(Py_UNICODE*)s;
+ if (ch2 >= 0xDC00 && ch2 <= 0xDFFF)
+ {
+ ch = (((ch & 0x3FF)<<10) | (ch2 & 0x3FF)) + 0x10000;
+ s += Py_UNICODE_SIZE;
}
}
- else {
- if (unicode_putchar(&v, &outpos, ch) < 0)
- goto onError;
- s += Py_UNICODE_SIZE;
- }
+#endif
+
+ if (unicode_putchar(&v, &outpos, ch) < 0)
+ goto onError;
}
if (PyUnicode_Resize(&v, outpos) < 0)