blob: e0d9b678c2391264f50ac18ae9acf46e2f5e7bd9 [file] [log] [blame]
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +00001/*
2 * multibytecodec.c: Common Multibyte Codec Implementation
3 *
4 * Written by Hye-Shik Chang <perky@FreeBSD.org>
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +00005 */
6
Hye-Shik Chang4b96c132006-03-04 16:08:19 +00007#define PY_SSIZE_T_CLEAN
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +00008#include "Python.h"
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00009#include "structmember.h"
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +000010#include "multibytecodec.h"
11
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +000012typedef struct {
Victor Stinnerd9491262013-04-14 02:06:32 +020013 PyObject *inobj;
14 Py_ssize_t inpos, inlen;
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +000015 unsigned char *outbuf, *outbuf_end;
16 PyObject *excobj, *outobj;
17} MultibyteEncodeBuffer;
18
19typedef struct {
20 const unsigned char *inbuf, *inbuf_top, *inbuf_end;
Victor Stinnera0dd0212013-04-11 22:09:04 +020021 PyObject *excobj;
22 _PyUnicodeWriter writer;
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +000023} MultibyteDecodeBuffer;
24
25PyDoc_STRVAR(MultibyteCodec_Encode__doc__,
26"I.encode(unicode[, errors]) -> (string, length consumed)\n\
27\n\
28Return an encoded string version of `unicode'. errors may be given to\n\
29set a different error handling scheme. Default is 'strict' meaning that\n\
30encoding errors raise a UnicodeEncodeError. Other possible values are\n\
31'ignore', 'replace' and 'xmlcharrefreplace' as well as any other name\n\
32registered with codecs.register_error that can handle UnicodeEncodeErrors.");
33
34PyDoc_STRVAR(MultibyteCodec_Decode__doc__,
35"I.decode(string[, errors]) -> (unicodeobject, length consumed)\n\
36\n\
37Decodes `string' using I, an MultibyteCodec instance. errors may be given\n\
38to set a different error handling scheme. Default is 'strict' meaning\n\
39that encoding errors raise a UnicodeDecodeError. Other possible values\n\
Benjamin Petersonf10a79a2008-10-11 00:49:57 +000040are 'ignore' and 'replace' as well as any other name registered with\n\
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +000041codecs.register_error that is able to handle UnicodeDecodeErrors.");
42
Martin v. Löwis02cbf4a2006-02-27 17:20:04 +000043static char *codeckwarglist[] = {"input", "errors", NULL};
Thomas Wouters49fd7fa2006-04-21 10:40:58 +000044static char *incnewkwarglist[] = {"errors", NULL};
45static char *incrementalkwarglist[] = {"input", "final", NULL};
Martin v. Löwis02cbf4a2006-02-27 17:20:04 +000046static char *streamkwarglist[] = {"stream", "errors", NULL};
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +000047
48static PyObject *multibytecodec_encode(MultibyteCodec *,
Victor Stinnerd9491262013-04-14 02:06:32 +020049 MultibyteCodec_State *, PyObject *, Py_ssize_t *,
Antoine Pitrouf95a1b32010-05-09 15:52:27 +000050 PyObject *, int);
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +000051
Antoine Pitrouf95a1b32010-05-09 15:52:27 +000052#define MBENC_RESET MBENC_MAX<<1 /* reset after an encoding session */
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +000053
Victor Stinner3f36a572013-11-12 21:39:02 +010054_Py_IDENTIFIER(write);
55
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +000056static PyObject *
Hye-Shik Chang4b96c132006-03-04 16:08:19 +000057make_tuple(PyObject *object, Py_ssize_t len)
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +000058{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +000059 PyObject *v, *w;
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +000060
Antoine Pitrouf95a1b32010-05-09 15:52:27 +000061 if (object == NULL)
62 return NULL;
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +000063
Antoine Pitrouf95a1b32010-05-09 15:52:27 +000064 v = PyTuple_New(2);
65 if (v == NULL) {
66 Py_DECREF(object);
67 return NULL;
68 }
69 PyTuple_SET_ITEM(v, 0, object);
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +000070
Antoine Pitrouf95a1b32010-05-09 15:52:27 +000071 w = PyLong_FromSsize_t(len);
72 if (w == NULL) {
73 Py_DECREF(v);
74 return NULL;
75 }
76 PyTuple_SET_ITEM(v, 1, w);
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +000077
Antoine Pitrouf95a1b32010-05-09 15:52:27 +000078 return v;
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +000079}
80
81static PyObject *
Thomas Wouters49fd7fa2006-04-21 10:40:58 +000082internal_error_callback(const char *errors)
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +000083{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +000084 if (errors == NULL || strcmp(errors, "strict") == 0)
85 return ERROR_STRICT;
86 else if (strcmp(errors, "ignore") == 0)
87 return ERROR_IGNORE;
88 else if (strcmp(errors, "replace") == 0)
89 return ERROR_REPLACE;
90 else
91 return PyUnicode_FromString(errors);
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +000092}
93
Thomas Wouters49fd7fa2006-04-21 10:40:58 +000094static PyObject *
95call_error_callback(PyObject *errors, PyObject *exc)
96{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +000097 PyObject *args, *cb, *r;
98 const char *str;
Thomas Wouters49fd7fa2006-04-21 10:40:58 +000099
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000100 assert(PyUnicode_Check(errors));
101 str = _PyUnicode_AsString(errors);
102 if (str == NULL)
103 return NULL;
104 cb = PyCodec_LookupError(str);
105 if (cb == NULL)
106 return NULL;
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000107
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000108 args = PyTuple_New(1);
109 if (args == NULL) {
110 Py_DECREF(cb);
111 return NULL;
112 }
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000113
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000114 PyTuple_SET_ITEM(args, 0, exc);
115 Py_INCREF(exc);
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000116
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000117 r = PyObject_CallObject(cb, args);
118 Py_DECREF(args);
119 Py_DECREF(cb);
120 return r;
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000121}
122
123static PyObject *
124codecctx_errors_get(MultibyteStatefulCodecContext *self)
125{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000126 const char *errors;
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000127
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000128 if (self->errors == ERROR_STRICT)
129 errors = "strict";
130 else if (self->errors == ERROR_IGNORE)
131 errors = "ignore";
132 else if (self->errors == ERROR_REPLACE)
133 errors = "replace";
134 else {
135 Py_INCREF(self->errors);
136 return self->errors;
137 }
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000138
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000139 return PyUnicode_FromString(errors);
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000140}
141
142static int
143codecctx_errors_set(MultibyteStatefulCodecContext *self, PyObject *value,
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000144 void *closure)
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000145{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000146 PyObject *cb;
147 const char *str;
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000148
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000149 if (!PyUnicode_Check(value)) {
150 PyErr_SetString(PyExc_TypeError, "errors must be a string");
151 return -1;
152 }
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000153
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000154 str = _PyUnicode_AsString(value);
155 if (str == NULL)
156 return -1;
Neal Norwitz6ea45d32007-08-26 04:19:43 +0000157
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000158 cb = internal_error_callback(str);
159 if (cb == NULL)
160 return -1;
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000161
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000162 ERROR_DECREF(self->errors);
163 self->errors = cb;
164 return 0;
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000165}
166
167/* This getset handlers list is used by all the stateful codec objects */
168static PyGetSetDef codecctx_getsets[] = {
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000169 {"errors", (getter)codecctx_errors_get,
170 (setter)codecctx_errors_set,
171 PyDoc_STR("how to treat errors")},
172 {NULL,}
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000173};
174
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +0000175static int
Hye-Shik Chang4b96c132006-03-04 16:08:19 +0000176expand_encodebuffer(MultibyteEncodeBuffer *buf, Py_ssize_t esize)
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +0000177{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000178 Py_ssize_t orgpos, orgsize, incsize;
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +0000179
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000180 orgpos = (Py_ssize_t)((char *)buf->outbuf -
181 PyBytes_AS_STRING(buf->outobj));
182 orgsize = PyBytes_GET_SIZE(buf->outobj);
183 incsize = (esize < (orgsize >> 1) ? (orgsize >> 1) | 1 : esize);
Amaury Forgeot d'Arc9c74b142008-06-18 00:47:36 +0000184
Serhiy Storchakaa1543cd2015-02-21 01:19:58 +0200185 if (orgsize > PY_SSIZE_T_MAX - incsize) {
186 PyErr_NoMemory();
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000187 return -1;
Serhiy Storchakaa1543cd2015-02-21 01:19:58 +0200188 }
Amaury Forgeot d'Arc9c74b142008-06-18 00:47:36 +0000189
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000190 if (_PyBytes_Resize(&buf->outobj, orgsize + incsize) == -1)
191 return -1;
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +0000192
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000193 buf->outbuf = (unsigned char *)PyBytes_AS_STRING(buf->outobj) +orgpos;
194 buf->outbuf_end = (unsigned char *)PyBytes_AS_STRING(buf->outobj)
195 + PyBytes_GET_SIZE(buf->outobj);
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +0000196
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000197 return 0;
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +0000198}
Serhiy Storchakaa1543cd2015-02-21 01:19:58 +0200199#define REQUIRE_ENCODEBUFFER(buf, s) do { \
200 if ((s) < 0 || (s) > (buf)->outbuf_end - (buf)->outbuf) \
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000201 if (expand_encodebuffer(buf, s) == -1) \
202 goto errorexit; \
Serhiy Storchakaa1543cd2015-02-21 01:19:58 +0200203} while(0)
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +0000204
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000205
206/**
207 * MultibyteCodec object
208 */
209
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +0000210static int
211multibytecodec_encerror(MultibyteCodec *codec,
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000212 MultibyteCodec_State *state,
213 MultibyteEncodeBuffer *buf,
214 PyObject *errors, Py_ssize_t e)
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +0000215{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000216 PyObject *retobj = NULL, *retstr = NULL, *tobj;
217 Py_ssize_t retstrsize, newpos;
218 Py_ssize_t esize, start, end;
219 const char *reason;
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +0000220
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000221 if (e > 0) {
222 reason = "illegal multibyte sequence";
223 esize = e;
224 }
225 else {
226 switch (e) {
227 case MBERR_TOOSMALL:
228 REQUIRE_ENCODEBUFFER(buf, -1);
229 return 0; /* retry it */
230 case MBERR_TOOFEW:
231 reason = "incomplete multibyte sequence";
Victor Stinnerd9491262013-04-14 02:06:32 +0200232 esize = (Py_ssize_t)buf->inpos;
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000233 break;
234 case MBERR_INTERNAL:
235 PyErr_SetString(PyExc_RuntimeError,
236 "internal codec error");
237 return -1;
238 default:
239 PyErr_SetString(PyExc_RuntimeError,
240 "unknown runtime error");
241 return -1;
242 }
243 }
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +0000244
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000245 if (errors == ERROR_REPLACE) {
Victor Stinnerd9491262013-04-14 02:06:32 +0200246 PyObject *replchar;
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000247 Py_ssize_t r;
Victor Stinnerd9491262013-04-14 02:06:32 +0200248 Py_ssize_t inpos;
249 int kind;
250 void *data;
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +0000251
Victor Stinnerd9491262013-04-14 02:06:32 +0200252 replchar = PyUnicode_FromOrdinal('?');
253 if (replchar == NULL)
254 goto errorexit;
255 kind = PyUnicode_KIND(replchar);
256 data = PyUnicode_DATA(replchar);
257
258 inpos = 0;
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000259 for (;;) {
Victor Stinnerd9491262013-04-14 02:06:32 +0200260 Py_ssize_t outleft = (Py_ssize_t)(buf->outbuf_end - buf->outbuf);
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +0000261
Victor Stinnerd9491262013-04-14 02:06:32 +0200262 r = codec->encode(state, codec->config,
263 kind, data, &inpos, 1,
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000264 &buf->outbuf, outleft, 0);
265 if (r == MBERR_TOOSMALL) {
266 REQUIRE_ENCODEBUFFER(buf, -1);
267 continue;
268 }
269 else
270 break;
271 }
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +0000272
Victor Stinnerd9491262013-04-14 02:06:32 +0200273 Py_DECREF(replchar);
274
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000275 if (r != 0) {
276 REQUIRE_ENCODEBUFFER(buf, 1);
277 *buf->outbuf++ = '?';
278 }
279 }
280 if (errors == ERROR_IGNORE || errors == ERROR_REPLACE) {
Victor Stinnerd9491262013-04-14 02:06:32 +0200281 buf->inpos += esize;
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000282 return 0;
283 }
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +0000284
Victor Stinnerd9491262013-04-14 02:06:32 +0200285 start = (Py_ssize_t)buf->inpos;
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000286 end = start + esize;
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +0000287
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000288 /* use cached exception object if available */
289 if (buf->excobj == NULL) {
Victor Stinnerd9491262013-04-14 02:06:32 +0200290 buf->excobj = PyObject_CallFunction(PyExc_UnicodeEncodeError,
291 "sOnns",
292 codec->encoding, buf->inobj,
293 start, end, reason);
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000294 if (buf->excobj == NULL)
295 goto errorexit;
296 }
297 else
298 if (PyUnicodeEncodeError_SetStart(buf->excobj, start) != 0 ||
299 PyUnicodeEncodeError_SetEnd(buf->excobj, end) != 0 ||
300 PyUnicodeEncodeError_SetReason(buf->excobj, reason) != 0)
301 goto errorexit;
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +0000302
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000303 if (errors == ERROR_STRICT) {
304 PyCodec_StrictErrors(buf->excobj);
305 goto errorexit;
306 }
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +0000307
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000308 retobj = call_error_callback(errors, buf->excobj);
309 if (retobj == NULL)
310 goto errorexit;
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +0000311
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000312 if (!PyTuple_Check(retobj) || PyTuple_GET_SIZE(retobj) != 2 ||
Benjamin Peterson47a00f32012-12-02 11:20:28 -0500313 (!PyUnicode_Check((tobj = PyTuple_GET_ITEM(retobj, 0))) && !PyBytes_Check(tobj)) ||
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000314 !PyLong_Check(PyTuple_GET_ITEM(retobj, 1))) {
315 PyErr_SetString(PyExc_TypeError,
316 "encoding error handler must return "
Benjamin Petersonaff47232012-12-02 10:53:41 -0500317 "(str, int) tuple");
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000318 goto errorexit;
319 }
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +0000320
Benjamin Peterson47a00f32012-12-02 11:20:28 -0500321 if (PyUnicode_Check(tobj)) {
Victor Stinnerd9491262013-04-14 02:06:32 +0200322 Py_ssize_t inpos;
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +0000323
Victor Stinnerd9491262013-04-14 02:06:32 +0200324 retstr = multibytecodec_encode(codec, state, tobj,
325 &inpos, ERROR_STRICT,
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000326 MBENC_FLUSH);
327 if (retstr == NULL)
328 goto errorexit;
329 }
Benjamin Peterson47a00f32012-12-02 11:20:28 -0500330 else {
331 Py_INCREF(tobj);
332 retstr = tobj;
333 }
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +0000334
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000335 assert(PyBytes_Check(retstr));
336 retstrsize = PyBytes_GET_SIZE(retstr);
Serhiy Storchakaa1543cd2015-02-21 01:19:58 +0200337 if (retstrsize > 0) {
338 REQUIRE_ENCODEBUFFER(buf, retstrsize);
339 memcpy(buf->outbuf, PyBytes_AS_STRING(retstr), retstrsize);
340 buf->outbuf += retstrsize;
341 }
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +0000342
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000343 newpos = PyLong_AsSsize_t(PyTuple_GET_ITEM(retobj, 1));
344 if (newpos < 0 && !PyErr_Occurred())
Victor Stinnerd9491262013-04-14 02:06:32 +0200345 newpos += (Py_ssize_t)buf->inlen;
346 if (newpos < 0 || newpos > buf->inlen) {
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000347 PyErr_Clear();
348 PyErr_Format(PyExc_IndexError,
349 "position %zd from error handler out of bounds",
350 newpos);
351 goto errorexit;
352 }
Victor Stinnerd9491262013-04-14 02:06:32 +0200353 buf->inpos = newpos;
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +0000354
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000355 Py_DECREF(retobj);
356 Py_DECREF(retstr);
357 return 0;
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +0000358
359errorexit:
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000360 Py_XDECREF(retobj);
361 Py_XDECREF(retstr);
362 return -1;
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +0000363}
364
365static int
366multibytecodec_decerror(MultibyteCodec *codec,
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000367 MultibyteCodec_State *state,
368 MultibyteDecodeBuffer *buf,
369 PyObject *errors, Py_ssize_t e)
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +0000370{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000371 PyObject *retobj = NULL, *retuni = NULL;
Victor Stinnera0dd0212013-04-11 22:09:04 +0200372 Py_ssize_t newpos;
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000373 const char *reason;
374 Py_ssize_t esize, start, end;
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +0000375
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000376 if (e > 0) {
377 reason = "illegal multibyte sequence";
378 esize = e;
379 }
380 else {
381 switch (e) {
382 case MBERR_TOOSMALL:
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000383 return 0; /* retry it */
384 case MBERR_TOOFEW:
385 reason = "incomplete multibyte sequence";
386 esize = (Py_ssize_t)(buf->inbuf_end - buf->inbuf);
387 break;
388 case MBERR_INTERNAL:
389 PyErr_SetString(PyExc_RuntimeError,
390 "internal codec error");
391 return -1;
Victor Stinnerd1f99422013-07-16 21:41:43 +0200392 case MBERR_EXCEPTION:
393 return -1;
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000394 default:
395 PyErr_SetString(PyExc_RuntimeError,
396 "unknown runtime error");
397 return -1;
398 }
399 }
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +0000400
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000401 if (errors == ERROR_REPLACE) {
Victor Stinnera0dd0212013-04-11 22:09:04 +0200402 if (_PyUnicodeWriter_WriteChar(&buf->writer,
403 Py_UNICODE_REPLACEMENT_CHARACTER) < 0)
404 goto errorexit;
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000405 }
406 if (errors == ERROR_IGNORE || errors == ERROR_REPLACE) {
407 buf->inbuf += esize;
408 return 0;
409 }
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +0000410
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000411 start = (Py_ssize_t)(buf->inbuf - buf->inbuf_top);
412 end = start + esize;
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +0000413
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000414 /* use cached exception object if available */
415 if (buf->excobj == NULL) {
416 buf->excobj = PyUnicodeDecodeError_Create(codec->encoding,
417 (const char *)buf->inbuf_top,
418 (Py_ssize_t)(buf->inbuf_end - buf->inbuf_top),
419 start, end, reason);
420 if (buf->excobj == NULL)
421 goto errorexit;
422 }
423 else
424 if (PyUnicodeDecodeError_SetStart(buf->excobj, start) ||
425 PyUnicodeDecodeError_SetEnd(buf->excobj, end) ||
426 PyUnicodeDecodeError_SetReason(buf->excobj, reason))
427 goto errorexit;
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +0000428
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000429 if (errors == ERROR_STRICT) {
430 PyCodec_StrictErrors(buf->excobj);
431 goto errorexit;
432 }
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +0000433
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000434 retobj = call_error_callback(errors, buf->excobj);
435 if (retobj == NULL)
436 goto errorexit;
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +0000437
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000438 if (!PyTuple_Check(retobj) || PyTuple_GET_SIZE(retobj) != 2 ||
439 !PyUnicode_Check((retuni = PyTuple_GET_ITEM(retobj, 0))) ||
440 !PyLong_Check(PyTuple_GET_ITEM(retobj, 1))) {
441 PyErr_SetString(PyExc_TypeError,
442 "decoding error handler must return "
Benjamin Petersonaff47232012-12-02 10:53:41 -0500443 "(str, int) tuple");
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000444 goto errorexit;
445 }
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +0000446
Victor Stinnera0dd0212013-04-11 22:09:04 +0200447 if (_PyUnicodeWriter_WriteStr(&buf->writer, retuni) < 0)
Victor Stinner4eea8492011-11-21 03:01:27 +0100448 goto errorexit;
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +0000449
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000450 newpos = PyLong_AsSsize_t(PyTuple_GET_ITEM(retobj, 1));
451 if (newpos < 0 && !PyErr_Occurred())
452 newpos += (Py_ssize_t)(buf->inbuf_end - buf->inbuf_top);
453 if (newpos < 0 || buf->inbuf_top + newpos > buf->inbuf_end) {
454 PyErr_Clear();
455 PyErr_Format(PyExc_IndexError,
456 "position %zd from error handler out of bounds",
457 newpos);
458 goto errorexit;
459 }
460 buf->inbuf = buf->inbuf_top + newpos;
461 Py_DECREF(retobj);
462 return 0;
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +0000463
464errorexit:
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000465 Py_XDECREF(retobj);
466 return -1;
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +0000467}
468
469static PyObject *
470multibytecodec_encode(MultibyteCodec *codec,
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000471 MultibyteCodec_State *state,
Victor Stinnerd9491262013-04-14 02:06:32 +0200472 PyObject *text, Py_ssize_t *inpos_t,
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000473 PyObject *errors, int flags)
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +0000474{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000475 MultibyteEncodeBuffer buf;
476 Py_ssize_t finalsize, r = 0;
Victor Stinnerd9491262013-04-14 02:06:32 +0200477 Py_ssize_t datalen;
478 int kind;
479 void *data;
480
481 if (PyUnicode_READY(text) < 0)
482 return NULL;
483 datalen = PyUnicode_GET_LENGTH(text);
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +0000484
Victor Stinner6bcbef72011-05-24 22:17:55 +0200485 if (datalen == 0 && !(flags & MBENC_RESET))
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000486 return PyBytes_FromStringAndSize(NULL, 0);
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +0000487
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000488 buf.excobj = NULL;
Brett Cannonb94767f2011-02-22 20:15:44 +0000489 buf.outobj = NULL;
Victor Stinnerd9491262013-04-14 02:06:32 +0200490 buf.inobj = text; /* borrowed reference */
491 buf.inpos = 0;
492 buf.inlen = datalen;
493 kind = PyUnicode_KIND(buf.inobj);
494 data = PyUnicode_DATA(buf.inobj);
Amaury Forgeot d'Arc9c74b142008-06-18 00:47:36 +0000495
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000496 if (datalen > (PY_SSIZE_T_MAX - 16) / 2) {
497 PyErr_NoMemory();
498 goto errorexit;
499 }
Amaury Forgeot d'Arc9c74b142008-06-18 00:47:36 +0000500
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000501 buf.outobj = PyBytes_FromStringAndSize(NULL, datalen * 2 + 16);
502 if (buf.outobj == NULL)
503 goto errorexit;
504 buf.outbuf = (unsigned char *)PyBytes_AS_STRING(buf.outobj);
505 buf.outbuf_end = buf.outbuf + PyBytes_GET_SIZE(buf.outobj);
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +0000506
Victor Stinnerd9491262013-04-14 02:06:32 +0200507 while (buf.inpos < buf.inlen) {
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000508 /* we don't reuse inleft and outleft here.
509 * error callbacks can relocate the cursor anywhere on buffer*/
Victor Stinnerd9491262013-04-14 02:06:32 +0200510 Py_ssize_t outleft = (Py_ssize_t)(buf.outbuf_end - buf.outbuf);
511
512 r = codec->encode(state, codec->config,
513 kind, data,
514 &buf.inpos, buf.inlen,
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000515 &buf.outbuf, outleft, flags);
516 if ((r == 0) || (r == MBERR_TOOFEW && !(flags & MBENC_FLUSH)))
517 break;
518 else if (multibytecodec_encerror(codec, state, &buf, errors,r))
519 goto errorexit;
520 else if (r == MBERR_TOOFEW)
521 break;
522 }
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +0000523
Victor Stinner6bcbef72011-05-24 22:17:55 +0200524 if (codec->encreset != NULL && (flags & MBENC_RESET))
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000525 for (;;) {
526 Py_ssize_t outleft;
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +0000527
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000528 outleft = (Py_ssize_t)(buf.outbuf_end - buf.outbuf);
529 r = codec->encreset(state, codec->config, &buf.outbuf,
530 outleft);
531 if (r == 0)
532 break;
533 else if (multibytecodec_encerror(codec, state,
534 &buf, errors, r))
535 goto errorexit;
536 }
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +0000537
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000538 finalsize = (Py_ssize_t)((char *)buf.outbuf -
539 PyBytes_AS_STRING(buf.outobj));
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +0000540
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000541 if (finalsize != PyBytes_GET_SIZE(buf.outobj))
542 if (_PyBytes_Resize(&buf.outobj, finalsize) == -1)
543 goto errorexit;
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +0000544
Victor Stinnerd9491262013-04-14 02:06:32 +0200545 if (inpos_t)
546 *inpos_t = buf.inpos;
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000547 Py_XDECREF(buf.excobj);
548 return buf.outobj;
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +0000549
550errorexit:
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000551 Py_XDECREF(buf.excobj);
552 Py_XDECREF(buf.outobj);
553 return NULL;
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +0000554}
555
556static PyObject *
557MultibyteCodec_Encode(MultibyteCodecObject *self,
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000558 PyObject *args, PyObject *kwargs)
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +0000559{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000560 MultibyteCodec_State state;
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000561 PyObject *errorcb, *r, *arg, *ucvt;
562 const char *errors = NULL;
563 Py_ssize_t datalen;
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +0000564
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000565 if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|z:encode",
566 codeckwarglist, &arg, &errors))
567 return NULL;
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +0000568
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000569 if (PyUnicode_Check(arg))
570 ucvt = NULL;
571 else {
572 arg = ucvt = PyObject_Str(arg);
573 if (arg == NULL)
574 return NULL;
575 else if (!PyUnicode_Check(arg)) {
576 PyErr_SetString(PyExc_TypeError,
577 "couldn't convert the object to unicode.");
578 Py_DECREF(ucvt);
579 return NULL;
580 }
581 }
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +0000582
Victor Stinnerd9491262013-04-14 02:06:32 +0200583 if (PyUnicode_READY(arg) < 0) {
Victor Stinner9a80fab2011-11-21 02:50:14 +0100584 Py_XDECREF(ucvt);
585 return NULL;
586 }
Victor Stinnerd9491262013-04-14 02:06:32 +0200587 datalen = PyUnicode_GET_LENGTH(arg);
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +0000588
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000589 errorcb = internal_error_callback(errors);
590 if (errorcb == NULL) {
591 Py_XDECREF(ucvt);
592 return NULL;
593 }
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +0000594
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000595 if (self->codec->encinit != NULL &&
596 self->codec->encinit(&state, self->codec->config) != 0)
597 goto errorexit;
598 r = multibytecodec_encode(self->codec, &state,
Victor Stinnerd9491262013-04-14 02:06:32 +0200599 arg, NULL, errorcb,
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000600 MBENC_FLUSH | MBENC_RESET);
601 if (r == NULL)
602 goto errorexit;
Hye-Shik Chang2bb146f2004-07-18 03:06:29 +0000603
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000604 ERROR_DECREF(errorcb);
605 Py_XDECREF(ucvt);
606 return make_tuple(r, datalen);
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +0000607
608errorexit:
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000609 ERROR_DECREF(errorcb);
610 Py_XDECREF(ucvt);
611 return NULL;
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +0000612}
613
614static PyObject *
615MultibyteCodec_Decode(MultibyteCodecObject *self,
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000616 PyObject *args, PyObject *kwargs)
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +0000617{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000618 MultibyteCodec_State state;
619 MultibyteDecodeBuffer buf;
Victor Stinnera0dd0212013-04-11 22:09:04 +0200620 PyObject *errorcb, *res;
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000621 Py_buffer pdata;
622 const char *data, *errors = NULL;
Victor Stinnera0dd0212013-04-11 22:09:04 +0200623 Py_ssize_t datalen;
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +0000624
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000625 if (!PyArg_ParseTupleAndKeywords(args, kwargs, "y*|z:decode",
626 codeckwarglist, &pdata, &errors))
627 return NULL;
628 data = pdata.buf;
629 datalen = pdata.len;
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +0000630
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000631 errorcb = internal_error_callback(errors);
632 if (errorcb == NULL) {
633 PyBuffer_Release(&pdata);
634 return NULL;
635 }
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +0000636
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000637 if (datalen == 0) {
638 PyBuffer_Release(&pdata);
639 ERROR_DECREF(errorcb);
Victor Stinnerb37b1742011-12-01 03:18:59 +0100640 return make_tuple(PyUnicode_New(0, 0), 0);
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000641 }
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +0000642
Victor Stinner8f674cc2013-04-17 23:02:17 +0200643 _PyUnicodeWriter_Init(&buf.writer);
644 buf.writer.min_length = datalen;
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000645 buf.excobj = NULL;
646 buf.inbuf = buf.inbuf_top = (unsigned char *)data;
647 buf.inbuf_end = buf.inbuf_top + datalen;
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +0000648
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000649 if (self->codec->decinit != NULL &&
650 self->codec->decinit(&state, self->codec->config) != 0)
651 goto errorexit;
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +0000652
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000653 while (buf.inbuf < buf.inbuf_end) {
Victor Stinnera0dd0212013-04-11 22:09:04 +0200654 Py_ssize_t inleft, r;
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +0000655
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000656 inleft = (Py_ssize_t)(buf.inbuf_end - buf.inbuf);
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +0000657
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000658 r = self->codec->decode(&state, self->codec->config,
Victor Stinnera0dd0212013-04-11 22:09:04 +0200659 &buf.inbuf, inleft, &buf.writer);
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000660 if (r == 0)
661 break;
662 else if (multibytecodec_decerror(self->codec, &state,
663 &buf, errorcb, r))
664 goto errorexit;
665 }
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +0000666
Victor Stinnera0dd0212013-04-11 22:09:04 +0200667 res = _PyUnicodeWriter_Finish(&buf.writer);
668 if (res == NULL)
669 goto errorexit;
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +0000670
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000671 PyBuffer_Release(&pdata);
672 Py_XDECREF(buf.excobj);
673 ERROR_DECREF(errorcb);
Victor Stinnera0dd0212013-04-11 22:09:04 +0200674 return make_tuple(res, datalen);
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +0000675
676errorexit:
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000677 PyBuffer_Release(&pdata);
678 ERROR_DECREF(errorcb);
679 Py_XDECREF(buf.excobj);
Victor Stinnera0dd0212013-04-11 22:09:04 +0200680 _PyUnicodeWriter_Dealloc(&buf.writer);
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +0000681
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000682 return NULL;
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +0000683}
684
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +0000685static struct PyMethodDef multibytecodec_methods[] = {
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000686 {"encode", (PyCFunction)MultibyteCodec_Encode,
687 METH_VARARGS | METH_KEYWORDS,
688 MultibyteCodec_Encode__doc__},
689 {"decode", (PyCFunction)MultibyteCodec_Decode,
690 METH_VARARGS | METH_KEYWORDS,
691 MultibyteCodec_Decode__doc__},
692 {NULL, NULL},
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +0000693};
694
695static void
696multibytecodec_dealloc(MultibyteCodecObject *self)
697{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000698 PyObject_Del(self);
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +0000699}
700
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +0000701static PyTypeObject MultibyteCodec_Type = {
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000702 PyVarObject_HEAD_INIT(NULL, 0)
703 "MultibyteCodec", /* tp_name */
704 sizeof(MultibyteCodecObject), /* tp_basicsize */
705 0, /* tp_itemsize */
706 /* methods */
707 (destructor)multibytecodec_dealloc, /* tp_dealloc */
708 0, /* tp_print */
709 0, /* tp_getattr */
710 0, /* tp_setattr */
711 0, /* tp_reserved */
712 0, /* tp_repr */
713 0, /* tp_as_number */
714 0, /* tp_as_sequence */
715 0, /* tp_as_mapping */
716 0, /* tp_hash */
717 0, /* tp_call */
718 0, /* tp_str */
719 PyObject_GenericGetAttr, /* tp_getattro */
720 0, /* tp_setattro */
721 0, /* tp_as_buffer */
722 Py_TPFLAGS_DEFAULT, /* tp_flags */
723 0, /* tp_doc */
724 0, /* tp_traverse */
725 0, /* tp_clear */
726 0, /* tp_richcompare */
727 0, /* tp_weaklistoffset */
728 0, /* tp_iter */
729 0, /* tp_iterext */
730 multibytecodec_methods, /* tp_methods */
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +0000731};
732
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000733
734/**
735 * Utility functions for stateful codec mechanism
736 */
737
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000738#define STATEFUL_DCTX(o) ((MultibyteStatefulDecoderContext *)(o))
739#define STATEFUL_ECTX(o) ((MultibyteStatefulEncoderContext *)(o))
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000740
741static PyObject *
742encoder_encode_stateful(MultibyteStatefulEncoderContext *ctx,
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000743 PyObject *unistr, int final)
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000744{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000745 PyObject *ucvt, *r = NULL;
Victor Stinnerd9491262013-04-14 02:06:32 +0200746 PyObject *inbuf = NULL;
747 Py_ssize_t inpos, datalen;
748 PyObject *origpending = NULL;
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000749
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000750 if (PyUnicode_Check(unistr))
751 ucvt = NULL;
752 else {
753 unistr = ucvt = PyObject_Str(unistr);
754 if (unistr == NULL)
755 return NULL;
756 else if (!PyUnicode_Check(unistr)) {
757 PyErr_SetString(PyExc_TypeError,
Benjamin Petersonaff47232012-12-02 10:53:41 -0500758 "couldn't convert the object to str.");
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000759 Py_DECREF(ucvt);
760 return NULL;
761 }
762 }
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000763
Victor Stinnerd9491262013-04-14 02:06:32 +0200764 if (ctx->pending) {
765 PyObject *inbuf_tmp;
766
767 Py_INCREF(ctx->pending);
768 origpending = ctx->pending;
769
770 Py_INCREF(ctx->pending);
771 inbuf_tmp = ctx->pending;
772 PyUnicode_Append(&inbuf_tmp, unistr);
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000773 if (inbuf_tmp == NULL)
774 goto errorexit;
Victor Stinnerd9491262013-04-14 02:06:32 +0200775 Py_CLEAR(ctx->pending);
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000776 inbuf = inbuf_tmp;
777 }
Victor Stinnerd9491262013-04-14 02:06:32 +0200778 else {
779 origpending = NULL;
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000780
Victor Stinnerd9491262013-04-14 02:06:32 +0200781 Py_INCREF(unistr);
782 inbuf = unistr;
783 }
784 if (PyUnicode_READY(inbuf) < 0)
785 goto errorexit;
786 inpos = 0;
787 datalen = PyUnicode_GET_LENGTH(inbuf);
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000788
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000789 r = multibytecodec_encode(ctx->codec, &ctx->state,
Victor Stinnerd9491262013-04-14 02:06:32 +0200790 inbuf, &inpos,
791 ctx->errors, final ? MBENC_FLUSH | MBENC_RESET : 0);
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000792 if (r == NULL) {
793 /* recover the original pending buffer */
Victor Stinnerd9491262013-04-14 02:06:32 +0200794 Py_CLEAR(ctx->pending);
795 ctx->pending = origpending;
796 origpending = NULL;
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000797 goto errorexit;
798 }
Victor Stinner322cc742013-04-14 18:11:41 +0200799 Py_XDECREF(origpending);
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000800
Victor Stinnerd9491262013-04-14 02:06:32 +0200801 if (inpos < datalen) {
802 if (datalen - inpos > MAXENCPENDING) {
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000803 /* normal codecs can't reach here */
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000804 PyErr_SetString(PyExc_UnicodeError,
805 "pending buffer overflow");
806 goto errorexit;
807 }
Victor Stinnerd9491262013-04-14 02:06:32 +0200808 ctx->pending = PyUnicode_Substring(inbuf, inpos, datalen);
809 if (ctx->pending == NULL) {
810 /* normal codecs can't reach here */
811 goto errorexit;
812 }
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000813 }
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000814
Victor Stinner322cc742013-04-14 18:11:41 +0200815 Py_DECREF(inbuf);
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000816 Py_XDECREF(ucvt);
817 return r;
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000818
819errorexit:
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000820 Py_XDECREF(r);
821 Py_XDECREF(ucvt);
Victor Stinnerd9491262013-04-14 02:06:32 +0200822 Py_XDECREF(origpending);
Victor Stinner322cc742013-04-14 18:11:41 +0200823 Py_XDECREF(inbuf);
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000824 return NULL;
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000825}
826
827static int
828decoder_append_pending(MultibyteStatefulDecoderContext *ctx,
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000829 MultibyteDecodeBuffer *buf)
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000830{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000831 Py_ssize_t npendings;
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000832
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000833 npendings = (Py_ssize_t)(buf->inbuf_end - buf->inbuf);
834 if (npendings + ctx->pendingsize > MAXDECPENDING ||
835 npendings > PY_SSIZE_T_MAX - ctx->pendingsize) {
836 PyErr_SetString(PyExc_UnicodeError, "pending buffer overflow");
837 return -1;
838 }
839 memcpy(ctx->pending + ctx->pendingsize, buf->inbuf, npendings);
840 ctx->pendingsize += npendings;
841 return 0;
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000842}
843
844static int
845decoder_prepare_buffer(MultibyteDecodeBuffer *buf, const char *data,
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000846 Py_ssize_t size)
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000847{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000848 buf->inbuf = buf->inbuf_top = (const unsigned char *)data;
849 buf->inbuf_end = buf->inbuf_top + size;
Victor Stinner8f674cc2013-04-17 23:02:17 +0200850 buf->writer.min_length += size;
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000851 return 0;
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000852}
853
854static int
855decoder_feed_buffer(MultibyteStatefulDecoderContext *ctx,
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000856 MultibyteDecodeBuffer *buf)
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000857{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000858 while (buf->inbuf < buf->inbuf_end) {
Victor Stinnera0dd0212013-04-11 22:09:04 +0200859 Py_ssize_t inleft;
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000860 Py_ssize_t r;
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000861
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000862 inleft = (Py_ssize_t)(buf->inbuf_end - buf->inbuf);
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000863
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000864 r = ctx->codec->decode(&ctx->state, ctx->codec->config,
Victor Stinnera0dd0212013-04-11 22:09:04 +0200865 &buf->inbuf, inleft, &buf->writer);
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000866 if (r == 0 || r == MBERR_TOOFEW)
867 break;
868 else if (multibytecodec_decerror(ctx->codec, &ctx->state,
869 buf, ctx->errors, r))
870 return -1;
871 }
872 return 0;
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000873}
874
875
876/**
877 * MultibyteIncrementalEncoder object
878 */
879
880static PyObject *
881mbiencoder_encode(MultibyteIncrementalEncoderObject *self,
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000882 PyObject *args, PyObject *kwargs)
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000883{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000884 PyObject *data;
885 int final = 0;
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000886
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000887 if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|i:encode",
888 incrementalkwarglist, &data, &final))
889 return NULL;
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000890
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000891 return encoder_encode_stateful(STATEFUL_ECTX(self), data, final);
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000892}
893
894static PyObject *
895mbiencoder_reset(MultibyteIncrementalEncoderObject *self)
896{
Victor Stinnere15dce32011-05-30 22:56:00 +0200897 /* Longest output: 4 bytes (b'\x0F\x1F(B') with ISO 2022 */
898 unsigned char buffer[4], *outbuf;
899 Py_ssize_t r;
900 if (self->codec->encreset != NULL) {
901 outbuf = buffer;
902 r = self->codec->encreset(&self->state, self->codec->config,
903 &outbuf, sizeof(buffer));
904 if (r != 0)
905 return NULL;
906 }
Victor Stinnerd9491262013-04-14 02:06:32 +0200907 Py_CLEAR(self->pending);
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000908 Py_RETURN_NONE;
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000909}
910
911static struct PyMethodDef mbiencoder_methods[] = {
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000912 {"encode", (PyCFunction)mbiencoder_encode,
913 METH_VARARGS | METH_KEYWORDS, NULL},
914 {"reset", (PyCFunction)mbiencoder_reset,
915 METH_NOARGS, NULL},
916 {NULL, NULL},
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000917};
918
919static PyObject *
920mbiencoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
921{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000922 MultibyteIncrementalEncoderObject *self;
923 PyObject *codec = NULL;
924 char *errors = NULL;
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000925
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000926 if (!PyArg_ParseTupleAndKeywords(args, kwds, "|s:IncrementalEncoder",
927 incnewkwarglist, &errors))
928 return NULL;
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000929
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000930 self = (MultibyteIncrementalEncoderObject *)type->tp_alloc(type, 0);
931 if (self == NULL)
932 return NULL;
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000933
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000934 codec = PyObject_GetAttrString((PyObject *)type, "codec");
935 if (codec == NULL)
936 goto errorexit;
937 if (!MultibyteCodec_Check(codec)) {
938 PyErr_SetString(PyExc_TypeError, "codec is unexpected type");
939 goto errorexit;
940 }
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000941
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000942 self->codec = ((MultibyteCodecObject *)codec)->codec;
Victor Stinnerd9491262013-04-14 02:06:32 +0200943 self->pending = NULL;
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000944 self->errors = internal_error_callback(errors);
945 if (self->errors == NULL)
946 goto errorexit;
947 if (self->codec->encinit != NULL &&
948 self->codec->encinit(&self->state, self->codec->config) != 0)
949 goto errorexit;
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000950
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000951 Py_DECREF(codec);
952 return (PyObject *)self;
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000953
954errorexit:
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000955 Py_XDECREF(self);
956 Py_XDECREF(codec);
957 return NULL;
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000958}
959
960static int
Hye-Shik Chang13247bf2006-04-21 16:21:44 +0000961mbiencoder_init(PyObject *self, PyObject *args, PyObject *kwds)
962{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000963 return 0;
Hye-Shik Chang13247bf2006-04-21 16:21:44 +0000964}
965
966static int
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000967mbiencoder_traverse(MultibyteIncrementalEncoderObject *self,
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000968 visitproc visit, void *arg)
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000969{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000970 if (ERROR_ISCUSTOM(self->errors))
971 Py_VISIT(self->errors);
972 return 0;
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000973}
974
975static void
976mbiencoder_dealloc(MultibyteIncrementalEncoderObject *self)
977{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000978 PyObject_GC_UnTrack(self);
979 ERROR_DECREF(self->errors);
980 Py_TYPE(self)->tp_free(self);
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000981}
982
983static PyTypeObject MultibyteIncrementalEncoder_Type = {
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000984 PyVarObject_HEAD_INIT(NULL, 0)
985 "MultibyteIncrementalEncoder", /* tp_name */
986 sizeof(MultibyteIncrementalEncoderObject), /* tp_basicsize */
987 0, /* tp_itemsize */
988 /* methods */
989 (destructor)mbiencoder_dealloc, /* tp_dealloc */
990 0, /* tp_print */
991 0, /* tp_getattr */
992 0, /* tp_setattr */
993 0, /* tp_reserved */
994 0, /* tp_repr */
995 0, /* tp_as_number */
996 0, /* tp_as_sequence */
997 0, /* tp_as_mapping */
998 0, /* tp_hash */
999 0, /* tp_call */
1000 0, /* tp_str */
1001 PyObject_GenericGetAttr, /* tp_getattro */
1002 0, /* tp_setattro */
1003 0, /* tp_as_buffer */
1004 Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC
1005 | Py_TPFLAGS_BASETYPE, /* tp_flags */
1006 0, /* tp_doc */
1007 (traverseproc)mbiencoder_traverse, /* tp_traverse */
1008 0, /* tp_clear */
1009 0, /* tp_richcompare */
1010 0, /* tp_weaklistoffset */
1011 0, /* tp_iter */
1012 0, /* tp_iterext */
1013 mbiencoder_methods, /* tp_methods */
1014 0, /* tp_members */
1015 codecctx_getsets, /* tp_getset */
1016 0, /* tp_base */
1017 0, /* tp_dict */
1018 0, /* tp_descr_get */
1019 0, /* tp_descr_set */
1020 0, /* tp_dictoffset */
1021 mbiencoder_init, /* tp_init */
1022 0, /* tp_alloc */
1023 mbiencoder_new, /* tp_new */
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001024};
1025
1026
1027/**
1028 * MultibyteIncrementalDecoder object
1029 */
1030
1031static PyObject *
1032mbidecoder_decode(MultibyteIncrementalDecoderObject *self,
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001033 PyObject *args, PyObject *kwargs)
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001034{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001035 MultibyteDecodeBuffer buf;
1036 char *data, *wdata = NULL;
1037 Py_buffer pdata;
Victor Stinnera0dd0212013-04-11 22:09:04 +02001038 Py_ssize_t wsize, size, origpending;
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001039 int final = 0;
Victor Stinnera0dd0212013-04-11 22:09:04 +02001040 PyObject *res;
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001041
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001042 if (!PyArg_ParseTupleAndKeywords(args, kwargs, "y*|i:decode",
1043 incrementalkwarglist, &pdata, &final))
1044 return NULL;
1045 data = pdata.buf;
1046 size = pdata.len;
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001047
Victor Stinner8f674cc2013-04-17 23:02:17 +02001048 _PyUnicodeWriter_Init(&buf.writer);
Victor Stinnera0dd0212013-04-11 22:09:04 +02001049 buf.excobj = NULL;
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001050 origpending = self->pendingsize;
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001051
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001052 if (self->pendingsize == 0) {
1053 wsize = size;
1054 wdata = data;
1055 }
1056 else {
1057 if (size > PY_SSIZE_T_MAX - self->pendingsize) {
1058 PyErr_NoMemory();
1059 goto errorexit;
1060 }
1061 wsize = size + self->pendingsize;
1062 wdata = PyMem_Malloc(wsize);
Victor Stinner33283ba2013-07-15 17:47:39 +02001063 if (wdata == NULL) {
1064 PyErr_NoMemory();
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001065 goto errorexit;
Victor Stinner33283ba2013-07-15 17:47:39 +02001066 }
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001067 memcpy(wdata, self->pending, self->pendingsize);
1068 memcpy(wdata + self->pendingsize, data, size);
1069 self->pendingsize = 0;
1070 }
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001071
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001072 if (decoder_prepare_buffer(&buf, wdata, wsize) != 0)
1073 goto errorexit;
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001074
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001075 if (decoder_feed_buffer(STATEFUL_DCTX(self), &buf))
1076 goto errorexit;
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001077
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001078 if (final && buf.inbuf < buf.inbuf_end) {
1079 if (multibytecodec_decerror(self->codec, &self->state,
1080 &buf, self->errors, MBERR_TOOFEW)) {
1081 /* recover the original pending buffer */
1082 memcpy(self->pending, wdata, origpending);
1083 self->pendingsize = origpending;
1084 goto errorexit;
1085 }
1086 }
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001087
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001088 if (buf.inbuf < buf.inbuf_end) { /* pending sequence still exists */
1089 if (decoder_append_pending(STATEFUL_DCTX(self), &buf) != 0)
1090 goto errorexit;
1091 }
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001092
Victor Stinnera0dd0212013-04-11 22:09:04 +02001093 res = _PyUnicodeWriter_Finish(&buf.writer);
1094 if (res == NULL)
1095 goto errorexit;
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001096
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001097 PyBuffer_Release(&pdata);
1098 if (wdata != data)
1099 PyMem_Del(wdata);
1100 Py_XDECREF(buf.excobj);
Victor Stinnera0dd0212013-04-11 22:09:04 +02001101 return res;
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001102
1103errorexit:
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001104 PyBuffer_Release(&pdata);
1105 if (wdata != NULL && wdata != data)
1106 PyMem_Del(wdata);
1107 Py_XDECREF(buf.excobj);
Victor Stinnera0dd0212013-04-11 22:09:04 +02001108 _PyUnicodeWriter_Dealloc(&buf.writer);
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001109 return NULL;
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001110}
1111
1112static PyObject *
1113mbidecoder_reset(MultibyteIncrementalDecoderObject *self)
1114{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001115 if (self->codec->decreset != NULL &&
1116 self->codec->decreset(&self->state, self->codec->config) != 0)
1117 return NULL;
1118 self->pendingsize = 0;
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001119
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001120 Py_RETURN_NONE;
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001121}
1122
1123static struct PyMethodDef mbidecoder_methods[] = {
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001124 {"decode", (PyCFunction)mbidecoder_decode,
1125 METH_VARARGS | METH_KEYWORDS, NULL},
1126 {"reset", (PyCFunction)mbidecoder_reset,
1127 METH_NOARGS, NULL},
1128 {NULL, NULL},
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001129};
1130
1131static PyObject *
1132mbidecoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
1133{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001134 MultibyteIncrementalDecoderObject *self;
1135 PyObject *codec = NULL;
1136 char *errors = NULL;
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001137
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001138 if (!PyArg_ParseTupleAndKeywords(args, kwds, "|s:IncrementalDecoder",
1139 incnewkwarglist, &errors))
1140 return NULL;
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001141
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001142 self = (MultibyteIncrementalDecoderObject *)type->tp_alloc(type, 0);
1143 if (self == NULL)
1144 return NULL;
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001145
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001146 codec = PyObject_GetAttrString((PyObject *)type, "codec");
1147 if (codec == NULL)
1148 goto errorexit;
1149 if (!MultibyteCodec_Check(codec)) {
1150 PyErr_SetString(PyExc_TypeError, "codec is unexpected type");
1151 goto errorexit;
1152 }
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001153
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001154 self->codec = ((MultibyteCodecObject *)codec)->codec;
1155 self->pendingsize = 0;
1156 self->errors = internal_error_callback(errors);
1157 if (self->errors == NULL)
1158 goto errorexit;
1159 if (self->codec->decinit != NULL &&
1160 self->codec->decinit(&self->state, self->codec->config) != 0)
1161 goto errorexit;
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001162
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001163 Py_DECREF(codec);
1164 return (PyObject *)self;
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001165
1166errorexit:
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001167 Py_XDECREF(self);
1168 Py_XDECREF(codec);
1169 return NULL;
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001170}
1171
1172static int
Hye-Shik Chang13247bf2006-04-21 16:21:44 +00001173mbidecoder_init(PyObject *self, PyObject *args, PyObject *kwds)
1174{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001175 return 0;
Hye-Shik Chang13247bf2006-04-21 16:21:44 +00001176}
1177
1178static int
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001179mbidecoder_traverse(MultibyteIncrementalDecoderObject *self,
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001180 visitproc visit, void *arg)
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001181{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001182 if (ERROR_ISCUSTOM(self->errors))
1183 Py_VISIT(self->errors);
1184 return 0;
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001185}
1186
1187static void
1188mbidecoder_dealloc(MultibyteIncrementalDecoderObject *self)
1189{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001190 PyObject_GC_UnTrack(self);
1191 ERROR_DECREF(self->errors);
1192 Py_TYPE(self)->tp_free(self);
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001193}
1194
1195static PyTypeObject MultibyteIncrementalDecoder_Type = {
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001196 PyVarObject_HEAD_INIT(NULL, 0)
1197 "MultibyteIncrementalDecoder", /* tp_name */
1198 sizeof(MultibyteIncrementalDecoderObject), /* tp_basicsize */
1199 0, /* tp_itemsize */
1200 /* methods */
1201 (destructor)mbidecoder_dealloc, /* tp_dealloc */
1202 0, /* tp_print */
1203 0, /* tp_getattr */
1204 0, /* tp_setattr */
1205 0, /* tp_reserved */
1206 0, /* tp_repr */
1207 0, /* tp_as_number */
1208 0, /* tp_as_sequence */
1209 0, /* tp_as_mapping */
1210 0, /* tp_hash */
1211 0, /* tp_call */
1212 0, /* tp_str */
1213 PyObject_GenericGetAttr, /* tp_getattro */
1214 0, /* tp_setattro */
1215 0, /* tp_as_buffer */
1216 Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC
1217 | Py_TPFLAGS_BASETYPE, /* tp_flags */
1218 0, /* tp_doc */
1219 (traverseproc)mbidecoder_traverse, /* tp_traverse */
1220 0, /* tp_clear */
1221 0, /* tp_richcompare */
1222 0, /* tp_weaklistoffset */
1223 0, /* tp_iter */
1224 0, /* tp_iterext */
1225 mbidecoder_methods, /* tp_methods */
1226 0, /* tp_members */
1227 codecctx_getsets, /* tp_getset */
1228 0, /* tp_base */
1229 0, /* tp_dict */
1230 0, /* tp_descr_get */
1231 0, /* tp_descr_set */
1232 0, /* tp_dictoffset */
1233 mbidecoder_init, /* tp_init */
1234 0, /* tp_alloc */
1235 mbidecoder_new, /* tp_new */
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001236};
1237
1238
1239/**
1240 * MultibyteStreamReader object
1241 */
1242
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +00001243static PyObject *
1244mbstreamreader_iread(MultibyteStreamReaderObject *self,
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001245 const char *method, Py_ssize_t sizehint)
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +00001246{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001247 MultibyteDecodeBuffer buf;
Victor Stinnera0dd0212013-04-11 22:09:04 +02001248 PyObject *cres, *res;
1249 Py_ssize_t rsize;
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +00001250
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001251 if (sizehint == 0)
Victor Stinnerb37b1742011-12-01 03:18:59 +01001252 return PyUnicode_New(0, 0);
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +00001253
Victor Stinner8f674cc2013-04-17 23:02:17 +02001254 _PyUnicodeWriter_Init(&buf.writer);
Victor Stinnera0dd0212013-04-11 22:09:04 +02001255 buf.excobj = NULL;
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001256 cres = NULL;
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +00001257
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001258 for (;;) {
1259 int endoffile;
Guido van Rossumcd16bf62007-06-13 18:07:49 +00001260
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001261 if (sizehint < 0)
1262 cres = PyObject_CallMethod(self->stream,
1263 (char *)method, NULL);
1264 else
1265 cres = PyObject_CallMethod(self->stream,
1266 (char *)method, "i", sizehint);
1267 if (cres == NULL)
1268 goto errorexit;
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +00001269
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001270 if (!PyBytes_Check(cres)) {
1271 PyErr_Format(PyExc_TypeError,
1272 "stream function returned a "
1273 "non-bytes object (%.100s)",
1274 cres->ob_type->tp_name);
1275 goto errorexit;
1276 }
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +00001277
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001278 endoffile = (PyBytes_GET_SIZE(cres) == 0);
Guido van Rossumcd16bf62007-06-13 18:07:49 +00001279
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001280 if (self->pendingsize > 0) {
1281 PyObject *ctr;
1282 char *ctrdata;
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +00001283
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001284 if (PyBytes_GET_SIZE(cres) > PY_SSIZE_T_MAX - self->pendingsize) {
1285 PyErr_NoMemory();
1286 goto errorexit;
Victor Stinner064bbdc2013-07-08 22:28:27 +02001287 }
1288 rsize = PyBytes_GET_SIZE(cres) + self->pendingsize;
1289 ctr = PyBytes_FromStringAndSize(NULL, rsize);
1290 if (ctr == NULL)
1291 goto errorexit;
1292 ctrdata = PyBytes_AS_STRING(ctr);
1293 memcpy(ctrdata, self->pending, self->pendingsize);
1294 memcpy(ctrdata + self->pendingsize,
1295 PyBytes_AS_STRING(cres),
1296 PyBytes_GET_SIZE(cres));
1297 Py_DECREF(cres);
1298 cres = ctr;
1299 self->pendingsize = 0;
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001300 }
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +00001301
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001302 rsize = PyBytes_GET_SIZE(cres);
1303 if (decoder_prepare_buffer(&buf, PyBytes_AS_STRING(cres),
1304 rsize) != 0)
1305 goto errorexit;
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +00001306
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001307 if (rsize > 0 && decoder_feed_buffer(
1308 (MultibyteStatefulDecoderContext *)self, &buf))
1309 goto errorexit;
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +00001310
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001311 if (endoffile || sizehint < 0) {
1312 if (buf.inbuf < buf.inbuf_end &&
1313 multibytecodec_decerror(self->codec, &self->state,
1314 &buf, self->errors, MBERR_TOOFEW))
1315 goto errorexit;
1316 }
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +00001317
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001318 if (buf.inbuf < buf.inbuf_end) { /* pending sequence exists */
1319 if (decoder_append_pending(STATEFUL_DCTX(self),
1320 &buf) != 0)
1321 goto errorexit;
1322 }
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +00001323
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001324 Py_DECREF(cres);
1325 cres = NULL;
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +00001326
Victor Stinnera0dd0212013-04-11 22:09:04 +02001327 if (sizehint < 0 || buf.writer.pos != 0 || rsize == 0)
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001328 break;
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +00001329
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001330 sizehint = 1; /* read 1 more byte and retry */
1331 }
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +00001332
Victor Stinnera0dd0212013-04-11 22:09:04 +02001333 res = _PyUnicodeWriter_Finish(&buf.writer);
1334 if (res == NULL)
1335 goto errorexit;
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +00001336
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001337 Py_XDECREF(cres);
1338 Py_XDECREF(buf.excobj);
Victor Stinnera0dd0212013-04-11 22:09:04 +02001339 return res;
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +00001340
1341errorexit:
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001342 Py_XDECREF(cres);
1343 Py_XDECREF(buf.excobj);
Victor Stinnera0dd0212013-04-11 22:09:04 +02001344 _PyUnicodeWriter_Dealloc(&buf.writer);
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001345 return NULL;
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +00001346}
1347
1348static PyObject *
1349mbstreamreader_read(MultibyteStreamReaderObject *self, PyObject *args)
1350{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001351 PyObject *sizeobj = NULL;
1352 Py_ssize_t size;
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +00001353
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001354 if (!PyArg_UnpackTuple(args, "read", 0, 1, &sizeobj))
1355 return NULL;
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +00001356
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001357 if (sizeobj == Py_None || sizeobj == NULL)
1358 size = -1;
1359 else if (PyLong_Check(sizeobj))
1360 size = PyLong_AsSsize_t(sizeobj);
1361 else {
1362 PyErr_SetString(PyExc_TypeError, "arg 1 must be an integer");
1363 return NULL;
1364 }
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +00001365
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001366 if (size == -1 && PyErr_Occurred())
1367 return NULL;
Guido van Rossumddefaf32007-01-14 03:31:43 +00001368
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001369 return mbstreamreader_iread(self, "read", size);
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +00001370}
1371
1372static PyObject *
1373mbstreamreader_readline(MultibyteStreamReaderObject *self, PyObject *args)
1374{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001375 PyObject *sizeobj = NULL;
1376 Py_ssize_t size;
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +00001377
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001378 if (!PyArg_UnpackTuple(args, "readline", 0, 1, &sizeobj))
1379 return NULL;
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +00001380
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001381 if (sizeobj == Py_None || sizeobj == NULL)
1382 size = -1;
1383 else if (PyLong_Check(sizeobj))
1384 size = PyLong_AsSsize_t(sizeobj);
1385 else {
1386 PyErr_SetString(PyExc_TypeError, "arg 1 must be an integer");
1387 return NULL;
1388 }
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +00001389
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001390 if (size == -1 && PyErr_Occurred())
1391 return NULL;
Guido van Rossumddefaf32007-01-14 03:31:43 +00001392
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001393 return mbstreamreader_iread(self, "readline", size);
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +00001394}
1395
1396static PyObject *
1397mbstreamreader_readlines(MultibyteStreamReaderObject *self, PyObject *args)
1398{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001399 PyObject *sizehintobj = NULL, *r, *sr;
1400 Py_ssize_t sizehint;
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +00001401
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001402 if (!PyArg_UnpackTuple(args, "readlines", 0, 1, &sizehintobj))
1403 return NULL;
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +00001404
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001405 if (sizehintobj == Py_None || sizehintobj == NULL)
1406 sizehint = -1;
1407 else if (PyLong_Check(sizehintobj))
1408 sizehint = PyLong_AsSsize_t(sizehintobj);
1409 else {
1410 PyErr_SetString(PyExc_TypeError, "arg 1 must be an integer");
1411 return NULL;
1412 }
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +00001413
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001414 if (sizehint == -1 && PyErr_Occurred())
1415 return NULL;
Guido van Rossumddefaf32007-01-14 03:31:43 +00001416
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001417 r = mbstreamreader_iread(self, "read", sizehint);
1418 if (r == NULL)
1419 return NULL;
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +00001420
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001421 sr = PyUnicode_Splitlines(r, 1);
1422 Py_DECREF(r);
1423 return sr;
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +00001424}
1425
1426static PyObject *
1427mbstreamreader_reset(MultibyteStreamReaderObject *self)
1428{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001429 if (self->codec->decreset != NULL &&
1430 self->codec->decreset(&self->state, self->codec->config) != 0)
1431 return NULL;
1432 self->pendingsize = 0;
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +00001433
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001434 Py_RETURN_NONE;
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +00001435}
1436
1437static struct PyMethodDef mbstreamreader_methods[] = {
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001438 {"read", (PyCFunction)mbstreamreader_read,
1439 METH_VARARGS, NULL},
1440 {"readline", (PyCFunction)mbstreamreader_readline,
1441 METH_VARARGS, NULL},
1442 {"readlines", (PyCFunction)mbstreamreader_readlines,
1443 METH_VARARGS, NULL},
1444 {"reset", (PyCFunction)mbstreamreader_reset,
1445 METH_NOARGS, NULL},
1446 {NULL, NULL},
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +00001447};
1448
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001449static PyMemberDef mbstreamreader_members[] = {
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001450 {"stream", T_OBJECT,
1451 offsetof(MultibyteStreamReaderObject, stream),
1452 READONLY, NULL},
1453 {NULL,}
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001454};
1455
1456static PyObject *
1457mbstreamreader_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
1458{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001459 MultibyteStreamReaderObject *self;
1460 PyObject *stream, *codec = NULL;
1461 char *errors = NULL;
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001462
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001463 if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|s:StreamReader",
1464 streamkwarglist, &stream, &errors))
1465 return NULL;
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001466
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001467 self = (MultibyteStreamReaderObject *)type->tp_alloc(type, 0);
1468 if (self == NULL)
1469 return NULL;
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001470
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001471 codec = PyObject_GetAttrString((PyObject *)type, "codec");
1472 if (codec == NULL)
1473 goto errorexit;
1474 if (!MultibyteCodec_Check(codec)) {
1475 PyErr_SetString(PyExc_TypeError, "codec is unexpected type");
1476 goto errorexit;
1477 }
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001478
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001479 self->codec = ((MultibyteCodecObject *)codec)->codec;
1480 self->stream = stream;
1481 Py_INCREF(stream);
1482 self->pendingsize = 0;
1483 self->errors = internal_error_callback(errors);
1484 if (self->errors == NULL)
1485 goto errorexit;
1486 if (self->codec->decinit != NULL &&
1487 self->codec->decinit(&self->state, self->codec->config) != 0)
1488 goto errorexit;
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001489
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001490 Py_DECREF(codec);
1491 return (PyObject *)self;
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001492
1493errorexit:
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001494 Py_XDECREF(self);
1495 Py_XDECREF(codec);
1496 return NULL;
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001497}
1498
1499static int
Hye-Shik Chang13247bf2006-04-21 16:21:44 +00001500mbstreamreader_init(PyObject *self, PyObject *args, PyObject *kwds)
1501{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001502 return 0;
Hye-Shik Chang13247bf2006-04-21 16:21:44 +00001503}
1504
1505static int
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001506mbstreamreader_traverse(MultibyteStreamReaderObject *self,
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001507 visitproc visit, void *arg)
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001508{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001509 if (ERROR_ISCUSTOM(self->errors))
1510 Py_VISIT(self->errors);
1511 Py_VISIT(self->stream);
1512 return 0;
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001513}
1514
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +00001515static void
1516mbstreamreader_dealloc(MultibyteStreamReaderObject *self)
1517{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001518 PyObject_GC_UnTrack(self);
1519 ERROR_DECREF(self->errors);
1520 Py_XDECREF(self->stream);
1521 Py_TYPE(self)->tp_free(self);
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +00001522}
1523
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +00001524static PyTypeObject MultibyteStreamReader_Type = {
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001525 PyVarObject_HEAD_INIT(NULL, 0)
1526 "MultibyteStreamReader", /* tp_name */
1527 sizeof(MultibyteStreamReaderObject), /* tp_basicsize */
1528 0, /* tp_itemsize */
1529 /* methods */
1530 (destructor)mbstreamreader_dealloc, /* tp_dealloc */
1531 0, /* tp_print */
1532 0, /* tp_getattr */
1533 0, /* tp_setattr */
1534 0, /* tp_reserved */
1535 0, /* tp_repr */
1536 0, /* tp_as_number */
1537 0, /* tp_as_sequence */
1538 0, /* tp_as_mapping */
1539 0, /* tp_hash */
1540 0, /* tp_call */
1541 0, /* tp_str */
1542 PyObject_GenericGetAttr, /* tp_getattro */
1543 0, /* tp_setattro */
1544 0, /* tp_as_buffer */
1545 Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC
1546 | Py_TPFLAGS_BASETYPE, /* tp_flags */
1547 0, /* tp_doc */
1548 (traverseproc)mbstreamreader_traverse, /* tp_traverse */
1549 0, /* tp_clear */
1550 0, /* tp_richcompare */
1551 0, /* tp_weaklistoffset */
1552 0, /* tp_iter */
1553 0, /* tp_iterext */
1554 mbstreamreader_methods, /* tp_methods */
1555 mbstreamreader_members, /* tp_members */
1556 codecctx_getsets, /* tp_getset */
1557 0, /* tp_base */
1558 0, /* tp_dict */
1559 0, /* tp_descr_get */
1560 0, /* tp_descr_set */
1561 0, /* tp_dictoffset */
1562 mbstreamreader_init, /* tp_init */
1563 0, /* tp_alloc */
1564 mbstreamreader_new, /* tp_new */
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +00001565};
1566
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001567
1568/**
1569 * MultibyteStreamWriter object
1570 */
1571
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +00001572static int
1573mbstreamwriter_iwrite(MultibyteStreamWriterObject *self,
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001574 PyObject *unistr)
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +00001575{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001576 PyObject *str, *wr;
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +00001577
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001578 str = encoder_encode_stateful(STATEFUL_ECTX(self), unistr, 0);
1579 if (str == NULL)
1580 return -1;
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +00001581
Martin v. Löwisafe55bb2011-10-09 10:38:36 +02001582 wr = _PyObject_CallMethodId(self->stream, &PyId_write, "O", str);
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001583 Py_DECREF(str);
1584 if (wr == NULL)
1585 return -1;
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +00001586
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001587 Py_DECREF(wr);
1588 return 0;
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +00001589}
1590
1591static PyObject *
Thomas Wouters4d70c3d2006-06-08 14:42:34 +00001592mbstreamwriter_write(MultibyteStreamWriterObject *self, PyObject *strobj)
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +00001593{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001594 if (mbstreamwriter_iwrite(self, strobj))
1595 return NULL;
1596 else
1597 Py_RETURN_NONE;
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +00001598}
1599
1600static PyObject *
Thomas Wouters4d70c3d2006-06-08 14:42:34 +00001601mbstreamwriter_writelines(MultibyteStreamWriterObject *self, PyObject *lines)
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +00001602{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001603 PyObject *strobj;
1604 int i, r;
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +00001605
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001606 if (!PySequence_Check(lines)) {
1607 PyErr_SetString(PyExc_TypeError,
1608 "arg must be a sequence object");
1609 return NULL;
1610 }
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +00001611
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001612 for (i = 0; i < PySequence_Length(lines); i++) {
1613 /* length can be changed even within this loop */
1614 strobj = PySequence_GetItem(lines, i);
1615 if (strobj == NULL)
1616 return NULL;
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +00001617
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001618 r = mbstreamwriter_iwrite(self, strobj);
1619 Py_DECREF(strobj);
1620 if (r == -1)
1621 return NULL;
1622 }
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +00001623
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001624 Py_RETURN_NONE;
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +00001625}
1626
1627static PyObject *
1628mbstreamwriter_reset(MultibyteStreamWriterObject *self)
1629{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001630 PyObject *pwrt;
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +00001631
Victor Stinner579db162015-07-16 22:17:31 +02001632 if (!self->pending)
1633 Py_RETURN_NONE;
1634
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001635 pwrt = multibytecodec_encode(self->codec, &self->state,
Victor Stinnerd9491262013-04-14 02:06:32 +02001636 self->pending, NULL, self->errors,
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001637 MBENC_FLUSH | MBENC_RESET);
1638 /* some pending buffer can be truncated when UnicodeEncodeError is
1639 * raised on 'strict' mode. but, 'reset' method is designed to
1640 * reset the pending buffer or states so failed string sequence
1641 * ought to be missed */
Victor Stinnerd9491262013-04-14 02:06:32 +02001642 Py_CLEAR(self->pending);
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001643 if (pwrt == NULL)
1644 return NULL;
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +00001645
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001646 assert(PyBytes_Check(pwrt));
1647 if (PyBytes_Size(pwrt) > 0) {
1648 PyObject *wr;
Martin v. Löwisafe55bb2011-10-09 10:38:36 +02001649
1650 wr = _PyObject_CallMethodId(self->stream, &PyId_write, "O", pwrt);
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001651 if (wr == NULL) {
1652 Py_DECREF(pwrt);
1653 return NULL;
1654 }
1655 }
1656 Py_DECREF(pwrt);
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +00001657
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001658 Py_RETURN_NONE;
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001659}
1660
1661static PyObject *
1662mbstreamwriter_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
1663{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001664 MultibyteStreamWriterObject *self;
1665 PyObject *stream, *codec = NULL;
1666 char *errors = NULL;
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001667
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001668 if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|s:StreamWriter",
1669 streamkwarglist, &stream, &errors))
1670 return NULL;
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001671
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001672 self = (MultibyteStreamWriterObject *)type->tp_alloc(type, 0);
1673 if (self == NULL)
1674 return NULL;
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001675
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001676 codec = PyObject_GetAttrString((PyObject *)type, "codec");
1677 if (codec == NULL)
1678 goto errorexit;
1679 if (!MultibyteCodec_Check(codec)) {
1680 PyErr_SetString(PyExc_TypeError, "codec is unexpected type");
1681 goto errorexit;
1682 }
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001683
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001684 self->codec = ((MultibyteCodecObject *)codec)->codec;
1685 self->stream = stream;
1686 Py_INCREF(stream);
Victor Stinnerd9491262013-04-14 02:06:32 +02001687 self->pending = NULL;
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001688 self->errors = internal_error_callback(errors);
1689 if (self->errors == NULL)
1690 goto errorexit;
1691 if (self->codec->encinit != NULL &&
1692 self->codec->encinit(&self->state, self->codec->config) != 0)
1693 goto errorexit;
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001694
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001695 Py_DECREF(codec);
1696 return (PyObject *)self;
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001697
1698errorexit:
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001699 Py_XDECREF(self);
1700 Py_XDECREF(codec);
1701 return NULL;
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001702}
1703
1704static int
Hye-Shik Chang13247bf2006-04-21 16:21:44 +00001705mbstreamwriter_init(PyObject *self, PyObject *args, PyObject *kwds)
1706{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001707 return 0;
Hye-Shik Chang13247bf2006-04-21 16:21:44 +00001708}
1709
1710static int
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001711mbstreamwriter_traverse(MultibyteStreamWriterObject *self,
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001712 visitproc visit, void *arg)
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001713{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001714 if (ERROR_ISCUSTOM(self->errors))
1715 Py_VISIT(self->errors);
1716 Py_VISIT(self->stream);
1717 return 0;
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +00001718}
1719
1720static void
1721mbstreamwriter_dealloc(MultibyteStreamWriterObject *self)
1722{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001723 PyObject_GC_UnTrack(self);
1724 ERROR_DECREF(self->errors);
1725 Py_XDECREF(self->stream);
1726 Py_TYPE(self)->tp_free(self);
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +00001727}
1728
1729static struct PyMethodDef mbstreamwriter_methods[] = {
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001730 {"write", (PyCFunction)mbstreamwriter_write,
1731 METH_O, NULL},
1732 {"writelines", (PyCFunction)mbstreamwriter_writelines,
1733 METH_O, NULL},
1734 {"reset", (PyCFunction)mbstreamwriter_reset,
1735 METH_NOARGS, NULL},
1736 {NULL, NULL},
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +00001737};
1738
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001739static PyMemberDef mbstreamwriter_members[] = {
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001740 {"stream", T_OBJECT,
1741 offsetof(MultibyteStreamWriterObject, stream),
1742 READONLY, NULL},
1743 {NULL,}
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001744};
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +00001745
1746static PyTypeObject MultibyteStreamWriter_Type = {
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001747 PyVarObject_HEAD_INIT(NULL, 0)
1748 "MultibyteStreamWriter", /* tp_name */
1749 sizeof(MultibyteStreamWriterObject), /* tp_basicsize */
1750 0, /* tp_itemsize */
1751 /* methods */
1752 (destructor)mbstreamwriter_dealloc, /* tp_dealloc */
1753 0, /* tp_print */
1754 0, /* tp_getattr */
1755 0, /* tp_setattr */
1756 0, /* tp_reserved */
1757 0, /* tp_repr */
1758 0, /* tp_as_number */
1759 0, /* tp_as_sequence */
1760 0, /* tp_as_mapping */
1761 0, /* tp_hash */
1762 0, /* tp_call */
1763 0, /* tp_str */
1764 PyObject_GenericGetAttr, /* tp_getattro */
1765 0, /* tp_setattro */
1766 0, /* tp_as_buffer */
1767 Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC
1768 | Py_TPFLAGS_BASETYPE, /* tp_flags */
1769 0, /* tp_doc */
1770 (traverseproc)mbstreamwriter_traverse, /* tp_traverse */
1771 0, /* tp_clear */
1772 0, /* tp_richcompare */
1773 0, /* tp_weaklistoffset */
1774 0, /* tp_iter */
1775 0, /* tp_iterext */
1776 mbstreamwriter_methods, /* tp_methods */
1777 mbstreamwriter_members, /* tp_members */
1778 codecctx_getsets, /* tp_getset */
1779 0, /* tp_base */
1780 0, /* tp_dict */
1781 0, /* tp_descr_get */
1782 0, /* tp_descr_set */
1783 0, /* tp_dictoffset */
1784 mbstreamwriter_init, /* tp_init */
1785 0, /* tp_alloc */
1786 mbstreamwriter_new, /* tp_new */
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +00001787};
1788
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001789
1790/**
1791 * Exposed factory function
1792 */
1793
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +00001794static PyObject *
1795__create_codec(PyObject *ignore, PyObject *arg)
1796{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001797 MultibyteCodecObject *self;
1798 MultibyteCodec *codec;
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +00001799
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001800 if (!PyCapsule_IsValid(arg, PyMultibyteCodec_CAPSULE_NAME)) {
1801 PyErr_SetString(PyExc_ValueError, "argument type invalid");
1802 return NULL;
1803 }
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +00001804
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001805 codec = PyCapsule_GetPointer(arg, PyMultibyteCodec_CAPSULE_NAME);
1806 if (codec->codecinit != NULL && codec->codecinit(codec->config) != 0)
1807 return NULL;
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +00001808
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001809 self = PyObject_New(MultibyteCodecObject, &MultibyteCodec_Type);
1810 if (self == NULL)
1811 return NULL;
1812 self->codec = codec;
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +00001813
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001814 return (PyObject *)self;
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +00001815}
1816
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +00001817static struct PyMethodDef __methods[] = {
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001818 {"__create_codec", (PyCFunction)__create_codec, METH_O},
1819 {NULL, NULL},
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +00001820};
1821
Martin v. Löwis1a214512008-06-11 05:26:20 +00001822
1823static struct PyModuleDef _multibytecodecmodule = {
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001824 PyModuleDef_HEAD_INIT,
1825 "_multibytecodec",
1826 NULL,
1827 -1,
1828 __methods,
1829 NULL,
1830 NULL,
1831 NULL,
1832 NULL
Martin v. Löwis1a214512008-06-11 05:26:20 +00001833};
1834
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001835PyMODINIT_FUNC
Martin v. Löwis1a214512008-06-11 05:26:20 +00001836PyInit__multibytecodec(void)
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +00001837{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001838 int i;
1839 PyObject *m;
1840 PyTypeObject *typelist[] = {
1841 &MultibyteIncrementalEncoder_Type,
1842 &MultibyteIncrementalDecoder_Type,
1843 &MultibyteStreamReader_Type,
1844 &MultibyteStreamWriter_Type,
1845 NULL
1846 };
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001847
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001848 if (PyType_Ready(&MultibyteCodec_Type) < 0)
1849 return NULL;
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001850
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001851 m = PyModule_Create(&_multibytecodecmodule);
1852 if (m == NULL)
1853 return NULL;
Neal Norwitz058bde12005-09-21 06:44:25 +00001854
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001855 for (i = 0; typelist[i] != NULL; i++) {
1856 if (PyType_Ready(typelist[i]) < 0)
1857 return NULL;
1858 Py_INCREF(typelist[i]);
1859 PyModule_AddObject(m, typelist[i]->tp_name,
1860 (PyObject *)typelist[i]);
1861 }
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +00001862
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001863 if (PyErr_Occurred()) {
1864 Py_FatalError("can't initialize the _multibytecodec module");
1865 Py_DECREF(m);
1866 m = NULL;
1867 }
1868 return m;
Hye-Shik Chang3e2a3062004-01-17 14:29:29 +00001869}