blob: 69498c4b0c95de03e5b15f411febde26fa1d0d94 [file] [log] [blame]
Guido van Rossumfeee4b92000-03-10 22:57:27 +00001/* ------------------------------------------------------------------------
2
3 Python Codec Registry and support functions
4
5Written by Marc-Andre Lemburg (mal@lemburg.com).
6
Guido van Rossum16b1ad92000-08-03 16:24:25 +00007Copyright (c) Corporation for National Research Initiatives.
Guido van Rossumfeee4b92000-03-10 22:57:27 +00008
9 ------------------------------------------------------------------------ */
10
11#include "Python.h"
12#include <ctype.h>
13
Guido van Rossumfeee4b92000-03-10 22:57:27 +000014/* --- Codec Registry ----------------------------------------------------- */
15
16/* Import the standard encodings package which will register the first
Antoine Pitrouc83ea132010-05-09 14:46:46 +000017 codec search function.
Guido van Rossumfeee4b92000-03-10 22:57:27 +000018
19 This is done in a lazy way so that the Unicode implementation does
20 not downgrade startup time of scripts not needing it.
21
Guido van Rossumb95de4f2000-03-31 17:25:23 +000022 ImportErrors are silently ignored by this function. Only one try is
23 made.
Guido van Rossumfeee4b92000-03-10 22:57:27 +000024
25*/
26
Gustavo Niemeyer5ddd4c32003-03-19 00:35:36 +000027static int _PyCodecRegistry_Init(void); /* Forward */
Guido van Rossumfeee4b92000-03-10 22:57:27 +000028
Guido van Rossumfeee4b92000-03-10 22:57:27 +000029int PyCodec_Register(PyObject *search_function)
30{
Nicholas Bastine5662ae2004-03-24 22:22:12 +000031 PyInterpreterState *interp = PyThreadState_GET()->interp;
Gustavo Niemeyer5ddd4c32003-03-19 00:35:36 +000032 if (interp->codec_search_path == NULL && _PyCodecRegistry_Init())
Antoine Pitrouc83ea132010-05-09 14:46:46 +000033 goto onError;
Guido van Rossumfeee4b92000-03-10 22:57:27 +000034 if (search_function == NULL) {
Antoine Pitrouc83ea132010-05-09 14:46:46 +000035 PyErr_BadArgument();
36 goto onError;
Guido van Rossumfeee4b92000-03-10 22:57:27 +000037 }
38 if (!PyCallable_Check(search_function)) {
Antoine Pitrouc83ea132010-05-09 14:46:46 +000039 PyErr_SetString(PyExc_TypeError, "argument must be callable");
40 goto onError;
Guido van Rossumfeee4b92000-03-10 22:57:27 +000041 }
Gustavo Niemeyer5ddd4c32003-03-19 00:35:36 +000042 return PyList_Append(interp->codec_search_path, search_function);
Guido van Rossumb95de4f2000-03-31 17:25:23 +000043
44 onError:
45 return -1;
Guido van Rossumfeee4b92000-03-10 22:57:27 +000046}
47
Guido van Rossum9e896b32000-04-05 20:11:21 +000048/* Convert a string to a normalized Python string: all characters are
49 converted to lower case, spaces are replaced with underscores. */
50
Guido van Rossumfeee4b92000-03-10 22:57:27 +000051static
Guido van Rossum9e896b32000-04-05 20:11:21 +000052PyObject *normalizestring(const char *string)
Guido van Rossumfeee4b92000-03-10 22:57:27 +000053{
Guido van Rossum33831132000-06-29 14:50:15 +000054 register size_t i;
Guido van Rossum582acec2000-06-28 22:07:35 +000055 size_t len = strlen(string);
Guido van Rossumfeee4b92000-03-10 22:57:27 +000056 char *p;
57 PyObject *v;
Antoine Pitrouc83ea132010-05-09 14:46:46 +000058
Martin v. Löwisb1ed7fa2006-04-13 07:52:27 +000059 if (len > PY_SSIZE_T_MAX) {
Antoine Pitrouc83ea132010-05-09 14:46:46 +000060 PyErr_SetString(PyExc_OverflowError, "string is too large");
61 return NULL;
Martin v. Löwisb1ed7fa2006-04-13 07:52:27 +000062 }
Antoine Pitrouc83ea132010-05-09 14:46:46 +000063
Gregory P. Smithdd96db62008-06-09 04:58:54 +000064 v = PyString_FromStringAndSize(NULL, len);
Guido van Rossumfeee4b92000-03-10 22:57:27 +000065 if (v == NULL)
Antoine Pitrouc83ea132010-05-09 14:46:46 +000066 return NULL;
Gregory P. Smithdd96db62008-06-09 04:58:54 +000067 p = PyString_AS_STRING(v);
Guido van Rossum9e896b32000-04-05 20:11:21 +000068 for (i = 0; i < len; i++) {
69 register char ch = string[i];
70 if (ch == ' ')
71 ch = '-';
72 else
Antoine Pitrou4cfae022011-07-24 02:51:01 +020073 ch = Py_TOLOWER(Py_CHARMASK(ch));
Antoine Pitrouc83ea132010-05-09 14:46:46 +000074 p[i] = ch;
Guido van Rossum9e896b32000-04-05 20:11:21 +000075 }
Guido van Rossumfeee4b92000-03-10 22:57:27 +000076 return v;
77}
78
79/* Lookup the given encoding and return a tuple providing the codec
80 facilities.
81
82 The encoding string is looked up converted to all lower-case
83 characters. This makes encodings looked up through this mechanism
84 effectively case-insensitive.
85
Antoine Pitrouc83ea132010-05-09 14:46:46 +000086 If no codec is found, a LookupError is set and NULL returned.
Guido van Rossumb95de4f2000-03-31 17:25:23 +000087
88 As side effect, this tries to load the encodings package, if not
89 yet done. This is part of the lazy load strategy for the encodings
90 package.
91
92*/
Guido van Rossumfeee4b92000-03-10 22:57:27 +000093
94PyObject *_PyCodec_Lookup(const char *encoding)
95{
Gustavo Niemeyer5ddd4c32003-03-19 00:35:36 +000096 PyInterpreterState *interp;
Guido van Rossum5ba3c842000-03-24 20:52:23 +000097 PyObject *result, *args = NULL, *v;
Martin v. Löwis66851282006-04-22 11:40:03 +000098 Py_ssize_t i, len;
Guido van Rossumfeee4b92000-03-10 22:57:27 +000099
Fred Drake766de832000-05-09 19:55:59 +0000100 if (encoding == NULL) {
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000101 PyErr_BadArgument();
102 goto onError;
Fred Drake766de832000-05-09 19:55:59 +0000103 }
Gustavo Niemeyer5ddd4c32003-03-19 00:35:36 +0000104
Nicholas Bastine5662ae2004-03-24 22:22:12 +0000105 interp = PyThreadState_GET()->interp;
Gustavo Niemeyer5ddd4c32003-03-19 00:35:36 +0000106 if (interp->codec_search_path == NULL && _PyCodecRegistry_Init())
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000107 goto onError;
Guido van Rossumfeee4b92000-03-10 22:57:27 +0000108
Guido van Rossum9e896b32000-04-05 20:11:21 +0000109 /* Convert the encoding to a normalized Python string: all
Thomas Wouters7e474022000-07-16 12:04:32 +0000110 characters are converted to lower case, spaces and hyphens are
Guido van Rossum9e896b32000-04-05 20:11:21 +0000111 replaced with underscores. */
112 v = normalizestring(encoding);
Guido van Rossumfeee4b92000-03-10 22:57:27 +0000113 if (v == NULL)
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000114 goto onError;
Gregory P. Smithdd96db62008-06-09 04:58:54 +0000115 PyString_InternInPlace(&v);
Guido van Rossumfeee4b92000-03-10 22:57:27 +0000116
117 /* First, try to lookup the name in the registry dictionary */
Gustavo Niemeyer5ddd4c32003-03-19 00:35:36 +0000118 result = PyDict_GetItem(interp->codec_search_cache, v);
Guido van Rossumfeee4b92000-03-10 22:57:27 +0000119 if (result != NULL) {
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000120 Py_INCREF(result);
121 Py_DECREF(v);
122 return result;
Guido van Rossumfeee4b92000-03-10 22:57:27 +0000123 }
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000124
Guido van Rossumfeee4b92000-03-10 22:57:27 +0000125 /* Next, scan the search functions in order of registration */
Guido van Rossumfeee4b92000-03-10 22:57:27 +0000126 args = PyTuple_New(1);
127 if (args == NULL)
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000128 goto onError;
Guido van Rossumfeee4b92000-03-10 22:57:27 +0000129 PyTuple_SET_ITEM(args,0,v);
Guido van Rossum5ba3c842000-03-24 20:52:23 +0000130
Gustavo Niemeyer5ddd4c32003-03-19 00:35:36 +0000131 len = PyList_Size(interp->codec_search_path);
Guido van Rossum5ba3c842000-03-24 20:52:23 +0000132 if (len < 0)
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000133 goto onError;
Guido van Rossumb95de4f2000-03-31 17:25:23 +0000134 if (len == 0) {
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000135 PyErr_SetString(PyExc_LookupError,
136 "no codec search functions registered: "
137 "can't find encoding");
138 goto onError;
Guido van Rossumb95de4f2000-03-31 17:25:23 +0000139 }
Guido van Rossumfeee4b92000-03-10 22:57:27 +0000140
141 for (i = 0; i < len; i++) {
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000142 PyObject *func;
143
144 func = PyList_GetItem(interp->codec_search_path, i);
145 if (func == NULL)
146 goto onError;
147 result = PyEval_CallObject(func, args);
148 if (result == NULL)
149 goto onError;
150 if (result == Py_None) {
151 Py_DECREF(result);
152 continue;
153 }
154 if (!PyTuple_Check(result) || PyTuple_GET_SIZE(result) != 4) {
155 PyErr_SetString(PyExc_TypeError,
156 "codec search functions must return 4-tuples");
157 Py_DECREF(result);
158 goto onError;
159 }
160 break;
Guido van Rossumfeee4b92000-03-10 22:57:27 +0000161 }
162 if (i == len) {
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000163 /* XXX Perhaps we should cache misses too ? */
164 PyErr_Format(PyExc_LookupError,
Martin v. Löwiseb42b022002-09-26 16:01:24 +0000165 "unknown encoding: %s", encoding);
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000166 goto onError;
Guido van Rossumfeee4b92000-03-10 22:57:27 +0000167 }
168
169 /* Cache and return the result */
Gustavo Niemeyer5ddd4c32003-03-19 00:35:36 +0000170 PyDict_SetItem(interp->codec_search_cache, v, result);
Guido van Rossumfeee4b92000-03-10 22:57:27 +0000171 Py_DECREF(args);
172 return result;
173
174 onError:
175 Py_XDECREF(args);
176 return NULL;
177}
178
179static
180PyObject *args_tuple(PyObject *object,
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000181 const char *errors)
Guido van Rossumfeee4b92000-03-10 22:57:27 +0000182{
183 PyObject *args;
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000184
Guido van Rossumfeee4b92000-03-10 22:57:27 +0000185 args = PyTuple_New(1 + (errors != NULL));
186 if (args == NULL)
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000187 return NULL;
Guido van Rossumfeee4b92000-03-10 22:57:27 +0000188 Py_INCREF(object);
189 PyTuple_SET_ITEM(args,0,object);
190 if (errors) {
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000191 PyObject *v;
192
193 v = PyString_FromString(errors);
194 if (v == NULL) {
195 Py_DECREF(args);
196 return NULL;
197 }
198 PyTuple_SET_ITEM(args, 1, v);
Guido van Rossumfeee4b92000-03-10 22:57:27 +0000199 }
200 return args;
201}
202
Walter Dörwaldd53850a2006-03-16 21:46:40 +0000203/* Helper function to get a codec item */
Guido van Rossumfeee4b92000-03-10 22:57:27 +0000204
205static
Walter Dörwaldd53850a2006-03-16 21:46:40 +0000206PyObject *codec_getitem(const char *encoding, int index)
Guido van Rossumfeee4b92000-03-10 22:57:27 +0000207{
Walter Dörwaldd53850a2006-03-16 21:46:40 +0000208 PyObject *codecs;
209 PyObject *v;
Guido van Rossumfeee4b92000-03-10 22:57:27 +0000210
Walter Dörwaldd53850a2006-03-16 21:46:40 +0000211 codecs = _PyCodec_Lookup(encoding);
212 if (codecs == NULL)
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000213 return NULL;
Walter Dörwaldd53850a2006-03-16 21:46:40 +0000214 v = PyTuple_GET_ITEM(codecs, index);
215 Py_DECREF(codecs);
216 Py_INCREF(v);
217 return v;
218}
219
220/* Helper function to create an incremental codec. */
221
222static
223PyObject *codec_getincrementalcodec(const char *encoding,
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000224 const char *errors,
225 const char *attrname)
Walter Dörwaldd53850a2006-03-16 21:46:40 +0000226{
227 PyObject *codecs, *ret, *inccodec;
228
229 codecs = _PyCodec_Lookup(encoding);
230 if (codecs == NULL)
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000231 return NULL;
Walter Dörwaldd53850a2006-03-16 21:46:40 +0000232 inccodec = PyObject_GetAttrString(codecs, attrname);
Walter Dörwaldba8e1802006-03-18 14:05:43 +0000233 Py_DECREF(codecs);
234 if (inccodec == NULL)
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000235 return NULL;
Walter Dörwaldd53850a2006-03-16 21:46:40 +0000236 if (errors)
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000237 ret = PyObject_CallFunction(inccodec, "s", errors);
Walter Dörwaldd53850a2006-03-16 21:46:40 +0000238 else
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000239 ret = PyObject_CallFunction(inccodec, NULL);
Walter Dörwaldd53850a2006-03-16 21:46:40 +0000240 Py_DECREF(inccodec);
Walter Dörwaldd53850a2006-03-16 21:46:40 +0000241 return ret;
242}
243
244/* Helper function to create a stream codec. */
245
246static
247PyObject *codec_getstreamcodec(const char *encoding,
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000248 PyObject *stream,
249 const char *errors,
250 const int index)
Walter Dörwaldd53850a2006-03-16 21:46:40 +0000251{
Hye-Shik Change6a1cb92006-06-23 21:16:18 +0000252 PyObject *codecs, *streamcodec, *codeccls;
Walter Dörwaldd53850a2006-03-16 21:46:40 +0000253
254 codecs = _PyCodec_Lookup(encoding);
255 if (codecs == NULL)
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000256 return NULL;
Walter Dörwaldd53850a2006-03-16 21:46:40 +0000257
Hye-Shik Change6a1cb92006-06-23 21:16:18 +0000258 codeccls = PyTuple_GET_ITEM(codecs, index);
259 if (errors != NULL)
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000260 streamcodec = PyObject_CallFunction(codeccls, "Os", stream, errors);
Hye-Shik Change6a1cb92006-06-23 21:16:18 +0000261 else
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000262 streamcodec = PyObject_CallFunction(codeccls, "O", stream);
Walter Dörwaldd53850a2006-03-16 21:46:40 +0000263 Py_DECREF(codecs);
264 return streamcodec;
Guido van Rossumfeee4b92000-03-10 22:57:27 +0000265}
266
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000267/* Convenience APIs to query the Codec registry.
268
Guido van Rossumfeee4b92000-03-10 22:57:27 +0000269 All APIs return a codec object with incremented refcount.
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000270
Guido van Rossumfeee4b92000-03-10 22:57:27 +0000271 */
272
273PyObject *PyCodec_Encoder(const char *encoding)
274{
Walter Dörwaldd53850a2006-03-16 21:46:40 +0000275 return codec_getitem(encoding, 0);
Guido van Rossumfeee4b92000-03-10 22:57:27 +0000276}
277
278PyObject *PyCodec_Decoder(const char *encoding)
279{
Walter Dörwaldd53850a2006-03-16 21:46:40 +0000280 return codec_getitem(encoding, 1);
Guido van Rossumfeee4b92000-03-10 22:57:27 +0000281}
282
Walter Dörwaldabb02e52006-03-15 11:35:15 +0000283PyObject *PyCodec_IncrementalEncoder(const char *encoding,
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000284 const char *errors)
Walter Dörwaldabb02e52006-03-15 11:35:15 +0000285{
Walter Dörwaldd53850a2006-03-16 21:46:40 +0000286 return codec_getincrementalcodec(encoding, errors, "incrementalencoder");
Walter Dörwaldabb02e52006-03-15 11:35:15 +0000287}
288
289PyObject *PyCodec_IncrementalDecoder(const char *encoding,
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000290 const char *errors)
Walter Dörwaldabb02e52006-03-15 11:35:15 +0000291{
Walter Dörwaldd53850a2006-03-16 21:46:40 +0000292 return codec_getincrementalcodec(encoding, errors, "incrementaldecoder");
Walter Dörwaldabb02e52006-03-15 11:35:15 +0000293}
294
Guido van Rossumfeee4b92000-03-10 22:57:27 +0000295PyObject *PyCodec_StreamReader(const char *encoding,
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000296 PyObject *stream,
297 const char *errors)
Guido van Rossumfeee4b92000-03-10 22:57:27 +0000298{
Walter Dörwaldd53850a2006-03-16 21:46:40 +0000299 return codec_getstreamcodec(encoding, stream, errors, 2);
Guido van Rossumfeee4b92000-03-10 22:57:27 +0000300}
301
302PyObject *PyCodec_StreamWriter(const char *encoding,
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000303 PyObject *stream,
304 const char *errors)
Guido van Rossumfeee4b92000-03-10 22:57:27 +0000305{
Walter Dörwaldd53850a2006-03-16 21:46:40 +0000306 return codec_getstreamcodec(encoding, stream, errors, 3);
Guido van Rossumfeee4b92000-03-10 22:57:27 +0000307}
308
309/* Encode an object (e.g. an Unicode object) using the given encoding
310 and return the resulting encoded object (usually a Python string).
311
312 errors is passed to the encoder factory as argument if non-NULL. */
313
314PyObject *PyCodec_Encode(PyObject *object,
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000315 const char *encoding,
316 const char *errors)
Guido van Rossumfeee4b92000-03-10 22:57:27 +0000317{
318 PyObject *encoder = NULL;
Neal Norwitz3715c3e2005-11-24 22:09:18 +0000319 PyObject *args = NULL, *result = NULL;
Guido van Rossumfeee4b92000-03-10 22:57:27 +0000320 PyObject *v;
321
322 encoder = PyCodec_Encoder(encoding);
323 if (encoder == NULL)
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000324 goto onError;
Guido van Rossumfeee4b92000-03-10 22:57:27 +0000325
326 args = args_tuple(object, errors);
327 if (args == NULL)
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000328 goto onError;
329
Guido van Rossumfeee4b92000-03-10 22:57:27 +0000330 result = PyEval_CallObject(encoder,args);
331 if (result == NULL)
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000332 goto onError;
Guido van Rossumfeee4b92000-03-10 22:57:27 +0000333
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000334 if (!PyTuple_Check(result) ||
335 PyTuple_GET_SIZE(result) != 2) {
336 PyErr_SetString(PyExc_TypeError,
337 "encoder must return a tuple (object,integer)");
338 goto onError;
Guido van Rossumfeee4b92000-03-10 22:57:27 +0000339 }
340 v = PyTuple_GET_ITEM(result,0);
341 Py_INCREF(v);
342 /* We don't check or use the second (integer) entry. */
343
344 Py_DECREF(args);
345 Py_DECREF(encoder);
346 Py_DECREF(result);
347 return v;
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000348
Guido van Rossumfeee4b92000-03-10 22:57:27 +0000349 onError:
Neal Norwitz3715c3e2005-11-24 22:09:18 +0000350 Py_XDECREF(result);
Guido van Rossumfeee4b92000-03-10 22:57:27 +0000351 Py_XDECREF(args);
352 Py_XDECREF(encoder);
353 return NULL;
354}
355
356/* Decode an object (usually a Python string) using the given encoding
357 and return an equivalent object (e.g. an Unicode object).
358
359 errors is passed to the decoder factory as argument if non-NULL. */
360
361PyObject *PyCodec_Decode(PyObject *object,
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000362 const char *encoding,
363 const char *errors)
Guido van Rossumfeee4b92000-03-10 22:57:27 +0000364{
365 PyObject *decoder = NULL;
366 PyObject *args = NULL, *result = NULL;
367 PyObject *v;
368
369 decoder = PyCodec_Decoder(encoding);
370 if (decoder == NULL)
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000371 goto onError;
Guido van Rossumfeee4b92000-03-10 22:57:27 +0000372
373 args = args_tuple(object, errors);
374 if (args == NULL)
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000375 goto onError;
376
Guido van Rossumfeee4b92000-03-10 22:57:27 +0000377 result = PyEval_CallObject(decoder,args);
378 if (result == NULL)
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000379 goto onError;
380 if (!PyTuple_Check(result) ||
381 PyTuple_GET_SIZE(result) != 2) {
382 PyErr_SetString(PyExc_TypeError,
383 "decoder must return a tuple (object,integer)");
384 goto onError;
Guido van Rossumfeee4b92000-03-10 22:57:27 +0000385 }
386 v = PyTuple_GET_ITEM(result,0);
387 Py_INCREF(v);
388 /* We don't check or use the second (integer) entry. */
389
390 Py_DECREF(args);
391 Py_DECREF(decoder);
392 Py_DECREF(result);
393 return v;
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000394
Guido van Rossumfeee4b92000-03-10 22:57:27 +0000395 onError:
396 Py_XDECREF(args);
397 Py_XDECREF(decoder);
398 Py_XDECREF(result);
399 return NULL;
400}
401
Walter Dörwald3aeb6322002-09-02 13:14:32 +0000402/* Register the error handling callback function error under the name
403 name. This function will be called by the codec when it encounters
404 an unencodable characters/undecodable bytes and doesn't know the
405 callback name, when name is specified as the error parameter
406 in the call to the encode/decode function.
407 Return 0 on success, -1 on error */
408int PyCodec_RegisterError(const char *name, PyObject *error)
409{
Nicholas Bastine5662ae2004-03-24 22:22:12 +0000410 PyInterpreterState *interp = PyThreadState_GET()->interp;
Gustavo Niemeyer5ddd4c32003-03-19 00:35:36 +0000411 if (interp->codec_search_path == NULL && _PyCodecRegistry_Init())
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000412 return -1;
Walter Dörwald3aeb6322002-09-02 13:14:32 +0000413 if (!PyCallable_Check(error)) {
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000414 PyErr_SetString(PyExc_TypeError, "handler must be callable");
415 return -1;
Walter Dörwald3aeb6322002-09-02 13:14:32 +0000416 }
Gustavo Niemeyer5ddd4c32003-03-19 00:35:36 +0000417 return PyDict_SetItemString(interp->codec_error_registry,
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000418 (char *)name, error);
Walter Dörwald3aeb6322002-09-02 13:14:32 +0000419}
420
421/* Lookup the error handling callback function registered under the
422 name error. As a special case NULL can be passed, in which case
423 the error handling callback for strict encoding will be returned. */
424PyObject *PyCodec_LookupError(const char *name)
425{
426 PyObject *handler = NULL;
427
Nicholas Bastine5662ae2004-03-24 22:22:12 +0000428 PyInterpreterState *interp = PyThreadState_GET()->interp;
Gustavo Niemeyer5ddd4c32003-03-19 00:35:36 +0000429 if (interp->codec_search_path == NULL && _PyCodecRegistry_Init())
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000430 return NULL;
Gustavo Niemeyer5ddd4c32003-03-19 00:35:36 +0000431
Walter Dörwald3aeb6322002-09-02 13:14:32 +0000432 if (name==NULL)
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000433 name = "strict";
Gustavo Niemeyer5ddd4c32003-03-19 00:35:36 +0000434 handler = PyDict_GetItemString(interp->codec_error_registry, (char *)name);
Walter Dörwald3aeb6322002-09-02 13:14:32 +0000435 if (!handler)
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000436 PyErr_Format(PyExc_LookupError, "unknown error handler name '%.400s'", name);
Walter Dörwald3aeb6322002-09-02 13:14:32 +0000437 else
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000438 Py_INCREF(handler);
Walter Dörwald3aeb6322002-09-02 13:14:32 +0000439 return handler;
440}
441
442static void wrong_exception_type(PyObject *exc)
443{
444 PyObject *type = PyObject_GetAttrString(exc, "__class__");
445 if (type != NULL) {
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000446 PyObject *name = PyObject_GetAttrString(type, "__name__");
447 Py_DECREF(type);
448 if (name != NULL) {
449 PyObject *string = PyObject_Str(name);
450 Py_DECREF(name);
451 if (string != NULL) {
452 PyErr_Format(PyExc_TypeError,
453 "don't know how to handle %.400s in error callback",
454 PyString_AS_STRING(string));
455 Py_DECREF(string);
456 }
457 }
Walter Dörwald3aeb6322002-09-02 13:14:32 +0000458 }
459}
460
461PyObject *PyCodec_StrictErrors(PyObject *exc)
462{
Brett Cannonbf364092006-03-01 04:25:17 +0000463 if (PyExceptionInstance_Check(exc))
464 PyErr_SetObject(PyExceptionInstance_Class(exc), exc);
Walter Dörwald3aeb6322002-09-02 13:14:32 +0000465 else
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000466 PyErr_SetString(PyExc_TypeError, "codec must pass exception instance");
Walter Dörwald3aeb6322002-09-02 13:14:32 +0000467 return NULL;
468}
469
470
Walter Dörwaldbf73db82002-11-21 20:08:33 +0000471#ifdef Py_USING_UNICODE
Walter Dörwald3aeb6322002-09-02 13:14:32 +0000472PyObject *PyCodec_IgnoreErrors(PyObject *exc)
473{
Martin v. Löwis18e16552006-02-15 17:27:45 +0000474 Py_ssize_t end;
Walter Dörwald3aeb6322002-09-02 13:14:32 +0000475 if (PyObject_IsInstance(exc, PyExc_UnicodeEncodeError)) {
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000476 if (PyUnicodeEncodeError_GetEnd(exc, &end))
477 return NULL;
Walter Dörwald3aeb6322002-09-02 13:14:32 +0000478 }
479 else if (PyObject_IsInstance(exc, PyExc_UnicodeDecodeError)) {
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000480 if (PyUnicodeDecodeError_GetEnd(exc, &end))
481 return NULL;
Walter Dörwald3aeb6322002-09-02 13:14:32 +0000482 }
483 else if (PyObject_IsInstance(exc, PyExc_UnicodeTranslateError)) {
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000484 if (PyUnicodeTranslateError_GetEnd(exc, &end))
485 return NULL;
Walter Dörwald3aeb6322002-09-02 13:14:32 +0000486 }
487 else {
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000488 wrong_exception_type(exc);
489 return NULL;
Walter Dörwald3aeb6322002-09-02 13:14:32 +0000490 }
491 /* ouch: passing NULL, 0, pos gives None instead of u'' */
Martin v. Löwis18e16552006-02-15 17:27:45 +0000492 return Py_BuildValue("(u#n)", &end, 0, end);
Walter Dörwald3aeb6322002-09-02 13:14:32 +0000493}
494
495
496PyObject *PyCodec_ReplaceErrors(PyObject *exc)
497{
498 PyObject *restuple;
Martin v. Löwis18e16552006-02-15 17:27:45 +0000499 Py_ssize_t start;
500 Py_ssize_t end;
501 Py_ssize_t i;
Walter Dörwald3aeb6322002-09-02 13:14:32 +0000502
503 if (PyObject_IsInstance(exc, PyExc_UnicodeEncodeError)) {
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000504 PyObject *res;
505 Py_UNICODE *p;
506 if (PyUnicodeEncodeError_GetStart(exc, &start))
507 return NULL;
508 if (PyUnicodeEncodeError_GetEnd(exc, &end))
509 return NULL;
510 res = PyUnicode_FromUnicode(NULL, end-start);
511 if (res == NULL)
512 return NULL;
513 for (p = PyUnicode_AS_UNICODE(res), i = start;
514 i<end; ++p, ++i)
515 *p = '?';
516 restuple = Py_BuildValue("(On)", res, end);
517 Py_DECREF(res);
518 return restuple;
Walter Dörwald3aeb6322002-09-02 13:14:32 +0000519 }
520 else if (PyObject_IsInstance(exc, PyExc_UnicodeDecodeError)) {
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000521 Py_UNICODE res = Py_UNICODE_REPLACEMENT_CHARACTER;
522 if (PyUnicodeDecodeError_GetEnd(exc, &end))
523 return NULL;
Serhiy Storchakaa9885e92013-08-20 20:08:53 +0300524 return Py_BuildValue("(u#n)", &res, (Py_ssize_t)1, end);
Walter Dörwald3aeb6322002-09-02 13:14:32 +0000525 }
526 else if (PyObject_IsInstance(exc, PyExc_UnicodeTranslateError)) {
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000527 PyObject *res;
528 Py_UNICODE *p;
529 if (PyUnicodeTranslateError_GetStart(exc, &start))
530 return NULL;
531 if (PyUnicodeTranslateError_GetEnd(exc, &end))
532 return NULL;
533 res = PyUnicode_FromUnicode(NULL, end-start);
534 if (res == NULL)
535 return NULL;
536 for (p = PyUnicode_AS_UNICODE(res), i = start;
537 i<end; ++p, ++i)
538 *p = Py_UNICODE_REPLACEMENT_CHARACTER;
539 restuple = Py_BuildValue("(On)", res, end);
540 Py_DECREF(res);
541 return restuple;
Walter Dörwald3aeb6322002-09-02 13:14:32 +0000542 }
543 else {
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000544 wrong_exception_type(exc);
545 return NULL;
Walter Dörwald3aeb6322002-09-02 13:14:32 +0000546 }
547}
548
549PyObject *PyCodec_XMLCharRefReplaceErrors(PyObject *exc)
550{
551 if (PyObject_IsInstance(exc, PyExc_UnicodeEncodeError)) {
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000552 PyObject *restuple;
553 PyObject *object;
554 Py_ssize_t start;
555 Py_ssize_t end;
556 PyObject *res;
557 Py_UNICODE *p;
558 Py_UNICODE *startp;
Serhiy Storchakae822b032013-08-06 16:56:26 +0300559 Py_UNICODE *e;
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000560 Py_UNICODE *outp;
561 int ressize;
562 if (PyUnicodeEncodeError_GetStart(exc, &start))
563 return NULL;
564 if (PyUnicodeEncodeError_GetEnd(exc, &end))
565 return NULL;
566 if (!(object = PyUnicodeEncodeError_GetObject(exc)))
567 return NULL;
568 startp = PyUnicode_AS_UNICODE(object);
Serhiy Storchakae822b032013-08-06 16:56:26 +0300569 e = startp + end;
570 for (p = startp+start, ressize = 0; p < e;) {
571 Py_UCS4 ch = *p++;
Hye-Shik Chang7db07e62003-12-29 01:36:01 +0000572#ifndef Py_UNICODE_WIDE
Serhiy Storchakae822b032013-08-06 16:56:26 +0300573 if ((0xD800 <= ch && ch <= 0xDBFF) &&
574 (p < e) &&
575 (0xDC00 <= *p && *p <= 0xDFFF)) {
576 ch = ((((ch & 0x03FF) << 10) |
577 ((Py_UCS4)*p++ & 0x03FF)) + 0x10000);
578 }
579#endif
580 if (ch < 10)
581 ressize += 2+1+1;
582 else if (ch < 100)
583 ressize += 2+2+1;
584 else if (ch < 1000)
585 ressize += 2+3+1;
586 else if (ch < 10000)
587 ressize += 2+4+1;
588 else if (ch < 100000)
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000589 ressize += 2+5+1;
Serhiy Storchakae822b032013-08-06 16:56:26 +0300590 else if (ch < 1000000)
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000591 ressize += 2+6+1;
592 else
593 ressize += 2+7+1;
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000594 }
595 /* allocate replacement */
596 res = PyUnicode_FromUnicode(NULL, ressize);
597 if (res == NULL) {
598 Py_DECREF(object);
599 return NULL;
600 }
601 /* generate replacement */
Serhiy Storchakae822b032013-08-06 16:56:26 +0300602 for (p = startp+start, outp = PyUnicode_AS_UNICODE(res); p < e;) {
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000603 int digits;
604 int base;
Serhiy Storchakae822b032013-08-06 16:56:26 +0300605 Py_UCS4 ch = *p++;
606#ifndef Py_UNICODE_WIDE
607 if ((0xD800 <= ch && ch <= 0xDBFF) &&
608 (p < startp+end) &&
609 (0xDC00 <= *p && *p <= 0xDFFF)) {
610 ch = ((((ch & 0x03FF) << 10) |
611 ((Py_UCS4)*p++ & 0x03FF)) + 0x10000);
612 }
613#endif
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000614 *outp++ = '&';
615 *outp++ = '#';
Serhiy Storchakae822b032013-08-06 16:56:26 +0300616 if (ch < 10) {
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000617 digits = 1;
618 base = 1;
619 }
Serhiy Storchakae822b032013-08-06 16:56:26 +0300620 else if (ch < 100) {
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000621 digits = 2;
622 base = 10;
623 }
Serhiy Storchakae822b032013-08-06 16:56:26 +0300624 else if (ch < 1000) {
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000625 digits = 3;
626 base = 100;
627 }
Serhiy Storchakae822b032013-08-06 16:56:26 +0300628 else if (ch < 10000) {
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000629 digits = 4;
630 base = 1000;
631 }
Serhiy Storchakae822b032013-08-06 16:56:26 +0300632 else if (ch < 100000) {
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000633 digits = 5;
634 base = 10000;
635 }
Serhiy Storchakae822b032013-08-06 16:56:26 +0300636 else if (ch < 1000000) {
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000637 digits = 6;
638 base = 100000;
639 }
640 else {
641 digits = 7;
642 base = 1000000;
643 }
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000644 while (digits-->0) {
Serhiy Storchakae822b032013-08-06 16:56:26 +0300645 *outp++ = '0' + ch/base;
646 ch %= base;
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000647 base /= 10;
648 }
649 *outp++ = ';';
650 }
651 restuple = Py_BuildValue("(On)", res, end);
652 Py_DECREF(res);
653 Py_DECREF(object);
654 return restuple;
Walter Dörwald3aeb6322002-09-02 13:14:32 +0000655 }
656 else {
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000657 wrong_exception_type(exc);
658 return NULL;
Walter Dörwald3aeb6322002-09-02 13:14:32 +0000659 }
660}
661
662static Py_UNICODE hexdigits[] = {
663 '0', '1', '2', '3', '4', '5', '6', '7',
664 '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'
665};
666
667PyObject *PyCodec_BackslashReplaceErrors(PyObject *exc)
668{
669 if (PyObject_IsInstance(exc, PyExc_UnicodeEncodeError)) {
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000670 PyObject *restuple;
671 PyObject *object;
672 Py_ssize_t start;
673 Py_ssize_t end;
674 PyObject *res;
675 Py_UNICODE *p;
676 Py_UNICODE *startp;
677 Py_UNICODE *outp;
678 int ressize;
679 if (PyUnicodeEncodeError_GetStart(exc, &start))
680 return NULL;
681 if (PyUnicodeEncodeError_GetEnd(exc, &end))
682 return NULL;
683 if (!(object = PyUnicodeEncodeError_GetObject(exc)))
684 return NULL;
685 startp = PyUnicode_AS_UNICODE(object);
686 for (p = startp+start, ressize = 0; p < startp+end; ++p) {
Hye-Shik Chang7db07e62003-12-29 01:36:01 +0000687#ifdef Py_UNICODE_WIDE
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000688 if (*p >= 0x00010000)
689 ressize += 1+1+8;
690 else
Hye-Shik Chang7db07e62003-12-29 01:36:01 +0000691#endif
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000692 if (*p >= 0x100) {
693 ressize += 1+1+4;
694 }
695 else
696 ressize += 1+1+2;
697 }
698 res = PyUnicode_FromUnicode(NULL, ressize);
699 if (res==NULL)
700 return NULL;
701 for (p = startp+start, outp = PyUnicode_AS_UNICODE(res);
702 p < startp+end; ++p) {
703 Py_UNICODE c = *p;
704 *outp++ = '\\';
Hye-Shik Chang7db07e62003-12-29 01:36:01 +0000705#ifdef Py_UNICODE_WIDE
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000706 if (c >= 0x00010000) {
707 *outp++ = 'U';
708 *outp++ = hexdigits[(c>>28)&0xf];
709 *outp++ = hexdigits[(c>>24)&0xf];
710 *outp++ = hexdigits[(c>>20)&0xf];
711 *outp++ = hexdigits[(c>>16)&0xf];
712 *outp++ = hexdigits[(c>>12)&0xf];
713 *outp++ = hexdigits[(c>>8)&0xf];
714 }
715 else
Hye-Shik Chang7db07e62003-12-29 01:36:01 +0000716#endif
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000717 if (c >= 0x100) {
718 *outp++ = 'u';
719 *outp++ = hexdigits[(c>>12)&0xf];
720 *outp++ = hexdigits[(c>>8)&0xf];
721 }
722 else
723 *outp++ = 'x';
724 *outp++ = hexdigits[(c>>4)&0xf];
725 *outp++ = hexdigits[c&0xf];
726 }
Walter Dörwald3aeb6322002-09-02 13:14:32 +0000727
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000728 restuple = Py_BuildValue("(On)", res, end);
729 Py_DECREF(res);
730 Py_DECREF(object);
731 return restuple;
Walter Dörwald3aeb6322002-09-02 13:14:32 +0000732 }
733 else {
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000734 wrong_exception_type(exc);
735 return NULL;
Walter Dörwald3aeb6322002-09-02 13:14:32 +0000736 }
737}
Walter Dörwaldbf73db82002-11-21 20:08:33 +0000738#endif
Walter Dörwald3aeb6322002-09-02 13:14:32 +0000739
740static PyObject *strict_errors(PyObject *self, PyObject *exc)
741{
742 return PyCodec_StrictErrors(exc);
743}
744
745
Walter Dörwaldbf73db82002-11-21 20:08:33 +0000746#ifdef Py_USING_UNICODE
Walter Dörwald3aeb6322002-09-02 13:14:32 +0000747static PyObject *ignore_errors(PyObject *self, PyObject *exc)
748{
749 return PyCodec_IgnoreErrors(exc);
750}
751
752
753static PyObject *replace_errors(PyObject *self, PyObject *exc)
754{
755 return PyCodec_ReplaceErrors(exc);
756}
757
758
759static PyObject *xmlcharrefreplace_errors(PyObject *self, PyObject *exc)
760{
761 return PyCodec_XMLCharRefReplaceErrors(exc);
762}
763
764
765static PyObject *backslashreplace_errors(PyObject *self, PyObject *exc)
766{
767 return PyCodec_BackslashReplaceErrors(exc);
768}
Walter Dörwaldbf73db82002-11-21 20:08:33 +0000769#endif
Walter Dörwald3aeb6322002-09-02 13:14:32 +0000770
Gustavo Niemeyer5ddd4c32003-03-19 00:35:36 +0000771static int _PyCodecRegistry_Init(void)
Guido van Rossumfeee4b92000-03-10 22:57:27 +0000772{
Walter Dörwald3aeb6322002-09-02 13:14:32 +0000773 static struct {
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000774 char *name;
775 PyMethodDef def;
Walter Dörwald3aeb6322002-09-02 13:14:32 +0000776 } methods[] =
777 {
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000778 {
779 "strict",
780 {
781 "strict_errors",
782 strict_errors,
783 METH_O,
784 PyDoc_STR("Implements the 'strict' error handling, which "
785 "raises a UnicodeError on coding errors.")
786 }
787 },
Walter Dörwaldbf73db82002-11-21 20:08:33 +0000788#ifdef Py_USING_UNICODE
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000789 {
790 "ignore",
791 {
792 "ignore_errors",
793 ignore_errors,
794 METH_O,
795 PyDoc_STR("Implements the 'ignore' error handling, which "
796 "ignores malformed data and continues.")
797 }
798 },
799 {
800 "replace",
801 {
802 "replace_errors",
803 replace_errors,
804 METH_O,
805 PyDoc_STR("Implements the 'replace' error handling, which "
806 "replaces malformed data with a replacement marker.")
807 }
808 },
809 {
810 "xmlcharrefreplace",
811 {
812 "xmlcharrefreplace_errors",
813 xmlcharrefreplace_errors,
814 METH_O,
815 PyDoc_STR("Implements the 'xmlcharrefreplace' error handling, "
816 "which replaces an unencodable character with the "
817 "appropriate XML character reference.")
818 }
819 },
820 {
821 "backslashreplace",
822 {
823 "backslashreplace_errors",
824 backslashreplace_errors,
825 METH_O,
826 PyDoc_STR("Implements the 'backslashreplace' error handling, "
827 "which replaces an unencodable character with a "
828 "backslashed escape sequence.")
829 }
830 }
Walter Dörwaldbf73db82002-11-21 20:08:33 +0000831#endif
Walter Dörwald3aeb6322002-09-02 13:14:32 +0000832 };
Walter Dörwald3aeb6322002-09-02 13:14:32 +0000833
Nicholas Bastine5662ae2004-03-24 22:22:12 +0000834 PyInterpreterState *interp = PyThreadState_GET()->interp;
Gustavo Niemeyer5ddd4c32003-03-19 00:35:36 +0000835 PyObject *mod;
Neal Norwitz739a8f82004-07-08 01:55:58 +0000836 unsigned i;
Gustavo Niemeyer5ddd4c32003-03-19 00:35:36 +0000837
838 if (interp->codec_search_path != NULL)
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000839 return 0;
Gustavo Niemeyer5ddd4c32003-03-19 00:35:36 +0000840
841 interp->codec_search_path = PyList_New(0);
842 interp->codec_search_cache = PyDict_New();
843 interp->codec_error_registry = PyDict_New();
844
845 if (interp->codec_error_registry) {
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000846 for (i = 0; i < sizeof(methods)/sizeof(methods[0]); ++i) {
847 PyObject *func = PyCFunction_New(&methods[i].def, NULL);
848 int res;
849 if (!func)
850 Py_FatalError("can't initialize codec error registry");
851 res = PyCodec_RegisterError(methods[i].name, func);
852 Py_DECREF(func);
853 if (res)
854 Py_FatalError("can't initialize codec error registry");
855 }
Walter Dörwald3aeb6322002-09-02 13:14:32 +0000856 }
Guido van Rossumfeee4b92000-03-10 22:57:27 +0000857
Gustavo Niemeyer5ddd4c32003-03-19 00:35:36 +0000858 if (interp->codec_search_path == NULL ||
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000859 interp->codec_search_cache == NULL ||
860 interp->codec_error_registry == NULL)
861 Py_FatalError("can't initialize codec registry");
Gustavo Niemeyer5ddd4c32003-03-19 00:35:36 +0000862
Thomas Woutersf7f438b2006-02-28 16:09:29 +0000863 mod = PyImport_ImportModuleLevel("encodings", NULL, NULL, NULL, 0);
Gustavo Niemeyer5ddd4c32003-03-19 00:35:36 +0000864 if (mod == NULL) {
Antoine Pitrouc83ea132010-05-09 14:46:46 +0000865 if (PyErr_ExceptionMatches(PyExc_ImportError)) {
866 /* Ignore ImportErrors... this is done so that
867 distributions can disable the encodings package. Note
868 that other errors are not masked, e.g. SystemErrors
869 raised to inform the user of an error in the Python
870 configuration are still reported back to the user. */
871 PyErr_Clear();
872 return 0;
873 }
874 return -1;
Gustavo Niemeyer5ddd4c32003-03-19 00:35:36 +0000875 }
876 Py_DECREF(mod);
877 return 0;
Guido van Rossumfeee4b92000-03-10 22:57:27 +0000878}