blob: 0f791c06024db5525964f64235114fa90026ff29 [file] [log] [blame]
Tim Petersced69f82003-09-16 20:30:58 +00001/*
Guido van Rossumd57fd912000-03-10 22:53:23 +00002
3Unicode implementation based on original code by Fredrik Lundh,
Fred Drake785d14f2000-05-09 19:54:43 +00004modified by Marc-Andre Lemburg <mal@lemburg.com> according to the
Guido van Rossumd57fd912000-03-10 22:53:23 +00005Unicode Integration Proposal (see file Misc/unicode.txt).
6
Fredrik Lundh06a69dd2006-05-26 08:54:28 +00007Major speed upgrades to the method implementations at the Reykjavik
8NeedForSpeed sprint, by Fredrik Lundh and Andrew Dalke.
9
Guido van Rossum16b1ad92000-08-03 16:24:25 +000010Copyright (c) Corporation for National Research Initiatives.
Guido van Rossumd57fd912000-03-10 22:53:23 +000011
Fredrik Lundh0fdb90c2001-01-19 09:45:02 +000012--------------------------------------------------------------------
13The original string type implementation is:
Guido van Rossumd57fd912000-03-10 22:53:23 +000014
Fredrik Lundh0fdb90c2001-01-19 09:45:02 +000015 Copyright (c) 1999 by Secret Labs AB
16 Copyright (c) 1999 by Fredrik Lundh
Guido van Rossumd57fd912000-03-10 22:53:23 +000017
Fredrik Lundh0fdb90c2001-01-19 09:45:02 +000018By obtaining, using, and/or copying this software and/or its
19associated documentation, you agree that you have read, understood,
20and will comply with the following terms and conditions:
21
22Permission to use, copy, modify, and distribute this software and its
23associated documentation for any purpose and without fee is hereby
24granted, provided that the above copyright notice appears in all
25copies, and that both that copyright notice and this permission notice
26appear in supporting documentation, and that the name of Secret Labs
27AB or the author not be used in advertising or publicity pertaining to
28distribution of the software without specific, written prior
29permission.
30
31SECRET LABS AB AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO
32THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
33FITNESS. IN NO EVENT SHALL SECRET LABS AB OR THE AUTHOR BE LIABLE FOR
34ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
35WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
36ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT
37OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
38--------------------------------------------------------------------
39
40*/
Guido van Rossumd57fd912000-03-10 22:53:23 +000041
Martin v. Löwis5cb69362006-04-14 09:08:42 +000042#define PY_SSIZE_T_CLEAN
Guido van Rossumd57fd912000-03-10 22:53:23 +000043#include "Python.h"
44
Guido van Rossumd57fd912000-03-10 22:53:23 +000045#include "unicodeobject.h"
Marc-André Lemburgd49e5b42000-06-30 14:58:20 +000046#include "ucnhash.h"
Guido van Rossumd57fd912000-03-10 22:53:23 +000047
Martin v. Löwis6238d2b2002-06-30 15:26:10 +000048#ifdef MS_WINDOWS
Guido van Rossumb7a40ba2000-03-28 02:01:52 +000049#include <windows.h>
50#endif
Guido van Rossumfd4b9572000-04-10 13:51:10 +000051
Guido van Rossumd57fd912000-03-10 22:53:23 +000052/* Limit for the Unicode object free list */
53
54#define MAX_UNICODE_FREELIST_SIZE 1024
55
56/* Limit for the Unicode object free list stay alive optimization.
57
58 The implementation will keep allocated Unicode memory intact for
59 all objects on the free list having a size less than this
Tim Petersced69f82003-09-16 20:30:58 +000060 limit. This reduces malloc() overhead for small Unicode objects.
Guido van Rossumd57fd912000-03-10 22:53:23 +000061
Barry Warsaw51ac5802000-03-20 16:36:48 +000062 At worst this will result in MAX_UNICODE_FREELIST_SIZE *
Guido van Rossumfd4b9572000-04-10 13:51:10 +000063 (sizeof(PyUnicodeObject) + KEEPALIVE_SIZE_LIMIT +
Guido van Rossumd57fd912000-03-10 22:53:23 +000064 malloc()-overhead) bytes of unused garbage.
65
66 Setting the limit to 0 effectively turns the feature off.
67
Guido van Rossumfd4b9572000-04-10 13:51:10 +000068 Note: This is an experimental feature ! If you get core dumps when
69 using Unicode objects, turn this feature off.
Guido van Rossumd57fd912000-03-10 22:53:23 +000070
71*/
72
Guido van Rossumfd4b9572000-04-10 13:51:10 +000073#define KEEPALIVE_SIZE_LIMIT 9
Guido van Rossumd57fd912000-03-10 22:53:23 +000074
75/* Endianness switches; defaults to little endian */
76
77#ifdef WORDS_BIGENDIAN
78# define BYTEORDER_IS_BIG_ENDIAN
79#else
80# define BYTEORDER_IS_LITTLE_ENDIAN
81#endif
82
Marc-André Lemburgd4ab4a52000-06-08 17:54:00 +000083/* --- Globals ------------------------------------------------------------
84
85 The globals are initialized by the _PyUnicode_Init() API and should
86 not be used before calling that API.
87
88*/
Guido van Rossumd57fd912000-03-10 22:53:23 +000089
Anthony Baxterac6bd462006-04-13 02:06:09 +000090
91#ifdef __cplusplus
92extern "C" {
93#endif
94
Guido van Rossumd57fd912000-03-10 22:53:23 +000095/* Free list for Unicode objects */
Marc-André Lemburgd4ab4a52000-06-08 17:54:00 +000096static PyUnicodeObject *unicode_freelist;
97static int unicode_freelist_size;
Guido van Rossumd57fd912000-03-10 22:53:23 +000098
Marc-André Lemburg8155e0e2001-04-23 14:44:21 +000099/* The empty Unicode object is shared to improve performance. */
100static PyUnicodeObject *unicode_empty;
101
102/* Single character Unicode strings in the Latin-1 range are being
103 shared as well. */
104static PyUnicodeObject *unicode_latin1[256];
105
Fred Drakee4315f52000-05-09 19:53:39 +0000106/* Default encoding to use and assume when NULL is passed as encoding
107 parameter; it is initialized by _PyUnicode_Init().
108
109 Always use the PyUnicode_SetDefaultEncoding() and
Tim Petersced69f82003-09-16 20:30:58 +0000110 PyUnicode_GetDefaultEncoding() APIs to access this global.
Fred Drakee4315f52000-05-09 19:53:39 +0000111
112*/
Fred Drakee4315f52000-05-09 19:53:39 +0000113static char unicode_default_encoding[100];
114
Martin v. Löwisce9b5a52001-06-27 06:28:56 +0000115Py_UNICODE
Marc-André Lemburg6c6bfb72001-07-20 17:39:11 +0000116PyUnicode_GetMax(void)
Martin v. Löwisce9b5a52001-06-27 06:28:56 +0000117{
Fredrik Lundh8f455852001-06-27 18:59:43 +0000118#ifdef Py_UNICODE_WIDE
Martin v. Löwisce9b5a52001-06-27 06:28:56 +0000119 return 0x10FFFF;
120#else
121 /* This is actually an illegal character, so it should
122 not be passed to unichr. */
123 return 0xFFFF;
124#endif
125}
126
Fredrik Lundhb63588c2006-05-23 18:44:25 +0000127/* --- Bloom Filters ----------------------------------------------------- */
128
129/* stuff to implement simple "bloom filters" for Unicode characters.
130 to keep things simple, we use a single bitmask, using the least 5
131 bits from each unicode characters as the bit index. */
132
133/* the linebreak mask is set up by Unicode_Init below */
134
135#define BLOOM_MASK unsigned long
136
137static BLOOM_MASK bloom_linebreak;
138
139#define BLOOM(mask, ch) ((mask & (1 << ((ch) & 0x1F))))
140
141#define BLOOM_LINEBREAK(ch)\
142 (BLOOM(bloom_linebreak, (ch)) && Py_UNICODE_ISLINEBREAK((ch)))
143
Fredrik Lundh95e2a912006-05-26 11:38:15 +0000144Py_LOCAL(BLOOM_MASK) make_bloom_mask(Py_UNICODE* ptr, Py_ssize_t len)
Fredrik Lundhb63588c2006-05-23 18:44:25 +0000145{
146 /* calculate simple bloom-style bitmask for a given unicode string */
147
148 long mask;
149 Py_ssize_t i;
150
151 mask = 0;
152 for (i = 0; i < len; i++)
153 mask |= (1 << (ptr[i] & 0x1F));
154
155 return mask;
156}
157
Fredrik Lundh95e2a912006-05-26 11:38:15 +0000158Py_LOCAL(int) unicode_member(Py_UNICODE chr, Py_UNICODE* set, Py_ssize_t setlen)
Fredrik Lundhb63588c2006-05-23 18:44:25 +0000159{
160 Py_ssize_t i;
161
162 for (i = 0; i < setlen; i++)
163 if (set[i] == chr)
164 return 1;
165
Fredrik Lundh77633512006-05-23 19:47:35 +0000166 return 0;
Fredrik Lundhb63588c2006-05-23 18:44:25 +0000167}
168
169#define BLOOM_MEMBER(mask, chr, set, setlen)\
170 BLOOM(mask, chr) && unicode_member(chr, set, setlen)
171
Guido van Rossumd57fd912000-03-10 22:53:23 +0000172/* --- Unicode Object ----------------------------------------------------- */
173
174static
Marc-André Lemburg8155e0e2001-04-23 14:44:21 +0000175int unicode_resize(register PyUnicodeObject *unicode,
Martin v. Löwis18e16552006-02-15 17:27:45 +0000176 Py_ssize_t length)
Guido van Rossumd57fd912000-03-10 22:53:23 +0000177{
178 void *oldstr;
Tim Petersced69f82003-09-16 20:30:58 +0000179
Guido van Rossumfd4b9572000-04-10 13:51:10 +0000180 /* Shortcut if there's nothing much to do. */
Guido van Rossumd57fd912000-03-10 22:53:23 +0000181 if (unicode->length == length)
Guido van Rossumfd4b9572000-04-10 13:51:10 +0000182 goto reset;
Guido van Rossumd57fd912000-03-10 22:53:23 +0000183
Marc-André Lemburg8155e0e2001-04-23 14:44:21 +0000184 /* Resizing shared object (unicode_empty or single character
185 objects) in-place is not allowed. Use PyUnicode_Resize()
186 instead ! */
Fredrik Lundh06a69dd2006-05-26 08:54:28 +0000187
Martin v. Löwis80d2e592006-04-13 06:06:08 +0000188 if (unicode == unicode_empty ||
189 (unicode->length == 1 &&
190 unicode->str[0] < 256U &&
Marc-André Lemburg8155e0e2001-04-23 14:44:21 +0000191 unicode_latin1[unicode->str[0]] == unicode)) {
Guido van Rossumd57fd912000-03-10 22:53:23 +0000192 PyErr_SetString(PyExc_SystemError,
Marc-André Lemburg8155e0e2001-04-23 14:44:21 +0000193 "can't resize shared unicode objects");
Guido van Rossumd57fd912000-03-10 22:53:23 +0000194 return -1;
195 }
196
Fredrik Lundh06a69dd2006-05-26 08:54:28 +0000197 /* We allocate one more byte to make sure the string is Ux0000 terminated.
198 The overallocation is also used by fastsearch, which assumes that it's
199 safe to look at str[length] (without makeing any assumptions about what
200 it contains). */
201
Guido van Rossumd57fd912000-03-10 22:53:23 +0000202 oldstr = unicode->str;
203 PyMem_RESIZE(unicode->str, Py_UNICODE, length + 1);
204 if (!unicode->str) {
Anthony Baxtera6286212006-04-11 07:42:36 +0000205 unicode->str = (Py_UNICODE *)oldstr;
Guido van Rossumd57fd912000-03-10 22:53:23 +0000206 PyErr_NoMemory();
207 return -1;
208 }
209 unicode->str[length] = 0;
Martin v. Löwis412fb672006-04-13 06:34:32 +0000210 unicode->length = length;
Guido van Rossumd57fd912000-03-10 22:53:23 +0000211
Guido van Rossumfd4b9572000-04-10 13:51:10 +0000212 reset:
Guido van Rossumd57fd912000-03-10 22:53:23 +0000213 /* Reset the object caches */
Marc-André Lemburgbff879c2000-08-03 18:46:08 +0000214 if (unicode->defenc) {
215 Py_DECREF(unicode->defenc);
216 unicode->defenc = NULL;
Guido van Rossumd57fd912000-03-10 22:53:23 +0000217 }
218 unicode->hash = -1;
Tim Petersced69f82003-09-16 20:30:58 +0000219
Guido van Rossumd57fd912000-03-10 22:53:23 +0000220 return 0;
221}
222
223/* We allocate one more byte to make sure the string is
Tim Petersced69f82003-09-16 20:30:58 +0000224 Ux0000 terminated -- XXX is this needed ?
Guido van Rossumd57fd912000-03-10 22:53:23 +0000225
226 XXX This allocator could further be enhanced by assuring that the
227 free list never reduces its size below 1.
228
229*/
230
231static
Martin v. Löwis18e16552006-02-15 17:27:45 +0000232PyUnicodeObject *_PyUnicode_New(Py_ssize_t length)
Guido van Rossumd57fd912000-03-10 22:53:23 +0000233{
234 register PyUnicodeObject *unicode;
235
Tim Petersced69f82003-09-16 20:30:58 +0000236 /* Optimization fo empty strings */
Guido van Rossumd57fd912000-03-10 22:53:23 +0000237 if (length == 0 && unicode_empty != NULL) {
238 Py_INCREF(unicode_empty);
239 return unicode_empty;
240 }
241
242 /* Unicode freelist & memory allocation */
243 if (unicode_freelist) {
244 unicode = unicode_freelist;
Marc-André Lemburgbea47e72000-06-17 20:31:17 +0000245 unicode_freelist = *(PyUnicodeObject **)unicode;
Guido van Rossumd57fd912000-03-10 22:53:23 +0000246 unicode_freelist_size--;
Guido van Rossumd57fd912000-03-10 22:53:23 +0000247 if (unicode->str) {
Guido van Rossumfd4b9572000-04-10 13:51:10 +0000248 /* Keep-Alive optimization: we only upsize the buffer,
249 never downsize it. */
Tim Petersced69f82003-09-16 20:30:58 +0000250 if ((unicode->length < length) &&
Jeremy Hyltondeb2dc62003-09-16 03:41:45 +0000251 unicode_resize(unicode, length) < 0) {
Guido van Rossumb18618d2000-05-03 23:44:39 +0000252 PyMem_DEL(unicode->str);
Guido van Rossum3c1bb802000-04-27 20:13:50 +0000253 goto onError;
Guido van Rossumd57fd912000-03-10 22:53:23 +0000254 }
255 }
Guido van Rossumad98db12001-06-14 17:52:02 +0000256 else {
Guido van Rossumd57fd912000-03-10 22:53:23 +0000257 unicode->str = PyMem_NEW(Py_UNICODE, length + 1);
Guido van Rossumad98db12001-06-14 17:52:02 +0000258 }
259 PyObject_INIT(unicode, &PyUnicode_Type);
Guido van Rossumd57fd912000-03-10 22:53:23 +0000260 }
261 else {
Neil Schemenauer58aa8612002-04-12 03:07:20 +0000262 unicode = PyObject_New(PyUnicodeObject, &PyUnicode_Type);
Guido van Rossumd57fd912000-03-10 22:53:23 +0000263 if (unicode == NULL)
264 return NULL;
265 unicode->str = PyMem_NEW(Py_UNICODE, length + 1);
266 }
267
Guido van Rossum3c1bb802000-04-27 20:13:50 +0000268 if (!unicode->str) {
269 PyErr_NoMemory();
Barry Warsaw51ac5802000-03-20 16:36:48 +0000270 goto onError;
Guido van Rossum3c1bb802000-04-27 20:13:50 +0000271 }
Jeremy Hyltond8082792003-09-16 19:41:39 +0000272 /* Initialize the first element to guard against cases where
Tim Petersced69f82003-09-16 20:30:58 +0000273 * the caller fails before initializing str -- unicode_resize()
274 * reads str[0], and the Keep-Alive optimization can keep memory
275 * allocated for str alive across a call to unicode_dealloc(unicode).
276 * We don't want unicode_resize to read uninitialized memory in
277 * that case.
278 */
Jeremy Hyltond8082792003-09-16 19:41:39 +0000279 unicode->str[0] = 0;
Guido van Rossumd57fd912000-03-10 22:53:23 +0000280 unicode->str[length] = 0;
Martin v. Löwisf15da692006-04-13 07:24:50 +0000281 unicode->length = length;
Guido van Rossumd57fd912000-03-10 22:53:23 +0000282 unicode->hash = -1;
Marc-André Lemburgbff879c2000-08-03 18:46:08 +0000283 unicode->defenc = NULL;
Guido van Rossumd57fd912000-03-10 22:53:23 +0000284 return unicode;
Barry Warsaw51ac5802000-03-20 16:36:48 +0000285
286 onError:
287 _Py_ForgetReference((PyObject *)unicode);
Neil Schemenauer58aa8612002-04-12 03:07:20 +0000288 PyObject_Del(unicode);
Barry Warsaw51ac5802000-03-20 16:36:48 +0000289 return NULL;
Guido van Rossumd57fd912000-03-10 22:53:23 +0000290}
291
292static
Guido van Rossum9475a232001-10-05 20:51:39 +0000293void unicode_dealloc(register PyUnicodeObject *unicode)
Guido van Rossumd57fd912000-03-10 22:53:23 +0000294{
Guido van Rossum604ddf82001-12-06 20:03:56 +0000295 if (PyUnicode_CheckExact(unicode) &&
296 unicode_freelist_size < MAX_UNICODE_FREELIST_SIZE) {
Guido van Rossumfd4b9572000-04-10 13:51:10 +0000297 /* Keep-Alive optimization */
298 if (unicode->length >= KEEPALIVE_SIZE_LIMIT) {
Guido van Rossumb18618d2000-05-03 23:44:39 +0000299 PyMem_DEL(unicode->str);
Guido van Rossumd57fd912000-03-10 22:53:23 +0000300 unicode->str = NULL;
301 unicode->length = 0;
302 }
Marc-André Lemburgbff879c2000-08-03 18:46:08 +0000303 if (unicode->defenc) {
304 Py_DECREF(unicode->defenc);
305 unicode->defenc = NULL;
Guido van Rossumfd4b9572000-04-10 13:51:10 +0000306 }
307 /* Add to free list */
Guido van Rossumd57fd912000-03-10 22:53:23 +0000308 *(PyUnicodeObject **)unicode = unicode_freelist;
309 unicode_freelist = unicode;
310 unicode_freelist_size++;
Guido van Rossumd57fd912000-03-10 22:53:23 +0000311 }
312 else {
Guido van Rossumb18618d2000-05-03 23:44:39 +0000313 PyMem_DEL(unicode->str);
Marc-André Lemburgbff879c2000-08-03 18:46:08 +0000314 Py_XDECREF(unicode->defenc);
Guido van Rossum604ddf82001-12-06 20:03:56 +0000315 unicode->ob_type->tp_free((PyObject *)unicode);
Guido van Rossumd57fd912000-03-10 22:53:23 +0000316 }
317}
318
Martin v. Löwis18e16552006-02-15 17:27:45 +0000319int PyUnicode_Resize(PyObject **unicode, Py_ssize_t length)
Marc-André Lemburg8155e0e2001-04-23 14:44:21 +0000320{
321 register PyUnicodeObject *v;
322
323 /* Argument checks */
324 if (unicode == NULL) {
325 PyErr_BadInternalCall();
326 return -1;
327 }
328 v = (PyUnicodeObject *)*unicode;
Guido van Rossum049cd6b2002-10-11 00:43:48 +0000329 if (v == NULL || !PyUnicode_Check(v) || v->ob_refcnt != 1 || length < 0) {
Marc-André Lemburg8155e0e2001-04-23 14:44:21 +0000330 PyErr_BadInternalCall();
331 return -1;
332 }
333
334 /* Resizing unicode_empty and single character objects is not
335 possible since these are being shared. We simply return a fresh
336 copy with the same Unicode content. */
Tim Petersced69f82003-09-16 20:30:58 +0000337 if (v->length != length &&
Marc-André Lemburg8155e0e2001-04-23 14:44:21 +0000338 (v == unicode_empty || v->length == 1)) {
339 PyUnicodeObject *w = _PyUnicode_New(length);
340 if (w == NULL)
341 return -1;
342 Py_UNICODE_COPY(w->str, v->str,
343 length < v->length ? length : v->length);
Raymond Hettingerc8df5782003-03-09 07:30:43 +0000344 Py_DECREF(*unicode);
Marc-André Lemburg8155e0e2001-04-23 14:44:21 +0000345 *unicode = (PyObject *)w;
346 return 0;
347 }
348
349 /* Note that we don't have to modify *unicode for unshared Unicode
350 objects, since we can modify them in-place. */
351 return unicode_resize(v, length);
352}
353
354/* Internal API for use in unicodeobject.c only ! */
355#define _PyUnicode_Resize(unicodevar, length) \
356 PyUnicode_Resize(((PyObject **)(unicodevar)), length)
357
Guido van Rossumd57fd912000-03-10 22:53:23 +0000358PyObject *PyUnicode_FromUnicode(const Py_UNICODE *u,
Martin v. Löwis18e16552006-02-15 17:27:45 +0000359 Py_ssize_t size)
Guido van Rossumd57fd912000-03-10 22:53:23 +0000360{
361 PyUnicodeObject *unicode;
362
Marc-André Lemburg8155e0e2001-04-23 14:44:21 +0000363 /* If the Unicode data is known at construction time, we can apply
364 some optimizations which share commonly used objects. */
365 if (u != NULL) {
366
367 /* Optimization for empty strings */
368 if (size == 0 && unicode_empty != NULL) {
369 Py_INCREF(unicode_empty);
370 return (PyObject *)unicode_empty;
371 }
372
373 /* Single character Unicode objects in the Latin-1 range are
374 shared when using this constructor */
375 if (size == 1 && *u < 256) {
376 unicode = unicode_latin1[*u];
377 if (!unicode) {
378 unicode = _PyUnicode_New(1);
Marc-André Lemburg8155e0e2001-04-23 14:44:21 +0000379 if (!unicode)
380 return NULL;
Marc-André Lemburg8879a332001-06-07 12:26:56 +0000381 unicode->str[0] = *u;
Marc-André Lemburg8155e0e2001-04-23 14:44:21 +0000382 unicode_latin1[*u] = unicode;
383 }
384 Py_INCREF(unicode);
385 return (PyObject *)unicode;
386 }
387 }
Tim Petersced69f82003-09-16 20:30:58 +0000388
Guido van Rossumd57fd912000-03-10 22:53:23 +0000389 unicode = _PyUnicode_New(size);
390 if (!unicode)
391 return NULL;
392
393 /* Copy the Unicode data into the new object */
394 if (u != NULL)
Marc-André Lemburg8155e0e2001-04-23 14:44:21 +0000395 Py_UNICODE_COPY(unicode->str, u, size);
Guido van Rossumd57fd912000-03-10 22:53:23 +0000396
397 return (PyObject *)unicode;
398}
399
400#ifdef HAVE_WCHAR_H
401
402PyObject *PyUnicode_FromWideChar(register const wchar_t *w,
Martin v. Löwis18e16552006-02-15 17:27:45 +0000403 Py_ssize_t size)
Guido van Rossumd57fd912000-03-10 22:53:23 +0000404{
405 PyUnicodeObject *unicode;
406
407 if (w == NULL) {
408 PyErr_BadInternalCall();
409 return NULL;
410 }
411
412 unicode = _PyUnicode_New(size);
413 if (!unicode)
414 return NULL;
415
416 /* Copy the wchar_t data into the new object */
417#ifdef HAVE_USABLE_WCHAR_T
418 memcpy(unicode->str, w, size * sizeof(wchar_t));
Tim Petersced69f82003-09-16 20:30:58 +0000419#else
Guido van Rossumd57fd912000-03-10 22:53:23 +0000420 {
421 register Py_UNICODE *u;
Martin v. Löwis412fb672006-04-13 06:34:32 +0000422 register Py_ssize_t i;
Guido van Rossumd57fd912000-03-10 22:53:23 +0000423 u = PyUnicode_AS_UNICODE(unicode);
Marc-André Lemburg204bd6d2004-10-15 07:45:05 +0000424 for (i = size; i > 0; i--)
Guido van Rossumd57fd912000-03-10 22:53:23 +0000425 *u++ = *w++;
426 }
427#endif
428
429 return (PyObject *)unicode;
430}
431
Martin v. Löwis18e16552006-02-15 17:27:45 +0000432Py_ssize_t PyUnicode_AsWideChar(PyUnicodeObject *unicode,
433 wchar_t *w,
434 Py_ssize_t size)
Guido van Rossumd57fd912000-03-10 22:53:23 +0000435{
436 if (unicode == NULL) {
437 PyErr_BadInternalCall();
438 return -1;
439 }
Marc-André Lemburga9cadcd2004-11-22 13:02:31 +0000440
441 /* If possible, try to copy the 0-termination as well */
Guido van Rossumd57fd912000-03-10 22:53:23 +0000442 if (size > PyUnicode_GET_SIZE(unicode))
Marc-André Lemburga9cadcd2004-11-22 13:02:31 +0000443 size = PyUnicode_GET_SIZE(unicode) + 1;
444
Guido van Rossumd57fd912000-03-10 22:53:23 +0000445#ifdef HAVE_USABLE_WCHAR_T
446 memcpy(w, unicode->str, size * sizeof(wchar_t));
447#else
448 {
449 register Py_UNICODE *u;
Martin v. Löwis412fb672006-04-13 06:34:32 +0000450 register Py_ssize_t i;
Guido van Rossumd57fd912000-03-10 22:53:23 +0000451 u = PyUnicode_AS_UNICODE(unicode);
Marc-André Lemburg204bd6d2004-10-15 07:45:05 +0000452 for (i = size; i > 0; i--)
Guido van Rossumd57fd912000-03-10 22:53:23 +0000453 *w++ = *u++;
454 }
455#endif
456
Marc-André Lemburga9cadcd2004-11-22 13:02:31 +0000457 if (size > PyUnicode_GET_SIZE(unicode))
458 return PyUnicode_GET_SIZE(unicode);
459 else
Guido van Rossumd57fd912000-03-10 22:53:23 +0000460 return size;
461}
462
463#endif
464
Marc-André Lemburgcc8764c2002-08-11 12:23:04 +0000465PyObject *PyUnicode_FromOrdinal(int ordinal)
466{
Hye-Shik Chang40574832004-04-06 07:24:51 +0000467 Py_UNICODE s[1];
Marc-André Lemburgcc8764c2002-08-11 12:23:04 +0000468
469#ifdef Py_UNICODE_WIDE
470 if (ordinal < 0 || ordinal > 0x10ffff) {
471 PyErr_SetString(PyExc_ValueError,
472 "unichr() arg not in range(0x110000) "
473 "(wide Python build)");
474 return NULL;
475 }
476#else
477 if (ordinal < 0 || ordinal > 0xffff) {
478 PyErr_SetString(PyExc_ValueError,
479 "unichr() arg not in range(0x10000) "
480 "(narrow Python build)");
481 return NULL;
482 }
483#endif
484
Hye-Shik Chang40574832004-04-06 07:24:51 +0000485 s[0] = (Py_UNICODE)ordinal;
486 return PyUnicode_FromUnicode(s, 1);
Marc-André Lemburgcc8764c2002-08-11 12:23:04 +0000487}
488
Guido van Rossumd57fd912000-03-10 22:53:23 +0000489PyObject *PyUnicode_FromObject(register PyObject *obj)
490{
Guido van Rossumb8c65bc2001-10-19 02:01:31 +0000491 /* XXX Perhaps we should make this API an alias of
492 PyObject_Unicode() instead ?! */
493 if (PyUnicode_CheckExact(obj)) {
494 Py_INCREF(obj);
495 return obj;
496 }
497 if (PyUnicode_Check(obj)) {
498 /* For a Unicode subtype that's not a Unicode object,
499 return a true Unicode object with the same data. */
500 return PyUnicode_FromUnicode(PyUnicode_AS_UNICODE(obj),
501 PyUnicode_GET_SIZE(obj));
502 }
Marc-André Lemburg5a5c81a2000-07-07 13:46:42 +0000503 return PyUnicode_FromEncodedObject(obj, NULL, "strict");
504}
505
506PyObject *PyUnicode_FromEncodedObject(register PyObject *obj,
507 const char *encoding,
508 const char *errors)
509{
Marc-André Lemburg6871f6a2001-09-20 12:53:16 +0000510 const char *s = NULL;
Martin v. Löwis18e16552006-02-15 17:27:45 +0000511 Py_ssize_t len;
Marc-André Lemburg5a5c81a2000-07-07 13:46:42 +0000512 PyObject *v;
Tim Petersced69f82003-09-16 20:30:58 +0000513
Guido van Rossumd57fd912000-03-10 22:53:23 +0000514 if (obj == NULL) {
515 PyErr_BadInternalCall();
516 return NULL;
517 }
Marc-André Lemburg5a5c81a2000-07-07 13:46:42 +0000518
Guido van Rossumb8c65bc2001-10-19 02:01:31 +0000519#if 0
520 /* For b/w compatibility we also accept Unicode objects provided
Marc-André Lemburgb5507ec2001-10-19 12:02:29 +0000521 that no encodings is given and then redirect to
522 PyObject_Unicode() which then applies the additional logic for
523 Unicode subclasses.
Marc-André Lemburg6871f6a2001-09-20 12:53:16 +0000524
Guido van Rossumb8c65bc2001-10-19 02:01:31 +0000525 NOTE: This API should really only be used for object which
526 represent *encoded* Unicode !
527
528 */
Marc-André Lemburg6871f6a2001-09-20 12:53:16 +0000529 if (PyUnicode_Check(obj)) {
530 if (encoding) {
531 PyErr_SetString(PyExc_TypeError,
532 "decoding Unicode is not supported");
Guido van Rossumb8c65bc2001-10-19 02:01:31 +0000533 return NULL;
Marc-André Lemburg6871f6a2001-09-20 12:53:16 +0000534 }
Guido van Rossumb8c65bc2001-10-19 02:01:31 +0000535 return PyObject_Unicode(obj);
Marc-André Lemburg6871f6a2001-09-20 12:53:16 +0000536 }
Guido van Rossumb8c65bc2001-10-19 02:01:31 +0000537#else
538 if (PyUnicode_Check(obj)) {
539 PyErr_SetString(PyExc_TypeError,
540 "decoding Unicode is not supported");
541 return NULL;
Marc-André Lemburg5a5c81a2000-07-07 13:46:42 +0000542 }
Guido van Rossumb8c65bc2001-10-19 02:01:31 +0000543#endif
544
545 /* Coerce object */
546 if (PyString_Check(obj)) {
Marc-André Lemburg6871f6a2001-09-20 12:53:16 +0000547 s = PyString_AS_STRING(obj);
548 len = PyString_GET_SIZE(obj);
Marc-André Lemburg6871f6a2001-09-20 12:53:16 +0000549 }
Guido van Rossumb8c65bc2001-10-19 02:01:31 +0000550 else if (PyObject_AsCharBuffer(obj, &s, &len)) {
551 /* Overwrite the error message with something more useful in
552 case of a TypeError. */
553 if (PyErr_ExceptionMatches(PyExc_TypeError))
Marc-André Lemburg6871f6a2001-09-20 12:53:16 +0000554 PyErr_Format(PyExc_TypeError,
Guido van Rossumb8c65bc2001-10-19 02:01:31 +0000555 "coercing to Unicode: need string or buffer, "
556 "%.80s found",
Marc-André Lemburg6871f6a2001-09-20 12:53:16 +0000557 obj->ob_type->tp_name);
558 goto onError;
559 }
Tim Petersced69f82003-09-16 20:30:58 +0000560
Marc-André Lemburg5a5c81a2000-07-07 13:46:42 +0000561 /* Convert to Unicode */
Guido van Rossumd57fd912000-03-10 22:53:23 +0000562 if (len == 0) {
563 Py_INCREF(unicode_empty);
Marc-André Lemburg5a5c81a2000-07-07 13:46:42 +0000564 v = (PyObject *)unicode_empty;
Guido van Rossumd57fd912000-03-10 22:53:23 +0000565 }
Tim Petersced69f82003-09-16 20:30:58 +0000566 else
Marc-André Lemburg5a5c81a2000-07-07 13:46:42 +0000567 v = PyUnicode_Decode(s, len, encoding, errors);
Marc-André Lemburgad7c98e2001-01-17 17:09:53 +0000568
Marc-André Lemburg5a5c81a2000-07-07 13:46:42 +0000569 return v;
570
571 onError:
Marc-André Lemburg5a5c81a2000-07-07 13:46:42 +0000572 return NULL;
Guido van Rossumd57fd912000-03-10 22:53:23 +0000573}
574
575PyObject *PyUnicode_Decode(const char *s,
Martin v. Löwis18e16552006-02-15 17:27:45 +0000576 Py_ssize_t size,
Guido van Rossumd57fd912000-03-10 22:53:23 +0000577 const char *encoding,
578 const char *errors)
579{
580 PyObject *buffer = NULL, *unicode;
Walter Dörwald3aeb6322002-09-02 13:14:32 +0000581
582 if (encoding == NULL)
Fred Drakee4315f52000-05-09 19:53:39 +0000583 encoding = PyUnicode_GetDefaultEncoding();
584
585 /* Shortcuts for common default encodings */
586 if (strcmp(encoding, "utf-8") == 0)
Guido van Rossumd57fd912000-03-10 22:53:23 +0000587 return PyUnicode_DecodeUTF8(s, size, errors);
Fred Drakee4315f52000-05-09 19:53:39 +0000588 else if (strcmp(encoding, "latin-1") == 0)
589 return PyUnicode_DecodeLatin1(s, size, errors);
Mark Hammond0ccda1e2003-07-01 00:13:27 +0000590#if defined(MS_WINDOWS) && defined(HAVE_USABLE_WCHAR_T)
591 else if (strcmp(encoding, "mbcs") == 0)
592 return PyUnicode_DecodeMBCS(s, size, errors);
593#endif
Fred Drakee4315f52000-05-09 19:53:39 +0000594 else if (strcmp(encoding, "ascii") == 0)
595 return PyUnicode_DecodeASCII(s, size, errors);
Guido van Rossumd57fd912000-03-10 22:53:23 +0000596
597 /* Decode via the codec registry */
598 buffer = PyBuffer_FromMemory((void *)s, size);
599 if (buffer == NULL)
600 goto onError;
601 unicode = PyCodec_Decode(buffer, encoding, errors);
602 if (unicode == NULL)
603 goto onError;
604 if (!PyUnicode_Check(unicode)) {
605 PyErr_Format(PyExc_TypeError,
Guido van Rossum5db862d2000-04-10 12:46:51 +0000606 "decoder did not return an unicode object (type=%.400s)",
Guido van Rossumd57fd912000-03-10 22:53:23 +0000607 unicode->ob_type->tp_name);
608 Py_DECREF(unicode);
609 goto onError;
610 }
611 Py_DECREF(buffer);
612 return unicode;
Tim Petersced69f82003-09-16 20:30:58 +0000613
Guido van Rossumd57fd912000-03-10 22:53:23 +0000614 onError:
615 Py_XDECREF(buffer);
616 return NULL;
617}
618
Marc-André Lemburgd2d45982004-07-08 17:57:32 +0000619PyObject *PyUnicode_AsDecodedObject(PyObject *unicode,
620 const char *encoding,
621 const char *errors)
622{
623 PyObject *v;
624
625 if (!PyUnicode_Check(unicode)) {
626 PyErr_BadArgument();
627 goto onError;
628 }
629
630 if (encoding == NULL)
631 encoding = PyUnicode_GetDefaultEncoding();
632
633 /* Decode via the codec registry */
634 v = PyCodec_Decode(unicode, encoding, errors);
635 if (v == NULL)
636 goto onError;
637 return v;
638
639 onError:
640 return NULL;
641}
642
Guido van Rossumd57fd912000-03-10 22:53:23 +0000643PyObject *PyUnicode_Encode(const Py_UNICODE *s,
Martin v. Löwis18e16552006-02-15 17:27:45 +0000644 Py_ssize_t size,
Guido van Rossumd57fd912000-03-10 22:53:23 +0000645 const char *encoding,
646 const char *errors)
647{
648 PyObject *v, *unicode;
Tim Petersced69f82003-09-16 20:30:58 +0000649
Guido van Rossumd57fd912000-03-10 22:53:23 +0000650 unicode = PyUnicode_FromUnicode(s, size);
651 if (unicode == NULL)
652 return NULL;
653 v = PyUnicode_AsEncodedString(unicode, encoding, errors);
654 Py_DECREF(unicode);
655 return v;
656}
657
Marc-André Lemburgd2d45982004-07-08 17:57:32 +0000658PyObject *PyUnicode_AsEncodedObject(PyObject *unicode,
659 const char *encoding,
660 const char *errors)
661{
662 PyObject *v;
663
664 if (!PyUnicode_Check(unicode)) {
665 PyErr_BadArgument();
666 goto onError;
667 }
668
669 if (encoding == NULL)
670 encoding = PyUnicode_GetDefaultEncoding();
671
672 /* Encode via the codec registry */
673 v = PyCodec_Encode(unicode, encoding, errors);
674 if (v == NULL)
675 goto onError;
676 return v;
677
678 onError:
679 return NULL;
680}
681
Guido van Rossumd57fd912000-03-10 22:53:23 +0000682PyObject *PyUnicode_AsEncodedString(PyObject *unicode,
683 const char *encoding,
684 const char *errors)
685{
686 PyObject *v;
Tim Petersced69f82003-09-16 20:30:58 +0000687
Guido van Rossumd57fd912000-03-10 22:53:23 +0000688 if (!PyUnicode_Check(unicode)) {
689 PyErr_BadArgument();
690 goto onError;
691 }
Fred Drakee4315f52000-05-09 19:53:39 +0000692
Tim Petersced69f82003-09-16 20:30:58 +0000693 if (encoding == NULL)
Fred Drakee4315f52000-05-09 19:53:39 +0000694 encoding = PyUnicode_GetDefaultEncoding();
695
696 /* Shortcuts for common default encodings */
697 if (errors == NULL) {
698 if (strcmp(encoding, "utf-8") == 0)
Jeremy Hylton9cea41c2001-05-29 17:13:15 +0000699 return PyUnicode_AsUTF8String(unicode);
Fred Drakee4315f52000-05-09 19:53:39 +0000700 else if (strcmp(encoding, "latin-1") == 0)
701 return PyUnicode_AsLatin1String(unicode);
Mark Hammond0ccda1e2003-07-01 00:13:27 +0000702#if defined(MS_WINDOWS) && defined(HAVE_USABLE_WCHAR_T)
703 else if (strcmp(encoding, "mbcs") == 0)
704 return PyUnicode_AsMBCSString(unicode);
705#endif
Fred Drakee4315f52000-05-09 19:53:39 +0000706 else if (strcmp(encoding, "ascii") == 0)
707 return PyUnicode_AsASCIIString(unicode);
708 }
Guido van Rossumd57fd912000-03-10 22:53:23 +0000709
710 /* Encode via the codec registry */
711 v = PyCodec_Encode(unicode, encoding, errors);
712 if (v == NULL)
713 goto onError;
Guido van Rossumd57fd912000-03-10 22:53:23 +0000714 if (!PyString_Check(v)) {
715 PyErr_Format(PyExc_TypeError,
Guido van Rossum5db862d2000-04-10 12:46:51 +0000716 "encoder did not return a string object (type=%.400s)",
Guido van Rossumd57fd912000-03-10 22:53:23 +0000717 v->ob_type->tp_name);
718 Py_DECREF(v);
719 goto onError;
720 }
721 return v;
Tim Petersced69f82003-09-16 20:30:58 +0000722
Guido van Rossumd57fd912000-03-10 22:53:23 +0000723 onError:
724 return NULL;
725}
726
Marc-André Lemburgbff879c2000-08-03 18:46:08 +0000727PyObject *_PyUnicode_AsDefaultEncodedString(PyObject *unicode,
728 const char *errors)
729{
730 PyObject *v = ((PyUnicodeObject *)unicode)->defenc;
731
732 if (v)
733 return v;
734 v = PyUnicode_AsEncodedString(unicode, NULL, errors);
735 if (v && errors == NULL)
736 ((PyUnicodeObject *)unicode)->defenc = v;
737 return v;
738}
739
Guido van Rossumd57fd912000-03-10 22:53:23 +0000740Py_UNICODE *PyUnicode_AsUnicode(PyObject *unicode)
741{
742 if (!PyUnicode_Check(unicode)) {
743 PyErr_BadArgument();
744 goto onError;
745 }
746 return PyUnicode_AS_UNICODE(unicode);
747
748 onError:
749 return NULL;
750}
751
Martin v. Löwis18e16552006-02-15 17:27:45 +0000752Py_ssize_t PyUnicode_GetSize(PyObject *unicode)
Guido van Rossumd57fd912000-03-10 22:53:23 +0000753{
754 if (!PyUnicode_Check(unicode)) {
755 PyErr_BadArgument();
756 goto onError;
757 }
758 return PyUnicode_GET_SIZE(unicode);
759
760 onError:
761 return -1;
762}
763
Thomas Wouters78890102000-07-22 19:25:51 +0000764const char *PyUnicode_GetDefaultEncoding(void)
Fred Drakee4315f52000-05-09 19:53:39 +0000765{
766 return unicode_default_encoding;
767}
768
769int PyUnicode_SetDefaultEncoding(const char *encoding)
770{
771 PyObject *v;
Tim Petersced69f82003-09-16 20:30:58 +0000772
Fred Drakee4315f52000-05-09 19:53:39 +0000773 /* Make sure the encoding is valid. As side effect, this also
774 loads the encoding into the codec registry cache. */
775 v = _PyCodec_Lookup(encoding);
776 if (v == NULL)
777 goto onError;
778 Py_DECREF(v);
779 strncpy(unicode_default_encoding,
Tim Petersced69f82003-09-16 20:30:58 +0000780 encoding,
Fred Drakee4315f52000-05-09 19:53:39 +0000781 sizeof(unicode_default_encoding));
782 return 0;
783
784 onError:
785 return -1;
786}
787
Walter Dörwald3aeb6322002-09-02 13:14:32 +0000788/* error handling callback helper:
789 build arguments, call the callback and check the arguments,
Fred Drakedb390c12005-10-28 14:39:47 +0000790 if no exception occurred, copy the replacement to the output
Walter Dörwald3aeb6322002-09-02 13:14:32 +0000791 and adjust various state variables.
792 return 0 on success, -1 on error
793*/
794
795static
796int unicode_decode_call_errorhandler(const char *errors, PyObject **errorHandler,
797 const char *encoding, const char *reason,
Martin v. Löwis18e16552006-02-15 17:27:45 +0000798 const char *input, Py_ssize_t insize, Py_ssize_t *startinpos, Py_ssize_t *endinpos, PyObject **exceptionObject, const char **inptr,
799 PyObject **output, Py_ssize_t *outpos, Py_UNICODE **outptr)
Walter Dörwald3aeb6322002-09-02 13:14:32 +0000800{
Martin v. Löwis18e16552006-02-15 17:27:45 +0000801 static char *argparse = "O!n;decoding error handler must return (unicode, int) tuple";
Walter Dörwald3aeb6322002-09-02 13:14:32 +0000802
803 PyObject *restuple = NULL;
804 PyObject *repunicode = NULL;
Martin v. Löwis18e16552006-02-15 17:27:45 +0000805 Py_ssize_t outsize = PyUnicode_GET_SIZE(*output);
806 Py_ssize_t requiredsize;
807 Py_ssize_t newpos;
Walter Dörwald3aeb6322002-09-02 13:14:32 +0000808 Py_UNICODE *repptr;
Martin v. Löwis18e16552006-02-15 17:27:45 +0000809 Py_ssize_t repsize;
Walter Dörwald3aeb6322002-09-02 13:14:32 +0000810 int res = -1;
811
812 if (*errorHandler == NULL) {
813 *errorHandler = PyCodec_LookupError(errors);
814 if (*errorHandler == NULL)
815 goto onError;
816 }
817
818 if (*exceptionObject == NULL) {
819 *exceptionObject = PyUnicodeDecodeError_Create(
820 encoding, input, insize, *startinpos, *endinpos, reason);
821 if (*exceptionObject == NULL)
822 goto onError;
823 }
824 else {
825 if (PyUnicodeDecodeError_SetStart(*exceptionObject, *startinpos))
826 goto onError;
827 if (PyUnicodeDecodeError_SetEnd(*exceptionObject, *endinpos))
828 goto onError;
829 if (PyUnicodeDecodeError_SetReason(*exceptionObject, reason))
830 goto onError;
831 }
832
833 restuple = PyObject_CallFunctionObjArgs(*errorHandler, *exceptionObject, NULL);
834 if (restuple == NULL)
835 goto onError;
836 if (!PyTuple_Check(restuple)) {
837 PyErr_Format(PyExc_TypeError, &argparse[4]);
838 goto onError;
839 }
840 if (!PyArg_ParseTuple(restuple, argparse, &PyUnicode_Type, &repunicode, &newpos))
841 goto onError;
842 if (newpos<0)
Walter Dörwald2e0b18a2003-01-31 17:19:08 +0000843 newpos = insize+newpos;
844 if (newpos<0 || newpos>insize) {
Martin v. Löwis2c95cc62006-02-16 06:54:25 +0000845 PyErr_Format(PyExc_IndexError, "position %zd from error handler out of bounds", newpos);
Walter Dörwald2e0b18a2003-01-31 17:19:08 +0000846 goto onError;
847 }
Walter Dörwald3aeb6322002-09-02 13:14:32 +0000848
849 /* need more space? (at least enough for what we
850 have+the replacement+the rest of the string (starting
851 at the new input position), so we won't have to check space
852 when there are no errors in the rest of the string) */
853 repptr = PyUnicode_AS_UNICODE(repunicode);
854 repsize = PyUnicode_GET_SIZE(repunicode);
855 requiredsize = *outpos + repsize + insize-newpos;
856 if (requiredsize > outsize) {
857 if (requiredsize<2*outsize)
858 requiredsize = 2*outsize;
Jeremy Hyltondeb2dc62003-09-16 03:41:45 +0000859 if (PyUnicode_Resize(output, requiredsize) < 0)
Walter Dörwald3aeb6322002-09-02 13:14:32 +0000860 goto onError;
861 *outptr = PyUnicode_AS_UNICODE(*output) + *outpos;
862 }
863 *endinpos = newpos;
864 *inptr = input + newpos;
865 Py_UNICODE_COPY(*outptr, repptr, repsize);
866 *outptr += repsize;
867 *outpos += repsize;
868 /* we made it! */
869 res = 0;
870
871 onError:
872 Py_XDECREF(restuple);
873 return res;
874}
875
Marc-André Lemburgc60e6f72001-09-20 10:35:46 +0000876/* --- UTF-7 Codec -------------------------------------------------------- */
877
878/* see RFC2152 for details */
879
Tim Petersced69f82003-09-16 20:30:58 +0000880static
Marc-André Lemburgc60e6f72001-09-20 10:35:46 +0000881char utf7_special[128] = {
882 /* indicate whether a UTF-7 character is special i.e. cannot be directly
883 encoded:
884 0 - not special
885 1 - special
886 2 - whitespace (optional)
887 3 - RFC2152 Set O (optional) */
888 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 1, 1, 2, 1, 1,
889 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
890 2, 3, 3, 3, 3, 3, 3, 0, 0, 0, 3, 1, 0, 0, 0, 1,
891 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 3, 3, 0,
892 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
893 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 3, 3, 3,
894 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
895 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 3, 1, 1,
896
897};
898
Marc-André Lemburge115ec82005-10-19 22:33:31 +0000899/* Note: The comparison (c) <= 0 is a trick to work-around gcc
900 warnings about the comparison always being false; since
901 utf7_special[0] is 1, we can safely make that one comparison
902 true */
903
Marc-André Lemburgc60e6f72001-09-20 10:35:46 +0000904#define SPECIAL(c, encodeO, encodeWS) \
Marc-André Lemburge115ec82005-10-19 22:33:31 +0000905 ((c) > 127 || (c) <= 0 || utf7_special[(c)] == 1 || \
Marc-André Lemburg5c4a9d62005-10-19 22:39:02 +0000906 (encodeWS && (utf7_special[(c)] == 2)) || \
Marc-André Lemburgc60e6f72001-09-20 10:35:46 +0000907 (encodeO && (utf7_special[(c)] == 3)))
908
Marc-André Lemburge115ec82005-10-19 22:33:31 +0000909#define B64(n) \
910 ("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"[(n) & 0x3f])
911#define B64CHAR(c) \
912 (isalnum(c) || (c) == '+' || (c) == '/')
913#define UB64(c) \
914 ((c) == '+' ? 62 : (c) == '/' ? 63 : (c) >= 'a' ? \
915 (c) - 71 : (c) >= 'A' ? (c) - 65 : (c) + 4 )
Marc-André Lemburgc60e6f72001-09-20 10:35:46 +0000916
Marc-André Lemburg5c4a9d62005-10-19 22:39:02 +0000917#define ENCODE(out, ch, bits) \
918 while (bits >= 6) { \
919 *out++ = B64(ch >> (bits-6)); \
920 bits -= 6; \
Marc-André Lemburgc60e6f72001-09-20 10:35:46 +0000921 }
922
Marc-André Lemburg5c4a9d62005-10-19 22:39:02 +0000923#define DECODE(out, ch, bits, surrogate) \
924 while (bits >= 16) { \
925 Py_UNICODE outCh = (Py_UNICODE) ((ch >> (bits-16)) & 0xffff); \
926 bits -= 16; \
927 if (surrogate) { \
Marc-André Lemburge115ec82005-10-19 22:33:31 +0000928 /* We have already generated an error for the high surrogate \
929 so let's not bother seeing if the low surrogate is correct or not */ \
Marc-André Lemburg5c4a9d62005-10-19 22:39:02 +0000930 surrogate = 0; \
931 } else if (0xDC00 <= outCh && outCh <= 0xDFFF) { \
Marc-André Lemburgc60e6f72001-09-20 10:35:46 +0000932 /* This is a surrogate pair. Unfortunately we can't represent \
Marc-André Lemburg5c4a9d62005-10-19 22:39:02 +0000933 it in a 16-bit character */ \
934 surrogate = 1; \
935 errmsg = "code pairs are not supported"; \
936 goto utf7Error; \
937 } else { \
938 *out++ = outCh; \
939 } \
Marc-André Lemburge115ec82005-10-19 22:33:31 +0000940 }
Marc-André Lemburgc60e6f72001-09-20 10:35:46 +0000941
Marc-André Lemburgc60e6f72001-09-20 10:35:46 +0000942PyObject *PyUnicode_DecodeUTF7(const char *s,
Martin v. Löwis18e16552006-02-15 17:27:45 +0000943 Py_ssize_t size,
Marc-André Lemburgc60e6f72001-09-20 10:35:46 +0000944 const char *errors)
945{
Walter Dörwald3aeb6322002-09-02 13:14:32 +0000946 const char *starts = s;
Martin v. Löwis18e16552006-02-15 17:27:45 +0000947 Py_ssize_t startinpos;
948 Py_ssize_t endinpos;
949 Py_ssize_t outpos;
Marc-André Lemburgc60e6f72001-09-20 10:35:46 +0000950 const char *e;
951 PyUnicodeObject *unicode;
952 Py_UNICODE *p;
953 const char *errmsg = "";
954 int inShift = 0;
955 unsigned int bitsleft = 0;
956 unsigned long charsleft = 0;
Walter Dörwald3aeb6322002-09-02 13:14:32 +0000957 int surrogate = 0;
958 PyObject *errorHandler = NULL;
959 PyObject *exc = NULL;
Marc-André Lemburgc60e6f72001-09-20 10:35:46 +0000960
961 unicode = _PyUnicode_New(size);
962 if (!unicode)
963 return NULL;
964 if (size == 0)
965 return (PyObject *)unicode;
966
967 p = unicode->str;
968 e = s + size;
969
970 while (s < e) {
Walter Dörwald3aeb6322002-09-02 13:14:32 +0000971 Py_UNICODE ch;
972 restart:
973 ch = *s;
Marc-André Lemburgc60e6f72001-09-20 10:35:46 +0000974
975 if (inShift) {
976 if ((ch == '-') || !B64CHAR(ch)) {
977 inShift = 0;
978 s++;
Tim Petersced69f82003-09-16 20:30:58 +0000979
Marc-André Lemburgc60e6f72001-09-20 10:35:46 +0000980 /* p, charsleft, bitsleft, surrogate = */ DECODE(p, charsleft, bitsleft, surrogate);
981 if (bitsleft >= 6) {
982 /* The shift sequence has a partial character in it. If
983 bitsleft < 6 then we could just classify it as padding
984 but that is not the case here */
985
986 errmsg = "partial character in shift sequence";
Tim Petersced69f82003-09-16 20:30:58 +0000987 goto utf7Error;
Marc-André Lemburgc60e6f72001-09-20 10:35:46 +0000988 }
989 /* According to RFC2152 the remaining bits should be zero. We
Tim Petersced69f82003-09-16 20:30:58 +0000990 choose to signal an error/insert a replacement character
Marc-André Lemburgc60e6f72001-09-20 10:35:46 +0000991 here so indicate the potential of a misencoded character. */
992
993 /* On x86, a << b == a << (b%32) so make sure that bitsleft != 0 */
994 if (bitsleft && charsleft << (sizeof(charsleft) * 8 - bitsleft)) {
995 errmsg = "non-zero padding bits in shift sequence";
Tim Petersced69f82003-09-16 20:30:58 +0000996 goto utf7Error;
Marc-André Lemburgc60e6f72001-09-20 10:35:46 +0000997 }
998
999 if (ch == '-') {
1000 if ((s < e) && (*(s) == '-')) {
Tim Petersced69f82003-09-16 20:30:58 +00001001 *p++ = '-';
Marc-André Lemburgc60e6f72001-09-20 10:35:46 +00001002 inShift = 1;
1003 }
1004 } else if (SPECIAL(ch,0,0)) {
1005 errmsg = "unexpected special character";
Tim Petersced69f82003-09-16 20:30:58 +00001006 goto utf7Error;
Marc-André Lemburgc60e6f72001-09-20 10:35:46 +00001007 } else {
1008 *p++ = ch;
1009 }
1010 } else {
1011 charsleft = (charsleft << 6) | UB64(ch);
1012 bitsleft += 6;
1013 s++;
1014 /* p, charsleft, bitsleft, surrogate = */ DECODE(p, charsleft, bitsleft, surrogate);
1015 }
1016 }
1017 else if ( ch == '+' ) {
Walter Dörwald3aeb6322002-09-02 13:14:32 +00001018 startinpos = s-starts;
Marc-André Lemburgc60e6f72001-09-20 10:35:46 +00001019 s++;
1020 if (s < e && *s == '-') {
1021 s++;
1022 *p++ = '+';
1023 } else
1024 {
1025 inShift = 1;
1026 bitsleft = 0;
1027 }
1028 }
1029 else if (SPECIAL(ch,0,0)) {
1030 errmsg = "unexpected special character";
1031 s++;
Tim Petersced69f82003-09-16 20:30:58 +00001032 goto utf7Error;
Marc-André Lemburgc60e6f72001-09-20 10:35:46 +00001033 }
1034 else {
1035 *p++ = ch;
1036 s++;
1037 }
1038 continue;
1039 utf7Error:
Walter Dörwald3aeb6322002-09-02 13:14:32 +00001040 outpos = p-PyUnicode_AS_UNICODE(unicode);
1041 endinpos = s-starts;
1042 if (unicode_decode_call_errorhandler(
1043 errors, &errorHandler,
1044 "utf7", errmsg,
1045 starts, size, &startinpos, &endinpos, &exc, &s,
1046 (PyObject **)&unicode, &outpos, &p))
1047 goto onError;
Marc-André Lemburgc60e6f72001-09-20 10:35:46 +00001048 }
1049
1050 if (inShift) {
Walter Dörwald3aeb6322002-09-02 13:14:32 +00001051 outpos = p-PyUnicode_AS_UNICODE(unicode);
1052 endinpos = size;
1053 if (unicode_decode_call_errorhandler(
1054 errors, &errorHandler,
1055 "utf7", "unterminated shift sequence",
1056 starts, size, &startinpos, &endinpos, &exc, &s,
1057 (PyObject **)&unicode, &outpos, &p))
Marc-André Lemburgc60e6f72001-09-20 10:35:46 +00001058 goto onError;
Walter Dörwald3aeb6322002-09-02 13:14:32 +00001059 if (s < e)
1060 goto restart;
Marc-André Lemburgc60e6f72001-09-20 10:35:46 +00001061 }
1062
Jeremy Hyltondeb2dc62003-09-16 03:41:45 +00001063 if (_PyUnicode_Resize(&unicode, p - PyUnicode_AS_UNICODE(unicode)) < 0)
Marc-André Lemburgc60e6f72001-09-20 10:35:46 +00001064 goto onError;
1065
Walter Dörwald3aeb6322002-09-02 13:14:32 +00001066 Py_XDECREF(errorHandler);
1067 Py_XDECREF(exc);
Marc-André Lemburgc60e6f72001-09-20 10:35:46 +00001068 return (PyObject *)unicode;
1069
1070onError:
Walter Dörwald3aeb6322002-09-02 13:14:32 +00001071 Py_XDECREF(errorHandler);
1072 Py_XDECREF(exc);
Marc-André Lemburgc60e6f72001-09-20 10:35:46 +00001073 Py_DECREF(unicode);
1074 return NULL;
1075}
1076
1077
1078PyObject *PyUnicode_EncodeUTF7(const Py_UNICODE *s,
Martin v. Löwis18e16552006-02-15 17:27:45 +00001079 Py_ssize_t size,
Marc-André Lemburgc60e6f72001-09-20 10:35:46 +00001080 int encodeSetO,
1081 int encodeWhiteSpace,
1082 const char *errors)
1083{
1084 PyObject *v;
1085 /* It might be possible to tighten this worst case */
Martin v. Löwis18e16552006-02-15 17:27:45 +00001086 Py_ssize_t cbAllocated = 5 * size;
Marc-André Lemburgc60e6f72001-09-20 10:35:46 +00001087 int inShift = 0;
Martin v. Löwis18e16552006-02-15 17:27:45 +00001088 Py_ssize_t i = 0;
Marc-André Lemburgc60e6f72001-09-20 10:35:46 +00001089 unsigned int bitsleft = 0;
1090 unsigned long charsleft = 0;
1091 char * out;
1092 char * start;
1093
1094 if (size == 0)
1095 return PyString_FromStringAndSize(NULL, 0);
1096
1097 v = PyString_FromStringAndSize(NULL, cbAllocated);
1098 if (v == NULL)
1099 return NULL;
1100
1101 start = out = PyString_AS_STRING(v);
1102 for (;i < size; ++i) {
1103 Py_UNICODE ch = s[i];
1104
1105 if (!inShift) {
Hye-Shik Chang1bc09b72004-01-03 19:35:43 +00001106 if (ch == '+') {
1107 *out++ = '+';
Marc-André Lemburgc60e6f72001-09-20 10:35:46 +00001108 *out++ = '-';
1109 } else if (SPECIAL(ch, encodeSetO, encodeWhiteSpace)) {
1110 charsleft = ch;
1111 bitsleft = 16;
1112 *out++ = '+';
Hye-Shik Chang1bc09b72004-01-03 19:35:43 +00001113 /* out, charsleft, bitsleft = */ ENCODE(out, charsleft, bitsleft);
Marc-André Lemburgc60e6f72001-09-20 10:35:46 +00001114 inShift = bitsleft > 0;
Hye-Shik Chang1bc09b72004-01-03 19:35:43 +00001115 } else {
1116 *out++ = (char) ch;
1117 }
1118 } else {
Marc-André Lemburgc60e6f72001-09-20 10:35:46 +00001119 if (!SPECIAL(ch, encodeSetO, encodeWhiteSpace)) {
1120 *out++ = B64(charsleft << (6-bitsleft));
1121 charsleft = 0;
1122 bitsleft = 0;
1123 /* Characters not in the BASE64 set implicitly unshift the sequence
1124 so no '-' is required, except if the character is itself a '-' */
1125 if (B64CHAR(ch) || ch == '-') {
1126 *out++ = '-';
1127 }
1128 inShift = 0;
1129 *out++ = (char) ch;
1130 } else {
1131 bitsleft += 16;
1132 charsleft = (charsleft << 16) | ch;
1133 /* out, charsleft, bitsleft = */ ENCODE(out, charsleft, bitsleft);
1134
1135 /* If the next character is special then we dont' need to terminate
Tim Petersced69f82003-09-16 20:30:58 +00001136 the shift sequence. If the next character is not a BASE64 character
Marc-André Lemburgc60e6f72001-09-20 10:35:46 +00001137 or '-' then the shift sequence will be terminated implicitly and we
1138 don't have to insert a '-'. */
1139
1140 if (bitsleft == 0) {
1141 if (i + 1 < size) {
1142 Py_UNICODE ch2 = s[i+1];
1143
1144 if (SPECIAL(ch2, encodeSetO, encodeWhiteSpace)) {
Tim Petersced69f82003-09-16 20:30:58 +00001145
Marc-André Lemburgc60e6f72001-09-20 10:35:46 +00001146 } else if (B64CHAR(ch2) || ch2 == '-') {
1147 *out++ = '-';
1148 inShift = 0;
1149 } else {
1150 inShift = 0;
1151 }
1152
1153 }
1154 else {
1155 *out++ = '-';
1156 inShift = 0;
1157 }
1158 }
Tim Petersced69f82003-09-16 20:30:58 +00001159 }
Marc-André Lemburgc60e6f72001-09-20 10:35:46 +00001160 }
Hye-Shik Chang1bc09b72004-01-03 19:35:43 +00001161 }
Marc-André Lemburgc60e6f72001-09-20 10:35:46 +00001162 if (bitsleft) {
1163 *out++= B64(charsleft << (6-bitsleft) );
1164 *out++ = '-';
1165 }
1166
Tim Peters5de98422002-04-27 18:44:32 +00001167 _PyString_Resize(&v, out - start);
Marc-André Lemburgc60e6f72001-09-20 10:35:46 +00001168 return v;
1169}
1170
1171#undef SPECIAL
1172#undef B64
1173#undef B64CHAR
1174#undef UB64
1175#undef ENCODE
1176#undef DECODE
1177
Guido van Rossumd57fd912000-03-10 22:53:23 +00001178/* --- UTF-8 Codec -------------------------------------------------------- */
1179
Tim Petersced69f82003-09-16 20:30:58 +00001180static
Guido van Rossumd57fd912000-03-10 22:53:23 +00001181char utf8_code_length[256] = {
1182 /* Map UTF-8 encoded prefix byte to sequence length. zero means
1183 illegal prefix. see RFC 2279 for details */
1184 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1185 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1186 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1187 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1188 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1189 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1190 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1191 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1192 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1193 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1194 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1195 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1196 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
1197 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
1198 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
1199 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 0, 0
1200};
1201
Guido van Rossumd57fd912000-03-10 22:53:23 +00001202PyObject *PyUnicode_DecodeUTF8(const char *s,
Martin v. Löwis18e16552006-02-15 17:27:45 +00001203 Py_ssize_t size,
Guido van Rossumd57fd912000-03-10 22:53:23 +00001204 const char *errors)
1205{
Walter Dörwald69652032004-09-07 20:24:22 +00001206 return PyUnicode_DecodeUTF8Stateful(s, size, errors, NULL);
1207}
1208
1209PyObject *PyUnicode_DecodeUTF8Stateful(const char *s,
Martin v. Löwis18e16552006-02-15 17:27:45 +00001210 Py_ssize_t size,
Walter Dörwald69652032004-09-07 20:24:22 +00001211 const char *errors,
Martin v. Löwis18e16552006-02-15 17:27:45 +00001212 Py_ssize_t *consumed)
Walter Dörwald69652032004-09-07 20:24:22 +00001213{
Walter Dörwald3aeb6322002-09-02 13:14:32 +00001214 const char *starts = s;
Guido van Rossumd57fd912000-03-10 22:53:23 +00001215 int n;
Martin v. Löwis18e16552006-02-15 17:27:45 +00001216 Py_ssize_t startinpos;
1217 Py_ssize_t endinpos;
1218 Py_ssize_t outpos;
Guido van Rossumd57fd912000-03-10 22:53:23 +00001219 const char *e;
1220 PyUnicodeObject *unicode;
1221 Py_UNICODE *p;
Marc-André Lemburg9542f482000-07-17 18:23:13 +00001222 const char *errmsg = "";
Walter Dörwald3aeb6322002-09-02 13:14:32 +00001223 PyObject *errorHandler = NULL;
1224 PyObject *exc = NULL;
Guido van Rossumd57fd912000-03-10 22:53:23 +00001225
1226 /* Note: size will always be longer than the resulting Unicode
1227 character count */
1228 unicode = _PyUnicode_New(size);
1229 if (!unicode)
1230 return NULL;
Walter Dörwald69652032004-09-07 20:24:22 +00001231 if (size == 0) {
1232 if (consumed)
1233 *consumed = 0;
Guido van Rossumd57fd912000-03-10 22:53:23 +00001234 return (PyObject *)unicode;
Walter Dörwald69652032004-09-07 20:24:22 +00001235 }
Guido van Rossumd57fd912000-03-10 22:53:23 +00001236
1237 /* Unpack UTF-8 encoded data */
1238 p = unicode->str;
1239 e = s + size;
1240
1241 while (s < e) {
Marc-André Lemburge12896e2000-07-07 17:51:08 +00001242 Py_UCS4 ch = (unsigned char)*s;
Guido van Rossumd57fd912000-03-10 22:53:23 +00001243
1244 if (ch < 0x80) {
Marc-André Lemburge12896e2000-07-07 17:51:08 +00001245 *p++ = (Py_UNICODE)ch;
Guido van Rossumd57fd912000-03-10 22:53:23 +00001246 s++;
1247 continue;
1248 }
1249
1250 n = utf8_code_length[ch];
1251
Marc-André Lemburg9542f482000-07-17 18:23:13 +00001252 if (s + n > e) {
Walter Dörwald69652032004-09-07 20:24:22 +00001253 if (consumed)
1254 break;
1255 else {
1256 errmsg = "unexpected end of data";
1257 startinpos = s-starts;
1258 endinpos = size;
1259 goto utf8Error;
1260 }
Marc-André Lemburg9542f482000-07-17 18:23:13 +00001261 }
Guido van Rossumd57fd912000-03-10 22:53:23 +00001262
1263 switch (n) {
1264
1265 case 0:
Marc-André Lemburg9542f482000-07-17 18:23:13 +00001266 errmsg = "unexpected code byte";
Walter Dörwald3aeb6322002-09-02 13:14:32 +00001267 startinpos = s-starts;
1268 endinpos = startinpos+1;
Marc-André Lemburg9542f482000-07-17 18:23:13 +00001269 goto utf8Error;
Guido van Rossumd57fd912000-03-10 22:53:23 +00001270
1271 case 1:
Marc-André Lemburg9542f482000-07-17 18:23:13 +00001272 errmsg = "internal error";
Walter Dörwald3aeb6322002-09-02 13:14:32 +00001273 startinpos = s-starts;
1274 endinpos = startinpos+1;
Marc-André Lemburg9542f482000-07-17 18:23:13 +00001275 goto utf8Error;
Guido van Rossumd57fd912000-03-10 22:53:23 +00001276
1277 case 2:
Marc-André Lemburg9542f482000-07-17 18:23:13 +00001278 if ((s[1] & 0xc0) != 0x80) {
1279 errmsg = "invalid data";
Walter Dörwald3aeb6322002-09-02 13:14:32 +00001280 startinpos = s-starts;
1281 endinpos = startinpos+2;
Marc-André Lemburg9542f482000-07-17 18:23:13 +00001282 goto utf8Error;
1283 }
Guido van Rossumd57fd912000-03-10 22:53:23 +00001284 ch = ((s[0] & 0x1f) << 6) + (s[1] & 0x3f);
Marc-André Lemburg9542f482000-07-17 18:23:13 +00001285 if (ch < 0x80) {
Walter Dörwald3aeb6322002-09-02 13:14:32 +00001286 startinpos = s-starts;
1287 endinpos = startinpos+2;
Marc-André Lemburg9542f482000-07-17 18:23:13 +00001288 errmsg = "illegal encoding";
1289 goto utf8Error;
1290 }
Guido van Rossumd57fd912000-03-10 22:53:23 +00001291 else
Marc-André Lemburg9542f482000-07-17 18:23:13 +00001292 *p++ = (Py_UNICODE)ch;
Guido van Rossumd57fd912000-03-10 22:53:23 +00001293 break;
1294
1295 case 3:
Tim Petersced69f82003-09-16 20:30:58 +00001296 if ((s[1] & 0xc0) != 0x80 ||
Marc-André Lemburg9542f482000-07-17 18:23:13 +00001297 (s[2] & 0xc0) != 0x80) {
1298 errmsg = "invalid data";
Walter Dörwald3aeb6322002-09-02 13:14:32 +00001299 startinpos = s-starts;
1300 endinpos = startinpos+3;
Marc-André Lemburg9542f482000-07-17 18:23:13 +00001301 goto utf8Error;
1302 }
Guido van Rossumd57fd912000-03-10 22:53:23 +00001303 ch = ((s[0] & 0x0f) << 12) + ((s[1] & 0x3f) << 6) + (s[2] & 0x3f);
Marc-André Lemburgbd3be8f2002-02-07 11:33:49 +00001304 if (ch < 0x0800) {
1305 /* Note: UTF-8 encodings of surrogates are considered
Tim Petersced69f82003-09-16 20:30:58 +00001306 legal UTF-8 sequences;
Marc-André Lemburgbd3be8f2002-02-07 11:33:49 +00001307
1308 XXX For wide builds (UCS-4) we should probably try
1309 to recombine the surrogates into a single code
1310 unit.
1311 */
Marc-André Lemburg9542f482000-07-17 18:23:13 +00001312 errmsg = "illegal encoding";
Walter Dörwald3aeb6322002-09-02 13:14:32 +00001313 startinpos = s-starts;
1314 endinpos = startinpos+3;
Marc-André Lemburg9542f482000-07-17 18:23:13 +00001315 goto utf8Error;
1316 }
Guido van Rossumd57fd912000-03-10 22:53:23 +00001317 else
Marc-André Lemburgbd3be8f2002-02-07 11:33:49 +00001318 *p++ = (Py_UNICODE)ch;
Marc-André Lemburge12896e2000-07-07 17:51:08 +00001319 break;
1320
1321 case 4:
1322 if ((s[1] & 0xc0) != 0x80 ||
1323 (s[2] & 0xc0) != 0x80 ||
Marc-André Lemburg9542f482000-07-17 18:23:13 +00001324 (s[3] & 0xc0) != 0x80) {
1325 errmsg = "invalid data";
Walter Dörwald3aeb6322002-09-02 13:14:32 +00001326 startinpos = s-starts;
1327 endinpos = startinpos+4;
Marc-André Lemburg9542f482000-07-17 18:23:13 +00001328 goto utf8Error;
1329 }
Marc-André Lemburge12896e2000-07-07 17:51:08 +00001330 ch = ((s[0] & 0x7) << 18) + ((s[1] & 0x3f) << 12) +
1331 ((s[2] & 0x3f) << 6) + (s[3] & 0x3f);
1332 /* validate and convert to UTF-16 */
Martin v. Löwis0ba70cc2001-06-26 22:22:37 +00001333 if ((ch < 0x10000) /* minimum value allowed for 4
Marc-André Lemburgbd3be8f2002-02-07 11:33:49 +00001334 byte encoding */
Martin v. Löwis0ba70cc2001-06-26 22:22:37 +00001335 || (ch > 0x10ffff)) /* maximum value allowed for
Marc-André Lemburgbd3be8f2002-02-07 11:33:49 +00001336 UTF-16 */
Martin v. Löwis0ba70cc2001-06-26 22:22:37 +00001337 {
Marc-André Lemburg9542f482000-07-17 18:23:13 +00001338 errmsg = "illegal encoding";
Walter Dörwald3aeb6322002-09-02 13:14:32 +00001339 startinpos = s-starts;
1340 endinpos = startinpos+4;
Marc-André Lemburg9542f482000-07-17 18:23:13 +00001341 goto utf8Error;
1342 }
Fredrik Lundh8f455852001-06-27 18:59:43 +00001343#ifdef Py_UNICODE_WIDE
Martin v. Löwis0ba70cc2001-06-26 22:22:37 +00001344 *p++ = (Py_UNICODE)ch;
1345#else
Marc-André Lemburge12896e2000-07-07 17:51:08 +00001346 /* compute and append the two surrogates: */
Tim Petersced69f82003-09-16 20:30:58 +00001347
Marc-André Lemburge12896e2000-07-07 17:51:08 +00001348 /* translate from 10000..10FFFF to 0..FFFF */
1349 ch -= 0x10000;
Tim Petersced69f82003-09-16 20:30:58 +00001350
Marc-André Lemburge12896e2000-07-07 17:51:08 +00001351 /* high surrogate = top 10 bits added to D800 */
1352 *p++ = (Py_UNICODE)(0xD800 + (ch >> 10));
Tim Petersced69f82003-09-16 20:30:58 +00001353
Marc-André Lemburge12896e2000-07-07 17:51:08 +00001354 /* low surrogate = bottom 10 bits added to DC00 */
Fredrik Lundh45714e92001-06-26 16:39:36 +00001355 *p++ = (Py_UNICODE)(0xDC00 + (ch & 0x03FF));
Martin v. Löwis0ba70cc2001-06-26 22:22:37 +00001356#endif
Guido van Rossumd57fd912000-03-10 22:53:23 +00001357 break;
1358
1359 default:
1360 /* Other sizes are only needed for UCS-4 */
Marc-André Lemburg9542f482000-07-17 18:23:13 +00001361 errmsg = "unsupported Unicode code range";
Walter Dörwald3aeb6322002-09-02 13:14:32 +00001362 startinpos = s-starts;
1363 endinpos = startinpos+n;
Marc-André Lemburg9542f482000-07-17 18:23:13 +00001364 goto utf8Error;
Guido van Rossumd57fd912000-03-10 22:53:23 +00001365 }
1366 s += n;
Marc-André Lemburg9542f482000-07-17 18:23:13 +00001367 continue;
Tim Petersced69f82003-09-16 20:30:58 +00001368
Marc-André Lemburg9542f482000-07-17 18:23:13 +00001369 utf8Error:
Walter Dörwald3aeb6322002-09-02 13:14:32 +00001370 outpos = p-PyUnicode_AS_UNICODE(unicode);
1371 if (unicode_decode_call_errorhandler(
1372 errors, &errorHandler,
1373 "utf8", errmsg,
1374 starts, size, &startinpos, &endinpos, &exc, &s,
1375 (PyObject **)&unicode, &outpos, &p))
1376 goto onError;
Guido van Rossumd57fd912000-03-10 22:53:23 +00001377 }
Walter Dörwald69652032004-09-07 20:24:22 +00001378 if (consumed)
1379 *consumed = s-starts;
Guido van Rossumd57fd912000-03-10 22:53:23 +00001380
1381 /* Adjust length */
Jeremy Hyltondeb2dc62003-09-16 03:41:45 +00001382 if (_PyUnicode_Resize(&unicode, p - unicode->str) < 0)
Guido van Rossumd57fd912000-03-10 22:53:23 +00001383 goto onError;
1384
Walter Dörwald3aeb6322002-09-02 13:14:32 +00001385 Py_XDECREF(errorHandler);
1386 Py_XDECREF(exc);
Guido van Rossumd57fd912000-03-10 22:53:23 +00001387 return (PyObject *)unicode;
1388
1389onError:
Walter Dörwald3aeb6322002-09-02 13:14:32 +00001390 Py_XDECREF(errorHandler);
1391 Py_XDECREF(exc);
Guido van Rossumd57fd912000-03-10 22:53:23 +00001392 Py_DECREF(unicode);
1393 return NULL;
1394}
1395
Tim Peters602f7402002-04-27 18:03:26 +00001396/* Allocation strategy: if the string is short, convert into a stack buffer
1397 and allocate exactly as much space needed at the end. Else allocate the
1398 maximum possible needed (4 result bytes per Unicode character), and return
1399 the excess memory at the end.
Martin v. Löwis2a7ff352002-04-21 09:59:45 +00001400*/
Tim Peters7e3d9612002-04-21 03:26:37 +00001401PyObject *
1402PyUnicode_EncodeUTF8(const Py_UNICODE *s,
Martin v. Löwis18e16552006-02-15 17:27:45 +00001403 Py_ssize_t size,
Tim Peters7e3d9612002-04-21 03:26:37 +00001404 const char *errors)
Guido van Rossumd57fd912000-03-10 22:53:23 +00001405{
Tim Peters602f7402002-04-27 18:03:26 +00001406#define MAX_SHORT_UNICHARS 300 /* largest size we'll do on the stack */
Tim Peters0eca65c2002-04-21 17:28:06 +00001407
Martin v. Löwis18e16552006-02-15 17:27:45 +00001408 Py_ssize_t i; /* index into s of next input byte */
Tim Peters602f7402002-04-27 18:03:26 +00001409 PyObject *v; /* result string object */
1410 char *p; /* next free byte in output buffer */
Martin v. Löwis18e16552006-02-15 17:27:45 +00001411 Py_ssize_t nallocated; /* number of result bytes allocated */
Martin v. Löwis412fb672006-04-13 06:34:32 +00001412 Py_ssize_t nneeded; /* number of result bytes needed */
Tim Peters602f7402002-04-27 18:03:26 +00001413 char stackbuf[MAX_SHORT_UNICHARS * 4];
Marc-André Lemburgbd3be8f2002-02-07 11:33:49 +00001414
Tim Peters602f7402002-04-27 18:03:26 +00001415 assert(s != NULL);
1416 assert(size >= 0);
Guido van Rossumd57fd912000-03-10 22:53:23 +00001417
Tim Peters602f7402002-04-27 18:03:26 +00001418 if (size <= MAX_SHORT_UNICHARS) {
1419 /* Write into the stack buffer; nallocated can't overflow.
1420 * At the end, we'll allocate exactly as much heap space as it
1421 * turns out we need.
1422 */
1423 nallocated = Py_SAFE_DOWNCAST(sizeof(stackbuf), size_t, int);
1424 v = NULL; /* will allocate after we're done */
1425 p = stackbuf;
1426 }
1427 else {
1428 /* Overallocate on the heap, and give the excess back at the end. */
1429 nallocated = size * 4;
1430 if (nallocated / 4 != size) /* overflow! */
1431 return PyErr_NoMemory();
1432 v = PyString_FromStringAndSize(NULL, nallocated);
1433 if (v == NULL)
1434 return NULL;
1435 p = PyString_AS_STRING(v);
1436 }
Martin v. Löwis2a7ff352002-04-21 09:59:45 +00001437
Tim Peters602f7402002-04-27 18:03:26 +00001438 for (i = 0; i < size;) {
Marc-André Lemburge12896e2000-07-07 17:51:08 +00001439 Py_UCS4 ch = s[i++];
Marc-André Lemburg3688a882002-02-06 18:09:02 +00001440
Martin v. Löwis2a7ff352002-04-21 09:59:45 +00001441 if (ch < 0x80)
Tim Peters602f7402002-04-27 18:03:26 +00001442 /* Encode ASCII */
Guido van Rossumd57fd912000-03-10 22:53:23 +00001443 *p++ = (char) ch;
Marc-André Lemburg3688a882002-02-06 18:09:02 +00001444
Guido van Rossumd57fd912000-03-10 22:53:23 +00001445 else if (ch < 0x0800) {
Tim Peters602f7402002-04-27 18:03:26 +00001446 /* Encode Latin-1 */
Marc-André Lemburgdc724d62002-02-06 18:20:19 +00001447 *p++ = (char)(0xc0 | (ch >> 6));
1448 *p++ = (char)(0x80 | (ch & 0x3f));
Marc-André Lemburge12896e2000-07-07 17:51:08 +00001449 }
Marc-André Lemburg3688a882002-02-06 18:09:02 +00001450 else {
Tim Peters602f7402002-04-27 18:03:26 +00001451 /* Encode UCS2 Unicode ordinals */
1452 if (ch < 0x10000) {
1453 /* Special case: check for high surrogate */
1454 if (0xD800 <= ch && ch <= 0xDBFF && i != size) {
1455 Py_UCS4 ch2 = s[i];
1456 /* Check for low surrogate and combine the two to
1457 form a UCS4 value */
1458 if (0xDC00 <= ch2 && ch2 <= 0xDFFF) {
Martin v. Löwis2a7ff352002-04-21 09:59:45 +00001459 ch = ((ch - 0xD800) << 10 | (ch2 - 0xDC00)) + 0x10000;
Tim Peters602f7402002-04-27 18:03:26 +00001460 i++;
1461 goto encodeUCS4;
Marc-André Lemburge12896e2000-07-07 17:51:08 +00001462 }
Tim Peters602f7402002-04-27 18:03:26 +00001463 /* Fall through: handles isolated high surrogates */
Marc-André Lemburge12896e2000-07-07 17:51:08 +00001464 }
Marc-André Lemburge12896e2000-07-07 17:51:08 +00001465 *p++ = (char)(0xe0 | (ch >> 12));
Tim Peters602f7402002-04-27 18:03:26 +00001466 *p++ = (char)(0x80 | ((ch >> 6) & 0x3f));
1467 *p++ = (char)(0x80 | (ch & 0x3f));
1468 continue;
1469 }
1470encodeUCS4:
1471 /* Encode UCS4 Unicode ordinals */
1472 *p++ = (char)(0xf0 | (ch >> 18));
1473 *p++ = (char)(0x80 | ((ch >> 12) & 0x3f));
1474 *p++ = (char)(0x80 | ((ch >> 6) & 0x3f));
1475 *p++ = (char)(0x80 | (ch & 0x3f));
1476 }
Guido van Rossumd57fd912000-03-10 22:53:23 +00001477 }
Tim Peters0eca65c2002-04-21 17:28:06 +00001478
Tim Peters602f7402002-04-27 18:03:26 +00001479 if (v == NULL) {
1480 /* This was stack allocated. */
Martin v. Löwis412fb672006-04-13 06:34:32 +00001481 nneeded = p - stackbuf;
Tim Peters602f7402002-04-27 18:03:26 +00001482 assert(nneeded <= nallocated);
1483 v = PyString_FromStringAndSize(stackbuf, nneeded);
1484 }
1485 else {
1486 /* Cut back to size actually needed. */
Martin v. Löwis412fb672006-04-13 06:34:32 +00001487 nneeded = p - PyString_AS_STRING(v);
Tim Peters602f7402002-04-27 18:03:26 +00001488 assert(nneeded <= nallocated);
1489 _PyString_Resize(&v, nneeded);
1490 }
Guido van Rossumd57fd912000-03-10 22:53:23 +00001491 return v;
Martin v. Löwis2a7ff352002-04-21 09:59:45 +00001492
Tim Peters602f7402002-04-27 18:03:26 +00001493#undef MAX_SHORT_UNICHARS
Guido van Rossumd57fd912000-03-10 22:53:23 +00001494}
1495
Guido van Rossumd57fd912000-03-10 22:53:23 +00001496PyObject *PyUnicode_AsUTF8String(PyObject *unicode)
1497{
Guido van Rossumd57fd912000-03-10 22:53:23 +00001498 if (!PyUnicode_Check(unicode)) {
1499 PyErr_BadArgument();
1500 return NULL;
1501 }
Barry Warsaw2dd4abf2000-08-18 06:58:15 +00001502 return PyUnicode_EncodeUTF8(PyUnicode_AS_UNICODE(unicode),
1503 PyUnicode_GET_SIZE(unicode),
1504 NULL);
Guido van Rossumd57fd912000-03-10 22:53:23 +00001505}
1506
1507/* --- UTF-16 Codec ------------------------------------------------------- */
1508
Tim Peters772747b2001-08-09 22:21:55 +00001509PyObject *
1510PyUnicode_DecodeUTF16(const char *s,
Martin v. Löwis18e16552006-02-15 17:27:45 +00001511 Py_ssize_t size,
Tim Peters772747b2001-08-09 22:21:55 +00001512 const char *errors,
1513 int *byteorder)
Guido van Rossumd57fd912000-03-10 22:53:23 +00001514{
Walter Dörwald69652032004-09-07 20:24:22 +00001515 return PyUnicode_DecodeUTF16Stateful(s, size, errors, byteorder, NULL);
1516}
1517
1518PyObject *
1519PyUnicode_DecodeUTF16Stateful(const char *s,
Martin v. Löwis18e16552006-02-15 17:27:45 +00001520 Py_ssize_t size,
Walter Dörwald69652032004-09-07 20:24:22 +00001521 const char *errors,
1522 int *byteorder,
Martin v. Löwis18e16552006-02-15 17:27:45 +00001523 Py_ssize_t *consumed)
Walter Dörwald69652032004-09-07 20:24:22 +00001524{
Walter Dörwald3aeb6322002-09-02 13:14:32 +00001525 const char *starts = s;
Martin v. Löwis18e16552006-02-15 17:27:45 +00001526 Py_ssize_t startinpos;
1527 Py_ssize_t endinpos;
1528 Py_ssize_t outpos;
Guido van Rossumd57fd912000-03-10 22:53:23 +00001529 PyUnicodeObject *unicode;
1530 Py_UNICODE *p;
Tim Peters772747b2001-08-09 22:21:55 +00001531 const unsigned char *q, *e;
1532 int bo = 0; /* assume native ordering by default */
Marc-André Lemburg9542f482000-07-17 18:23:13 +00001533 const char *errmsg = "";
Tim Peters772747b2001-08-09 22:21:55 +00001534 /* Offsets from q for retrieving byte pairs in the right order. */
1535#ifdef BYTEORDER_IS_LITTLE_ENDIAN
1536 int ihi = 1, ilo = 0;
1537#else
1538 int ihi = 0, ilo = 1;
1539#endif
Walter Dörwald3aeb6322002-09-02 13:14:32 +00001540 PyObject *errorHandler = NULL;
1541 PyObject *exc = NULL;
Guido van Rossumd57fd912000-03-10 22:53:23 +00001542
1543 /* Note: size will always be longer than the resulting Unicode
1544 character count */
1545 unicode = _PyUnicode_New(size);
1546 if (!unicode)
1547 return NULL;
1548 if (size == 0)
1549 return (PyObject *)unicode;
1550
1551 /* Unpack UTF-16 encoded data */
1552 p = unicode->str;
Tim Peters772747b2001-08-09 22:21:55 +00001553 q = (unsigned char *)s;
1554 e = q + size;
Guido van Rossumd57fd912000-03-10 22:53:23 +00001555
1556 if (byteorder)
Tim Peters772747b2001-08-09 22:21:55 +00001557 bo = *byteorder;
Guido van Rossumd57fd912000-03-10 22:53:23 +00001558
Marc-André Lemburg489b56e2001-05-21 20:30:15 +00001559 /* Check for BOM marks (U+FEFF) in the input and adjust current
1560 byte order setting accordingly. In native mode, the leading BOM
1561 mark is skipped, in all other modes, it is copied to the output
1562 stream as-is (giving a ZWNBSP character). */
1563 if (bo == 0) {
Walter Dörwald69652032004-09-07 20:24:22 +00001564 if (size >= 2) {
1565 const Py_UNICODE bom = (q[ihi] << 8) | q[ilo];
Marc-André Lemburg489b56e2001-05-21 20:30:15 +00001566#ifdef BYTEORDER_IS_LITTLE_ENDIAN
Walter Dörwald69652032004-09-07 20:24:22 +00001567 if (bom == 0xFEFF) {
1568 q += 2;
1569 bo = -1;
1570 }
1571 else if (bom == 0xFFFE) {
1572 q += 2;
1573 bo = 1;
1574 }
Tim Petersced69f82003-09-16 20:30:58 +00001575#else
Walter Dörwald69652032004-09-07 20:24:22 +00001576 if (bom == 0xFEFF) {
1577 q += 2;
1578 bo = 1;
1579 }
1580 else if (bom == 0xFFFE) {
1581 q += 2;
1582 bo = -1;
1583 }
Marc-André Lemburg489b56e2001-05-21 20:30:15 +00001584#endif
Walter Dörwald69652032004-09-07 20:24:22 +00001585 }
Marc-André Lemburg489b56e2001-05-21 20:30:15 +00001586 }
Guido van Rossumd57fd912000-03-10 22:53:23 +00001587
Tim Peters772747b2001-08-09 22:21:55 +00001588 if (bo == -1) {
1589 /* force LE */
1590 ihi = 1;
1591 ilo = 0;
1592 }
1593 else if (bo == 1) {
1594 /* force BE */
1595 ihi = 0;
1596 ilo = 1;
1597 }
1598
1599 while (q < e) {
Walter Dörwald3aeb6322002-09-02 13:14:32 +00001600 Py_UNICODE ch;
Walter Dörwald69652032004-09-07 20:24:22 +00001601 /* remaining bytes at the end? (size should be even) */
Walter Dörwald3aeb6322002-09-02 13:14:32 +00001602 if (e-q<2) {
Walter Dörwald69652032004-09-07 20:24:22 +00001603 if (consumed)
1604 break;
Walter Dörwald3aeb6322002-09-02 13:14:32 +00001605 errmsg = "truncated data";
1606 startinpos = ((const char *)q)-starts;
1607 endinpos = ((const char *)e)-starts;
1608 goto utf16Error;
1609 /* The remaining input chars are ignored if the callback
1610 chooses to skip the input */
1611 }
1612 ch = (q[ihi] << 8) | q[ilo];
1613
Tim Peters772747b2001-08-09 22:21:55 +00001614 q += 2;
1615
Guido van Rossumd57fd912000-03-10 22:53:23 +00001616 if (ch < 0xD800 || ch > 0xDFFF) {
1617 *p++ = ch;
1618 continue;
1619 }
1620
1621 /* UTF-16 code pair: */
Marc-André Lemburg9542f482000-07-17 18:23:13 +00001622 if (q >= e) {
1623 errmsg = "unexpected end of data";
Walter Dörwald3aeb6322002-09-02 13:14:32 +00001624 startinpos = (((const char *)q)-2)-starts;
1625 endinpos = ((const char *)e)-starts;
Marc-André Lemburg9542f482000-07-17 18:23:13 +00001626 goto utf16Error;
1627 }
Martin v. Löwisac93bc22001-06-26 22:43:40 +00001628 if (0xD800 <= ch && ch <= 0xDBFF) {
Tim Peters772747b2001-08-09 22:21:55 +00001629 Py_UNICODE ch2 = (q[ihi] << 8) | q[ilo];
1630 q += 2;
Martin v. Löwisac93bc22001-06-26 22:43:40 +00001631 if (0xDC00 <= ch2 && ch2 <= 0xDFFF) {
Fredrik Lundh8f455852001-06-27 18:59:43 +00001632#ifndef Py_UNICODE_WIDE
Marc-André Lemburg6c6bfb72001-07-20 17:39:11 +00001633 *p++ = ch;
1634 *p++ = ch2;
Martin v. Löwis0ba70cc2001-06-26 22:22:37 +00001635#else
1636 *p++ = (((ch & 0x3FF)<<10) | (ch2 & 0x3FF)) + 0x10000;
Martin v. Löwis0ba70cc2001-06-26 22:22:37 +00001637#endif
Marc-André Lemburg6c6bfb72001-07-20 17:39:11 +00001638 continue;
Martin v. Löwis0ba70cc2001-06-26 22:22:37 +00001639 }
1640 else {
1641 errmsg = "illegal UTF-16 surrogate";
Walter Dörwald3aeb6322002-09-02 13:14:32 +00001642 startinpos = (((const char *)q)-4)-starts;
1643 endinpos = startinpos+2;
Martin v. Löwis0ba70cc2001-06-26 22:22:37 +00001644 goto utf16Error;
1645 }
1646
Guido van Rossumd57fd912000-03-10 22:53:23 +00001647 }
Marc-André Lemburg9542f482000-07-17 18:23:13 +00001648 errmsg = "illegal encoding";
Walter Dörwald3aeb6322002-09-02 13:14:32 +00001649 startinpos = (((const char *)q)-2)-starts;
1650 endinpos = startinpos+2;
Marc-André Lemburg9542f482000-07-17 18:23:13 +00001651 /* Fall through to report the error */
1652
1653 utf16Error:
Walter Dörwald3aeb6322002-09-02 13:14:32 +00001654 outpos = p-PyUnicode_AS_UNICODE(unicode);
1655 if (unicode_decode_call_errorhandler(
1656 errors, &errorHandler,
1657 "utf16", errmsg,
1658 starts, size, &startinpos, &endinpos, &exc, (const char **)&q,
1659 (PyObject **)&unicode, &outpos, &p))
Marc-André Lemburg9542f482000-07-17 18:23:13 +00001660 goto onError;
Guido van Rossumd57fd912000-03-10 22:53:23 +00001661 }
1662
1663 if (byteorder)
1664 *byteorder = bo;
1665
Walter Dörwald69652032004-09-07 20:24:22 +00001666 if (consumed)
1667 *consumed = (const char *)q-starts;
1668
Guido van Rossumd57fd912000-03-10 22:53:23 +00001669 /* Adjust length */
Jeremy Hyltondeb2dc62003-09-16 03:41:45 +00001670 if (_PyUnicode_Resize(&unicode, p - unicode->str) < 0)
Guido van Rossumd57fd912000-03-10 22:53:23 +00001671 goto onError;
1672
Walter Dörwald3aeb6322002-09-02 13:14:32 +00001673 Py_XDECREF(errorHandler);
1674 Py_XDECREF(exc);
Guido van Rossumd57fd912000-03-10 22:53:23 +00001675 return (PyObject *)unicode;
1676
1677onError:
1678 Py_DECREF(unicode);
Walter Dörwald3aeb6322002-09-02 13:14:32 +00001679 Py_XDECREF(errorHandler);
1680 Py_XDECREF(exc);
Guido van Rossumd57fd912000-03-10 22:53:23 +00001681 return NULL;
1682}
1683
Tim Peters772747b2001-08-09 22:21:55 +00001684PyObject *
1685PyUnicode_EncodeUTF16(const Py_UNICODE *s,
Martin v. Löwis18e16552006-02-15 17:27:45 +00001686 Py_ssize_t size,
Tim Peters772747b2001-08-09 22:21:55 +00001687 const char *errors,
1688 int byteorder)
Guido van Rossumd57fd912000-03-10 22:53:23 +00001689{
1690 PyObject *v;
Tim Peters772747b2001-08-09 22:21:55 +00001691 unsigned char *p;
Hye-Shik Chang4a264fb2003-12-19 01:59:56 +00001692#ifdef Py_UNICODE_WIDE
Tim Peters772747b2001-08-09 22:21:55 +00001693 int i, pairs;
Hye-Shik Chang4a264fb2003-12-19 01:59:56 +00001694#else
1695 const int pairs = 0;
1696#endif
Tim Peters772747b2001-08-09 22:21:55 +00001697 /* Offsets from p for storing byte pairs in the right order. */
1698#ifdef BYTEORDER_IS_LITTLE_ENDIAN
1699 int ihi = 1, ilo = 0;
1700#else
1701 int ihi = 0, ilo = 1;
1702#endif
1703
1704#define STORECHAR(CH) \
1705 do { \
1706 p[ihi] = ((CH) >> 8) & 0xff; \
1707 p[ilo] = (CH) & 0xff; \
1708 p += 2; \
1709 } while(0)
Guido van Rossumd57fd912000-03-10 22:53:23 +00001710
Hye-Shik Chang4a264fb2003-12-19 01:59:56 +00001711#ifdef Py_UNICODE_WIDE
Martin v. Löwis0ba70cc2001-06-26 22:22:37 +00001712 for (i = pairs = 0; i < size; i++)
1713 if (s[i] >= 0x10000)
1714 pairs++;
Hye-Shik Chang4a264fb2003-12-19 01:59:56 +00001715#endif
Tim Petersced69f82003-09-16 20:30:58 +00001716 v = PyString_FromStringAndSize(NULL,
Tim Peters772747b2001-08-09 22:21:55 +00001717 2 * (size + pairs + (byteorder == 0)));
Guido van Rossumd57fd912000-03-10 22:53:23 +00001718 if (v == NULL)
1719 return NULL;
Guido van Rossumd57fd912000-03-10 22:53:23 +00001720
Tim Peters772747b2001-08-09 22:21:55 +00001721 p = (unsigned char *)PyString_AS_STRING(v);
Guido van Rossumd57fd912000-03-10 22:53:23 +00001722 if (byteorder == 0)
Tim Peters772747b2001-08-09 22:21:55 +00001723 STORECHAR(0xFEFF);
Marc-André Lemburg063e0cb2000-07-07 11:27:45 +00001724 if (size == 0)
Marc-André Lemburgb7520772000-08-14 11:29:19 +00001725 return v;
Tim Peters772747b2001-08-09 22:21:55 +00001726
1727 if (byteorder == -1) {
1728 /* force LE */
1729 ihi = 1;
1730 ilo = 0;
1731 }
1732 else if (byteorder == 1) {
1733 /* force BE */
1734 ihi = 0;
1735 ilo = 1;
1736 }
1737
Martin v. Löwis0ba70cc2001-06-26 22:22:37 +00001738 while (size-- > 0) {
1739 Py_UNICODE ch = *s++;
1740 Py_UNICODE ch2 = 0;
Hye-Shik Chang4a264fb2003-12-19 01:59:56 +00001741#ifdef Py_UNICODE_WIDE
Martin v. Löwis0ba70cc2001-06-26 22:22:37 +00001742 if (ch >= 0x10000) {
Tim Peters772747b2001-08-09 22:21:55 +00001743 ch2 = 0xDC00 | ((ch-0x10000) & 0x3FF);
1744 ch = 0xD800 | ((ch-0x10000) >> 10);
Guido van Rossumd57fd912000-03-10 22:53:23 +00001745 }
Hye-Shik Chang4a264fb2003-12-19 01:59:56 +00001746#endif
Tim Peters772747b2001-08-09 22:21:55 +00001747 STORECHAR(ch);
1748 if (ch2)
1749 STORECHAR(ch2);
Martin v. Löwis0ba70cc2001-06-26 22:22:37 +00001750 }
Guido van Rossumd57fd912000-03-10 22:53:23 +00001751 return v;
Tim Peters772747b2001-08-09 22:21:55 +00001752#undef STORECHAR
Guido van Rossumd57fd912000-03-10 22:53:23 +00001753}
1754
1755PyObject *PyUnicode_AsUTF16String(PyObject *unicode)
1756{
1757 if (!PyUnicode_Check(unicode)) {
1758 PyErr_BadArgument();
1759 return NULL;
1760 }
1761 return PyUnicode_EncodeUTF16(PyUnicode_AS_UNICODE(unicode),
1762 PyUnicode_GET_SIZE(unicode),
1763 NULL,
1764 0);
1765}
1766
1767/* --- Unicode Escape Codec ----------------------------------------------- */
1768
Fredrik Lundh06d12682001-01-24 07:59:11 +00001769static _PyUnicode_Name_CAPI *ucnhash_CAPI = NULL;
Marc-André Lemburg0f774e32000-06-28 16:43:35 +00001770
Guido van Rossumd57fd912000-03-10 22:53:23 +00001771PyObject *PyUnicode_DecodeUnicodeEscape(const char *s,
Martin v. Löwis18e16552006-02-15 17:27:45 +00001772 Py_ssize_t size,
Guido van Rossumd57fd912000-03-10 22:53:23 +00001773 const char *errors)
1774{
Walter Dörwald3aeb6322002-09-02 13:14:32 +00001775 const char *starts = s;
Martin v. Löwis18e16552006-02-15 17:27:45 +00001776 Py_ssize_t startinpos;
1777 Py_ssize_t endinpos;
1778 Py_ssize_t outpos;
Walter Dörwald3aeb6322002-09-02 13:14:32 +00001779 int i;
Guido van Rossumd57fd912000-03-10 22:53:23 +00001780 PyUnicodeObject *v;
Walter Dörwald3aeb6322002-09-02 13:14:32 +00001781 Py_UNICODE *p;
Guido van Rossumd57fd912000-03-10 22:53:23 +00001782 const char *end;
Fredrik Lundhccc74732001-02-18 22:13:49 +00001783 char* message;
1784 Py_UCS4 chr = 0xffffffff; /* in case 'getcode' messes up */
Walter Dörwald3aeb6322002-09-02 13:14:32 +00001785 PyObject *errorHandler = NULL;
1786 PyObject *exc = NULL;
Fredrik Lundhccc74732001-02-18 22:13:49 +00001787
Guido van Rossumd57fd912000-03-10 22:53:23 +00001788 /* Escaped strings will always be longer than the resulting
1789 Unicode string, so we start with size here and then reduce the
Walter Dörwald3aeb6322002-09-02 13:14:32 +00001790 length after conversion to the true value.
1791 (but if the error callback returns a long replacement string
1792 we'll have to allocate more space) */
Guido van Rossumd57fd912000-03-10 22:53:23 +00001793 v = _PyUnicode_New(size);
1794 if (v == NULL)
1795 goto onError;
1796 if (size == 0)
1797 return (PyObject *)v;
Fredrik Lundhccc74732001-02-18 22:13:49 +00001798
Walter Dörwald3aeb6322002-09-02 13:14:32 +00001799 p = PyUnicode_AS_UNICODE(v);
Guido van Rossumd57fd912000-03-10 22:53:23 +00001800 end = s + size;
Fredrik Lundhccc74732001-02-18 22:13:49 +00001801
Guido van Rossumd57fd912000-03-10 22:53:23 +00001802 while (s < end) {
1803 unsigned char c;
Marc-André Lemburg063e0cb2000-07-07 11:27:45 +00001804 Py_UNICODE x;
Walter Dörwald3aeb6322002-09-02 13:14:32 +00001805 int digits;
Guido van Rossumd57fd912000-03-10 22:53:23 +00001806
1807 /* Non-escape characters are interpreted as Unicode ordinals */
1808 if (*s != '\\') {
Fredrik Lundhccc74732001-02-18 22:13:49 +00001809 *p++ = (unsigned char) *s++;
Guido van Rossumd57fd912000-03-10 22:53:23 +00001810 continue;
1811 }
1812
Walter Dörwald3aeb6322002-09-02 13:14:32 +00001813 startinpos = s-starts;
Guido van Rossumd57fd912000-03-10 22:53:23 +00001814 /* \ - Escapes */
1815 s++;
1816 switch (*s++) {
1817
1818 /* \x escapes */
1819 case '\n': break;
1820 case '\\': *p++ = '\\'; break;
1821 case '\'': *p++ = '\''; break;
1822 case '\"': *p++ = '\"'; break;
1823 case 'b': *p++ = '\b'; break;
1824 case 'f': *p++ = '\014'; break; /* FF */
1825 case 't': *p++ = '\t'; break;
1826 case 'n': *p++ = '\n'; break;
1827 case 'r': *p++ = '\r'; break;
1828 case 'v': *p++ = '\013'; break; /* VT */
1829 case 'a': *p++ = '\007'; break; /* BEL, not classic C */
1830
1831 /* \OOO (octal) escapes */
1832 case '0': case '1': case '2': case '3':
1833 case '4': case '5': case '6': case '7':
Guido van Rossum0e4f6572000-05-01 21:27:20 +00001834 x = s[-1] - '0';
Guido van Rossumd57fd912000-03-10 22:53:23 +00001835 if ('0' <= *s && *s <= '7') {
Guido van Rossum0e4f6572000-05-01 21:27:20 +00001836 x = (x<<3) + *s++ - '0';
Guido van Rossumd57fd912000-03-10 22:53:23 +00001837 if ('0' <= *s && *s <= '7')
Guido van Rossum0e4f6572000-05-01 21:27:20 +00001838 x = (x<<3) + *s++ - '0';
Guido van Rossumd57fd912000-03-10 22:53:23 +00001839 }
Guido van Rossum0e4f6572000-05-01 21:27:20 +00001840 *p++ = x;
Guido van Rossumd57fd912000-03-10 22:53:23 +00001841 break;
1842
Fredrik Lundhccc74732001-02-18 22:13:49 +00001843 /* hex escapes */
1844 /* \xXX */
Guido van Rossumd57fd912000-03-10 22:53:23 +00001845 case 'x':
Fredrik Lundhccc74732001-02-18 22:13:49 +00001846 digits = 2;
1847 message = "truncated \\xXX escape";
1848 goto hexescape;
Guido van Rossumd57fd912000-03-10 22:53:23 +00001849
Fredrik Lundhccc74732001-02-18 22:13:49 +00001850 /* \uXXXX */
Guido van Rossumd57fd912000-03-10 22:53:23 +00001851 case 'u':
Fredrik Lundhccc74732001-02-18 22:13:49 +00001852 digits = 4;
1853 message = "truncated \\uXXXX escape";
1854 goto hexescape;
Guido van Rossumd57fd912000-03-10 22:53:23 +00001855
Fredrik Lundhccc74732001-02-18 22:13:49 +00001856 /* \UXXXXXXXX */
Fredrik Lundhdf846752000-09-03 11:29:49 +00001857 case 'U':
Fredrik Lundhccc74732001-02-18 22:13:49 +00001858 digits = 8;
1859 message = "truncated \\UXXXXXXXX escape";
1860 hexescape:
1861 chr = 0;
Walter Dörwald3aeb6322002-09-02 13:14:32 +00001862 outpos = p-PyUnicode_AS_UNICODE(v);
1863 if (s+digits>end) {
1864 endinpos = size;
1865 if (unicode_decode_call_errorhandler(
1866 errors, &errorHandler,
1867 "unicodeescape", "end of string in escape sequence",
1868 starts, size, &startinpos, &endinpos, &exc, &s,
1869 (PyObject **)&v, &outpos, &p))
1870 goto onError;
1871 goto nextByte;
1872 }
1873 for (i = 0; i < digits; ++i) {
Fredrik Lundhccc74732001-02-18 22:13:49 +00001874 c = (unsigned char) s[i];
Fredrik Lundhdf846752000-09-03 11:29:49 +00001875 if (!isxdigit(c)) {
Walter Dörwald3aeb6322002-09-02 13:14:32 +00001876 endinpos = (s+i+1)-starts;
1877 if (unicode_decode_call_errorhandler(
1878 errors, &errorHandler,
1879 "unicodeescape", message,
1880 starts, size, &startinpos, &endinpos, &exc, &s,
1881 (PyObject **)&v, &outpos, &p))
Fredrik Lundhdf846752000-09-03 11:29:49 +00001882 goto onError;
Walter Dörwald3aeb6322002-09-02 13:14:32 +00001883 goto nextByte;
Fredrik Lundhdf846752000-09-03 11:29:49 +00001884 }
1885 chr = (chr<<4) & ~0xF;
1886 if (c >= '0' && c <= '9')
1887 chr += c - '0';
1888 else if (c >= 'a' && c <= 'f')
1889 chr += 10 + c - 'a';
1890 else
1891 chr += 10 + c - 'A';
1892 }
1893 s += i;
Jeremy Hylton504de6b2003-10-06 05:08:26 +00001894 if (chr == 0xffffffff && PyErr_Occurred())
Walter Dörwald3aeb6322002-09-02 13:14:32 +00001895 /* _decoding_error will have already written into the
1896 target buffer. */
1897 break;
Fredrik Lundhccc74732001-02-18 22:13:49 +00001898 store:
Fredrik Lundhdf846752000-09-03 11:29:49 +00001899 /* when we get here, chr is a 32-bit unicode character */
1900 if (chr <= 0xffff)
1901 /* UCS-2 character */
1902 *p++ = (Py_UNICODE) chr;
1903 else if (chr <= 0x10ffff) {
Marc-André Lemburg6c6bfb72001-07-20 17:39:11 +00001904 /* UCS-4 character. Either store directly, or as
Walter Dörwald8c077222002-03-25 11:16:18 +00001905 surrogate pair. */
Fredrik Lundh8f455852001-06-27 18:59:43 +00001906#ifdef Py_UNICODE_WIDE
Martin v. Löwis0ba70cc2001-06-26 22:22:37 +00001907 *p++ = chr;
1908#else
Fredrik Lundhdf846752000-09-03 11:29:49 +00001909 chr -= 0x10000L;
1910 *p++ = 0xD800 + (Py_UNICODE) (chr >> 10);
Fredrik Lundh45714e92001-06-26 16:39:36 +00001911 *p++ = 0xDC00 + (Py_UNICODE) (chr & 0x03FF);
Martin v. Löwis0ba70cc2001-06-26 22:22:37 +00001912#endif
Fredrik Lundhdf846752000-09-03 11:29:49 +00001913 } else {
Walter Dörwald3aeb6322002-09-02 13:14:32 +00001914 endinpos = s-starts;
1915 outpos = p-PyUnicode_AS_UNICODE(v);
1916 if (unicode_decode_call_errorhandler(
1917 errors, &errorHandler,
1918 "unicodeescape", "illegal Unicode character",
1919 starts, size, &startinpos, &endinpos, &exc, &s,
1920 (PyObject **)&v, &outpos, &p))
Fredrik Lundhdf846752000-09-03 11:29:49 +00001921 goto onError;
1922 }
Fredrik Lundhccc74732001-02-18 22:13:49 +00001923 break;
1924
1925 /* \N{name} */
1926 case 'N':
1927 message = "malformed \\N character escape";
1928 if (ucnhash_CAPI == NULL) {
1929 /* load the unicode data module */
Hye-Shik Chang4af5c8c2006-03-07 15:39:21 +00001930 PyObject *m, *api;
Fredrik Lundhccc74732001-02-18 22:13:49 +00001931 m = PyImport_ImportModule("unicodedata");
1932 if (m == NULL)
1933 goto ucnhashError;
Hye-Shik Chang4af5c8c2006-03-07 15:39:21 +00001934 api = PyObject_GetAttrString(m, "ucnhash_CAPI");
Fredrik Lundhccc74732001-02-18 22:13:49 +00001935 Py_DECREF(m);
Hye-Shik Chang4af5c8c2006-03-07 15:39:21 +00001936 if (api == NULL)
Fredrik Lundhccc74732001-02-18 22:13:49 +00001937 goto ucnhashError;
Anthony Baxtera6286212006-04-11 07:42:36 +00001938 ucnhash_CAPI = (_PyUnicode_Name_CAPI *)PyCObject_AsVoidPtr(api);
Hye-Shik Chang4af5c8c2006-03-07 15:39:21 +00001939 Py_DECREF(api);
Fredrik Lundhccc74732001-02-18 22:13:49 +00001940 if (ucnhash_CAPI == NULL)
1941 goto ucnhashError;
1942 }
1943 if (*s == '{') {
1944 const char *start = s+1;
1945 /* look for the closing brace */
1946 while (*s != '}' && s < end)
1947 s++;
1948 if (s > start && s < end && *s == '}') {
1949 /* found a name. look it up in the unicode database */
1950 message = "unknown Unicode character name";
1951 s++;
Martin v. Löwis480f1bb2006-03-09 23:38:20 +00001952 if (ucnhash_CAPI->getcode(NULL, start, (int)(s-start-1), &chr))
Fredrik Lundhccc74732001-02-18 22:13:49 +00001953 goto store;
1954 }
1955 }
Walter Dörwald3aeb6322002-09-02 13:14:32 +00001956 endinpos = s-starts;
1957 outpos = p-PyUnicode_AS_UNICODE(v);
1958 if (unicode_decode_call_errorhandler(
1959 errors, &errorHandler,
1960 "unicodeescape", message,
1961 starts, size, &startinpos, &endinpos, &exc, &s,
1962 (PyObject **)&v, &outpos, &p))
Fredrik Lundhccc74732001-02-18 22:13:49 +00001963 goto onError;
Fredrik Lundhccc74732001-02-18 22:13:49 +00001964 break;
1965
1966 default:
Walter Dörwald8c077222002-03-25 11:16:18 +00001967 if (s > end) {
Walter Dörwald3aeb6322002-09-02 13:14:32 +00001968 message = "\\ at end of string";
1969 s--;
1970 endinpos = s-starts;
1971 outpos = p-PyUnicode_AS_UNICODE(v);
1972 if (unicode_decode_call_errorhandler(
1973 errors, &errorHandler,
1974 "unicodeescape", message,
1975 starts, size, &startinpos, &endinpos, &exc, &s,
1976 (PyObject **)&v, &outpos, &p))
Walter Dörwald8c077222002-03-25 11:16:18 +00001977 goto onError;
1978 }
1979 else {
1980 *p++ = '\\';
1981 *p++ = (unsigned char)s[-1];
1982 }
Fredrik Lundhccc74732001-02-18 22:13:49 +00001983 break;
Guido van Rossumd57fd912000-03-10 22:53:23 +00001984 }
Walter Dörwald3aeb6322002-09-02 13:14:32 +00001985 nextByte:
1986 ;
Guido van Rossumd57fd912000-03-10 22:53:23 +00001987 }
Martin v. Löwis412fb672006-04-13 06:34:32 +00001988 if (_PyUnicode_Resize(&v, p - PyUnicode_AS_UNICODE(v)) < 0)
Walter Dörwald3aeb6322002-09-02 13:14:32 +00001989 goto onError;
Walter Dörwaldd4ade082003-08-15 15:00:26 +00001990 Py_XDECREF(errorHandler);
1991 Py_XDECREF(exc);
Guido van Rossumd57fd912000-03-10 22:53:23 +00001992 return (PyObject *)v;
Walter Dörwald8c077222002-03-25 11:16:18 +00001993
Fredrik Lundhccc74732001-02-18 22:13:49 +00001994ucnhashError:
Fredrik Lundh06d12682001-01-24 07:59:11 +00001995 PyErr_SetString(
1996 PyExc_UnicodeError,
1997 "\\N escapes not supported (can't load unicodedata module)"
1998 );
Hye-Shik Chang4af5c8c2006-03-07 15:39:21 +00001999 Py_XDECREF(v);
Walter Dörwald3aeb6322002-09-02 13:14:32 +00002000 Py_XDECREF(errorHandler);
2001 Py_XDECREF(exc);
Fredrik Lundhf6056062001-01-20 11:15:25 +00002002 return NULL;
2003
Fredrik Lundhccc74732001-02-18 22:13:49 +00002004onError:
Guido van Rossumd57fd912000-03-10 22:53:23 +00002005 Py_XDECREF(v);
Walter Dörwald3aeb6322002-09-02 13:14:32 +00002006 Py_XDECREF(errorHandler);
2007 Py_XDECREF(exc);
Guido van Rossumd57fd912000-03-10 22:53:23 +00002008 return NULL;
2009}
2010
2011/* Return a Unicode-Escape string version of the Unicode object.
2012
2013 If quotes is true, the string is enclosed in u"" or u'' quotes as
2014 appropriate.
2015
2016*/
2017
Fredrik Lundh95e2a912006-05-26 11:38:15 +00002018Py_LOCAL(const Py_UNICODE *) findchar(const Py_UNICODE *s,
2019 Py_ssize_t size,
2020 Py_UNICODE ch)
Fredrik Lundh347ee272006-05-24 16:35:18 +00002021{
2022 /* like wcschr, but doesn't stop at NULL characters */
2023
2024 while (size-- > 0) {
2025 if (*s == ch)
2026 return s;
2027 s++;
2028 }
2029
2030 return NULL;
2031}
Barry Warsaw51ac5802000-03-20 16:36:48 +00002032
Guido van Rossumd57fd912000-03-10 22:53:23 +00002033static
2034PyObject *unicodeescape_string(const Py_UNICODE *s,
Martin v. Löwis18e16552006-02-15 17:27:45 +00002035 Py_ssize_t size,
Guido van Rossumd57fd912000-03-10 22:53:23 +00002036 int quotes)
2037{
2038 PyObject *repr;
2039 char *p;
Guido van Rossumd57fd912000-03-10 22:53:23 +00002040
Ka-Ping Yeefa004ad2001-01-24 17:19:08 +00002041 static const char *hexdigit = "0123456789abcdef";
Guido van Rossumd57fd912000-03-10 22:53:23 +00002042
2043 repr = PyString_FromStringAndSize(NULL, 2 + 6*size + 1);
2044 if (repr == NULL)
2045 return NULL;
2046
Marc-André Lemburg80d1dd52001-07-25 16:05:59 +00002047 p = PyString_AS_STRING(repr);
Guido van Rossumd57fd912000-03-10 22:53:23 +00002048
2049 if (quotes) {
Guido van Rossumd57fd912000-03-10 22:53:23 +00002050 *p++ = 'u';
Tim Petersced69f82003-09-16 20:30:58 +00002051 *p++ = (findchar(s, size, '\'') &&
Guido van Rossumd57fd912000-03-10 22:53:23 +00002052 !findchar(s, size, '"')) ? '"' : '\'';
2053 }
2054 while (size-- > 0) {
2055 Py_UNICODE ch = *s++;
Marc-André Lemburg80d1dd52001-07-25 16:05:59 +00002056
Hye-Shik Chang835b2432005-12-17 04:38:31 +00002057 /* Escape quotes and backslashes */
2058 if ((quotes &&
2059 ch == (Py_UNICODE) PyString_AS_STRING(repr)[1]) || ch == '\\') {
Guido van Rossumd57fd912000-03-10 22:53:23 +00002060 *p++ = '\\';
2061 *p++ = (char) ch;
Guido van Rossumad9744a2001-09-21 15:38:17 +00002062 continue;
Tim Petersced69f82003-09-16 20:30:58 +00002063 }
Marc-André Lemburg80d1dd52001-07-25 16:05:59 +00002064
Guido van Rossum0d42e0c2001-07-20 16:36:21 +00002065#ifdef Py_UNICODE_WIDE
Martin v. Löwis0ba70cc2001-06-26 22:22:37 +00002066 /* Map 21-bit characters to '\U00xxxxxx' */
2067 else if (ch >= 0x10000) {
Martin v. Löwis412fb672006-04-13 06:34:32 +00002068 Py_ssize_t offset = p - PyString_AS_STRING(repr);
Tim Petersced69f82003-09-16 20:30:58 +00002069
Marc-André Lemburg80d1dd52001-07-25 16:05:59 +00002070 /* Resize the string if necessary */
2071 if (offset + 12 > PyString_GET_SIZE(repr)) {
2072 if (_PyString_Resize(&repr, PyString_GET_SIZE(repr) + 100))
Tim Peters5de98422002-04-27 18:44:32 +00002073 return NULL;
Marc-André Lemburg80d1dd52001-07-25 16:05:59 +00002074 p = PyString_AS_STRING(repr) + offset;
2075 }
2076
Martin v. Löwis0ba70cc2001-06-26 22:22:37 +00002077 *p++ = '\\';
2078 *p++ = 'U';
Marc-André Lemburg6c6bfb72001-07-20 17:39:11 +00002079 *p++ = hexdigit[(ch >> 28) & 0x0000000F];
2080 *p++ = hexdigit[(ch >> 24) & 0x0000000F];
2081 *p++ = hexdigit[(ch >> 20) & 0x0000000F];
2082 *p++ = hexdigit[(ch >> 16) & 0x0000000F];
2083 *p++ = hexdigit[(ch >> 12) & 0x0000000F];
2084 *p++ = hexdigit[(ch >> 8) & 0x0000000F];
2085 *p++ = hexdigit[(ch >> 4) & 0x0000000F];
Marc-André Lemburg80d1dd52001-07-25 16:05:59 +00002086 *p++ = hexdigit[ch & 0x0000000F];
2087 continue;
Martin v. Löwis0ba70cc2001-06-26 22:22:37 +00002088 }
Guido van Rossum0d42e0c2001-07-20 16:36:21 +00002089#endif
Marc-André Lemburg6c6bfb72001-07-20 17:39:11 +00002090 /* Map UTF-16 surrogate pairs to Unicode \UXXXXXXXX escapes */
2091 else if (ch >= 0xD800 && ch < 0xDC00) {
2092 Py_UNICODE ch2;
2093 Py_UCS4 ucs;
Tim Petersced69f82003-09-16 20:30:58 +00002094
Marc-André Lemburg6c6bfb72001-07-20 17:39:11 +00002095 ch2 = *s++;
2096 size--;
2097 if (ch2 >= 0xDC00 && ch2 <= 0xDFFF) {
2098 ucs = (((ch & 0x03FF) << 10) | (ch2 & 0x03FF)) + 0x00010000;
2099 *p++ = '\\';
2100 *p++ = 'U';
2101 *p++ = hexdigit[(ucs >> 28) & 0x0000000F];
2102 *p++ = hexdigit[(ucs >> 24) & 0x0000000F];
2103 *p++ = hexdigit[(ucs >> 20) & 0x0000000F];
2104 *p++ = hexdigit[(ucs >> 16) & 0x0000000F];
2105 *p++ = hexdigit[(ucs >> 12) & 0x0000000F];
2106 *p++ = hexdigit[(ucs >> 8) & 0x0000000F];
2107 *p++ = hexdigit[(ucs >> 4) & 0x0000000F];
2108 *p++ = hexdigit[ucs & 0x0000000F];
2109 continue;
2110 }
2111 /* Fall through: isolated surrogates are copied as-is */
2112 s--;
2113 size++;
2114 }
2115
Guido van Rossumd57fd912000-03-10 22:53:23 +00002116 /* Map 16-bit characters to '\uxxxx' */
Marc-André Lemburg6c6bfb72001-07-20 17:39:11 +00002117 if (ch >= 256) {
Guido van Rossumd57fd912000-03-10 22:53:23 +00002118 *p++ = '\\';
2119 *p++ = 'u';
Marc-André Lemburg6c6bfb72001-07-20 17:39:11 +00002120 *p++ = hexdigit[(ch >> 12) & 0x000F];
2121 *p++ = hexdigit[(ch >> 8) & 0x000F];
2122 *p++ = hexdigit[(ch >> 4) & 0x000F];
2123 *p++ = hexdigit[ch & 0x000F];
Guido van Rossumd57fd912000-03-10 22:53:23 +00002124 }
Marc-André Lemburg80d1dd52001-07-25 16:05:59 +00002125
Ka-Ping Yeefa004ad2001-01-24 17:19:08 +00002126 /* Map special whitespace to '\t', \n', '\r' */
2127 else if (ch == '\t') {
2128 *p++ = '\\';
2129 *p++ = 't';
2130 }
2131 else if (ch == '\n') {
2132 *p++ = '\\';
2133 *p++ = 'n';
2134 }
2135 else if (ch == '\r') {
2136 *p++ = '\\';
2137 *p++ = 'r';
2138 }
Marc-André Lemburg80d1dd52001-07-25 16:05:59 +00002139
Ka-Ping Yeefa004ad2001-01-24 17:19:08 +00002140 /* Map non-printable US ASCII to '\xhh' */
Marc-André Lemburg11326de2001-11-28 12:56:20 +00002141 else if (ch < ' ' || ch >= 0x7F) {
Guido van Rossumd57fd912000-03-10 22:53:23 +00002142 *p++ = '\\';
Ka-Ping Yeefa004ad2001-01-24 17:19:08 +00002143 *p++ = 'x';
Marc-André Lemburg6c6bfb72001-07-20 17:39:11 +00002144 *p++ = hexdigit[(ch >> 4) & 0x000F];
2145 *p++ = hexdigit[ch & 0x000F];
Tim Petersced69f82003-09-16 20:30:58 +00002146 }
Marc-André Lemburg80d1dd52001-07-25 16:05:59 +00002147
Guido van Rossumd57fd912000-03-10 22:53:23 +00002148 /* Copy everything else as-is */
2149 else
2150 *p++ = (char) ch;
2151 }
2152 if (quotes)
Marc-André Lemburg80d1dd52001-07-25 16:05:59 +00002153 *p++ = PyString_AS_STRING(repr)[1];
Guido van Rossumd57fd912000-03-10 22:53:23 +00002154
2155 *p = '\0';
Tim Peters5de98422002-04-27 18:44:32 +00002156 _PyString_Resize(&repr, p - PyString_AS_STRING(repr));
Guido van Rossumd57fd912000-03-10 22:53:23 +00002157 return repr;
2158}
2159
2160PyObject *PyUnicode_EncodeUnicodeEscape(const Py_UNICODE *s,
Martin v. Löwis18e16552006-02-15 17:27:45 +00002161 Py_ssize_t size)
Guido van Rossumd57fd912000-03-10 22:53:23 +00002162{
2163 return unicodeescape_string(s, size, 0);
2164}
2165
2166PyObject *PyUnicode_AsUnicodeEscapeString(PyObject *unicode)
2167{
2168 if (!PyUnicode_Check(unicode)) {
2169 PyErr_BadArgument();
2170 return NULL;
2171 }
2172 return PyUnicode_EncodeUnicodeEscape(PyUnicode_AS_UNICODE(unicode),
2173 PyUnicode_GET_SIZE(unicode));
2174}
2175
2176/* --- Raw Unicode Escape Codec ------------------------------------------- */
2177
2178PyObject *PyUnicode_DecodeRawUnicodeEscape(const char *s,
Martin v. Löwis18e16552006-02-15 17:27:45 +00002179 Py_ssize_t size,
Guido van Rossumd57fd912000-03-10 22:53:23 +00002180 const char *errors)
2181{
Walter Dörwald3aeb6322002-09-02 13:14:32 +00002182 const char *starts = s;
Martin v. Löwis18e16552006-02-15 17:27:45 +00002183 Py_ssize_t startinpos;
2184 Py_ssize_t endinpos;
2185 Py_ssize_t outpos;
Guido van Rossumd57fd912000-03-10 22:53:23 +00002186 PyUnicodeObject *v;
Walter Dörwald3aeb6322002-09-02 13:14:32 +00002187 Py_UNICODE *p;
Guido van Rossumd57fd912000-03-10 22:53:23 +00002188 const char *end;
2189 const char *bs;
Walter Dörwald3aeb6322002-09-02 13:14:32 +00002190 PyObject *errorHandler = NULL;
2191 PyObject *exc = NULL;
Tim Petersced69f82003-09-16 20:30:58 +00002192
Guido van Rossumd57fd912000-03-10 22:53:23 +00002193 /* Escaped strings will always be longer than the resulting
2194 Unicode string, so we start with size here and then reduce the
Walter Dörwald3aeb6322002-09-02 13:14:32 +00002195 length after conversion to the true value. (But decoding error
2196 handler might have to resize the string) */
Guido van Rossumd57fd912000-03-10 22:53:23 +00002197 v = _PyUnicode_New(size);
2198 if (v == NULL)
2199 goto onError;
2200 if (size == 0)
2201 return (PyObject *)v;
Walter Dörwald3aeb6322002-09-02 13:14:32 +00002202 p = PyUnicode_AS_UNICODE(v);
Guido van Rossumd57fd912000-03-10 22:53:23 +00002203 end = s + size;
2204 while (s < end) {
2205 unsigned char c;
Martin v. Löwis047c05e2002-03-21 08:55:28 +00002206 Py_UCS4 x;
Guido van Rossumd57fd912000-03-10 22:53:23 +00002207 int i;
Martin v. Löwis9a3a9f72003-05-18 12:31:09 +00002208 int count;
Guido van Rossumd57fd912000-03-10 22:53:23 +00002209
2210 /* Non-escape characters are interpreted as Unicode ordinals */
2211 if (*s != '\\') {
2212 *p++ = (unsigned char)*s++;
2213 continue;
2214 }
Walter Dörwald3aeb6322002-09-02 13:14:32 +00002215 startinpos = s-starts;
Guido van Rossumd57fd912000-03-10 22:53:23 +00002216
2217 /* \u-escapes are only interpreted iff the number of leading
2218 backslashes if odd */
2219 bs = s;
2220 for (;s < end;) {
2221 if (*s != '\\')
2222 break;
2223 *p++ = (unsigned char)*s++;
2224 }
2225 if (((s - bs) & 1) == 0 ||
2226 s >= end ||
Martin v. Löwis9a3a9f72003-05-18 12:31:09 +00002227 (*s != 'u' && *s != 'U')) {
Guido van Rossumd57fd912000-03-10 22:53:23 +00002228 continue;
2229 }
2230 p--;
Martin v. Löwis9a3a9f72003-05-18 12:31:09 +00002231 count = *s=='u' ? 4 : 8;
Guido van Rossumd57fd912000-03-10 22:53:23 +00002232 s++;
2233
Martin v. Löwis9a3a9f72003-05-18 12:31:09 +00002234 /* \uXXXX with 4 hex digits, \Uxxxxxxxx with 8 */
Walter Dörwald3aeb6322002-09-02 13:14:32 +00002235 outpos = p-PyUnicode_AS_UNICODE(v);
Martin v. Löwis9a3a9f72003-05-18 12:31:09 +00002236 for (x = 0, i = 0; i < count; ++i, ++s) {
Walter Dörwald3aeb6322002-09-02 13:14:32 +00002237 c = (unsigned char)*s;
Guido van Rossumd57fd912000-03-10 22:53:23 +00002238 if (!isxdigit(c)) {
Walter Dörwald3aeb6322002-09-02 13:14:32 +00002239 endinpos = s-starts;
2240 if (unicode_decode_call_errorhandler(
2241 errors, &errorHandler,
2242 "rawunicodeescape", "truncated \\uXXXX",
2243 starts, size, &startinpos, &endinpos, &exc, &s,
2244 (PyObject **)&v, &outpos, &p))
Guido van Rossumd57fd912000-03-10 22:53:23 +00002245 goto onError;
Walter Dörwald3aeb6322002-09-02 13:14:32 +00002246 goto nextByte;
Guido van Rossumd57fd912000-03-10 22:53:23 +00002247 }
2248 x = (x<<4) & ~0xF;
2249 if (c >= '0' && c <= '9')
2250 x += c - '0';
2251 else if (c >= 'a' && c <= 'f')
2252 x += 10 + c - 'a';
2253 else
2254 x += 10 + c - 'A';
2255 }
Martin v. Löwis9a3a9f72003-05-18 12:31:09 +00002256#ifndef Py_UNICODE_WIDE
2257 if (x > 0x10000) {
2258 if (unicode_decode_call_errorhandler(
2259 errors, &errorHandler,
2260 "rawunicodeescape", "\\Uxxxxxxxx out of range",
2261 starts, size, &startinpos, &endinpos, &exc, &s,
2262 (PyObject **)&v, &outpos, &p))
2263 goto onError;
2264 }
2265#endif
Walter Dörwald3aeb6322002-09-02 13:14:32 +00002266 *p++ = x;
2267 nextByte:
2268 ;
Guido van Rossumd57fd912000-03-10 22:53:23 +00002269 }
Martin v. Löwis412fb672006-04-13 06:34:32 +00002270 if (_PyUnicode_Resize(&v, p - PyUnicode_AS_UNICODE(v)) < 0)
Guido van Rossumfd4b9572000-04-10 13:51:10 +00002271 goto onError;
Walter Dörwald3aeb6322002-09-02 13:14:32 +00002272 Py_XDECREF(errorHandler);
2273 Py_XDECREF(exc);
Guido van Rossumd57fd912000-03-10 22:53:23 +00002274 return (PyObject *)v;
Tim Petersced69f82003-09-16 20:30:58 +00002275
Guido van Rossumd57fd912000-03-10 22:53:23 +00002276 onError:
2277 Py_XDECREF(v);
Walter Dörwald3aeb6322002-09-02 13:14:32 +00002278 Py_XDECREF(errorHandler);
2279 Py_XDECREF(exc);
Guido van Rossumd57fd912000-03-10 22:53:23 +00002280 return NULL;
2281}
2282
2283PyObject *PyUnicode_EncodeRawUnicodeEscape(const Py_UNICODE *s,
Martin v. Löwis18e16552006-02-15 17:27:45 +00002284 Py_ssize_t size)
Guido van Rossumd57fd912000-03-10 22:53:23 +00002285{
2286 PyObject *repr;
2287 char *p;
2288 char *q;
2289
Ka-Ping Yeefa004ad2001-01-24 17:19:08 +00002290 static const char *hexdigit = "0123456789abcdef";
Guido van Rossumd57fd912000-03-10 22:53:23 +00002291
Martin v. Löwis9a3a9f72003-05-18 12:31:09 +00002292#ifdef Py_UNICODE_WIDE
2293 repr = PyString_FromStringAndSize(NULL, 10 * size);
2294#else
Guido van Rossumd57fd912000-03-10 22:53:23 +00002295 repr = PyString_FromStringAndSize(NULL, 6 * size);
Martin v. Löwis9a3a9f72003-05-18 12:31:09 +00002296#endif
Guido van Rossumd57fd912000-03-10 22:53:23 +00002297 if (repr == NULL)
2298 return NULL;
Marc-André Lemburgb7520772000-08-14 11:29:19 +00002299 if (size == 0)
2300 return repr;
Guido van Rossumd57fd912000-03-10 22:53:23 +00002301
2302 p = q = PyString_AS_STRING(repr);
2303 while (size-- > 0) {
2304 Py_UNICODE ch = *s++;
Martin v. Löwis9a3a9f72003-05-18 12:31:09 +00002305#ifdef Py_UNICODE_WIDE
2306 /* Map 32-bit characters to '\Uxxxxxxxx' */
2307 if (ch >= 0x10000) {
2308 *p++ = '\\';
2309 *p++ = 'U';
2310 *p++ = hexdigit[(ch >> 28) & 0xf];
2311 *p++ = hexdigit[(ch >> 24) & 0xf];
2312 *p++ = hexdigit[(ch >> 20) & 0xf];
2313 *p++ = hexdigit[(ch >> 16) & 0xf];
2314 *p++ = hexdigit[(ch >> 12) & 0xf];
2315 *p++ = hexdigit[(ch >> 8) & 0xf];
2316 *p++ = hexdigit[(ch >> 4) & 0xf];
2317 *p++ = hexdigit[ch & 15];
Tim Petersced69f82003-09-16 20:30:58 +00002318 }
Martin v. Löwis9a3a9f72003-05-18 12:31:09 +00002319 else
2320#endif
Guido van Rossumd57fd912000-03-10 22:53:23 +00002321 /* Map 16-bit characters to '\uxxxx' */
2322 if (ch >= 256) {
2323 *p++ = '\\';
2324 *p++ = 'u';
2325 *p++ = hexdigit[(ch >> 12) & 0xf];
2326 *p++ = hexdigit[(ch >> 8) & 0xf];
2327 *p++ = hexdigit[(ch >> 4) & 0xf];
2328 *p++ = hexdigit[ch & 15];
2329 }
2330 /* Copy everything else as-is */
2331 else
2332 *p++ = (char) ch;
2333 }
2334 *p = '\0';
Tim Peters5de98422002-04-27 18:44:32 +00002335 _PyString_Resize(&repr, p - q);
Guido van Rossumd57fd912000-03-10 22:53:23 +00002336 return repr;
2337}
2338
2339PyObject *PyUnicode_AsRawUnicodeEscapeString(PyObject *unicode)
2340{
2341 if (!PyUnicode_Check(unicode)) {
2342 PyErr_BadArgument();
2343 return NULL;
2344 }
2345 return PyUnicode_EncodeRawUnicodeEscape(PyUnicode_AS_UNICODE(unicode),
2346 PyUnicode_GET_SIZE(unicode));
2347}
2348
Walter Dörwalda47d1c02005-08-30 10:23:14 +00002349/* --- Unicode Internal Codec ------------------------------------------- */
2350
2351PyObject *_PyUnicode_DecodeUnicodeInternal(const char *s,
Martin v. Löwis18e16552006-02-15 17:27:45 +00002352 Py_ssize_t size,
Walter Dörwalda47d1c02005-08-30 10:23:14 +00002353 const char *errors)
2354{
2355 const char *starts = s;
Martin v. Löwis18e16552006-02-15 17:27:45 +00002356 Py_ssize_t startinpos;
2357 Py_ssize_t endinpos;
2358 Py_ssize_t outpos;
Walter Dörwalda47d1c02005-08-30 10:23:14 +00002359 PyUnicodeObject *v;
2360 Py_UNICODE *p;
2361 const char *end;
2362 const char *reason;
2363 PyObject *errorHandler = NULL;
2364 PyObject *exc = NULL;
2365
Neal Norwitzd43069c2006-01-08 01:12:10 +00002366#ifdef Py_UNICODE_WIDE
2367 Py_UNICODE unimax = PyUnicode_GetMax();
2368#endif
2369
Walter Dörwalda47d1c02005-08-30 10:23:14 +00002370 v = _PyUnicode_New((size+Py_UNICODE_SIZE-1)/ Py_UNICODE_SIZE);
2371 if (v == NULL)
2372 goto onError;
2373 if (PyUnicode_GetSize((PyObject *)v) == 0)
2374 return (PyObject *)v;
2375 p = PyUnicode_AS_UNICODE(v);
2376 end = s + size;
2377
2378 while (s < end) {
Neal Norwitz1004a532006-05-15 07:17:23 +00002379 memcpy(p, s, sizeof(Py_UNICODE));
Walter Dörwalda47d1c02005-08-30 10:23:14 +00002380 /* We have to sanity check the raw data, otherwise doom looms for
2381 some malformed UCS-4 data. */
2382 if (
2383 #ifdef Py_UNICODE_WIDE
2384 *p > unimax || *p < 0 ||
2385 #endif
2386 end-s < Py_UNICODE_SIZE
2387 )
2388 {
2389 startinpos = s - starts;
2390 if (end-s < Py_UNICODE_SIZE) {
2391 endinpos = end-starts;
2392 reason = "truncated input";
2393 }
2394 else {
2395 endinpos = s - starts + Py_UNICODE_SIZE;
2396 reason = "illegal code point (> 0x10FFFF)";
2397 }
2398 outpos = p - PyUnicode_AS_UNICODE(v);
2399 if (unicode_decode_call_errorhandler(
2400 errors, &errorHandler,
2401 "unicode_internal", reason,
2402 starts, size, &startinpos, &endinpos, &exc, &s,
2403 (PyObject **)&v, &outpos, &p)) {
2404 goto onError;
2405 }
2406 }
2407 else {
2408 p++;
2409 s += Py_UNICODE_SIZE;
2410 }
2411 }
2412
Martin v. Löwis412fb672006-04-13 06:34:32 +00002413 if (_PyUnicode_Resize(&v, p - PyUnicode_AS_UNICODE(v)) < 0)
Walter Dörwalda47d1c02005-08-30 10:23:14 +00002414 goto onError;
2415 Py_XDECREF(errorHandler);
2416 Py_XDECREF(exc);
2417 return (PyObject *)v;
2418
2419 onError:
2420 Py_XDECREF(v);
2421 Py_XDECREF(errorHandler);
2422 Py_XDECREF(exc);
2423 return NULL;
2424}
2425
Guido van Rossumd57fd912000-03-10 22:53:23 +00002426/* --- Latin-1 Codec ------------------------------------------------------ */
2427
2428PyObject *PyUnicode_DecodeLatin1(const char *s,
Martin v. Löwis18e16552006-02-15 17:27:45 +00002429 Py_ssize_t size,
Guido van Rossumd57fd912000-03-10 22:53:23 +00002430 const char *errors)
2431{
2432 PyUnicodeObject *v;
2433 Py_UNICODE *p;
Tim Petersced69f82003-09-16 20:30:58 +00002434
Guido van Rossumd57fd912000-03-10 22:53:23 +00002435 /* Latin-1 is equivalent to the first 256 ordinals in Unicode. */
Hye-Shik Chang4a264fb2003-12-19 01:59:56 +00002436 if (size == 1) {
Marc-André Lemburg8155e0e2001-04-23 14:44:21 +00002437 Py_UNICODE r = *(unsigned char*)s;
2438 return PyUnicode_FromUnicode(&r, 1);
2439 }
2440
Guido van Rossumd57fd912000-03-10 22:53:23 +00002441 v = _PyUnicode_New(size);
2442 if (v == NULL)
2443 goto onError;
2444 if (size == 0)
2445 return (PyObject *)v;
2446 p = PyUnicode_AS_UNICODE(v);
2447 while (size-- > 0)
2448 *p++ = (unsigned char)*s++;
2449 return (PyObject *)v;
Tim Petersced69f82003-09-16 20:30:58 +00002450
Guido van Rossumd57fd912000-03-10 22:53:23 +00002451 onError:
2452 Py_XDECREF(v);
2453 return NULL;
2454}
2455
Walter Dörwald3aeb6322002-09-02 13:14:32 +00002456/* create or adjust a UnicodeEncodeError */
2457static void make_encode_exception(PyObject **exceptionObject,
2458 const char *encoding,
Martin v. Löwis18e16552006-02-15 17:27:45 +00002459 const Py_UNICODE *unicode, Py_ssize_t size,
2460 Py_ssize_t startpos, Py_ssize_t endpos,
Walter Dörwald3aeb6322002-09-02 13:14:32 +00002461 const char *reason)
Guido van Rossumd57fd912000-03-10 22:53:23 +00002462{
Walter Dörwald3aeb6322002-09-02 13:14:32 +00002463 if (*exceptionObject == NULL) {
2464 *exceptionObject = PyUnicodeEncodeError_Create(
2465 encoding, unicode, size, startpos, endpos, reason);
Guido van Rossumd57fd912000-03-10 22:53:23 +00002466 }
2467 else {
Walter Dörwald3aeb6322002-09-02 13:14:32 +00002468 if (PyUnicodeEncodeError_SetStart(*exceptionObject, startpos))
2469 goto onError;
2470 if (PyUnicodeEncodeError_SetEnd(*exceptionObject, endpos))
2471 goto onError;
2472 if (PyUnicodeEncodeError_SetReason(*exceptionObject, reason))
2473 goto onError;
2474 return;
2475 onError:
2476 Py_DECREF(*exceptionObject);
2477 *exceptionObject = NULL;
Guido van Rossumd57fd912000-03-10 22:53:23 +00002478 }
2479}
2480
Walter Dörwald3aeb6322002-09-02 13:14:32 +00002481/* raises a UnicodeEncodeError */
2482static void raise_encode_exception(PyObject **exceptionObject,
2483 const char *encoding,
Martin v. Löwis18e16552006-02-15 17:27:45 +00002484 const Py_UNICODE *unicode, Py_ssize_t size,
2485 Py_ssize_t startpos, Py_ssize_t endpos,
Walter Dörwald3aeb6322002-09-02 13:14:32 +00002486 const char *reason)
2487{
2488 make_encode_exception(exceptionObject,
2489 encoding, unicode, size, startpos, endpos, reason);
2490 if (*exceptionObject != NULL)
2491 PyCodec_StrictErrors(*exceptionObject);
2492}
2493
2494/* error handling callback helper:
2495 build arguments, call the callback and check the arguments,
2496 put the result into newpos and return the replacement string, which
2497 has to be freed by the caller */
2498static PyObject *unicode_encode_call_errorhandler(const char *errors,
2499 PyObject **errorHandler,
2500 const char *encoding, const char *reason,
Martin v. Löwis18e16552006-02-15 17:27:45 +00002501 const Py_UNICODE *unicode, Py_ssize_t size, PyObject **exceptionObject,
2502 Py_ssize_t startpos, Py_ssize_t endpos,
2503 Py_ssize_t *newpos)
Walter Dörwald3aeb6322002-09-02 13:14:32 +00002504{
Martin v. Löwis18e16552006-02-15 17:27:45 +00002505 static char *argparse = "O!n;encoding error handler must return (unicode, int) tuple";
Walter Dörwald3aeb6322002-09-02 13:14:32 +00002506
2507 PyObject *restuple;
2508 PyObject *resunicode;
2509
2510 if (*errorHandler == NULL) {
2511 *errorHandler = PyCodec_LookupError(errors);
2512 if (*errorHandler == NULL)
2513 return NULL;
2514 }
2515
2516 make_encode_exception(exceptionObject,
2517 encoding, unicode, size, startpos, endpos, reason);
2518 if (*exceptionObject == NULL)
2519 return NULL;
2520
2521 restuple = PyObject_CallFunctionObjArgs(
2522 *errorHandler, *exceptionObject, NULL);
2523 if (restuple == NULL)
2524 return NULL;
2525 if (!PyTuple_Check(restuple)) {
2526 PyErr_Format(PyExc_TypeError, &argparse[4]);
2527 Py_DECREF(restuple);
2528 return NULL;
2529 }
2530 if (!PyArg_ParseTuple(restuple, argparse, &PyUnicode_Type,
2531 &resunicode, newpos)) {
2532 Py_DECREF(restuple);
2533 return NULL;
2534 }
2535 if (*newpos<0)
Walter Dörwald2e0b18a2003-01-31 17:19:08 +00002536 *newpos = size+*newpos;
2537 if (*newpos<0 || *newpos>size) {
Martin v. Löwis2c95cc62006-02-16 06:54:25 +00002538 PyErr_Format(PyExc_IndexError, "position %zd from error handler out of bounds", *newpos);
Walter Dörwald2e0b18a2003-01-31 17:19:08 +00002539 Py_DECREF(restuple);
2540 return NULL;
2541 }
Walter Dörwald3aeb6322002-09-02 13:14:32 +00002542 Py_INCREF(resunicode);
2543 Py_DECREF(restuple);
2544 return resunicode;
2545}
2546
2547static PyObject *unicode_encode_ucs1(const Py_UNICODE *p,
Martin v. Löwis18e16552006-02-15 17:27:45 +00002548 Py_ssize_t size,
Walter Dörwald3aeb6322002-09-02 13:14:32 +00002549 const char *errors,
2550 int limit)
2551{
2552 /* output object */
2553 PyObject *res;
2554 /* pointers to the beginning and end+1 of input */
2555 const Py_UNICODE *startp = p;
2556 const Py_UNICODE *endp = p + size;
2557 /* pointer to the beginning of the unencodable characters */
2558 /* const Py_UNICODE *badp = NULL; */
2559 /* pointer into the output */
2560 char *str;
2561 /* current output position */
Martin v. Löwis18e16552006-02-15 17:27:45 +00002562 Py_ssize_t respos = 0;
2563 Py_ssize_t ressize;
Anthony Baxtera6286212006-04-11 07:42:36 +00002564 const char *encoding = (limit == 256) ? "latin-1" : "ascii";
2565 const char *reason = (limit == 256) ? "ordinal not in range(256)" : "ordinal not in range(128)";
Walter Dörwald3aeb6322002-09-02 13:14:32 +00002566 PyObject *errorHandler = NULL;
2567 PyObject *exc = NULL;
2568 /* the following variable is used for caching string comparisons
2569 * -1=not initialized, 0=unknown, 1=strict, 2=replace, 3=ignore, 4=xmlcharrefreplace */
2570 int known_errorHandler = -1;
2571
2572 /* allocate enough for a simple encoding without
2573 replacements, if we need more, we'll resize */
2574 res = PyString_FromStringAndSize(NULL, size);
2575 if (res == NULL)
2576 goto onError;
2577 if (size == 0)
2578 return res;
2579 str = PyString_AS_STRING(res);
2580 ressize = size;
2581
2582 while (p<endp) {
2583 Py_UNICODE c = *p;
2584
2585 /* can we encode this? */
2586 if (c<limit) {
2587 /* no overflow check, because we know that the space is enough */
2588 *str++ = (char)c;
2589 ++p;
2590 }
2591 else {
Martin v. Löwis18e16552006-02-15 17:27:45 +00002592 Py_ssize_t unicodepos = p-startp;
2593 Py_ssize_t requiredsize;
Walter Dörwald3aeb6322002-09-02 13:14:32 +00002594 PyObject *repunicode;
Martin v. Löwis18e16552006-02-15 17:27:45 +00002595 Py_ssize_t repsize;
2596 Py_ssize_t newpos;
2597 Py_ssize_t respos;
Walter Dörwald3aeb6322002-09-02 13:14:32 +00002598 Py_UNICODE *uni2;
2599 /* startpos for collecting unencodable chars */
2600 const Py_UNICODE *collstart = p;
2601 const Py_UNICODE *collend = p;
2602 /* find all unecodable characters */
2603 while ((collend < endp) && ((*collend)>=limit))
2604 ++collend;
2605 /* cache callback name lookup (if not done yet, i.e. it's the first error) */
2606 if (known_errorHandler==-1) {
2607 if ((errors==NULL) || (!strcmp(errors, "strict")))
2608 known_errorHandler = 1;
2609 else if (!strcmp(errors, "replace"))
2610 known_errorHandler = 2;
2611 else if (!strcmp(errors, "ignore"))
2612 known_errorHandler = 3;
2613 else if (!strcmp(errors, "xmlcharrefreplace"))
2614 known_errorHandler = 4;
2615 else
2616 known_errorHandler = 0;
2617 }
2618 switch (known_errorHandler) {
2619 case 1: /* strict */
2620 raise_encode_exception(&exc, encoding, startp, size, collstart-startp, collend-startp, reason);
2621 goto onError;
2622 case 2: /* replace */
2623 while (collstart++<collend)
2624 *str++ = '?'; /* fall through */
2625 case 3: /* ignore */
2626 p = collend;
2627 break;
2628 case 4: /* xmlcharrefreplace */
2629 respos = str-PyString_AS_STRING(res);
2630 /* determine replacement size (temporarily (mis)uses p) */
2631 for (p = collstart, repsize = 0; p < collend; ++p) {
2632 if (*p<10)
2633 repsize += 2+1+1;
2634 else if (*p<100)
2635 repsize += 2+2+1;
2636 else if (*p<1000)
2637 repsize += 2+3+1;
2638 else if (*p<10000)
2639 repsize += 2+4+1;
Hye-Shik Chang40e95092003-12-22 01:31:13 +00002640#ifndef Py_UNICODE_WIDE
2641 else
2642 repsize += 2+5+1;
2643#else
Walter Dörwald3aeb6322002-09-02 13:14:32 +00002644 else if (*p<100000)
2645 repsize += 2+5+1;
2646 else if (*p<1000000)
2647 repsize += 2+6+1;
2648 else
2649 repsize += 2+7+1;
Hye-Shik Chang4a264fb2003-12-19 01:59:56 +00002650#endif
Walter Dörwald3aeb6322002-09-02 13:14:32 +00002651 }
2652 requiredsize = respos+repsize+(endp-collend);
2653 if (requiredsize > ressize) {
2654 if (requiredsize<2*ressize)
2655 requiredsize = 2*ressize;
2656 if (_PyString_Resize(&res, requiredsize))
2657 goto onError;
2658 str = PyString_AS_STRING(res) + respos;
2659 ressize = requiredsize;
2660 }
2661 /* generate replacement (temporarily (mis)uses p) */
2662 for (p = collstart; p < collend; ++p) {
2663 str += sprintf(str, "&#%d;", (int)*p);
2664 }
2665 p = collend;
2666 break;
2667 default:
2668 repunicode = unicode_encode_call_errorhandler(errors, &errorHandler,
2669 encoding, reason, startp, size, &exc,
2670 collstart-startp, collend-startp, &newpos);
2671 if (repunicode == NULL)
2672 goto onError;
2673 /* need more space? (at least enough for what we
2674 have+the replacement+the rest of the string, so
2675 we won't have to check space for encodable characters) */
2676 respos = str-PyString_AS_STRING(res);
2677 repsize = PyUnicode_GET_SIZE(repunicode);
2678 requiredsize = respos+repsize+(endp-collend);
2679 if (requiredsize > ressize) {
2680 if (requiredsize<2*ressize)
2681 requiredsize = 2*ressize;
2682 if (_PyString_Resize(&res, requiredsize)) {
2683 Py_DECREF(repunicode);
2684 goto onError;
2685 }
2686 str = PyString_AS_STRING(res) + respos;
2687 ressize = requiredsize;
2688 }
2689 /* check if there is anything unencodable in the replacement
2690 and copy it to the output */
2691 for (uni2 = PyUnicode_AS_UNICODE(repunicode);repsize-->0; ++uni2, ++str) {
2692 c = *uni2;
2693 if (c >= limit) {
2694 raise_encode_exception(&exc, encoding, startp, size,
2695 unicodepos, unicodepos+1, reason);
2696 Py_DECREF(repunicode);
2697 goto onError;
2698 }
2699 *str = (char)c;
2700 }
2701 p = startp + newpos;
2702 Py_DECREF(repunicode);
2703 }
2704 }
2705 }
2706 /* Resize if we allocated to much */
2707 respos = str-PyString_AS_STRING(res);
2708 if (respos<ressize)
2709 /* If this falls res will be NULL */
2710 _PyString_Resize(&res, respos);
2711 Py_XDECREF(errorHandler);
2712 Py_XDECREF(exc);
2713 return res;
2714
2715 onError:
2716 Py_XDECREF(res);
2717 Py_XDECREF(errorHandler);
2718 Py_XDECREF(exc);
2719 return NULL;
2720}
2721
Guido van Rossumd57fd912000-03-10 22:53:23 +00002722PyObject *PyUnicode_EncodeLatin1(const Py_UNICODE *p,
Martin v. Löwis18e16552006-02-15 17:27:45 +00002723 Py_ssize_t size,
Guido van Rossumd57fd912000-03-10 22:53:23 +00002724 const char *errors)
2725{
Walter Dörwald3aeb6322002-09-02 13:14:32 +00002726 return unicode_encode_ucs1(p, size, errors, 256);
Guido van Rossumd57fd912000-03-10 22:53:23 +00002727}
2728
2729PyObject *PyUnicode_AsLatin1String(PyObject *unicode)
2730{
2731 if (!PyUnicode_Check(unicode)) {
2732 PyErr_BadArgument();
2733 return NULL;
2734 }
2735 return PyUnicode_EncodeLatin1(PyUnicode_AS_UNICODE(unicode),
2736 PyUnicode_GET_SIZE(unicode),
2737 NULL);
2738}
2739
2740/* --- 7-bit ASCII Codec -------------------------------------------------- */
2741
Guido van Rossumd57fd912000-03-10 22:53:23 +00002742PyObject *PyUnicode_DecodeASCII(const char *s,
Martin v. Löwis18e16552006-02-15 17:27:45 +00002743 Py_ssize_t size,
Guido van Rossumd57fd912000-03-10 22:53:23 +00002744 const char *errors)
2745{
Walter Dörwald3aeb6322002-09-02 13:14:32 +00002746 const char *starts = s;
Guido van Rossumd57fd912000-03-10 22:53:23 +00002747 PyUnicodeObject *v;
2748 Py_UNICODE *p;
Martin v. Löwis18e16552006-02-15 17:27:45 +00002749 Py_ssize_t startinpos;
2750 Py_ssize_t endinpos;
2751 Py_ssize_t outpos;
Walter Dörwald3aeb6322002-09-02 13:14:32 +00002752 const char *e;
2753 PyObject *errorHandler = NULL;
2754 PyObject *exc = NULL;
Tim Petersced69f82003-09-16 20:30:58 +00002755
Guido van Rossumd57fd912000-03-10 22:53:23 +00002756 /* ASCII is equivalent to the first 128 ordinals in Unicode. */
Marc-André Lemburg8155e0e2001-04-23 14:44:21 +00002757 if (size == 1 && *(unsigned char*)s < 128) {
2758 Py_UNICODE r = *(unsigned char*)s;
2759 return PyUnicode_FromUnicode(&r, 1);
2760 }
Tim Petersced69f82003-09-16 20:30:58 +00002761
Guido van Rossumd57fd912000-03-10 22:53:23 +00002762 v = _PyUnicode_New(size);
2763 if (v == NULL)
2764 goto onError;
2765 if (size == 0)
2766 return (PyObject *)v;
2767 p = PyUnicode_AS_UNICODE(v);
Walter Dörwald3aeb6322002-09-02 13:14:32 +00002768 e = s + size;
2769 while (s < e) {
2770 register unsigned char c = (unsigned char)*s;
2771 if (c < 128) {
Guido van Rossumd57fd912000-03-10 22:53:23 +00002772 *p++ = c;
Walter Dörwald3aeb6322002-09-02 13:14:32 +00002773 ++s;
2774 }
2775 else {
2776 startinpos = s-starts;
2777 endinpos = startinpos + 1;
Jeremy Hyltond8082792003-09-16 19:41:39 +00002778 outpos = p - (Py_UNICODE *)PyUnicode_AS_UNICODE(v);
Walter Dörwald3aeb6322002-09-02 13:14:32 +00002779 if (unicode_decode_call_errorhandler(
2780 errors, &errorHandler,
2781 "ascii", "ordinal not in range(128)",
2782 starts, size, &startinpos, &endinpos, &exc, &s,
2783 (PyObject **)&v, &outpos, &p))
Guido van Rossumd57fd912000-03-10 22:53:23 +00002784 goto onError;
Walter Dörwald3aeb6322002-09-02 13:14:32 +00002785 }
Guido van Rossumd57fd912000-03-10 22:53:23 +00002786 }
Guido van Rossumfd4b9572000-04-10 13:51:10 +00002787 if (p - PyUnicode_AS_UNICODE(v) < PyString_GET_SIZE(v))
Martin v. Löwis412fb672006-04-13 06:34:32 +00002788 if (_PyUnicode_Resize(&v, p - PyUnicode_AS_UNICODE(v)) < 0)
Guido van Rossumfd4b9572000-04-10 13:51:10 +00002789 goto onError;
Walter Dörwald3aeb6322002-09-02 13:14:32 +00002790 Py_XDECREF(errorHandler);
2791 Py_XDECREF(exc);
Guido van Rossumd57fd912000-03-10 22:53:23 +00002792 return (PyObject *)v;
Tim Petersced69f82003-09-16 20:30:58 +00002793
Guido van Rossumd57fd912000-03-10 22:53:23 +00002794 onError:
2795 Py_XDECREF(v);
Walter Dörwald3aeb6322002-09-02 13:14:32 +00002796 Py_XDECREF(errorHandler);
2797 Py_XDECREF(exc);
Guido van Rossumd57fd912000-03-10 22:53:23 +00002798 return NULL;
2799}
2800
Guido van Rossumd57fd912000-03-10 22:53:23 +00002801PyObject *PyUnicode_EncodeASCII(const Py_UNICODE *p,
Martin v. Löwis18e16552006-02-15 17:27:45 +00002802 Py_ssize_t size,
Guido van Rossumd57fd912000-03-10 22:53:23 +00002803 const char *errors)
2804{
Walter Dörwald3aeb6322002-09-02 13:14:32 +00002805 return unicode_encode_ucs1(p, size, errors, 128);
Guido van Rossumd57fd912000-03-10 22:53:23 +00002806}
2807
2808PyObject *PyUnicode_AsASCIIString(PyObject *unicode)
2809{
2810 if (!PyUnicode_Check(unicode)) {
2811 PyErr_BadArgument();
2812 return NULL;
2813 }
2814 return PyUnicode_EncodeASCII(PyUnicode_AS_UNICODE(unicode),
2815 PyUnicode_GET_SIZE(unicode),
2816 NULL);
2817}
2818
Martin v. Löwis6238d2b2002-06-30 15:26:10 +00002819#if defined(MS_WINDOWS) && defined(HAVE_USABLE_WCHAR_T)
Guido van Rossum2ea3e142000-03-31 17:24:09 +00002820
Guido van Rossumb7a40ba2000-03-28 02:01:52 +00002821/* --- MBCS codecs for Windows -------------------------------------------- */
Guido van Rossum2ea3e142000-03-31 17:24:09 +00002822
Guido van Rossumb7a40ba2000-03-28 02:01:52 +00002823PyObject *PyUnicode_DecodeMBCS(const char *s,
Martin v. Löwis18e16552006-02-15 17:27:45 +00002824 Py_ssize_t size,
Guido van Rossumb7a40ba2000-03-28 02:01:52 +00002825 const char *errors)
2826{
2827 PyUnicodeObject *v;
2828 Py_UNICODE *p;
Martin v. Löwis18e16552006-02-15 17:27:45 +00002829 DWORD usize;
Guido van Rossumb7a40ba2000-03-28 02:01:52 +00002830
2831 /* First get the size of the result */
Martin v. Löwis18e16552006-02-15 17:27:45 +00002832 assert(size < INT_MAX);
2833 usize = MultiByteToWideChar(CP_ACP, 0, s, (int)size, NULL, 0);
Guido van Rossum03e29f12000-05-04 15:52:20 +00002834 if (size > 0 && usize==0)
Guido van Rossumb7a40ba2000-03-28 02:01:52 +00002835 return PyErr_SetFromWindowsErrWithFilename(0, NULL);
2836
2837 v = _PyUnicode_New(usize);
2838 if (v == NULL)
2839 return NULL;
2840 if (usize == 0)
2841 return (PyObject *)v;
2842 p = PyUnicode_AS_UNICODE(v);
Martin v. Löwis18e16552006-02-15 17:27:45 +00002843 if (0 == MultiByteToWideChar(CP_ACP, 0, s, (int)size, p, usize)) {
Guido van Rossumb7a40ba2000-03-28 02:01:52 +00002844 Py_DECREF(v);
2845 return PyErr_SetFromWindowsErrWithFilename(0, NULL);
2846 }
2847
2848 return (PyObject *)v;
2849}
2850
2851PyObject *PyUnicode_EncodeMBCS(const Py_UNICODE *p,
Martin v. Löwis18e16552006-02-15 17:27:45 +00002852 Py_ssize_t size,
Guido van Rossumb7a40ba2000-03-28 02:01:52 +00002853 const char *errors)
2854{
2855 PyObject *repr;
2856 char *s;
Guido van Rossum03e29f12000-05-04 15:52:20 +00002857 DWORD mbcssize;
2858
2859 /* If there are no characters, bail now! */
2860 if (size==0)
2861 return PyString_FromString("");
Guido van Rossumb7a40ba2000-03-28 02:01:52 +00002862
2863 /* First get the size of the result */
Martin v. Löwis18e16552006-02-15 17:27:45 +00002864 assert(size<INT_MAX);
2865 mbcssize = WideCharToMultiByte(CP_ACP, 0, p, (int)size, NULL, 0, NULL, NULL);
Guido van Rossumb7a40ba2000-03-28 02:01:52 +00002866 if (mbcssize==0)
2867 return PyErr_SetFromWindowsErrWithFilename(0, NULL);
2868
2869 repr = PyString_FromStringAndSize(NULL, mbcssize);
2870 if (repr == NULL)
2871 return NULL;
Marc-André Lemburgb7520772000-08-14 11:29:19 +00002872 if (mbcssize == 0)
Guido van Rossumb7a40ba2000-03-28 02:01:52 +00002873 return repr;
2874
2875 /* Do the conversion */
2876 s = PyString_AS_STRING(repr);
Martin v. Löwis18e16552006-02-15 17:27:45 +00002877 assert(size < INT_MAX);
2878 if (0 == WideCharToMultiByte(CP_ACP, 0, p, (int)size, s, mbcssize, NULL, NULL)) {
Guido van Rossumb7a40ba2000-03-28 02:01:52 +00002879 Py_DECREF(repr);
2880 return PyErr_SetFromWindowsErrWithFilename(0, NULL);
2881 }
2882 return repr;
2883}
Guido van Rossum2ea3e142000-03-31 17:24:09 +00002884
Mark Hammond0ccda1e2003-07-01 00:13:27 +00002885PyObject *PyUnicode_AsMBCSString(PyObject *unicode)
2886{
2887 if (!PyUnicode_Check(unicode)) {
2888 PyErr_BadArgument();
2889 return NULL;
2890 }
2891 return PyUnicode_EncodeMBCS(PyUnicode_AS_UNICODE(unicode),
2892 PyUnicode_GET_SIZE(unicode),
2893 NULL);
2894}
2895
Martin v. Löwis6238d2b2002-06-30 15:26:10 +00002896#endif /* MS_WINDOWS */
Guido van Rossumb7a40ba2000-03-28 02:01:52 +00002897
Guido van Rossumd57fd912000-03-10 22:53:23 +00002898/* --- Character Mapping Codec -------------------------------------------- */
2899
Guido van Rossumd57fd912000-03-10 22:53:23 +00002900PyObject *PyUnicode_DecodeCharmap(const char *s,
Martin v. Löwis18e16552006-02-15 17:27:45 +00002901 Py_ssize_t size,
Guido van Rossumd57fd912000-03-10 22:53:23 +00002902 PyObject *mapping,
2903 const char *errors)
2904{
Walter Dörwald3aeb6322002-09-02 13:14:32 +00002905 const char *starts = s;
Martin v. Löwis18e16552006-02-15 17:27:45 +00002906 Py_ssize_t startinpos;
2907 Py_ssize_t endinpos;
2908 Py_ssize_t outpos;
Walter Dörwald3aeb6322002-09-02 13:14:32 +00002909 const char *e;
Guido van Rossumd57fd912000-03-10 22:53:23 +00002910 PyUnicodeObject *v;
2911 Py_UNICODE *p;
Martin v. Löwis18e16552006-02-15 17:27:45 +00002912 Py_ssize_t extrachars = 0;
Walter Dörwald3aeb6322002-09-02 13:14:32 +00002913 PyObject *errorHandler = NULL;
2914 PyObject *exc = NULL;
Walter Dörwaldd1c1e102005-10-06 20:29:57 +00002915 Py_UNICODE *mapstring = NULL;
Martin v. Löwis18e16552006-02-15 17:27:45 +00002916 Py_ssize_t maplen = 0;
Tim Petersced69f82003-09-16 20:30:58 +00002917
Guido van Rossumd57fd912000-03-10 22:53:23 +00002918 /* Default to Latin-1 */
2919 if (mapping == NULL)
2920 return PyUnicode_DecodeLatin1(s, size, errors);
2921
2922 v = _PyUnicode_New(size);
2923 if (v == NULL)
2924 goto onError;
2925 if (size == 0)
2926 return (PyObject *)v;
2927 p = PyUnicode_AS_UNICODE(v);
Walter Dörwald3aeb6322002-09-02 13:14:32 +00002928 e = s + size;
Walter Dörwaldd1c1e102005-10-06 20:29:57 +00002929 if (PyUnicode_CheckExact(mapping)) {
2930 mapstring = PyUnicode_AS_UNICODE(mapping);
2931 maplen = PyUnicode_GET_SIZE(mapping);
2932 while (s < e) {
2933 unsigned char ch = *s;
2934 Py_UNICODE x = 0xfffe; /* illegal value */
Guido van Rossumd57fd912000-03-10 22:53:23 +00002935
Walter Dörwaldd1c1e102005-10-06 20:29:57 +00002936 if (ch < maplen)
2937 x = mapstring[ch];
Guido van Rossumd57fd912000-03-10 22:53:23 +00002938
Walter Dörwaldd1c1e102005-10-06 20:29:57 +00002939 if (x == 0xfffe) {
2940 /* undefined mapping */
2941 outpos = p-PyUnicode_AS_UNICODE(v);
2942 startinpos = s-starts;
2943 endinpos = startinpos+1;
2944 if (unicode_decode_call_errorhandler(
2945 errors, &errorHandler,
2946 "charmap", "character maps to <undefined>",
2947 starts, size, &startinpos, &endinpos, &exc, &s,
2948 (PyObject **)&v, &outpos, &p)) {
2949 goto onError;
Marc-André Lemburgec233e52001-01-06 14:59:58 +00002950 }
Walter Dörwaldd1c1e102005-10-06 20:29:57 +00002951 continue;
Marc-André Lemburgec233e52001-01-06 14:59:58 +00002952 }
Walter Dörwaldd1c1e102005-10-06 20:29:57 +00002953 *p++ = x;
2954 ++s;
Guido van Rossumd57fd912000-03-10 22:53:23 +00002955 }
Walter Dörwaldd1c1e102005-10-06 20:29:57 +00002956 }
2957 else {
2958 while (s < e) {
2959 unsigned char ch = *s;
2960 PyObject *w, *x;
2961
2962 /* Get mapping (char ordinal -> integer, Unicode char or None) */
2963 w = PyInt_FromLong((long)ch);
2964 if (w == NULL)
2965 goto onError;
2966 x = PyObject_GetItem(mapping, w);
2967 Py_DECREF(w);
2968 if (x == NULL) {
2969 if (PyErr_ExceptionMatches(PyExc_LookupError)) {
2970 /* No mapping found means: mapping is undefined. */
2971 PyErr_Clear();
2972 x = Py_None;
2973 Py_INCREF(x);
2974 } else
2975 goto onError;
2976 }
2977
2978 /* Apply mapping */
2979 if (PyInt_Check(x)) {
2980 long value = PyInt_AS_LONG(x);
2981 if (value < 0 || value > 65535) {
2982 PyErr_SetString(PyExc_TypeError,
2983 "character mapping must be in range(65536)");
2984 Py_DECREF(x);
2985 goto onError;
2986 }
2987 *p++ = (Py_UNICODE)value;
2988 }
2989 else if (x == Py_None) {
2990 /* undefined mapping */
2991 outpos = p-PyUnicode_AS_UNICODE(v);
2992 startinpos = s-starts;
2993 endinpos = startinpos+1;
2994 if (unicode_decode_call_errorhandler(
2995 errors, &errorHandler,
2996 "charmap", "character maps to <undefined>",
2997 starts, size, &startinpos, &endinpos, &exc, &s,
2998 (PyObject **)&v, &outpos, &p)) {
2999 Py_DECREF(x);
3000 goto onError;
3001 }
Walter Dörwaldd4fff172005-11-28 22:15:56 +00003002 Py_DECREF(x);
Walter Dörwaldd1c1e102005-10-06 20:29:57 +00003003 continue;
3004 }
3005 else if (PyUnicode_Check(x)) {
Martin v. Löwis18e16552006-02-15 17:27:45 +00003006 Py_ssize_t targetsize = PyUnicode_GET_SIZE(x);
Walter Dörwaldd1c1e102005-10-06 20:29:57 +00003007
3008 if (targetsize == 1)
3009 /* 1-1 mapping */
3010 *p++ = *PyUnicode_AS_UNICODE(x);
3011
3012 else if (targetsize > 1) {
3013 /* 1-n mapping */
3014 if (targetsize > extrachars) {
3015 /* resize first */
Martin v. Löwis18e16552006-02-15 17:27:45 +00003016 Py_ssize_t oldpos = p - PyUnicode_AS_UNICODE(v);
3017 Py_ssize_t needed = (targetsize - extrachars) + \
Walter Dörwaldd1c1e102005-10-06 20:29:57 +00003018 (targetsize << 2);
3019 extrachars += needed;
3020 if (_PyUnicode_Resize(&v,
3021 PyUnicode_GET_SIZE(v) + needed) < 0) {
3022 Py_DECREF(x);
3023 goto onError;
3024 }
3025 p = PyUnicode_AS_UNICODE(v) + oldpos;
3026 }
3027 Py_UNICODE_COPY(p,
3028 PyUnicode_AS_UNICODE(x),
3029 targetsize);
3030 p += targetsize;
3031 extrachars -= targetsize;
3032 }
3033 /* 1-0 mapping: skip the character */
3034 }
3035 else {
3036 /* wrong return value */
3037 PyErr_SetString(PyExc_TypeError,
3038 "character mapping must return integer, None or unicode");
3039 Py_DECREF(x);
3040 goto onError;
3041 }
Guido van Rossumd57fd912000-03-10 22:53:23 +00003042 Py_DECREF(x);
Walter Dörwaldd1c1e102005-10-06 20:29:57 +00003043 ++s;
Guido van Rossumd57fd912000-03-10 22:53:23 +00003044 }
Guido van Rossumd57fd912000-03-10 22:53:23 +00003045 }
3046 if (p - PyUnicode_AS_UNICODE(v) < PyUnicode_GET_SIZE(v))
Martin v. Löwis412fb672006-04-13 06:34:32 +00003047 if (_PyUnicode_Resize(&v, p - PyUnicode_AS_UNICODE(v)) < 0)
Guido van Rossumd57fd912000-03-10 22:53:23 +00003048 goto onError;
Walter Dörwald3aeb6322002-09-02 13:14:32 +00003049 Py_XDECREF(errorHandler);
3050 Py_XDECREF(exc);
Guido van Rossumd57fd912000-03-10 22:53:23 +00003051 return (PyObject *)v;
Tim Petersced69f82003-09-16 20:30:58 +00003052
Guido van Rossumd57fd912000-03-10 22:53:23 +00003053 onError:
Walter Dörwald3aeb6322002-09-02 13:14:32 +00003054 Py_XDECREF(errorHandler);
3055 Py_XDECREF(exc);
Guido van Rossumd57fd912000-03-10 22:53:23 +00003056 Py_XDECREF(v);
3057 return NULL;
3058}
3059
Walter Dörwald3aeb6322002-09-02 13:14:32 +00003060/* Lookup the character ch in the mapping. If the character
3061 can't be found, Py_None is returned (or NULL, if another
Fred Drakedb390c12005-10-28 14:39:47 +00003062 error occurred). */
Walter Dörwald3aeb6322002-09-02 13:14:32 +00003063static PyObject *charmapencode_lookup(Py_UNICODE c, PyObject *mapping)
Guido van Rossumd57fd912000-03-10 22:53:23 +00003064{
Walter Dörwald3aeb6322002-09-02 13:14:32 +00003065 PyObject *w = PyInt_FromLong((long)c);
3066 PyObject *x;
3067
3068 if (w == NULL)
3069 return NULL;
3070 x = PyObject_GetItem(mapping, w);
3071 Py_DECREF(w);
3072 if (x == NULL) {
3073 if (PyErr_ExceptionMatches(PyExc_LookupError)) {
3074 /* No mapping found means: mapping is undefined. */
3075 PyErr_Clear();
3076 x = Py_None;
3077 Py_INCREF(x);
3078 return x;
3079 } else
3080 return NULL;
Guido van Rossumd57fd912000-03-10 22:53:23 +00003081 }
Walter Dörwaldadc72742003-01-08 22:01:33 +00003082 else if (x == Py_None)
3083 return x;
Walter Dörwald3aeb6322002-09-02 13:14:32 +00003084 else if (PyInt_Check(x)) {
3085 long value = PyInt_AS_LONG(x);
3086 if (value < 0 || value > 255) {
3087 PyErr_SetString(PyExc_TypeError,
3088 "character mapping must be in range(256)");
3089 Py_DECREF(x);
3090 return NULL;
3091 }
3092 return x;
Guido van Rossumd57fd912000-03-10 22:53:23 +00003093 }
Walter Dörwald3aeb6322002-09-02 13:14:32 +00003094 else if (PyString_Check(x))
3095 return x;
Guido van Rossumd57fd912000-03-10 22:53:23 +00003096 else {
Walter Dörwald3aeb6322002-09-02 13:14:32 +00003097 /* wrong return value */
3098 PyErr_SetString(PyExc_TypeError,
3099 "character mapping must return integer, None or str");
3100 Py_DECREF(x);
3101 return NULL;
Guido van Rossumd57fd912000-03-10 22:53:23 +00003102 }
3103}
3104
Walter Dörwald3aeb6322002-09-02 13:14:32 +00003105/* lookup the character, put the result in the output string and adjust
3106 various state variables. Reallocate the output string if not enough
3107 space is available. Return a new reference to the object that
3108 was put in the output buffer, or Py_None, if the mapping was undefined
3109 (in which case no character was written) or NULL, if a
Andrew M. Kuchling8294de52005-11-02 16:36:12 +00003110 reallocation error occurred. The caller must decref the result */
Walter Dörwald3aeb6322002-09-02 13:14:32 +00003111static
3112PyObject *charmapencode_output(Py_UNICODE c, PyObject *mapping,
Martin v. Löwis18e16552006-02-15 17:27:45 +00003113 PyObject **outobj, Py_ssize_t *outpos)
Walter Dörwald3aeb6322002-09-02 13:14:32 +00003114{
3115 PyObject *rep = charmapencode_lookup(c, mapping);
3116
3117 if (rep==NULL)
3118 return NULL;
3119 else if (rep==Py_None)
3120 return rep;
3121 else {
3122 char *outstart = PyString_AS_STRING(*outobj);
Martin v. Löwis18e16552006-02-15 17:27:45 +00003123 Py_ssize_t outsize = PyString_GET_SIZE(*outobj);
Walter Dörwald3aeb6322002-09-02 13:14:32 +00003124 if (PyInt_Check(rep)) {
Martin v. Löwis18e16552006-02-15 17:27:45 +00003125 Py_ssize_t requiredsize = *outpos+1;
Walter Dörwald3aeb6322002-09-02 13:14:32 +00003126 if (outsize<requiredsize) {
3127 /* exponentially overallocate to minimize reallocations */
3128 if (requiredsize < 2*outsize)
3129 requiredsize = 2*outsize;
3130 if (_PyString_Resize(outobj, requiredsize)) {
3131 Py_DECREF(rep);
3132 return NULL;
3133 }
3134 outstart = PyString_AS_STRING(*outobj);
3135 }
3136 outstart[(*outpos)++] = (char)PyInt_AS_LONG(rep);
3137 }
3138 else {
3139 const char *repchars = PyString_AS_STRING(rep);
Martin v. Löwis18e16552006-02-15 17:27:45 +00003140 Py_ssize_t repsize = PyString_GET_SIZE(rep);
3141 Py_ssize_t requiredsize = *outpos+repsize;
Walter Dörwald3aeb6322002-09-02 13:14:32 +00003142 if (outsize<requiredsize) {
3143 /* exponentially overallocate to minimize reallocations */
3144 if (requiredsize < 2*outsize)
3145 requiredsize = 2*outsize;
3146 if (_PyString_Resize(outobj, requiredsize)) {
3147 Py_DECREF(rep);
3148 return NULL;
3149 }
3150 outstart = PyString_AS_STRING(*outobj);
3151 }
3152 memcpy(outstart + *outpos, repchars, repsize);
3153 *outpos += repsize;
3154 }
3155 }
3156 return rep;
3157}
3158
3159/* handle an error in PyUnicode_EncodeCharmap
3160 Return 0 on success, -1 on error */
3161static
3162int charmap_encoding_error(
Martin v. Löwis18e16552006-02-15 17:27:45 +00003163 const Py_UNICODE *p, Py_ssize_t size, Py_ssize_t *inpos, PyObject *mapping,
Walter Dörwald3aeb6322002-09-02 13:14:32 +00003164 PyObject **exceptionObject,
Walter Dörwalde5402fb2003-08-14 20:25:29 +00003165 int *known_errorHandler, PyObject **errorHandler, const char *errors,
Martin v. Löwis18e16552006-02-15 17:27:45 +00003166 PyObject **res, Py_ssize_t *respos)
Walter Dörwald3aeb6322002-09-02 13:14:32 +00003167{
3168 PyObject *repunicode = NULL; /* initialize to prevent gcc warning */
Martin v. Löwis18e16552006-02-15 17:27:45 +00003169 Py_ssize_t repsize;
3170 Py_ssize_t newpos;
Walter Dörwald3aeb6322002-09-02 13:14:32 +00003171 Py_UNICODE *uni2;
3172 /* startpos for collecting unencodable chars */
Martin v. Löwis18e16552006-02-15 17:27:45 +00003173 Py_ssize_t collstartpos = *inpos;
3174 Py_ssize_t collendpos = *inpos+1;
3175 Py_ssize_t collpos;
Walter Dörwald3aeb6322002-09-02 13:14:32 +00003176 char *encoding = "charmap";
3177 char *reason = "character maps to <undefined>";
3178
3179 PyObject *x;
3180 /* find all unencodable characters */
3181 while (collendpos < size) {
3182 x = charmapencode_lookup(p[collendpos], mapping);
3183 if (x==NULL)
3184 return -1;
3185 else if (x!=Py_None) {
3186 Py_DECREF(x);
3187 break;
3188 }
3189 Py_DECREF(x);
3190 ++collendpos;
3191 }
3192 /* cache callback name lookup
3193 * (if not done yet, i.e. it's the first error) */
3194 if (*known_errorHandler==-1) {
3195 if ((errors==NULL) || (!strcmp(errors, "strict")))
3196 *known_errorHandler = 1;
3197 else if (!strcmp(errors, "replace"))
3198 *known_errorHandler = 2;
3199 else if (!strcmp(errors, "ignore"))
3200 *known_errorHandler = 3;
3201 else if (!strcmp(errors, "xmlcharrefreplace"))
3202 *known_errorHandler = 4;
3203 else
3204 *known_errorHandler = 0;
3205 }
3206 switch (*known_errorHandler) {
3207 case 1: /* strict */
3208 raise_encode_exception(exceptionObject, encoding, p, size, collstartpos, collendpos, reason);
3209 return -1;
3210 case 2: /* replace */
3211 for (collpos = collstartpos; collpos<collendpos; ++collpos) {
3212 x = charmapencode_output('?', mapping, res, respos);
3213 if (x==NULL) {
3214 return -1;
3215 }
3216 else if (x==Py_None) {
3217 Py_DECREF(x);
3218 raise_encode_exception(exceptionObject, encoding, p, size, collstartpos, collendpos, reason);
3219 return -1;
3220 }
3221 Py_DECREF(x);
3222 }
3223 /* fall through */
3224 case 3: /* ignore */
3225 *inpos = collendpos;
3226 break;
3227 case 4: /* xmlcharrefreplace */
3228 /* generate replacement (temporarily (mis)uses p) */
3229 for (collpos = collstartpos; collpos < collendpos; ++collpos) {
3230 char buffer[2+29+1+1];
3231 char *cp;
3232 sprintf(buffer, "&#%d;", (int)p[collpos]);
3233 for (cp = buffer; *cp; ++cp) {
3234 x = charmapencode_output(*cp, mapping, res, respos);
3235 if (x==NULL)
3236 return -1;
3237 else if (x==Py_None) {
3238 Py_DECREF(x);
3239 raise_encode_exception(exceptionObject, encoding, p, size, collstartpos, collendpos, reason);
3240 return -1;
3241 }
3242 Py_DECREF(x);
3243 }
3244 }
3245 *inpos = collendpos;
3246 break;
3247 default:
Walter Dörwalde5402fb2003-08-14 20:25:29 +00003248 repunicode = unicode_encode_call_errorhandler(errors, errorHandler,
Walter Dörwald3aeb6322002-09-02 13:14:32 +00003249 encoding, reason, p, size, exceptionObject,
3250 collstartpos, collendpos, &newpos);
3251 if (repunicode == NULL)
3252 return -1;
3253 /* generate replacement */
3254 repsize = PyUnicode_GET_SIZE(repunicode);
3255 for (uni2 = PyUnicode_AS_UNICODE(repunicode); repsize-->0; ++uni2) {
3256 x = charmapencode_output(*uni2, mapping, res, respos);
3257 if (x==NULL) {
3258 Py_DECREF(repunicode);
3259 return -1;
3260 }
3261 else if (x==Py_None) {
3262 Py_DECREF(repunicode);
3263 Py_DECREF(x);
3264 raise_encode_exception(exceptionObject, encoding, p, size, collstartpos, collendpos, reason);
3265 return -1;
3266 }
3267 Py_DECREF(x);
3268 }
3269 *inpos = newpos;
3270 Py_DECREF(repunicode);
3271 }
3272 return 0;
3273}
3274
Guido van Rossumd57fd912000-03-10 22:53:23 +00003275PyObject *PyUnicode_EncodeCharmap(const Py_UNICODE *p,
Martin v. Löwis18e16552006-02-15 17:27:45 +00003276 Py_ssize_t size,
Guido van Rossumd57fd912000-03-10 22:53:23 +00003277 PyObject *mapping,
3278 const char *errors)
3279{
Walter Dörwald3aeb6322002-09-02 13:14:32 +00003280 /* output object */
3281 PyObject *res = NULL;
3282 /* current input position */
Martin v. Löwis18e16552006-02-15 17:27:45 +00003283 Py_ssize_t inpos = 0;
Walter Dörwald3aeb6322002-09-02 13:14:32 +00003284 /* current output position */
Martin v. Löwis18e16552006-02-15 17:27:45 +00003285 Py_ssize_t respos = 0;
Walter Dörwald3aeb6322002-09-02 13:14:32 +00003286 PyObject *errorHandler = NULL;
3287 PyObject *exc = NULL;
3288 /* the following variable is used for caching string comparisons
3289 * -1=not initialized, 0=unknown, 1=strict, 2=replace,
3290 * 3=ignore, 4=xmlcharrefreplace */
3291 int known_errorHandler = -1;
Guido van Rossumd57fd912000-03-10 22:53:23 +00003292
3293 /* Default to Latin-1 */
3294 if (mapping == NULL)
3295 return PyUnicode_EncodeLatin1(p, size, errors);
3296
Walter Dörwald3aeb6322002-09-02 13:14:32 +00003297 /* allocate enough for a simple encoding without
3298 replacements, if we need more, we'll resize */
3299 res = PyString_FromStringAndSize(NULL, size);
3300 if (res == NULL)
3301 goto onError;
Marc-André Lemburgb7520772000-08-14 11:29:19 +00003302 if (size == 0)
Walter Dörwald3aeb6322002-09-02 13:14:32 +00003303 return res;
Guido van Rossumd57fd912000-03-10 22:53:23 +00003304
Walter Dörwald3aeb6322002-09-02 13:14:32 +00003305 while (inpos<size) {
3306 /* try to encode it */
3307 PyObject *x = charmapencode_output(p[inpos], mapping, &res, &respos);
3308 if (x==NULL) /* error */
Guido van Rossumd57fd912000-03-10 22:53:23 +00003309 goto onError;
Walter Dörwald3aeb6322002-09-02 13:14:32 +00003310 if (x==Py_None) { /* unencodable character */
3311 if (charmap_encoding_error(p, size, &inpos, mapping,
3312 &exc,
Walter Dörwalde5402fb2003-08-14 20:25:29 +00003313 &known_errorHandler, &errorHandler, errors,
Walter Dörwald9b30f202003-08-15 16:26:34 +00003314 &res, &respos)) {
3315 Py_DECREF(x);
Marc-André Lemburg3a645e42001-01-16 11:54:12 +00003316 goto onError;
Walter Dörwald9b30f202003-08-15 16:26:34 +00003317 }
Guido van Rossumd57fd912000-03-10 22:53:23 +00003318 }
Walter Dörwald3aeb6322002-09-02 13:14:32 +00003319 else
3320 /* done with this character => adjust input position */
3321 ++inpos;
Guido van Rossumd57fd912000-03-10 22:53:23 +00003322 Py_DECREF(x);
3323 }
Guido van Rossumd57fd912000-03-10 22:53:23 +00003324
Walter Dörwald3aeb6322002-09-02 13:14:32 +00003325 /* Resize if we allocated to much */
3326 if (respos<PyString_GET_SIZE(res)) {
3327 if (_PyString_Resize(&res, respos))
3328 goto onError;
3329 }
3330 Py_XDECREF(exc);
3331 Py_XDECREF(errorHandler);
3332 return res;
3333
3334 onError:
3335 Py_XDECREF(res);
3336 Py_XDECREF(exc);
3337 Py_XDECREF(errorHandler);
Guido van Rossumd57fd912000-03-10 22:53:23 +00003338 return NULL;
3339}
3340
3341PyObject *PyUnicode_AsCharmapString(PyObject *unicode,
3342 PyObject *mapping)
3343{
3344 if (!PyUnicode_Check(unicode) || mapping == NULL) {
3345 PyErr_BadArgument();
3346 return NULL;
3347 }
3348 return PyUnicode_EncodeCharmap(PyUnicode_AS_UNICODE(unicode),
3349 PyUnicode_GET_SIZE(unicode),
3350 mapping,
3351 NULL);
3352}
3353
Walter Dörwald3aeb6322002-09-02 13:14:32 +00003354/* create or adjust a UnicodeTranslateError */
3355static void make_translate_exception(PyObject **exceptionObject,
Martin v. Löwis18e16552006-02-15 17:27:45 +00003356 const Py_UNICODE *unicode, Py_ssize_t size,
3357 Py_ssize_t startpos, Py_ssize_t endpos,
Walter Dörwald3aeb6322002-09-02 13:14:32 +00003358 const char *reason)
Guido van Rossumd57fd912000-03-10 22:53:23 +00003359{
Walter Dörwald3aeb6322002-09-02 13:14:32 +00003360 if (*exceptionObject == NULL) {
3361 *exceptionObject = PyUnicodeTranslateError_Create(
3362 unicode, size, startpos, endpos, reason);
Guido van Rossumd57fd912000-03-10 22:53:23 +00003363 }
3364 else {
Walter Dörwald3aeb6322002-09-02 13:14:32 +00003365 if (PyUnicodeTranslateError_SetStart(*exceptionObject, startpos))
3366 goto onError;
3367 if (PyUnicodeTranslateError_SetEnd(*exceptionObject, endpos))
3368 goto onError;
3369 if (PyUnicodeTranslateError_SetReason(*exceptionObject, reason))
3370 goto onError;
3371 return;
3372 onError:
3373 Py_DECREF(*exceptionObject);
3374 *exceptionObject = NULL;
Guido van Rossumd57fd912000-03-10 22:53:23 +00003375 }
3376}
3377
Walter Dörwald3aeb6322002-09-02 13:14:32 +00003378/* raises a UnicodeTranslateError */
3379static void raise_translate_exception(PyObject **exceptionObject,
Martin v. Löwis18e16552006-02-15 17:27:45 +00003380 const Py_UNICODE *unicode, Py_ssize_t size,
3381 Py_ssize_t startpos, Py_ssize_t endpos,
Walter Dörwald3aeb6322002-09-02 13:14:32 +00003382 const char *reason)
3383{
3384 make_translate_exception(exceptionObject,
3385 unicode, size, startpos, endpos, reason);
3386 if (*exceptionObject != NULL)
3387 PyCodec_StrictErrors(*exceptionObject);
3388}
3389
3390/* error handling callback helper:
3391 build arguments, call the callback and check the arguments,
3392 put the result into newpos and return the replacement string, which
3393 has to be freed by the caller */
3394static PyObject *unicode_translate_call_errorhandler(const char *errors,
3395 PyObject **errorHandler,
3396 const char *reason,
Martin v. Löwis18e16552006-02-15 17:27:45 +00003397 const Py_UNICODE *unicode, Py_ssize_t size, PyObject **exceptionObject,
3398 Py_ssize_t startpos, Py_ssize_t endpos,
3399 Py_ssize_t *newpos)
Walter Dörwald3aeb6322002-09-02 13:14:32 +00003400{
Martin v. Löwis412fb672006-04-13 06:34:32 +00003401 static char *argparse = "O!n;translating error handler must return (unicode, int) tuple";
Walter Dörwald3aeb6322002-09-02 13:14:32 +00003402
Martin v. Löwis412fb672006-04-13 06:34:32 +00003403 Py_ssize_t i_newpos;
Walter Dörwald3aeb6322002-09-02 13:14:32 +00003404 PyObject *restuple;
3405 PyObject *resunicode;
3406
3407 if (*errorHandler == NULL) {
3408 *errorHandler = PyCodec_LookupError(errors);
3409 if (*errorHandler == NULL)
3410 return NULL;
3411 }
3412
3413 make_translate_exception(exceptionObject,
3414 unicode, size, startpos, endpos, reason);
3415 if (*exceptionObject == NULL)
3416 return NULL;
3417
3418 restuple = PyObject_CallFunctionObjArgs(
3419 *errorHandler, *exceptionObject, NULL);
3420 if (restuple == NULL)
3421 return NULL;
3422 if (!PyTuple_Check(restuple)) {
3423 PyErr_Format(PyExc_TypeError, &argparse[4]);
3424 Py_DECREF(restuple);
3425 return NULL;
3426 }
3427 if (!PyArg_ParseTuple(restuple, argparse, &PyUnicode_Type,
Martin v. Löwis18e16552006-02-15 17:27:45 +00003428 &resunicode, &i_newpos)) {
Walter Dörwald3aeb6322002-09-02 13:14:32 +00003429 Py_DECREF(restuple);
3430 return NULL;
3431 }
Martin v. Löwis18e16552006-02-15 17:27:45 +00003432 if (i_newpos<0)
3433 *newpos = size+i_newpos;
3434 else
3435 *newpos = i_newpos;
Walter Dörwald2e0b18a2003-01-31 17:19:08 +00003436 if (*newpos<0 || *newpos>size) {
Martin v. Löwis2c95cc62006-02-16 06:54:25 +00003437 PyErr_Format(PyExc_IndexError, "position %zd from error handler out of bounds", *newpos);
Walter Dörwald2e0b18a2003-01-31 17:19:08 +00003438 Py_DECREF(restuple);
3439 return NULL;
3440 }
Walter Dörwald3aeb6322002-09-02 13:14:32 +00003441 Py_INCREF(resunicode);
3442 Py_DECREF(restuple);
3443 return resunicode;
3444}
3445
3446/* Lookup the character ch in the mapping and put the result in result,
3447 which must be decrefed by the caller.
3448 Return 0 on success, -1 on error */
3449static
3450int charmaptranslate_lookup(Py_UNICODE c, PyObject *mapping, PyObject **result)
3451{
3452 PyObject *w = PyInt_FromLong((long)c);
3453 PyObject *x;
3454
3455 if (w == NULL)
3456 return -1;
3457 x = PyObject_GetItem(mapping, w);
3458 Py_DECREF(w);
3459 if (x == NULL) {
3460 if (PyErr_ExceptionMatches(PyExc_LookupError)) {
3461 /* No mapping found means: use 1:1 mapping. */
3462 PyErr_Clear();
3463 *result = NULL;
3464 return 0;
3465 } else
3466 return -1;
3467 }
3468 else if (x == Py_None) {
3469 *result = x;
3470 return 0;
3471 }
3472 else if (PyInt_Check(x)) {
3473 long value = PyInt_AS_LONG(x);
3474 long max = PyUnicode_GetMax();
3475 if (value < 0 || value > max) {
3476 PyErr_Format(PyExc_TypeError,
3477 "character mapping must be in range(0x%lx)", max+1);
3478 Py_DECREF(x);
3479 return -1;
3480 }
3481 *result = x;
3482 return 0;
3483 }
3484 else if (PyUnicode_Check(x)) {
3485 *result = x;
3486 return 0;
3487 }
3488 else {
3489 /* wrong return value */
3490 PyErr_SetString(PyExc_TypeError,
3491 "character mapping must return integer, None or unicode");
Walter Dörwald150523e2003-08-15 16:52:19 +00003492 Py_DECREF(x);
Walter Dörwald3aeb6322002-09-02 13:14:32 +00003493 return -1;
3494 }
3495}
3496/* ensure that *outobj is at least requiredsize characters long,
3497if not reallocate and adjust various state variables.
3498Return 0 on success, -1 on error */
3499static
Walter Dörwald4894c302003-10-24 14:25:28 +00003500int charmaptranslate_makespace(PyObject **outobj, Py_UNICODE **outp,
Martin v. Löwis18e16552006-02-15 17:27:45 +00003501 Py_ssize_t requiredsize)
Walter Dörwald3aeb6322002-09-02 13:14:32 +00003502{
Martin v. Löwis18e16552006-02-15 17:27:45 +00003503 Py_ssize_t oldsize = PyUnicode_GET_SIZE(*outobj);
Walter Dörwald4894c302003-10-24 14:25:28 +00003504 if (requiredsize > oldsize) {
Walter Dörwald3aeb6322002-09-02 13:14:32 +00003505 /* remember old output position */
Martin v. Löwis18e16552006-02-15 17:27:45 +00003506 Py_ssize_t outpos = *outp-PyUnicode_AS_UNICODE(*outobj);
Walter Dörwald3aeb6322002-09-02 13:14:32 +00003507 /* exponentially overallocate to minimize reallocations */
Walter Dörwald4894c302003-10-24 14:25:28 +00003508 if (requiredsize < 2 * oldsize)
3509 requiredsize = 2 * oldsize;
Jeremy Hyltondeb2dc62003-09-16 03:41:45 +00003510 if (_PyUnicode_Resize(outobj, requiredsize) < 0)
Walter Dörwald3aeb6322002-09-02 13:14:32 +00003511 return -1;
3512 *outp = PyUnicode_AS_UNICODE(*outobj) + outpos;
Walter Dörwald3aeb6322002-09-02 13:14:32 +00003513 }
3514 return 0;
3515}
3516/* lookup the character, put the result in the output string and adjust
3517 various state variables. Return a new reference to the object that
3518 was put in the output buffer in *result, or Py_None, if the mapping was
3519 undefined (in which case no character was written).
3520 The called must decref result.
3521 Return 0 on success, -1 on error. */
3522static
Walter Dörwald4894c302003-10-24 14:25:28 +00003523int charmaptranslate_output(const Py_UNICODE *startinp, const Py_UNICODE *curinp,
Martin v. Löwis18e16552006-02-15 17:27:45 +00003524 Py_ssize_t insize, PyObject *mapping, PyObject **outobj, Py_UNICODE **outp,
Walter Dörwald4894c302003-10-24 14:25:28 +00003525 PyObject **res)
Walter Dörwald3aeb6322002-09-02 13:14:32 +00003526{
Walter Dörwald4894c302003-10-24 14:25:28 +00003527 if (charmaptranslate_lookup(*curinp, mapping, res))
Walter Dörwald3aeb6322002-09-02 13:14:32 +00003528 return -1;
3529 if (*res==NULL) {
3530 /* not found => default to 1:1 mapping */
Walter Dörwald4894c302003-10-24 14:25:28 +00003531 *(*outp)++ = *curinp;
Walter Dörwald3aeb6322002-09-02 13:14:32 +00003532 }
3533 else if (*res==Py_None)
3534 ;
3535 else if (PyInt_Check(*res)) {
3536 /* no overflow check, because we know that the space is enough */
3537 *(*outp)++ = (Py_UNICODE)PyInt_AS_LONG(*res);
3538 }
3539 else if (PyUnicode_Check(*res)) {
Martin v. Löwis18e16552006-02-15 17:27:45 +00003540 Py_ssize_t repsize = PyUnicode_GET_SIZE(*res);
Walter Dörwald3aeb6322002-09-02 13:14:32 +00003541 if (repsize==1) {
3542 /* no overflow check, because we know that the space is enough */
3543 *(*outp)++ = *PyUnicode_AS_UNICODE(*res);
3544 }
3545 else if (repsize!=0) {
3546 /* more than one character */
Martin v. Löwis18e16552006-02-15 17:27:45 +00003547 Py_ssize_t requiredsize = (*outp-PyUnicode_AS_UNICODE(*outobj)) +
Walter Dörwaldcd736e72004-02-05 17:36:00 +00003548 (insize - (curinp-startinp)) +
Walter Dörwald4894c302003-10-24 14:25:28 +00003549 repsize - 1;
3550 if (charmaptranslate_makespace(outobj, outp, requiredsize))
Walter Dörwald3aeb6322002-09-02 13:14:32 +00003551 return -1;
3552 memcpy(*outp, PyUnicode_AS_UNICODE(*res), sizeof(Py_UNICODE)*repsize);
3553 *outp += repsize;
3554 }
3555 }
3556 else
3557 return -1;
3558 return 0;
3559}
3560
3561PyObject *PyUnicode_TranslateCharmap(const Py_UNICODE *p,
Martin v. Löwis18e16552006-02-15 17:27:45 +00003562 Py_ssize_t size,
Guido van Rossumd57fd912000-03-10 22:53:23 +00003563 PyObject *mapping,
3564 const char *errors)
3565{
Walter Dörwald3aeb6322002-09-02 13:14:32 +00003566 /* output object */
3567 PyObject *res = NULL;
3568 /* pointers to the beginning and end+1 of input */
3569 const Py_UNICODE *startp = p;
3570 const Py_UNICODE *endp = p + size;
3571 /* pointer into the output */
3572 Py_UNICODE *str;
3573 /* current output position */
Martin v. Löwis18e16552006-02-15 17:27:45 +00003574 Py_ssize_t respos = 0;
Walter Dörwald3aeb6322002-09-02 13:14:32 +00003575 char *reason = "character maps to <undefined>";
3576 PyObject *errorHandler = NULL;
3577 PyObject *exc = NULL;
3578 /* the following variable is used for caching string comparisons
3579 * -1=not initialized, 0=unknown, 1=strict, 2=replace,
3580 * 3=ignore, 4=xmlcharrefreplace */
3581 int known_errorHandler = -1;
3582
Guido van Rossumd57fd912000-03-10 22:53:23 +00003583 if (mapping == NULL) {
3584 PyErr_BadArgument();
3585 return NULL;
3586 }
Walter Dörwald3aeb6322002-09-02 13:14:32 +00003587
3588 /* allocate enough for a simple 1:1 translation without
3589 replacements, if we need more, we'll resize */
3590 res = PyUnicode_FromUnicode(NULL, size);
3591 if (res == NULL)
Walter Dörwald4894c302003-10-24 14:25:28 +00003592 goto onError;
Guido van Rossumd57fd912000-03-10 22:53:23 +00003593 if (size == 0)
Walter Dörwald3aeb6322002-09-02 13:14:32 +00003594 return res;
3595 str = PyUnicode_AS_UNICODE(res);
Guido van Rossumd57fd912000-03-10 22:53:23 +00003596
Walter Dörwald3aeb6322002-09-02 13:14:32 +00003597 while (p<endp) {
3598 /* try to encode it */
3599 PyObject *x = NULL;
Walter Dörwald4894c302003-10-24 14:25:28 +00003600 if (charmaptranslate_output(startp, p, size, mapping, &res, &str, &x)) {
Walter Dörwald3aeb6322002-09-02 13:14:32 +00003601 Py_XDECREF(x);
Guido van Rossumd57fd912000-03-10 22:53:23 +00003602 goto onError;
3603 }
Walter Dörwaldf6b56ae2003-02-09 23:42:56 +00003604 Py_XDECREF(x);
Walter Dörwald3aeb6322002-09-02 13:14:32 +00003605 if (x!=Py_None) /* it worked => adjust input pointer */
3606 ++p;
3607 else { /* untranslatable character */
3608 PyObject *repunicode = NULL; /* initialize to prevent gcc warning */
Martin v. Löwis18e16552006-02-15 17:27:45 +00003609 Py_ssize_t repsize;
3610 Py_ssize_t newpos;
Walter Dörwald3aeb6322002-09-02 13:14:32 +00003611 Py_UNICODE *uni2;
3612 /* startpos for collecting untranslatable chars */
3613 const Py_UNICODE *collstart = p;
3614 const Py_UNICODE *collend = p+1;
3615 const Py_UNICODE *coll;
Guido van Rossumd57fd912000-03-10 22:53:23 +00003616
Walter Dörwald3aeb6322002-09-02 13:14:32 +00003617 /* find all untranslatable characters */
3618 while (collend < endp) {
Walter Dörwald4894c302003-10-24 14:25:28 +00003619 if (charmaptranslate_lookup(*collend, mapping, &x))
Walter Dörwald3aeb6322002-09-02 13:14:32 +00003620 goto onError;
3621 Py_XDECREF(x);
3622 if (x!=Py_None)
3623 break;
3624 ++collend;
3625 }
3626 /* cache callback name lookup
3627 * (if not done yet, i.e. it's the first error) */
3628 if (known_errorHandler==-1) {
3629 if ((errors==NULL) || (!strcmp(errors, "strict")))
3630 known_errorHandler = 1;
3631 else if (!strcmp(errors, "replace"))
3632 known_errorHandler = 2;
3633 else if (!strcmp(errors, "ignore"))
3634 known_errorHandler = 3;
3635 else if (!strcmp(errors, "xmlcharrefreplace"))
3636 known_errorHandler = 4;
3637 else
3638 known_errorHandler = 0;
3639 }
3640 switch (known_errorHandler) {
3641 case 1: /* strict */
3642 raise_translate_exception(&exc, startp, size, collstart-startp, collend-startp, reason);
3643 goto onError;
3644 case 2: /* replace */
3645 /* No need to check for space, this is a 1:1 replacement */
3646 for (coll = collstart; coll<collend; ++coll)
3647 *str++ = '?';
3648 /* fall through */
3649 case 3: /* ignore */
3650 p = collend;
3651 break;
3652 case 4: /* xmlcharrefreplace */
3653 /* generate replacement (temporarily (mis)uses p) */
3654 for (p = collstart; p < collend; ++p) {
3655 char buffer[2+29+1+1];
3656 char *cp;
3657 sprintf(buffer, "&#%d;", (int)*p);
Walter Dörwald4894c302003-10-24 14:25:28 +00003658 if (charmaptranslate_makespace(&res, &str,
Walter Dörwald3aeb6322002-09-02 13:14:32 +00003659 (str-PyUnicode_AS_UNICODE(res))+strlen(buffer)+(endp-collend)))
3660 goto onError;
3661 for (cp = buffer; *cp; ++cp)
3662 *str++ = *cp;
3663 }
3664 p = collend;
3665 break;
3666 default:
3667 repunicode = unicode_translate_call_errorhandler(errors, &errorHandler,
3668 reason, startp, size, &exc,
3669 collstart-startp, collend-startp, &newpos);
3670 if (repunicode == NULL)
3671 goto onError;
3672 /* generate replacement */
3673 repsize = PyUnicode_GET_SIZE(repunicode);
Walter Dörwald4894c302003-10-24 14:25:28 +00003674 if (charmaptranslate_makespace(&res, &str,
Walter Dörwald3aeb6322002-09-02 13:14:32 +00003675 (str-PyUnicode_AS_UNICODE(res))+repsize+(endp-collend))) {
3676 Py_DECREF(repunicode);
3677 goto onError;
3678 }
3679 for (uni2 = PyUnicode_AS_UNICODE(repunicode); repsize-->0; ++uni2)
3680 *str++ = *uni2;
3681 p = startp + newpos;
3682 Py_DECREF(repunicode);
Guido van Rossumd57fd912000-03-10 22:53:23 +00003683 }
3684 }
Guido van Rossumd57fd912000-03-10 22:53:23 +00003685 }
Walter Dörwald3aeb6322002-09-02 13:14:32 +00003686 /* Resize if we allocated to much */
3687 respos = str-PyUnicode_AS_UNICODE(res);
Walter Dörwald4894c302003-10-24 14:25:28 +00003688 if (respos<PyUnicode_GET_SIZE(res)) {
Jeremy Hyltondeb2dc62003-09-16 03:41:45 +00003689 if (_PyUnicode_Resize(&res, respos) < 0)
Guido van Rossumfd4b9572000-04-10 13:51:10 +00003690 goto onError;
Walter Dörwald3aeb6322002-09-02 13:14:32 +00003691 }
3692 Py_XDECREF(exc);
3693 Py_XDECREF(errorHandler);
3694 return res;
Guido van Rossumd57fd912000-03-10 22:53:23 +00003695
Walter Dörwald3aeb6322002-09-02 13:14:32 +00003696 onError:
3697 Py_XDECREF(res);
3698 Py_XDECREF(exc);
3699 Py_XDECREF(errorHandler);
Guido van Rossumd57fd912000-03-10 22:53:23 +00003700 return NULL;
3701}
3702
3703PyObject *PyUnicode_Translate(PyObject *str,
3704 PyObject *mapping,
3705 const char *errors)
3706{
3707 PyObject *result;
Tim Petersced69f82003-09-16 20:30:58 +00003708
Guido van Rossumd57fd912000-03-10 22:53:23 +00003709 str = PyUnicode_FromObject(str);
3710 if (str == NULL)
3711 goto onError;
3712 result = PyUnicode_TranslateCharmap(PyUnicode_AS_UNICODE(str),
3713 PyUnicode_GET_SIZE(str),
3714 mapping,
3715 errors);
3716 Py_DECREF(str);
3717 return result;
Tim Petersced69f82003-09-16 20:30:58 +00003718
Guido van Rossumd57fd912000-03-10 22:53:23 +00003719 onError:
3720 Py_XDECREF(str);
3721 return NULL;
3722}
Tim Petersced69f82003-09-16 20:30:58 +00003723
Guido van Rossum9e896b32000-04-05 20:11:21 +00003724/* --- Decimal Encoder ---------------------------------------------------- */
3725
3726int PyUnicode_EncodeDecimal(Py_UNICODE *s,
Martin v. Löwis18e16552006-02-15 17:27:45 +00003727 Py_ssize_t length,
Guido van Rossum9e896b32000-04-05 20:11:21 +00003728 char *output,
3729 const char *errors)
3730{
3731 Py_UNICODE *p, *end;
Walter Dörwald3aeb6322002-09-02 13:14:32 +00003732 PyObject *errorHandler = NULL;
3733 PyObject *exc = NULL;
3734 const char *encoding = "decimal";
3735 const char *reason = "invalid decimal Unicode string";
3736 /* the following variable is used for caching string comparisons
3737 * -1=not initialized, 0=unknown, 1=strict, 2=replace, 3=ignore, 4=xmlcharrefreplace */
3738 int known_errorHandler = -1;
Guido van Rossum9e896b32000-04-05 20:11:21 +00003739
3740 if (output == NULL) {
3741 PyErr_BadArgument();
3742 return -1;
3743 }
3744
3745 p = s;
3746 end = s + length;
3747 while (p < end) {
Walter Dörwald3aeb6322002-09-02 13:14:32 +00003748 register Py_UNICODE ch = *p;
Guido van Rossum9e896b32000-04-05 20:11:21 +00003749 int decimal;
Walter Dörwald3aeb6322002-09-02 13:14:32 +00003750 PyObject *repunicode;
Martin v. Löwis18e16552006-02-15 17:27:45 +00003751 Py_ssize_t repsize;
3752 Py_ssize_t newpos;
Walter Dörwald3aeb6322002-09-02 13:14:32 +00003753 Py_UNICODE *uni2;
3754 Py_UNICODE *collstart;
3755 Py_UNICODE *collend;
Tim Petersced69f82003-09-16 20:30:58 +00003756
Guido van Rossum9e896b32000-04-05 20:11:21 +00003757 if (Py_UNICODE_ISSPACE(ch)) {
3758 *output++ = ' ';
Walter Dörwald3aeb6322002-09-02 13:14:32 +00003759 ++p;
Guido van Rossum9e896b32000-04-05 20:11:21 +00003760 continue;
3761 }
3762 decimal = Py_UNICODE_TODECIMAL(ch);
3763 if (decimal >= 0) {
3764 *output++ = '0' + decimal;
Walter Dörwald3aeb6322002-09-02 13:14:32 +00003765 ++p;
Guido van Rossum9e896b32000-04-05 20:11:21 +00003766 continue;
3767 }
Guido van Rossumba477042000-04-06 18:18:10 +00003768 if (0 < ch && ch < 256) {
Guido van Rossum42c29aa2000-05-03 23:58:29 +00003769 *output++ = (char)ch;
Walter Dörwald3aeb6322002-09-02 13:14:32 +00003770 ++p;
Guido van Rossum9e896b32000-04-05 20:11:21 +00003771 continue;
3772 }
Walter Dörwald3aeb6322002-09-02 13:14:32 +00003773 /* All other characters are considered unencodable */
3774 collstart = p;
3775 collend = p+1;
3776 while (collend < end) {
3777 if ((0 < *collend && *collend < 256) ||
3778 !Py_UNICODE_ISSPACE(*collend) ||
3779 Py_UNICODE_TODECIMAL(*collend))
3780 break;
Guido van Rossum9e896b32000-04-05 20:11:21 +00003781 }
Walter Dörwald3aeb6322002-09-02 13:14:32 +00003782 /* cache callback name lookup
3783 * (if not done yet, i.e. it's the first error) */
3784 if (known_errorHandler==-1) {
3785 if ((errors==NULL) || (!strcmp(errors, "strict")))
3786 known_errorHandler = 1;
3787 else if (!strcmp(errors, "replace"))
3788 known_errorHandler = 2;
3789 else if (!strcmp(errors, "ignore"))
3790 known_errorHandler = 3;
3791 else if (!strcmp(errors, "xmlcharrefreplace"))
3792 known_errorHandler = 4;
3793 else
3794 known_errorHandler = 0;
3795 }
3796 switch (known_errorHandler) {
3797 case 1: /* strict */
3798 raise_encode_exception(&exc, encoding, s, length, collstart-s, collend-s, reason);
3799 goto onError;
3800 case 2: /* replace */
3801 for (p = collstart; p < collend; ++p)
3802 *output++ = '?';
3803 /* fall through */
3804 case 3: /* ignore */
3805 p = collend;
3806 break;
3807 case 4: /* xmlcharrefreplace */
3808 /* generate replacement (temporarily (mis)uses p) */
3809 for (p = collstart; p < collend; ++p)
3810 output += sprintf(output, "&#%d;", (int)*p);
3811 p = collend;
3812 break;
3813 default:
3814 repunicode = unicode_encode_call_errorhandler(errors, &errorHandler,
3815 encoding, reason, s, length, &exc,
3816 collstart-s, collend-s, &newpos);
3817 if (repunicode == NULL)
3818 goto onError;
3819 /* generate replacement */
3820 repsize = PyUnicode_GET_SIZE(repunicode);
3821 for (uni2 = PyUnicode_AS_UNICODE(repunicode); repsize-->0; ++uni2) {
3822 Py_UNICODE ch = *uni2;
3823 if (Py_UNICODE_ISSPACE(ch))
3824 *output++ = ' ';
3825 else {
3826 decimal = Py_UNICODE_TODECIMAL(ch);
3827 if (decimal >= 0)
3828 *output++ = '0' + decimal;
3829 else if (0 < ch && ch < 256)
3830 *output++ = (char)ch;
3831 else {
3832 Py_DECREF(repunicode);
3833 raise_encode_exception(&exc, encoding,
3834 s, length, collstart-s, collend-s, reason);
3835 goto onError;
3836 }
3837 }
3838 }
3839 p = s + newpos;
3840 Py_DECREF(repunicode);
Guido van Rossum9e896b32000-04-05 20:11:21 +00003841 }
3842 }
3843 /* 0-terminate the output string */
3844 *output++ = '\0';
Walter Dörwald3aeb6322002-09-02 13:14:32 +00003845 Py_XDECREF(exc);
3846 Py_XDECREF(errorHandler);
Guido van Rossum9e896b32000-04-05 20:11:21 +00003847 return 0;
3848
3849 onError:
Walter Dörwald3aeb6322002-09-02 13:14:32 +00003850 Py_XDECREF(exc);
3851 Py_XDECREF(errorHandler);
Guido van Rossum9e896b32000-04-05 20:11:21 +00003852 return -1;
3853}
3854
Guido van Rossumd57fd912000-03-10 22:53:23 +00003855/* --- Helpers ------------------------------------------------------------ */
3856
Fredrik Lundha50d2012006-05-26 17:04:58 +00003857#define STRINGLIB_CHAR Py_UNICODE
Fredrik Lundh6471ee42006-05-24 14:28:11 +00003858
Fredrik Lundhb9479482006-05-26 17:22:38 +00003859#define STRINGLIB_NEW PyUnicode_FromUnicode
3860
Fredrik Lundhb3167cb2006-05-26 18:15:38 +00003861Py_LOCAL(int)
3862STRINGLIB_CMP(const Py_UNICODE* str, const Py_UNICODE* other, Py_ssize_t len)
3863{
Fredrik Lundh9c0e9c02006-05-26 18:24:15 +00003864 if (str[0] != other[0])
3865 return 1;
Fredrik Lundhb3167cb2006-05-26 18:15:38 +00003866 return memcmp((void*) str, (void*) other, len * sizeof(Py_UNICODE));
3867}
3868
Fredrik Lundhb9479482006-05-26 17:22:38 +00003869#define STRINGLIB_EMPTY unicode_empty
3870
Fredrik Lundha50d2012006-05-26 17:04:58 +00003871#include "stringlib/fastsearch.h"
Fredrik Lundh58b5e842006-05-26 19:24:53 +00003872
3873#include "stringlib/count.h"
3874#include "stringlib/find.h"
Fredrik Lundhb9479482006-05-26 17:22:38 +00003875#include "stringlib/partition.h"
3876
Martin v. Löwis18e16552006-02-15 17:27:45 +00003877Py_ssize_t PyUnicode_Count(PyObject *str,
Fredrik Lundh58b5e842006-05-26 19:24:53 +00003878 PyObject *substr,
3879 Py_ssize_t start,
3880 Py_ssize_t end)
Guido van Rossumd57fd912000-03-10 22:53:23 +00003881{
Martin v. Löwis18e16552006-02-15 17:27:45 +00003882 Py_ssize_t result;
Fredrik Lundh58b5e842006-05-26 19:24:53 +00003883 PyUnicodeObject* str_obj;
3884 PyUnicodeObject* sub_obj;
Tim Petersced69f82003-09-16 20:30:58 +00003885
Fredrik Lundh58b5e842006-05-26 19:24:53 +00003886 str_obj = (PyUnicodeObject*) PyUnicode_FromObject(str);
3887 if (!str_obj)
Guido van Rossumd57fd912000-03-10 22:53:23 +00003888 return -1;
Fredrik Lundh58b5e842006-05-26 19:24:53 +00003889 sub_obj = (PyUnicodeObject*) PyUnicode_FromObject(substr);
3890 if (!sub_obj) {
3891 Py_DECREF(str_obj);
Guido van Rossumd57fd912000-03-10 22:53:23 +00003892 return -1;
3893 }
Tim Petersced69f82003-09-16 20:30:58 +00003894
Fredrik Lundh58b5e842006-05-26 19:24:53 +00003895 if (start < 0)
3896 start += str_obj->length;
3897 if (start < 0)
3898 start = 0;
3899 if (end > str_obj->length)
3900 end = str_obj->length;
3901 if (end < 0)
3902 end += str_obj->length;
3903 if (end < 0)
3904 end = 0;
Tim Petersced69f82003-09-16 20:30:58 +00003905
Fredrik Lundh58b5e842006-05-26 19:24:53 +00003906 result = stringlib_count(
3907 str_obj->str + start, end - start, sub_obj->str, sub_obj->length
3908 );
3909
3910 Py_DECREF(sub_obj);
3911 Py_DECREF(str_obj);
3912
Guido van Rossumd57fd912000-03-10 22:53:23 +00003913 return result;
3914}
3915
Fredrik Lundhb63588c2006-05-23 18:44:25 +00003916static Py_ssize_t findstring(PyUnicodeObject *self,
Guido van Rossumd57fd912000-03-10 22:53:23 +00003917 PyUnicodeObject *substring,
Martin v. Löwis18e16552006-02-15 17:27:45 +00003918 Py_ssize_t start,
3919 Py_ssize_t end,
Guido van Rossumd57fd912000-03-10 22:53:23 +00003920 int direction)
3921{
3922 if (start < 0)
3923 start += self->length;
3924 if (start < 0)
3925 start = 0;
3926
Guido van Rossumd57fd912000-03-10 22:53:23 +00003927 if (end > self->length)
3928 end = self->length;
3929 if (end < 0)
3930 end += self->length;
3931 if (end < 0)
3932 end = 0;
3933
Guido van Rossum76afbd92002-08-20 17:29:29 +00003934 if (substring->length == 0)
3935 return (direction > 0) ? start : end;
3936
Fredrik Lundh6471ee42006-05-24 14:28:11 +00003937 if (direction > 0) {
3938 Py_ssize_t pos = fastsearch(
3939 PyUnicode_AS_UNICODE(self) + start, end - start,
3940 substring->str, substring->length, FAST_SEARCH
3941 );
Fredrik Lundh06a69dd2006-05-26 08:54:28 +00003942 if (pos >= 0)
3943 return pos + start;
3944 } else {
3945 end -= substring->length;
Guido van Rossumd57fd912000-03-10 22:53:23 +00003946 for (; end >= start; end--)
3947 if (Py_UNICODE_MATCH(self, end, substring))
3948 return end;
Guido van Rossumd57fd912000-03-10 22:53:23 +00003949 }
Guido van Rossumd57fd912000-03-10 22:53:23 +00003950 return -1;
3951}
3952
Martin v. Löwis18e16552006-02-15 17:27:45 +00003953Py_ssize_t PyUnicode_Find(PyObject *str,
Fredrik Lundhce4eccb2006-05-26 19:29:05 +00003954 PyObject *substr,
3955 Py_ssize_t start,
3956 Py_ssize_t end,
3957 int direction)
Guido van Rossumd57fd912000-03-10 22:53:23 +00003958{
Martin v. Löwis18e16552006-02-15 17:27:45 +00003959 Py_ssize_t result;
Fredrik Lundhce4eccb2006-05-26 19:29:05 +00003960 PyUnicodeObject* str_obj;
3961 PyUnicodeObject* sub_obj;
Tim Petersced69f82003-09-16 20:30:58 +00003962
Fredrik Lundhce4eccb2006-05-26 19:29:05 +00003963 str_obj = (PyUnicodeObject*) PyUnicode_FromObject(str);
3964 if (!str)
Marc-André Lemburg4da6fd62002-05-29 11:33:13 +00003965 return -2;
Fredrik Lundhce4eccb2006-05-26 19:29:05 +00003966 sub_obj = (PyUnicodeObject*) PyUnicode_FromObject(substr);
3967 if (!sub_obj) {
3968 Py_DECREF(str_obj);
Marc-André Lemburg4da6fd62002-05-29 11:33:13 +00003969 return -2;
Guido van Rossumd57fd912000-03-10 22:53:23 +00003970 }
Tim Petersced69f82003-09-16 20:30:58 +00003971
Fredrik Lundhce4eccb2006-05-26 19:29:05 +00003972 if (start < 0)
3973 start += str_obj->length;
3974 if (start < 0)
3975 start = 0;
3976 if (end > str_obj->length)
3977 end = str_obj->length;
3978 if (end < 0)
3979 end += str_obj->length;
3980 if (end < 0)
3981 end = 0;
3982
3983 if (direction > 0)
3984 result = stringlib_find(
3985 str_obj->str + start, end - start, sub_obj->str, sub_obj->length
3986 );
3987 else
3988 result = stringlib_rfind(
3989 str_obj->str + start, end - start, sub_obj->str, sub_obj->length
3990 );
3991
3992 if (result >= 0)
3993 result += start;
3994
3995 Py_DECREF(str_obj);
3996 Py_DECREF(sub_obj);
Guido van Rossumd57fd912000-03-10 22:53:23 +00003997 return result;
3998}
3999
Tim Petersced69f82003-09-16 20:30:58 +00004000static
Guido van Rossumd57fd912000-03-10 22:53:23 +00004001int tailmatch(PyUnicodeObject *self,
4002 PyUnicodeObject *substring,
Martin v. Löwis18e16552006-02-15 17:27:45 +00004003 Py_ssize_t start,
4004 Py_ssize_t end,
Guido van Rossumd57fd912000-03-10 22:53:23 +00004005 int direction)
4006{
4007 if (start < 0)
4008 start += self->length;
4009 if (start < 0)
4010 start = 0;
4011
4012 if (substring->length == 0)
4013 return 1;
4014
4015 if (end > self->length)
4016 end = self->length;
4017 if (end < 0)
4018 end += self->length;
4019 if (end < 0)
4020 end = 0;
4021
4022 end -= substring->length;
4023 if (end < start)
4024 return 0;
4025
4026 if (direction > 0) {
4027 if (Py_UNICODE_MATCH(self, end, substring))
4028 return 1;
4029 } else {
4030 if (Py_UNICODE_MATCH(self, start, substring))
4031 return 1;
4032 }
4033
4034 return 0;
4035}
4036
Martin v. Löwis18e16552006-02-15 17:27:45 +00004037Py_ssize_t PyUnicode_Tailmatch(PyObject *str,
Guido van Rossumd57fd912000-03-10 22:53:23 +00004038 PyObject *substr,
Martin v. Löwis18e16552006-02-15 17:27:45 +00004039 Py_ssize_t start,
4040 Py_ssize_t end,
Guido van Rossumd57fd912000-03-10 22:53:23 +00004041 int direction)
4042{
Martin v. Löwis18e16552006-02-15 17:27:45 +00004043 Py_ssize_t result;
Tim Petersced69f82003-09-16 20:30:58 +00004044
Guido van Rossumd57fd912000-03-10 22:53:23 +00004045 str = PyUnicode_FromObject(str);
4046 if (str == NULL)
4047 return -1;
4048 substr = PyUnicode_FromObject(substr);
4049 if (substr == NULL) {
Hye-Shik Chang4af5c8c2006-03-07 15:39:21 +00004050 Py_DECREF(str);
Guido van Rossumd57fd912000-03-10 22:53:23 +00004051 return -1;
4052 }
Tim Petersced69f82003-09-16 20:30:58 +00004053
Guido van Rossumd57fd912000-03-10 22:53:23 +00004054 result = tailmatch((PyUnicodeObject *)str,
4055 (PyUnicodeObject *)substr,
4056 start, end, direction);
4057 Py_DECREF(str);
4058 Py_DECREF(substr);
4059 return result;
4060}
4061
Guido van Rossumd57fd912000-03-10 22:53:23 +00004062/* Apply fixfct filter to the Unicode object self and return a
4063 reference to the modified object */
4064
Tim Petersced69f82003-09-16 20:30:58 +00004065static
Guido van Rossumd57fd912000-03-10 22:53:23 +00004066PyObject *fixup(PyUnicodeObject *self,
4067 int (*fixfct)(PyUnicodeObject *s))
4068{
4069
4070 PyUnicodeObject *u;
4071
Marc-André Lemburg8155e0e2001-04-23 14:44:21 +00004072 u = (PyUnicodeObject*) PyUnicode_FromUnicode(NULL, self->length);
Guido van Rossumd57fd912000-03-10 22:53:23 +00004073 if (u == NULL)
4074 return NULL;
Marc-André Lemburg8155e0e2001-04-23 14:44:21 +00004075
4076 Py_UNICODE_COPY(u->str, self->str, self->length);
4077
Tim Peters7a29bd52001-09-12 03:03:31 +00004078 if (!fixfct(u) && PyUnicode_CheckExact(self)) {
Guido van Rossumd57fd912000-03-10 22:53:23 +00004079 /* fixfct should return TRUE if it modified the buffer. If
4080 FALSE, return a reference to the original buffer instead
4081 (to save space, not time) */
4082 Py_INCREF(self);
4083 Py_DECREF(u);
4084 return (PyObject*) self;
4085 }
4086 return (PyObject*) u;
4087}
4088
Tim Petersced69f82003-09-16 20:30:58 +00004089static
Guido van Rossumd57fd912000-03-10 22:53:23 +00004090int fixupper(PyUnicodeObject *self)
4091{
Martin v. Löwis18e16552006-02-15 17:27:45 +00004092 Py_ssize_t len = self->length;
Guido van Rossumd57fd912000-03-10 22:53:23 +00004093 Py_UNICODE *s = self->str;
4094 int status = 0;
Tim Petersced69f82003-09-16 20:30:58 +00004095
Guido van Rossumd57fd912000-03-10 22:53:23 +00004096 while (len-- > 0) {
4097 register Py_UNICODE ch;
Tim Petersced69f82003-09-16 20:30:58 +00004098
Guido van Rossumd57fd912000-03-10 22:53:23 +00004099 ch = Py_UNICODE_TOUPPER(*s);
4100 if (ch != *s) {
4101 status = 1;
4102 *s = ch;
4103 }
4104 s++;
4105 }
4106
4107 return status;
4108}
4109
Tim Petersced69f82003-09-16 20:30:58 +00004110static
Guido van Rossumd57fd912000-03-10 22:53:23 +00004111int fixlower(PyUnicodeObject *self)
4112{
Martin v. Löwis18e16552006-02-15 17:27:45 +00004113 Py_ssize_t len = self->length;
Guido van Rossumd57fd912000-03-10 22:53:23 +00004114 Py_UNICODE *s = self->str;
4115 int status = 0;
Tim Petersced69f82003-09-16 20:30:58 +00004116
Guido van Rossumd57fd912000-03-10 22:53:23 +00004117 while (len-- > 0) {
4118 register Py_UNICODE ch;
Tim Petersced69f82003-09-16 20:30:58 +00004119
Guido van Rossumd57fd912000-03-10 22:53:23 +00004120 ch = Py_UNICODE_TOLOWER(*s);
4121 if (ch != *s) {
4122 status = 1;
4123 *s = ch;
4124 }
4125 s++;
4126 }
4127
4128 return status;
4129}
4130
Tim Petersced69f82003-09-16 20:30:58 +00004131static
Guido van Rossumd57fd912000-03-10 22:53:23 +00004132int fixswapcase(PyUnicodeObject *self)
4133{
Martin v. Löwis18e16552006-02-15 17:27:45 +00004134 Py_ssize_t len = self->length;
Guido van Rossumd57fd912000-03-10 22:53:23 +00004135 Py_UNICODE *s = self->str;
4136 int status = 0;
Tim Petersced69f82003-09-16 20:30:58 +00004137
Guido van Rossumd57fd912000-03-10 22:53:23 +00004138 while (len-- > 0) {
4139 if (Py_UNICODE_ISUPPER(*s)) {
4140 *s = Py_UNICODE_TOLOWER(*s);
4141 status = 1;
4142 } else if (Py_UNICODE_ISLOWER(*s)) {
4143 *s = Py_UNICODE_TOUPPER(*s);
4144 status = 1;
4145 }
4146 s++;
4147 }
4148
4149 return status;
4150}
4151
Tim Petersced69f82003-09-16 20:30:58 +00004152static
Guido van Rossumd57fd912000-03-10 22:53:23 +00004153int fixcapitalize(PyUnicodeObject *self)
4154{
Martin v. Löwis18e16552006-02-15 17:27:45 +00004155 Py_ssize_t len = self->length;
Marc-André Lemburgfde66e12001-01-29 11:14:16 +00004156 Py_UNICODE *s = self->str;
4157 int status = 0;
Tim Petersced69f82003-09-16 20:30:58 +00004158
Marc-André Lemburgfde66e12001-01-29 11:14:16 +00004159 if (len == 0)
4160 return 0;
4161 if (Py_UNICODE_ISLOWER(*s)) {
4162 *s = Py_UNICODE_TOUPPER(*s);
4163 status = 1;
Guido van Rossumd57fd912000-03-10 22:53:23 +00004164 }
Marc-André Lemburgfde66e12001-01-29 11:14:16 +00004165 s++;
4166 while (--len > 0) {
4167 if (Py_UNICODE_ISUPPER(*s)) {
4168 *s = Py_UNICODE_TOLOWER(*s);
4169 status = 1;
4170 }
4171 s++;
4172 }
4173 return status;
Guido van Rossumd57fd912000-03-10 22:53:23 +00004174}
4175
4176static
4177int fixtitle(PyUnicodeObject *self)
4178{
4179 register Py_UNICODE *p = PyUnicode_AS_UNICODE(self);
4180 register Py_UNICODE *e;
4181 int previous_is_cased;
4182
4183 /* Shortcut for single character strings */
4184 if (PyUnicode_GET_SIZE(self) == 1) {
4185 Py_UNICODE ch = Py_UNICODE_TOTITLE(*p);
4186 if (*p != ch) {
4187 *p = ch;
4188 return 1;
4189 }
4190 else
4191 return 0;
4192 }
Tim Petersced69f82003-09-16 20:30:58 +00004193
Guido van Rossumd57fd912000-03-10 22:53:23 +00004194 e = p + PyUnicode_GET_SIZE(self);
4195 previous_is_cased = 0;
4196 for (; p < e; p++) {
4197 register const Py_UNICODE ch = *p;
Tim Petersced69f82003-09-16 20:30:58 +00004198
Guido van Rossumd57fd912000-03-10 22:53:23 +00004199 if (previous_is_cased)
4200 *p = Py_UNICODE_TOLOWER(ch);
4201 else
4202 *p = Py_UNICODE_TOTITLE(ch);
Tim Petersced69f82003-09-16 20:30:58 +00004203
4204 if (Py_UNICODE_ISLOWER(ch) ||
4205 Py_UNICODE_ISUPPER(ch) ||
Guido van Rossumd57fd912000-03-10 22:53:23 +00004206 Py_UNICODE_ISTITLE(ch))
4207 previous_is_cased = 1;
4208 else
4209 previous_is_cased = 0;
4210 }
4211 return 1;
4212}
4213
Tim Peters8ce9f162004-08-27 01:49:32 +00004214PyObject *
4215PyUnicode_Join(PyObject *separator, PyObject *seq)
Guido van Rossumd57fd912000-03-10 22:53:23 +00004216{
Tim Peters8ce9f162004-08-27 01:49:32 +00004217 PyObject *internal_separator = NULL;
Skip Montanaro6543b452004-09-16 03:28:13 +00004218 const Py_UNICODE blank = ' ';
4219 const Py_UNICODE *sep = &blank;
Tim Peters286085c2006-05-22 19:17:04 +00004220 Py_ssize_t seplen = 1;
Tim Peters05eba1f2004-08-27 21:32:02 +00004221 PyUnicodeObject *res = NULL; /* the result */
Tim Peters286085c2006-05-22 19:17:04 +00004222 Py_ssize_t res_alloc = 100; /* # allocated bytes for string in res */
4223 Py_ssize_t res_used; /* # used bytes */
Tim Peters05eba1f2004-08-27 21:32:02 +00004224 Py_UNICODE *res_p; /* pointer to free byte in res's string area */
4225 PyObject *fseq; /* PySequence_Fast(seq) */
Martin v. Löwis18e16552006-02-15 17:27:45 +00004226 Py_ssize_t seqlen; /* len(fseq) -- number of items in sequence */
Tim Peters8ce9f162004-08-27 01:49:32 +00004227 PyObject *item;
Martin v. Löwis412fb672006-04-13 06:34:32 +00004228 Py_ssize_t i;
Guido van Rossumd57fd912000-03-10 22:53:23 +00004229
Tim Peters05eba1f2004-08-27 21:32:02 +00004230 fseq = PySequence_Fast(seq, "");
4231 if (fseq == NULL) {
Tim Peters05eba1f2004-08-27 21:32:02 +00004232 return NULL;
Tim Peters8ce9f162004-08-27 01:49:32 +00004233 }
4234
Tim Peters91879ab2004-08-27 22:35:44 +00004235 /* Grrrr. A codec may be invoked to convert str objects to
4236 * Unicode, and so it's possible to call back into Python code
4237 * during PyUnicode_FromObject(), and so it's possible for a sick
4238 * codec to change the size of fseq (if seq is a list). Therefore
4239 * we have to keep refetching the size -- can't assume seqlen
4240 * is invariant.
4241 */
Tim Peters05eba1f2004-08-27 21:32:02 +00004242 seqlen = PySequence_Fast_GET_SIZE(fseq);
4243 /* If empty sequence, return u"". */
4244 if (seqlen == 0) {
4245 res = _PyUnicode_New(0); /* empty sequence; return u"" */
4246 goto Done;
4247 }
4248 /* If singleton sequence with an exact Unicode, return that. */
4249 if (seqlen == 1) {
4250 item = PySequence_Fast_GET_ITEM(fseq, 0);
4251 if (PyUnicode_CheckExact(item)) {
4252 Py_INCREF(item);
4253 res = (PyUnicodeObject *)item;
4254 goto Done;
4255 }
Tim Peters8ce9f162004-08-27 01:49:32 +00004256 }
4257
Tim Peters05eba1f2004-08-27 21:32:02 +00004258 /* At least two items to join, or one that isn't exact Unicode. */
4259 if (seqlen > 1) {
4260 /* Set up sep and seplen -- they're needed. */
4261 if (separator == NULL) {
4262 sep = &blank;
4263 seplen = 1;
4264 }
4265 else {
4266 internal_separator = PyUnicode_FromObject(separator);
4267 if (internal_separator == NULL)
4268 goto onError;
4269 sep = PyUnicode_AS_UNICODE(internal_separator);
4270 seplen = PyUnicode_GET_SIZE(internal_separator);
Tim Peters91879ab2004-08-27 22:35:44 +00004271 /* In case PyUnicode_FromObject() mutated seq. */
4272 seqlen = PySequence_Fast_GET_SIZE(fseq);
Tim Peters05eba1f2004-08-27 21:32:02 +00004273 }
4274 }
4275
4276 /* Get space. */
Martin v. Löwis412fb672006-04-13 06:34:32 +00004277 res = _PyUnicode_New(res_alloc);
Tim Peters05eba1f2004-08-27 21:32:02 +00004278 if (res == NULL)
Tim Peters8ce9f162004-08-27 01:49:32 +00004279 goto onError;
Tim Peters05eba1f2004-08-27 21:32:02 +00004280 res_p = PyUnicode_AS_UNICODE(res);
4281 res_used = 0;
Tim Peters8ce9f162004-08-27 01:49:32 +00004282
Tim Peters05eba1f2004-08-27 21:32:02 +00004283 for (i = 0; i < seqlen; ++i) {
Tim Peters286085c2006-05-22 19:17:04 +00004284 Py_ssize_t itemlen;
4285 Py_ssize_t new_res_used;
Tim Peters05eba1f2004-08-27 21:32:02 +00004286
4287 item = PySequence_Fast_GET_ITEM(fseq, i);
4288 /* Convert item to Unicode. */
4289 if (! PyUnicode_Check(item) && ! PyString_Check(item)) {
4290 PyErr_Format(PyExc_TypeError,
Thomas Wouters715a4cd2006-04-16 22:04:49 +00004291 "sequence item %zd: expected string or Unicode,"
Tim Peters05eba1f2004-08-27 21:32:02 +00004292 " %.80s found",
4293 i, item->ob_type->tp_name);
Tim Peters2cfe3682001-05-05 05:36:48 +00004294 goto onError;
Tim Peters8ce9f162004-08-27 01:49:32 +00004295 }
Tim Peters05eba1f2004-08-27 21:32:02 +00004296 item = PyUnicode_FromObject(item);
4297 if (item == NULL)
4298 goto onError;
4299 /* We own a reference to item from here on. */
Tim Petersced69f82003-09-16 20:30:58 +00004300
Tim Peters91879ab2004-08-27 22:35:44 +00004301 /* In case PyUnicode_FromObject() mutated seq. */
4302 seqlen = PySequence_Fast_GET_SIZE(fseq);
4303
Tim Peters8ce9f162004-08-27 01:49:32 +00004304 /* Make sure we have enough space for the separator and the item. */
Guido van Rossumd57fd912000-03-10 22:53:23 +00004305 itemlen = PyUnicode_GET_SIZE(item);
Tim Peters05eba1f2004-08-27 21:32:02 +00004306 new_res_used = res_used + itemlen;
Tim Peters286085c2006-05-22 19:17:04 +00004307 if (new_res_used <= 0)
Tim Peters8ce9f162004-08-27 01:49:32 +00004308 goto Overflow;
Tim Peters05eba1f2004-08-27 21:32:02 +00004309 if (i < seqlen - 1) {
4310 new_res_used += seplen;
Tim Peters286085c2006-05-22 19:17:04 +00004311 if (new_res_used <= 0)
Tim Peters05eba1f2004-08-27 21:32:02 +00004312 goto Overflow;
4313 }
4314 if (new_res_used > res_alloc) {
4315 /* double allocated size until it's big enough */
Tim Peters8ce9f162004-08-27 01:49:32 +00004316 do {
Tim Peters05eba1f2004-08-27 21:32:02 +00004317 res_alloc += res_alloc;
Tim Peters286085c2006-05-22 19:17:04 +00004318 if (res_alloc <= 0)
Tim Peters8ce9f162004-08-27 01:49:32 +00004319 goto Overflow;
Tim Peters05eba1f2004-08-27 21:32:02 +00004320 } while (new_res_used > res_alloc);
Martin v. Löwis412fb672006-04-13 06:34:32 +00004321 if (_PyUnicode_Resize(&res, res_alloc) < 0) {
Marc-André Lemburg3508e302001-09-20 17:22:58 +00004322 Py_DECREF(item);
Guido van Rossumd57fd912000-03-10 22:53:23 +00004323 goto onError;
Marc-André Lemburg3508e302001-09-20 17:22:58 +00004324 }
Tim Peters05eba1f2004-08-27 21:32:02 +00004325 res_p = PyUnicode_AS_UNICODE(res) + res_used;
Guido van Rossumd57fd912000-03-10 22:53:23 +00004326 }
Tim Peters05eba1f2004-08-27 21:32:02 +00004327
4328 /* Copy item, and maybe the separator. */
Martin v. Löwis412fb672006-04-13 06:34:32 +00004329 Py_UNICODE_COPY(res_p, PyUnicode_AS_UNICODE(item), itemlen);
Tim Peters05eba1f2004-08-27 21:32:02 +00004330 res_p += itemlen;
4331 if (i < seqlen - 1) {
Martin v. Löwis412fb672006-04-13 06:34:32 +00004332 Py_UNICODE_COPY(res_p, sep, seplen);
Tim Peters05eba1f2004-08-27 21:32:02 +00004333 res_p += seplen;
4334 }
Guido van Rossumd57fd912000-03-10 22:53:23 +00004335 Py_DECREF(item);
Tim Peters05eba1f2004-08-27 21:32:02 +00004336 res_used = new_res_used;
4337 }
Tim Peters8ce9f162004-08-27 01:49:32 +00004338
Tim Peters05eba1f2004-08-27 21:32:02 +00004339 /* Shrink res to match the used area; this probably can't fail,
4340 * but it's cheap to check.
4341 */
Martin v. Löwis412fb672006-04-13 06:34:32 +00004342 if (_PyUnicode_Resize(&res, res_used) < 0)
Tim Peters8ce9f162004-08-27 01:49:32 +00004343 goto onError;
4344
4345 Done:
4346 Py_XDECREF(internal_separator);
Tim Peters05eba1f2004-08-27 21:32:02 +00004347 Py_DECREF(fseq);
Guido van Rossumd57fd912000-03-10 22:53:23 +00004348 return (PyObject *)res;
4349
Tim Peters8ce9f162004-08-27 01:49:32 +00004350 Overflow:
4351 PyErr_SetString(PyExc_OverflowError,
4352 "join() is too long for a Python string");
4353 Py_DECREF(item);
4354 /* fall through */
4355
Guido van Rossumd57fd912000-03-10 22:53:23 +00004356 onError:
Tim Peters8ce9f162004-08-27 01:49:32 +00004357 Py_XDECREF(internal_separator);
Tim Peters05eba1f2004-08-27 21:32:02 +00004358 Py_DECREF(fseq);
Tim Peters8ce9f162004-08-27 01:49:32 +00004359 Py_XDECREF(res);
Guido van Rossumd57fd912000-03-10 22:53:23 +00004360 return NULL;
4361}
4362
Tim Petersced69f82003-09-16 20:30:58 +00004363static
4364PyUnicodeObject *pad(PyUnicodeObject *self,
Martin v. Löwis18e16552006-02-15 17:27:45 +00004365 Py_ssize_t left,
4366 Py_ssize_t right,
Guido van Rossumd57fd912000-03-10 22:53:23 +00004367 Py_UNICODE fill)
4368{
4369 PyUnicodeObject *u;
4370
4371 if (left < 0)
4372 left = 0;
4373 if (right < 0)
4374 right = 0;
4375
Tim Peters7a29bd52001-09-12 03:03:31 +00004376 if (left == 0 && right == 0 && PyUnicode_CheckExact(self)) {
Guido van Rossumd57fd912000-03-10 22:53:23 +00004377 Py_INCREF(self);
4378 return self;
4379 }
4380
4381 u = _PyUnicode_New(left + self->length + right);
4382 if (u) {
4383 if (left)
4384 Py_UNICODE_FILL(u->str, fill, left);
4385 Py_UNICODE_COPY(u->str + left, self->str, self->length);
4386 if (right)
4387 Py_UNICODE_FILL(u->str + left + self->length, fill, right);
4388 }
4389
4390 return u;
4391}
4392
4393#define SPLIT_APPEND(data, left, right) \
Hye-Shik Chang3ae811b2003-12-15 18:49:53 +00004394 str = PyUnicode_FromUnicode((data) + (left), (right) - (left)); \
Guido van Rossumd57fd912000-03-10 22:53:23 +00004395 if (!str) \
4396 goto onError; \
4397 if (PyList_Append(list, str)) { \
4398 Py_DECREF(str); \
4399 goto onError; \
4400 } \
4401 else \
4402 Py_DECREF(str);
4403
4404static
4405PyObject *split_whitespace(PyUnicodeObject *self,
4406 PyObject *list,
Martin v. Löwis18e16552006-02-15 17:27:45 +00004407 Py_ssize_t maxcount)
Guido van Rossumd57fd912000-03-10 22:53:23 +00004408{
Martin v. Löwis18e16552006-02-15 17:27:45 +00004409 register Py_ssize_t i;
4410 register Py_ssize_t j;
4411 Py_ssize_t len = self->length;
Guido van Rossumd57fd912000-03-10 22:53:23 +00004412 PyObject *str;
4413
4414 for (i = j = 0; i < len; ) {
4415 /* find a token */
4416 while (i < len && Py_UNICODE_ISSPACE(self->str[i]))
4417 i++;
4418 j = i;
4419 while (i < len && !Py_UNICODE_ISSPACE(self->str[i]))
4420 i++;
4421 if (j < i) {
4422 if (maxcount-- <= 0)
4423 break;
4424 SPLIT_APPEND(self->str, j, i);
4425 while (i < len && Py_UNICODE_ISSPACE(self->str[i]))
4426 i++;
4427 j = i;
4428 }
4429 }
4430 if (j < len) {
4431 SPLIT_APPEND(self->str, j, len);
4432 }
4433 return list;
4434
4435 onError:
4436 Py_DECREF(list);
4437 return NULL;
4438}
4439
4440PyObject *PyUnicode_Splitlines(PyObject *string,
Guido van Rossum86662912000-04-11 15:38:46 +00004441 int keepends)
Guido van Rossumd57fd912000-03-10 22:53:23 +00004442{
Martin v. Löwis18e16552006-02-15 17:27:45 +00004443 register Py_ssize_t i;
4444 register Py_ssize_t j;
4445 Py_ssize_t len;
Guido van Rossumd57fd912000-03-10 22:53:23 +00004446 PyObject *list;
4447 PyObject *str;
4448 Py_UNICODE *data;
4449
4450 string = PyUnicode_FromObject(string);
4451 if (string == NULL)
4452 return NULL;
4453 data = PyUnicode_AS_UNICODE(string);
4454 len = PyUnicode_GET_SIZE(string);
4455
Guido van Rossumd57fd912000-03-10 22:53:23 +00004456 list = PyList_New(0);
4457 if (!list)
4458 goto onError;
4459
4460 for (i = j = 0; i < len; ) {
Martin v. Löwis18e16552006-02-15 17:27:45 +00004461 Py_ssize_t eol;
Tim Petersced69f82003-09-16 20:30:58 +00004462
Guido van Rossumd57fd912000-03-10 22:53:23 +00004463 /* Find a line and append it */
Fredrik Lundhb63588c2006-05-23 18:44:25 +00004464 while (i < len && !BLOOM_LINEBREAK(data[i]))
Guido van Rossumd57fd912000-03-10 22:53:23 +00004465 i++;
Guido van Rossumd57fd912000-03-10 22:53:23 +00004466
4467 /* Skip the line break reading CRLF as one line break */
Guido van Rossum86662912000-04-11 15:38:46 +00004468 eol = i;
Guido van Rossumd57fd912000-03-10 22:53:23 +00004469 if (i < len) {
4470 if (data[i] == '\r' && i + 1 < len &&
4471 data[i+1] == '\n')
4472 i += 2;
4473 else
4474 i++;
Guido van Rossum86662912000-04-11 15:38:46 +00004475 if (keepends)
4476 eol = i;
Guido van Rossumd57fd912000-03-10 22:53:23 +00004477 }
Guido van Rossum86662912000-04-11 15:38:46 +00004478 SPLIT_APPEND(data, j, eol);
Guido van Rossumd57fd912000-03-10 22:53:23 +00004479 j = i;
4480 }
4481 if (j < len) {
4482 SPLIT_APPEND(data, j, len);
4483 }
4484
4485 Py_DECREF(string);
4486 return list;
4487
4488 onError:
Hye-Shik Chang4af5c8c2006-03-07 15:39:21 +00004489 Py_XDECREF(list);
Guido van Rossumd57fd912000-03-10 22:53:23 +00004490 Py_DECREF(string);
4491 return NULL;
4492}
4493
Tim Petersced69f82003-09-16 20:30:58 +00004494static
Guido van Rossumd57fd912000-03-10 22:53:23 +00004495PyObject *split_char(PyUnicodeObject *self,
4496 PyObject *list,
4497 Py_UNICODE ch,
Martin v. Löwis18e16552006-02-15 17:27:45 +00004498 Py_ssize_t maxcount)
Guido van Rossumd57fd912000-03-10 22:53:23 +00004499{
Martin v. Löwis18e16552006-02-15 17:27:45 +00004500 register Py_ssize_t i;
4501 register Py_ssize_t j;
4502 Py_ssize_t len = self->length;
Guido van Rossumd57fd912000-03-10 22:53:23 +00004503 PyObject *str;
4504
4505 for (i = j = 0; i < len; ) {
4506 if (self->str[i] == ch) {
4507 if (maxcount-- <= 0)
4508 break;
4509 SPLIT_APPEND(self->str, j, i);
4510 i = j = i + 1;
4511 } else
4512 i++;
4513 }
4514 if (j <= len) {
4515 SPLIT_APPEND(self->str, j, len);
4516 }
4517 return list;
4518
4519 onError:
4520 Py_DECREF(list);
4521 return NULL;
4522}
4523
Tim Petersced69f82003-09-16 20:30:58 +00004524static
Guido van Rossumd57fd912000-03-10 22:53:23 +00004525PyObject *split_substring(PyUnicodeObject *self,
4526 PyObject *list,
4527 PyUnicodeObject *substring,
Martin v. Löwis18e16552006-02-15 17:27:45 +00004528 Py_ssize_t maxcount)
Guido van Rossumd57fd912000-03-10 22:53:23 +00004529{
Martin v. Löwis18e16552006-02-15 17:27:45 +00004530 register Py_ssize_t i;
4531 register Py_ssize_t j;
4532 Py_ssize_t len = self->length;
4533 Py_ssize_t sublen = substring->length;
Guido van Rossumd57fd912000-03-10 22:53:23 +00004534 PyObject *str;
4535
Guido van Rossumcda4f9a2000-12-19 02:23:19 +00004536 for (i = j = 0; i <= len - sublen; ) {
Guido van Rossumd57fd912000-03-10 22:53:23 +00004537 if (Py_UNICODE_MATCH(self, i, substring)) {
4538 if (maxcount-- <= 0)
4539 break;
4540 SPLIT_APPEND(self->str, j, i);
4541 i = j = i + sublen;
4542 } else
4543 i++;
4544 }
4545 if (j <= len) {
4546 SPLIT_APPEND(self->str, j, len);
4547 }
4548 return list;
4549
4550 onError:
4551 Py_DECREF(list);
4552 return NULL;
4553}
4554
Hye-Shik Chang3ae811b2003-12-15 18:49:53 +00004555static
4556PyObject *rsplit_whitespace(PyUnicodeObject *self,
4557 PyObject *list,
Martin v. Löwis18e16552006-02-15 17:27:45 +00004558 Py_ssize_t maxcount)
Hye-Shik Chang3ae811b2003-12-15 18:49:53 +00004559{
Martin v. Löwis18e16552006-02-15 17:27:45 +00004560 register Py_ssize_t i;
4561 register Py_ssize_t j;
4562 Py_ssize_t len = self->length;
Hye-Shik Chang3ae811b2003-12-15 18:49:53 +00004563 PyObject *str;
4564
4565 for (i = j = len - 1; i >= 0; ) {
4566 /* find a token */
4567 while (i >= 0 && Py_UNICODE_ISSPACE(self->str[i]))
4568 i--;
4569 j = i;
4570 while (i >= 0 && !Py_UNICODE_ISSPACE(self->str[i]))
4571 i--;
4572 if (j > i) {
4573 if (maxcount-- <= 0)
4574 break;
Fredrik Lundhb63588c2006-05-23 18:44:25 +00004575 SPLIT_APPEND(self->str, i + 1, j + 1);
Hye-Shik Chang3ae811b2003-12-15 18:49:53 +00004576 while (i >= 0 && Py_UNICODE_ISSPACE(self->str[i]))
4577 i--;
4578 j = i;
4579 }
4580 }
4581 if (j >= 0) {
Fredrik Lundhb63588c2006-05-23 18:44:25 +00004582 SPLIT_APPEND(self->str, 0, j + 1);
Hye-Shik Chang3ae811b2003-12-15 18:49:53 +00004583 }
Fredrik Lundhb63588c2006-05-23 18:44:25 +00004584 if (PyList_Reverse(list) < 0)
4585 goto onError;
Hye-Shik Chang3ae811b2003-12-15 18:49:53 +00004586 return list;
4587
4588 onError:
4589 Py_DECREF(list);
4590 return NULL;
4591}
4592
4593static
4594PyObject *rsplit_char(PyUnicodeObject *self,
4595 PyObject *list,
4596 Py_UNICODE ch,
Martin v. Löwis18e16552006-02-15 17:27:45 +00004597 Py_ssize_t maxcount)
Hye-Shik Chang3ae811b2003-12-15 18:49:53 +00004598{
Martin v. Löwis18e16552006-02-15 17:27:45 +00004599 register Py_ssize_t i;
4600 register Py_ssize_t j;
4601 Py_ssize_t len = self->length;
Hye-Shik Chang3ae811b2003-12-15 18:49:53 +00004602 PyObject *str;
4603
4604 for (i = j = len - 1; i >= 0; ) {
4605 if (self->str[i] == ch) {
4606 if (maxcount-- <= 0)
4607 break;
Fredrik Lundhb63588c2006-05-23 18:44:25 +00004608 SPLIT_APPEND(self->str, i + 1, j + 1);
Hye-Shik Chang3ae811b2003-12-15 18:49:53 +00004609 j = i = i - 1;
4610 } else
4611 i--;
4612 }
Hye-Shik Chang7fc4cf52003-12-23 09:10:16 +00004613 if (j >= -1) {
Fredrik Lundhb63588c2006-05-23 18:44:25 +00004614 SPLIT_APPEND(self->str, 0, j + 1);
Hye-Shik Chang3ae811b2003-12-15 18:49:53 +00004615 }
Fredrik Lundhb63588c2006-05-23 18:44:25 +00004616 if (PyList_Reverse(list) < 0)
4617 goto onError;
Hye-Shik Chang3ae811b2003-12-15 18:49:53 +00004618 return list;
4619
4620 onError:
4621 Py_DECREF(list);
4622 return NULL;
4623}
4624
4625static
4626PyObject *rsplit_substring(PyUnicodeObject *self,
4627 PyObject *list,
4628 PyUnicodeObject *substring,
Martin v. Löwis18e16552006-02-15 17:27:45 +00004629 Py_ssize_t maxcount)
Hye-Shik Chang3ae811b2003-12-15 18:49:53 +00004630{
Martin v. Löwis18e16552006-02-15 17:27:45 +00004631 register Py_ssize_t i;
4632 register Py_ssize_t j;
4633 Py_ssize_t len = self->length;
4634 Py_ssize_t sublen = substring->length;
Hye-Shik Chang3ae811b2003-12-15 18:49:53 +00004635 PyObject *str;
4636
4637 for (i = len - sublen, j = len; i >= 0; ) {
4638 if (Py_UNICODE_MATCH(self, i, substring)) {
4639 if (maxcount-- <= 0)
4640 break;
Fredrik Lundhb63588c2006-05-23 18:44:25 +00004641 SPLIT_APPEND(self->str, i + sublen, j);
Hye-Shik Chang3ae811b2003-12-15 18:49:53 +00004642 j = i;
4643 i -= sublen;
4644 } else
4645 i--;
4646 }
4647 if (j >= 0) {
Fredrik Lundhb63588c2006-05-23 18:44:25 +00004648 SPLIT_APPEND(self->str, 0, j);
Hye-Shik Chang3ae811b2003-12-15 18:49:53 +00004649 }
Fredrik Lundhb63588c2006-05-23 18:44:25 +00004650 if (PyList_Reverse(list) < 0)
4651 goto onError;
Hye-Shik Chang3ae811b2003-12-15 18:49:53 +00004652 return list;
4653
4654 onError:
4655 Py_DECREF(list);
4656 return NULL;
4657}
4658
Guido van Rossumd57fd912000-03-10 22:53:23 +00004659#undef SPLIT_APPEND
4660
4661static
4662PyObject *split(PyUnicodeObject *self,
4663 PyUnicodeObject *substring,
Martin v. Löwis18e16552006-02-15 17:27:45 +00004664 Py_ssize_t maxcount)
Guido van Rossumd57fd912000-03-10 22:53:23 +00004665{
4666 PyObject *list;
4667
4668 if (maxcount < 0)
Martin v. Löwis412fb672006-04-13 06:34:32 +00004669 maxcount = PY_SSIZE_T_MAX;
Guido van Rossumd57fd912000-03-10 22:53:23 +00004670
4671 list = PyList_New(0);
4672 if (!list)
4673 return NULL;
4674
4675 if (substring == NULL)
4676 return split_whitespace(self,list,maxcount);
4677
4678 else if (substring->length == 1)
4679 return split_char(self,list,substring->str[0],maxcount);
4680
4681 else if (substring->length == 0) {
4682 Py_DECREF(list);
4683 PyErr_SetString(PyExc_ValueError, "empty separator");
4684 return NULL;
4685 }
4686 else
4687 return split_substring(self,list,substring,maxcount);
4688}
4689
Tim Petersced69f82003-09-16 20:30:58 +00004690static
Hye-Shik Chang3ae811b2003-12-15 18:49:53 +00004691PyObject *rsplit(PyUnicodeObject *self,
4692 PyUnicodeObject *substring,
Martin v. Löwis18e16552006-02-15 17:27:45 +00004693 Py_ssize_t maxcount)
Hye-Shik Chang3ae811b2003-12-15 18:49:53 +00004694{
4695 PyObject *list;
4696
4697 if (maxcount < 0)
Martin v. Löwis412fb672006-04-13 06:34:32 +00004698 maxcount = PY_SSIZE_T_MAX;
Hye-Shik Chang3ae811b2003-12-15 18:49:53 +00004699
4700 list = PyList_New(0);
4701 if (!list)
4702 return NULL;
4703
4704 if (substring == NULL)
4705 return rsplit_whitespace(self,list,maxcount);
4706
4707 else if (substring->length == 1)
4708 return rsplit_char(self,list,substring->str[0],maxcount);
4709
4710 else if (substring->length == 0) {
4711 Py_DECREF(list);
4712 PyErr_SetString(PyExc_ValueError, "empty separator");
4713 return NULL;
4714 }
4715 else
4716 return rsplit_substring(self,list,substring,maxcount);
4717}
4718
4719static
Guido van Rossumd57fd912000-03-10 22:53:23 +00004720PyObject *replace(PyUnicodeObject *self,
4721 PyUnicodeObject *str1,
4722 PyUnicodeObject *str2,
Martin v. Löwis18e16552006-02-15 17:27:45 +00004723 Py_ssize_t maxcount)
Guido van Rossumd57fd912000-03-10 22:53:23 +00004724{
4725 PyUnicodeObject *u;
4726
4727 if (maxcount < 0)
Martin v. Löwis412fb672006-04-13 06:34:32 +00004728 maxcount = PY_SSIZE_T_MAX;
Guido van Rossumd57fd912000-03-10 22:53:23 +00004729
Fredrik Lundh347ee272006-05-24 16:35:18 +00004730 if (str1->length == str2->length) {
4731 /* same length */
Martin v. Löwis412fb672006-04-13 06:34:32 +00004732 Py_ssize_t i;
Fredrik Lundh347ee272006-05-24 16:35:18 +00004733 if (str1->length == 1) {
4734 /* replace characters */
4735 Py_UNICODE u1, u2;
4736 if (!findchar(self->str, self->length, str1->str[0]))
4737 goto nothing;
4738 u = (PyUnicodeObject*) PyUnicode_FromUnicode(NULL, self->length);
4739 if (!u)
4740 return NULL;
4741 Py_UNICODE_COPY(u->str, self->str, self->length);
4742 u1 = str1->str[0];
4743 u2 = str2->str[0];
4744 for (i = 0; i < u->length; i++)
4745 if (u->str[i] == u1) {
4746 if (--maxcount < 0)
4747 break;
4748 u->str[i] = u2;
4749 }
Guido van Rossumd57fd912000-03-10 22:53:23 +00004750 } else {
Fredrik Lundh347ee272006-05-24 16:35:18 +00004751 i = fastsearch(
4752 self->str, self->length, str1->str, str1->length, FAST_SEARCH
Guido van Rossumd57fd912000-03-10 22:53:23 +00004753 );
Fredrik Lundh347ee272006-05-24 16:35:18 +00004754 if (i < 0)
4755 goto nothing;
4756 u = (PyUnicodeObject*) PyUnicode_FromUnicode(NULL, self->length);
4757 if (!u)
4758 return NULL;
4759 Py_UNICODE_COPY(u->str, self->str, self->length);
4760 while (i <= self->length - str1->length)
4761 if (Py_UNICODE_MATCH(self, i, str1)) {
4762 if (--maxcount < 0)
4763 break;
4764 Py_UNICODE_COPY(u->str+i, str2->str, str2->length);
4765 i += str1->length;
4766 } else
4767 i++;
Guido van Rossumd57fd912000-03-10 22:53:23 +00004768 }
Guido van Rossumd57fd912000-03-10 22:53:23 +00004769 } else {
Fredrik Lundh347ee272006-05-24 16:35:18 +00004770
Martin v. Löwis18e16552006-02-15 17:27:45 +00004771 Py_ssize_t n, i;
Fredrik Lundh0c71f882006-05-25 16:46:54 +00004772 Py_ssize_t product, new_size, delta;
Guido van Rossumd57fd912000-03-10 22:53:23 +00004773 Py_UNICODE *p;
4774
4775 /* replace strings */
Fredrik Lundh58b5e842006-05-26 19:24:53 +00004776 n = stringlib_count(self->str, self->length, str1->str, str1->length);
Guido van Rossumd57fd912000-03-10 22:53:23 +00004777 if (n > maxcount)
4778 n = maxcount;
Fredrik Lundh347ee272006-05-24 16:35:18 +00004779 if (n == 0)
4780 goto nothing;
Fredrik Lundh0c71f882006-05-25 16:46:54 +00004781 /* new_size = self->length + n * (str2->length - str1->length)); */
4782 delta = (str2->length - str1->length);
4783 if (delta == 0) {
4784 new_size = self->length;
4785 } else {
4786 product = n * (str2->length - str1->length);
4787 if ((product / (str2->length - str1->length)) != n) {
4788 PyErr_SetString(PyExc_OverflowError,
4789 "replace string is too long");
4790 return NULL;
4791 }
4792 new_size = self->length + product;
4793 if (new_size < 0) {
4794 PyErr_SetString(PyExc_OverflowError,
4795 "replace string is too long");
4796 return NULL;
4797 }
4798 }
4799 u = _PyUnicode_New(new_size);
Fredrik Lundh347ee272006-05-24 16:35:18 +00004800 if (!u)
4801 return NULL;
4802 i = 0;
4803 p = u->str;
4804 if (str1->length > 0) {
4805 while (i <= self->length - str1->length)
4806 if (Py_UNICODE_MATCH(self, i, str1)) {
4807 /* replace string segment */
4808 Py_UNICODE_COPY(p, str2->str, str2->length);
4809 p += str2->length;
4810 i += str1->length;
4811 if (--n <= 0) {
4812 /* copy remaining part */
4813 Py_UNICODE_COPY(p, self->str+i, self->length-i);
4814 break;
Guido van Rossum8b1a6d62002-08-23 18:21:28 +00004815 }
Fredrik Lundh347ee272006-05-24 16:35:18 +00004816 } else
4817 *p++ = self->str[i++];
4818 } else {
4819 while (n > 0) {
4820 Py_UNICODE_COPY(p, str2->str, str2->length);
4821 p += str2->length;
4822 if (--n <= 0)
4823 break;
4824 *p++ = self->str[i++];
Guido van Rossumd57fd912000-03-10 22:53:23 +00004825 }
Fredrik Lundh347ee272006-05-24 16:35:18 +00004826 Py_UNICODE_COPY(p, self->str+i, self->length-i);
Guido van Rossumd57fd912000-03-10 22:53:23 +00004827 }
4828 }
Guido van Rossumd57fd912000-03-10 22:53:23 +00004829 return (PyObject *) u;
Fredrik Lundh347ee272006-05-24 16:35:18 +00004830
4831nothing:
4832 /* nothing to replace; return original string (when possible) */
4833 if (PyUnicode_CheckExact(self)) {
4834 Py_INCREF(self);
4835 return (PyObject *) self;
4836 }
4837 return PyUnicode_FromUnicode(self->str, self->length);
Guido van Rossumd57fd912000-03-10 22:53:23 +00004838}
4839
4840/* --- Unicode Object Methods --------------------------------------------- */
4841
Martin v. Löwis14f8b4c2002-06-13 20:33:02 +00004842PyDoc_STRVAR(title__doc__,
Guido van Rossumd57fd912000-03-10 22:53:23 +00004843"S.title() -> unicode\n\
4844\n\
4845Return a titlecased version of S, i.e. words start with title case\n\
Martin v. Löwis14f8b4c2002-06-13 20:33:02 +00004846characters, all remaining cased characters have lower case.");
Guido van Rossumd57fd912000-03-10 22:53:23 +00004847
4848static PyObject*
Martin v. Löwise3eb1f22001-08-16 13:15:00 +00004849unicode_title(PyUnicodeObject *self)
Guido van Rossumd57fd912000-03-10 22:53:23 +00004850{
Guido van Rossumd57fd912000-03-10 22:53:23 +00004851 return fixup(self, fixtitle);
4852}
4853
Martin v. Löwis14f8b4c2002-06-13 20:33:02 +00004854PyDoc_STRVAR(capitalize__doc__,
Guido van Rossumd57fd912000-03-10 22:53:23 +00004855"S.capitalize() -> unicode\n\
4856\n\
4857Return a capitalized version of S, i.e. make the first character\n\
Martin v. Löwis14f8b4c2002-06-13 20:33:02 +00004858have upper case.");
Guido van Rossumd57fd912000-03-10 22:53:23 +00004859
4860static PyObject*
Martin v. Löwise3eb1f22001-08-16 13:15:00 +00004861unicode_capitalize(PyUnicodeObject *self)
Guido van Rossumd57fd912000-03-10 22:53:23 +00004862{
Guido van Rossumd57fd912000-03-10 22:53:23 +00004863 return fixup(self, fixcapitalize);
4864}
4865
4866#if 0
Martin v. Löwis14f8b4c2002-06-13 20:33:02 +00004867PyDoc_STRVAR(capwords__doc__,
Guido van Rossumd57fd912000-03-10 22:53:23 +00004868"S.capwords() -> unicode\n\
4869\n\
4870Apply .capitalize() to all words in S and return the result with\n\
Martin v. Löwis14f8b4c2002-06-13 20:33:02 +00004871normalized whitespace (all whitespace strings are replaced by ' ').");
Guido van Rossumd57fd912000-03-10 22:53:23 +00004872
4873static PyObject*
Martin v. Löwise3eb1f22001-08-16 13:15:00 +00004874unicode_capwords(PyUnicodeObject *self)
Guido van Rossumd57fd912000-03-10 22:53:23 +00004875{
4876 PyObject *list;
4877 PyObject *item;
Martin v. Löwis18e16552006-02-15 17:27:45 +00004878 Py_ssize_t i;
Guido van Rossumd57fd912000-03-10 22:53:23 +00004879
Guido van Rossumd57fd912000-03-10 22:53:23 +00004880 /* Split into words */
4881 list = split(self, NULL, -1);
4882 if (!list)
4883 return NULL;
4884
4885 /* Capitalize each word */
4886 for (i = 0; i < PyList_GET_SIZE(list); i++) {
4887 item = fixup((PyUnicodeObject *)PyList_GET_ITEM(list, i),
4888 fixcapitalize);
4889 if (item == NULL)
4890 goto onError;
4891 Py_DECREF(PyList_GET_ITEM(list, i));
4892 PyList_SET_ITEM(list, i, item);
4893 }
4894
4895 /* Join the words to form a new string */
4896 item = PyUnicode_Join(NULL, list);
4897
4898onError:
4899 Py_DECREF(list);
4900 return (PyObject *)item;
4901}
4902#endif
4903
Raymond Hettinger4f8f9762003-11-26 08:21:35 +00004904/* Argument converter. Coerces to a single unicode character */
4905
4906static int
4907convert_uc(PyObject *obj, void *addr)
4908{
4909 Py_UNICODE *fillcharloc = (Py_UNICODE *)addr;
4910 PyObject *uniobj;
4911 Py_UNICODE *unistr;
4912
4913 uniobj = PyUnicode_FromObject(obj);
4914 if (uniobj == NULL) {
4915 PyErr_SetString(PyExc_TypeError,
4916 "The fill character cannot be converted to Unicode");
4917 return 0;
4918 }
4919 if (PyUnicode_GET_SIZE(uniobj) != 1) {
4920 PyErr_SetString(PyExc_TypeError,
4921 "The fill character must be exactly one character long");
4922 Py_DECREF(uniobj);
4923 return 0;
4924 }
4925 unistr = PyUnicode_AS_UNICODE(uniobj);
4926 *fillcharloc = unistr[0];
4927 Py_DECREF(uniobj);
4928 return 1;
4929}
4930
Martin v. Löwis14f8b4c2002-06-13 20:33:02 +00004931PyDoc_STRVAR(center__doc__,
Raymond Hettinger4f8f9762003-11-26 08:21:35 +00004932"S.center(width[, fillchar]) -> unicode\n\
Guido van Rossumd57fd912000-03-10 22:53:23 +00004933\n\
Raymond Hettinger4f8f9762003-11-26 08:21:35 +00004934Return S centered in a Unicode string of length width. Padding is\n\
4935done using the specified fill character (default is a space)");
Guido van Rossumd57fd912000-03-10 22:53:23 +00004936
4937static PyObject *
4938unicode_center(PyUnicodeObject *self, PyObject *args)
4939{
Martin v. Löwis18e16552006-02-15 17:27:45 +00004940 Py_ssize_t marg, left;
4941 Py_ssize_t width;
Raymond Hettinger4f8f9762003-11-26 08:21:35 +00004942 Py_UNICODE fillchar = ' ';
Guido van Rossumd57fd912000-03-10 22:53:23 +00004943
Thomas Woutersde017742006-02-16 19:34:37 +00004944 if (!PyArg_ParseTuple(args, "n|O&:center", &width, convert_uc, &fillchar))
Guido van Rossumd57fd912000-03-10 22:53:23 +00004945 return NULL;
4946
Tim Peters7a29bd52001-09-12 03:03:31 +00004947 if (self->length >= width && PyUnicode_CheckExact(self)) {
Guido van Rossumd57fd912000-03-10 22:53:23 +00004948 Py_INCREF(self);
4949 return (PyObject*) self;
4950 }
4951
4952 marg = width - self->length;
4953 left = marg / 2 + (marg & width & 1);
4954
Raymond Hettinger4f8f9762003-11-26 08:21:35 +00004955 return (PyObject*) pad(self, left, marg - left, fillchar);
Guido van Rossumd57fd912000-03-10 22:53:23 +00004956}
4957
Marc-André Lemburge5034372000-08-08 08:04:29 +00004958#if 0
4959
4960/* This code should go into some future Unicode collation support
4961 module. The basic comparison should compare ordinals on a naive
Trent Mick20abf572000-08-12 22:14:34 +00004962 basis (this is what Java does and thus JPython too). */
Marc-André Lemburge5034372000-08-08 08:04:29 +00004963
Marc-André Lemburg1e7205a2000-07-04 09:51:07 +00004964/* speedy UTF-16 code point order comparison */
4965/* gleaned from: */
4966/* http://www-4.ibm.com/software/developer/library/utf16.html?dwzone=unicode */
4967
Marc-André Lemburge12896e2000-07-07 17:51:08 +00004968static short utf16Fixup[32] =
Marc-André Lemburg1e7205a2000-07-04 09:51:07 +00004969{
Marc-André Lemburg1e7205a2000-07-04 09:51:07 +00004970 0, 0, 0, 0, 0, 0, 0, 0,
Tim Petersced69f82003-09-16 20:30:58 +00004971 0, 0, 0, 0, 0, 0, 0, 0,
4972 0, 0, 0, 0, 0, 0, 0, 0,
Marc-André Lemburge12896e2000-07-07 17:51:08 +00004973 0, 0, 0, 0x2000, -0x800, -0x800, -0x800, -0x800
Marc-André Lemburg1e7205a2000-07-04 09:51:07 +00004974};
4975
Guido van Rossumd57fd912000-03-10 22:53:23 +00004976static int
4977unicode_compare(PyUnicodeObject *str1, PyUnicodeObject *str2)
4978{
Martin v. Löwis18e16552006-02-15 17:27:45 +00004979 Py_ssize_t len1, len2;
Marc-André Lemburg1e7205a2000-07-04 09:51:07 +00004980
Guido van Rossumd57fd912000-03-10 22:53:23 +00004981 Py_UNICODE *s1 = str1->str;
4982 Py_UNICODE *s2 = str2->str;
4983
4984 len1 = str1->length;
4985 len2 = str2->length;
Tim Petersced69f82003-09-16 20:30:58 +00004986
Guido van Rossumd57fd912000-03-10 22:53:23 +00004987 while (len1 > 0 && len2 > 0) {
Tim Petersced69f82003-09-16 20:30:58 +00004988 Py_UNICODE c1, c2;
Marc-André Lemburg1e7205a2000-07-04 09:51:07 +00004989
4990 c1 = *s1++;
4991 c2 = *s2++;
Fredrik Lundh45714e92001-06-26 16:39:36 +00004992
Marc-André Lemburg1e7205a2000-07-04 09:51:07 +00004993 if (c1 > (1<<11) * 26)
4994 c1 += utf16Fixup[c1>>11];
4995 if (c2 > (1<<11) * 26)
4996 c2 += utf16Fixup[c2>>11];
Marc-André Lemburg1e7205a2000-07-04 09:51:07 +00004997 /* now c1 and c2 are in UTF-32-compatible order */
Fredrik Lundh45714e92001-06-26 16:39:36 +00004998
4999 if (c1 != c2)
5000 return (c1 < c2) ? -1 : 1;
Tim Petersced69f82003-09-16 20:30:58 +00005001
Marc-André Lemburg1e7205a2000-07-04 09:51:07 +00005002 len1--; len2--;
Guido van Rossumd57fd912000-03-10 22:53:23 +00005003 }
5004
5005 return (len1 < len2) ? -1 : (len1 != len2);
5006}
5007
Marc-André Lemburge5034372000-08-08 08:04:29 +00005008#else
5009
5010static int
5011unicode_compare(PyUnicodeObject *str1, PyUnicodeObject *str2)
5012{
Martin v. Löwis18e16552006-02-15 17:27:45 +00005013 register Py_ssize_t len1, len2;
Marc-André Lemburge5034372000-08-08 08:04:29 +00005014
5015 Py_UNICODE *s1 = str1->str;
5016 Py_UNICODE *s2 = str2->str;
5017
5018 len1 = str1->length;
5019 len2 = str2->length;
Tim Petersced69f82003-09-16 20:30:58 +00005020
Marc-André Lemburge5034372000-08-08 08:04:29 +00005021 while (len1 > 0 && len2 > 0) {
Tim Petersced69f82003-09-16 20:30:58 +00005022 Py_UNICODE c1, c2;
Marc-André Lemburge5034372000-08-08 08:04:29 +00005023
Fredrik Lundh45714e92001-06-26 16:39:36 +00005024 c1 = *s1++;
5025 c2 = *s2++;
5026
5027 if (c1 != c2)
5028 return (c1 < c2) ? -1 : 1;
5029
Marc-André Lemburge5034372000-08-08 08:04:29 +00005030 len1--; len2--;
5031 }
5032
5033 return (len1 < len2) ? -1 : (len1 != len2);
5034}
5035
5036#endif
5037
Guido van Rossumd57fd912000-03-10 22:53:23 +00005038int PyUnicode_Compare(PyObject *left,
5039 PyObject *right)
5040{
5041 PyUnicodeObject *u = NULL, *v = NULL;
5042 int result;
5043
5044 /* Coerce the two arguments */
5045 u = (PyUnicodeObject *)PyUnicode_FromObject(left);
5046 if (u == NULL)
5047 goto onError;
5048 v = (PyUnicodeObject *)PyUnicode_FromObject(right);
5049 if (v == NULL)
5050 goto onError;
5051
Thomas Wouters7e474022000-07-16 12:04:32 +00005052 /* Shortcut for empty or interned objects */
Guido van Rossumd57fd912000-03-10 22:53:23 +00005053 if (v == u) {
5054 Py_DECREF(u);
5055 Py_DECREF(v);
5056 return 0;
5057 }
5058
5059 result = unicode_compare(u, v);
5060
5061 Py_DECREF(u);
5062 Py_DECREF(v);
5063 return result;
5064
5065onError:
5066 Py_XDECREF(u);
5067 Py_XDECREF(v);
5068 return -1;
5069}
5070
Guido van Rossum403d68b2000-03-13 15:55:09 +00005071int PyUnicode_Contains(PyObject *container,
5072 PyObject *element)
5073{
Fredrik Lundh833bf942006-05-23 10:12:21 +00005074 PyUnicodeObject *u, *v;
Martin v. Löwis18e16552006-02-15 17:27:45 +00005075 Py_ssize_t size;
Fredrik Lundhd5e0dc52006-05-24 15:11:01 +00005076 Py_ssize_t pos;
Guido van Rossum403d68b2000-03-13 15:55:09 +00005077
5078 /* Coerce the two arguments */
Fredrik Lundh833bf942006-05-23 10:12:21 +00005079 v = (PyUnicodeObject *) PyUnicode_FromObject(element);
5080 if (!v) {
Marc-André Lemburg7c014682000-06-28 08:11:47 +00005081 PyErr_SetString(PyExc_TypeError,
Barry Warsaw817918c2002-08-06 16:58:21 +00005082 "'in <string>' requires string as left operand");
Fredrik Lundh833bf942006-05-23 10:12:21 +00005083 return -1;
Marc-André Lemburg7c014682000-06-28 08:11:47 +00005084 }
Fredrik Lundh833bf942006-05-23 10:12:21 +00005085
5086 u = (PyUnicodeObject *) PyUnicode_FromObject(container);
5087 if (!u) {
5088 Py_DECREF(v);
5089 return -1;
5090 }
Guido van Rossum403d68b2000-03-13 15:55:09 +00005091
Barry Warsaw817918c2002-08-06 16:58:21 +00005092 size = PyUnicode_GET_SIZE(v);
Fredrik Lundh833bf942006-05-23 10:12:21 +00005093 if (!size) {
Fredrik Lundh06a69dd2006-05-26 08:54:28 +00005094 pos = 0;
Fredrik Lundh833bf942006-05-23 10:12:21 +00005095 goto done;
5096 }
Barry Warsaw817918c2002-08-06 16:58:21 +00005097
Fredrik Lundhd5e0dc52006-05-24 15:11:01 +00005098 pos = fastsearch(
5099 PyUnicode_AS_UNICODE(u), PyUnicode_GET_SIZE(u),
5100 PyUnicode_AS_UNICODE(v), size, FAST_SEARCH
5101 );
Guido van Rossum403d68b2000-03-13 15:55:09 +00005102
Fredrik Lundh833bf942006-05-23 10:12:21 +00005103done:
Guido van Rossum403d68b2000-03-13 15:55:09 +00005104 Py_DECREF(u);
5105 Py_DECREF(v);
Fredrik Lundh06a69dd2006-05-26 08:54:28 +00005106 return (pos != -1);
Guido van Rossum403d68b2000-03-13 15:55:09 +00005107}
5108
Guido van Rossumd57fd912000-03-10 22:53:23 +00005109/* Concat to string or Unicode object giving a new Unicode object. */
5110
5111PyObject *PyUnicode_Concat(PyObject *left,
5112 PyObject *right)
5113{
5114 PyUnicodeObject *u = NULL, *v = NULL, *w;
5115
5116 /* Coerce the two arguments */
5117 u = (PyUnicodeObject *)PyUnicode_FromObject(left);
5118 if (u == NULL)
5119 goto onError;
5120 v = (PyUnicodeObject *)PyUnicode_FromObject(right);
5121 if (v == NULL)
5122 goto onError;
5123
5124 /* Shortcuts */
5125 if (v == unicode_empty) {
5126 Py_DECREF(v);
5127 return (PyObject *)u;
5128 }
5129 if (u == unicode_empty) {
5130 Py_DECREF(u);
5131 return (PyObject *)v;
5132 }
5133
5134 /* Concat the two Unicode strings */
5135 w = _PyUnicode_New(u->length + v->length);
5136 if (w == NULL)
5137 goto onError;
5138 Py_UNICODE_COPY(w->str, u->str, u->length);
5139 Py_UNICODE_COPY(w->str + u->length, v->str, v->length);
5140
5141 Py_DECREF(u);
5142 Py_DECREF(v);
5143 return (PyObject *)w;
5144
5145onError:
5146 Py_XDECREF(u);
5147 Py_XDECREF(v);
5148 return NULL;
5149}
5150
Martin v. Löwis14f8b4c2002-06-13 20:33:02 +00005151PyDoc_STRVAR(count__doc__,
Guido van Rossumd57fd912000-03-10 22:53:23 +00005152"S.count(sub[, start[, end]]) -> int\n\
5153\n\
Fredrik Lundh763b50f2006-05-22 15:35:12 +00005154Return the number of non-overlapping occurrences of substring sub in\n\
5155Unicode string S[start:end]. Optional arguments start and end are\n\
Martin v. Löwis14f8b4c2002-06-13 20:33:02 +00005156interpreted as in slice notation.");
Guido van Rossumd57fd912000-03-10 22:53:23 +00005157
5158static PyObject *
5159unicode_count(PyUnicodeObject *self, PyObject *args)
5160{
5161 PyUnicodeObject *substring;
Martin v. Löwis18e16552006-02-15 17:27:45 +00005162 Py_ssize_t start = 0;
Martin v. Löwis412fb672006-04-13 06:34:32 +00005163 Py_ssize_t end = PY_SSIZE_T_MAX;
Guido van Rossumd57fd912000-03-10 22:53:23 +00005164 PyObject *result;
5165
Guido van Rossumb8872e62000-05-09 14:14:27 +00005166 if (!PyArg_ParseTuple(args, "O|O&O&:count", &substring,
5167 _PyEval_SliceIndex, &start, _PyEval_SliceIndex, &end))
Guido van Rossumd57fd912000-03-10 22:53:23 +00005168 return NULL;
5169
5170 substring = (PyUnicodeObject *)PyUnicode_FromObject(
Fredrik Lundh58b5e842006-05-26 19:24:53 +00005171 (PyObject *)substring);
Guido van Rossumd57fd912000-03-10 22:53:23 +00005172 if (substring == NULL)
5173 return NULL;
Tim Petersced69f82003-09-16 20:30:58 +00005174
Guido van Rossumd57fd912000-03-10 22:53:23 +00005175 if (start < 0)
5176 start += self->length;
5177 if (start < 0)
5178 start = 0;
5179 if (end > self->length)
5180 end = self->length;
5181 if (end < 0)
5182 end += self->length;
5183 if (end < 0)
5184 end = 0;
5185
Fredrik Lundh58b5e842006-05-26 19:24:53 +00005186 result = PyInt_FromSsize_t(
5187 stringlib_count(self->str + start, end - start,
5188 substring->str, substring->length)
5189 );
Guido van Rossumd57fd912000-03-10 22:53:23 +00005190
5191 Py_DECREF(substring);
Fredrik Lundh58b5e842006-05-26 19:24:53 +00005192
Guido van Rossumd57fd912000-03-10 22:53:23 +00005193 return result;
5194}
5195
Martin v. Löwis14f8b4c2002-06-13 20:33:02 +00005196PyDoc_STRVAR(encode__doc__,
Marc-André Lemburgd2d45982004-07-08 17:57:32 +00005197"S.encode([encoding[,errors]]) -> string or unicode\n\
Guido van Rossumd57fd912000-03-10 22:53:23 +00005198\n\
Marc-André Lemburgd2d45982004-07-08 17:57:32 +00005199Encodes S using the codec registered for encoding. encoding defaults\n\
5200to the default encoding. errors may be given to set a different error\n\
Fred Drakee4315f52000-05-09 19:53:39 +00005201handling scheme. Default is 'strict' meaning that encoding errors raise\n\
Walter Dörwald3aeb6322002-09-02 13:14:32 +00005202a UnicodeEncodeError. Other possible values are 'ignore', 'replace' and\n\
5203'xmlcharrefreplace' as well as any other name registered with\n\
5204codecs.register_error that can handle UnicodeEncodeErrors.");
Guido van Rossumd57fd912000-03-10 22:53:23 +00005205
5206static PyObject *
5207unicode_encode(PyUnicodeObject *self, PyObject *args)
5208{
5209 char *encoding = NULL;
5210 char *errors = NULL;
Marc-André Lemburgd2d45982004-07-08 17:57:32 +00005211 PyObject *v;
5212
Guido van Rossumd57fd912000-03-10 22:53:23 +00005213 if (!PyArg_ParseTuple(args, "|ss:encode", &encoding, &errors))
5214 return NULL;
Marc-André Lemburgd2d45982004-07-08 17:57:32 +00005215 v = PyUnicode_AsEncodedObject((PyObject *)self, encoding, errors);
Marc-André Lemburg1dffb122004-07-08 19:13:55 +00005216 if (v == NULL)
5217 goto onError;
Marc-André Lemburgd2d45982004-07-08 17:57:32 +00005218 if (!PyString_Check(v) && !PyUnicode_Check(v)) {
5219 PyErr_Format(PyExc_TypeError,
5220 "encoder did not return a string/unicode object "
5221 "(type=%.400s)",
5222 v->ob_type->tp_name);
5223 Py_DECREF(v);
5224 return NULL;
5225 }
5226 return v;
Marc-André Lemburg1dffb122004-07-08 19:13:55 +00005227
5228 onError:
5229 return NULL;
Marc-André Lemburgd2d45982004-07-08 17:57:32 +00005230}
5231
5232PyDoc_STRVAR(decode__doc__,
5233"S.decode([encoding[,errors]]) -> string or unicode\n\
5234\n\
5235Decodes S using the codec registered for encoding. encoding defaults\n\
5236to the default encoding. errors may be given to set a different error\n\
5237handling scheme. Default is 'strict' meaning that encoding errors raise\n\
5238a UnicodeDecodeError. Other possible values are 'ignore' and 'replace'\n\
5239as well as any other name registerd with codecs.register_error that is\n\
5240able to handle UnicodeDecodeErrors.");
5241
5242static PyObject *
Marc-André Lemburg126b44c2004-07-10 12:04:20 +00005243unicode_decode(PyUnicodeObject *self, PyObject *args)
Marc-André Lemburgd2d45982004-07-08 17:57:32 +00005244{
5245 char *encoding = NULL;
5246 char *errors = NULL;
5247 PyObject *v;
5248
5249 if (!PyArg_ParseTuple(args, "|ss:decode", &encoding, &errors))
5250 return NULL;
5251 v = PyUnicode_AsDecodedObject((PyObject *)self, encoding, errors);
Marc-André Lemburg1dffb122004-07-08 19:13:55 +00005252 if (v == NULL)
5253 goto onError;
Marc-André Lemburgd2d45982004-07-08 17:57:32 +00005254 if (!PyString_Check(v) && !PyUnicode_Check(v)) {
5255 PyErr_Format(PyExc_TypeError,
5256 "decoder did not return a string/unicode object "
5257 "(type=%.400s)",
5258 v->ob_type->tp_name);
5259 Py_DECREF(v);
5260 return NULL;
5261 }
5262 return v;
Marc-André Lemburg1dffb122004-07-08 19:13:55 +00005263
5264 onError:
5265 return NULL;
Guido van Rossumd57fd912000-03-10 22:53:23 +00005266}
5267
Martin v. Löwis14f8b4c2002-06-13 20:33:02 +00005268PyDoc_STRVAR(expandtabs__doc__,
Guido van Rossumd57fd912000-03-10 22:53:23 +00005269"S.expandtabs([tabsize]) -> unicode\n\
5270\n\
5271Return a copy of S where all tab characters are expanded using spaces.\n\
Martin v. Löwis14f8b4c2002-06-13 20:33:02 +00005272If tabsize is not given, a tab size of 8 characters is assumed.");
Guido van Rossumd57fd912000-03-10 22:53:23 +00005273
5274static PyObject*
5275unicode_expandtabs(PyUnicodeObject *self, PyObject *args)
5276{
5277 Py_UNICODE *e;
5278 Py_UNICODE *p;
5279 Py_UNICODE *q;
Martin v. Löwis18e16552006-02-15 17:27:45 +00005280 Py_ssize_t i, j;
Guido van Rossumd57fd912000-03-10 22:53:23 +00005281 PyUnicodeObject *u;
5282 int tabsize = 8;
5283
5284 if (!PyArg_ParseTuple(args, "|i:expandtabs", &tabsize))
5285 return NULL;
5286
Thomas Wouters7e474022000-07-16 12:04:32 +00005287 /* First pass: determine size of output string */
Guido van Rossumd57fd912000-03-10 22:53:23 +00005288 i = j = 0;
5289 e = self->str + self->length;
5290 for (p = self->str; p < e; p++)
5291 if (*p == '\t') {
5292 if (tabsize > 0)
5293 j += tabsize - (j % tabsize);
5294 }
5295 else {
5296 j++;
5297 if (*p == '\n' || *p == '\r') {
5298 i += j;
5299 j = 0;
5300 }
5301 }
5302
5303 /* Second pass: create output string and fill it */
5304 u = _PyUnicode_New(i + j);
5305 if (!u)
5306 return NULL;
5307
5308 j = 0;
5309 q = u->str;
5310
5311 for (p = self->str; p < e; p++)
5312 if (*p == '\t') {
5313 if (tabsize > 0) {
5314 i = tabsize - (j % tabsize);
5315 j += i;
5316 while (i--)
5317 *q++ = ' ';
5318 }
5319 }
5320 else {
5321 j++;
5322 *q++ = *p;
5323 if (*p == '\n' || *p == '\r')
5324 j = 0;
5325 }
5326
5327 return (PyObject*) u;
5328}
5329
Martin v. Löwis14f8b4c2002-06-13 20:33:02 +00005330PyDoc_STRVAR(find__doc__,
Guido van Rossumd57fd912000-03-10 22:53:23 +00005331"S.find(sub [,start [,end]]) -> int\n\
5332\n\
5333Return the lowest index in S where substring sub is found,\n\
5334such that sub is contained within s[start,end]. Optional\n\
5335arguments start and end are interpreted as in slice notation.\n\
5336\n\
Martin v. Löwis14f8b4c2002-06-13 20:33:02 +00005337Return -1 on failure.");
Guido van Rossumd57fd912000-03-10 22:53:23 +00005338
5339static PyObject *
5340unicode_find(PyUnicodeObject *self, PyObject *args)
5341{
5342 PyUnicodeObject *substring;
Martin v. Löwis18e16552006-02-15 17:27:45 +00005343 Py_ssize_t start = 0;
Martin v. Löwis412fb672006-04-13 06:34:32 +00005344 Py_ssize_t end = PY_SSIZE_T_MAX;
Guido van Rossumd57fd912000-03-10 22:53:23 +00005345 PyObject *result;
5346
Guido van Rossumb8872e62000-05-09 14:14:27 +00005347 if (!PyArg_ParseTuple(args, "O|O&O&:find", &substring,
5348 _PyEval_SliceIndex, &start, _PyEval_SliceIndex, &end))
Guido van Rossumd57fd912000-03-10 22:53:23 +00005349 return NULL;
5350 substring = (PyUnicodeObject *)PyUnicode_FromObject(
5351 (PyObject *)substring);
5352 if (substring == NULL)
5353 return NULL;
5354
Martin v. Löwis18e16552006-02-15 17:27:45 +00005355 result = PyInt_FromSsize_t(findstring(self, substring, start, end, 1));
Guido van Rossumd57fd912000-03-10 22:53:23 +00005356
5357 Py_DECREF(substring);
5358 return result;
5359}
5360
5361static PyObject *
Martin v. Löwis18e16552006-02-15 17:27:45 +00005362unicode_getitem(PyUnicodeObject *self, Py_ssize_t index)
Guido van Rossumd57fd912000-03-10 22:53:23 +00005363{
5364 if (index < 0 || index >= self->length) {
5365 PyErr_SetString(PyExc_IndexError, "string index out of range");
5366 return NULL;
5367 }
5368
5369 return (PyObject*) PyUnicode_FromUnicode(&self->str[index], 1);
5370}
5371
5372static long
5373unicode_hash(PyUnicodeObject *self)
5374{
Fredrik Lundhdde61642000-07-10 18:27:47 +00005375 /* Since Unicode objects compare equal to their ASCII string
5376 counterparts, they should use the individual character values
5377 as basis for their hash value. This is needed to assure that
5378 strings and Unicode objects behave in the same way as
5379 dictionary keys. */
Guido van Rossumd57fd912000-03-10 22:53:23 +00005380
Martin v. Löwis18e16552006-02-15 17:27:45 +00005381 register Py_ssize_t len;
Fredrik Lundhdde61642000-07-10 18:27:47 +00005382 register Py_UNICODE *p;
5383 register long x;
5384
Guido van Rossumd57fd912000-03-10 22:53:23 +00005385 if (self->hash != -1)
5386 return self->hash;
Fredrik Lundhdde61642000-07-10 18:27:47 +00005387 len = PyUnicode_GET_SIZE(self);
5388 p = PyUnicode_AS_UNICODE(self);
5389 x = *p << 7;
5390 while (--len >= 0)
5391 x = (1000003*x) ^ *p++;
5392 x ^= PyUnicode_GET_SIZE(self);
5393 if (x == -1)
5394 x = -2;
5395 self->hash = x;
5396 return x;
Guido van Rossumd57fd912000-03-10 22:53:23 +00005397}
5398
Martin v. Löwis14f8b4c2002-06-13 20:33:02 +00005399PyDoc_STRVAR(index__doc__,
Guido van Rossumd57fd912000-03-10 22:53:23 +00005400"S.index(sub [,start [,end]]) -> int\n\
5401\n\
Martin v. Löwis14f8b4c2002-06-13 20:33:02 +00005402Like S.find() but raise ValueError when the substring is not found.");
Guido van Rossumd57fd912000-03-10 22:53:23 +00005403
5404static PyObject *
5405unicode_index(PyUnicodeObject *self, PyObject *args)
5406{
Martin v. Löwis18e16552006-02-15 17:27:45 +00005407 Py_ssize_t result;
Guido van Rossumd57fd912000-03-10 22:53:23 +00005408 PyUnicodeObject *substring;
Martin v. Löwis18e16552006-02-15 17:27:45 +00005409 Py_ssize_t start = 0;
Martin v. Löwis412fb672006-04-13 06:34:32 +00005410 Py_ssize_t end = PY_SSIZE_T_MAX;
Guido van Rossumd57fd912000-03-10 22:53:23 +00005411
Guido van Rossumb8872e62000-05-09 14:14:27 +00005412 if (!PyArg_ParseTuple(args, "O|O&O&:index", &substring,
5413 _PyEval_SliceIndex, &start, _PyEval_SliceIndex, &end))
Guido van Rossumd57fd912000-03-10 22:53:23 +00005414 return NULL;
Tim Petersced69f82003-09-16 20:30:58 +00005415
Guido van Rossumd57fd912000-03-10 22:53:23 +00005416 substring = (PyUnicodeObject *)PyUnicode_FromObject(
5417 (PyObject *)substring);
5418 if (substring == NULL)
5419 return NULL;
5420
5421 result = findstring(self, substring, start, end, 1);
5422
5423 Py_DECREF(substring);
5424 if (result < 0) {
5425 PyErr_SetString(PyExc_ValueError, "substring not found");
5426 return NULL;
5427 }
Martin v. Löwis18e16552006-02-15 17:27:45 +00005428 return PyInt_FromSsize_t(result);
Guido van Rossumd57fd912000-03-10 22:53:23 +00005429}
5430
Martin v. Löwis14f8b4c2002-06-13 20:33:02 +00005431PyDoc_STRVAR(islower__doc__,
Guido van Rossum77f6a652002-04-03 22:41:51 +00005432"S.islower() -> bool\n\
Guido van Rossumd57fd912000-03-10 22:53:23 +00005433\n\
Guido van Rossum77f6a652002-04-03 22:41:51 +00005434Return True if all cased characters in S are lowercase and there is\n\
Martin v. Löwis14f8b4c2002-06-13 20:33:02 +00005435at least one cased character in S, False otherwise.");
Guido van Rossumd57fd912000-03-10 22:53:23 +00005436
5437static PyObject*
Martin v. Löwise3eb1f22001-08-16 13:15:00 +00005438unicode_islower(PyUnicodeObject *self)
Guido van Rossumd57fd912000-03-10 22:53:23 +00005439{
5440 register const Py_UNICODE *p = PyUnicode_AS_UNICODE(self);
5441 register const Py_UNICODE *e;
5442 int cased;
5443
Guido van Rossumd57fd912000-03-10 22:53:23 +00005444 /* Shortcut for single character strings */
5445 if (PyUnicode_GET_SIZE(self) == 1)
Guido van Rossum77f6a652002-04-03 22:41:51 +00005446 return PyBool_FromLong(Py_UNICODE_ISLOWER(*p));
Guido van Rossumd57fd912000-03-10 22:53:23 +00005447
Marc-André Lemburg60bc8092000-06-14 09:18:32 +00005448 /* Special case for empty strings */
Martin v. Löwisdea59e52006-01-05 10:00:36 +00005449 if (PyUnicode_GET_SIZE(self) == 0)
Guido van Rossum77f6a652002-04-03 22:41:51 +00005450 return PyBool_FromLong(0);
Marc-André Lemburg60bc8092000-06-14 09:18:32 +00005451
Guido van Rossumd57fd912000-03-10 22:53:23 +00005452 e = p + PyUnicode_GET_SIZE(self);
5453 cased = 0;
5454 for (; p < e; p++) {
5455 register const Py_UNICODE ch = *p;
Tim Petersced69f82003-09-16 20:30:58 +00005456
Guido van Rossumd57fd912000-03-10 22:53:23 +00005457 if (Py_UNICODE_ISUPPER(ch) || Py_UNICODE_ISTITLE(ch))
Guido van Rossum77f6a652002-04-03 22:41:51 +00005458 return PyBool_FromLong(0);
Guido van Rossumd57fd912000-03-10 22:53:23 +00005459 else if (!cased && Py_UNICODE_ISLOWER(ch))
5460 cased = 1;
5461 }
Guido van Rossum77f6a652002-04-03 22:41:51 +00005462 return PyBool_FromLong(cased);
Guido van Rossumd57fd912000-03-10 22:53:23 +00005463}
5464
Martin v. Löwis14f8b4c2002-06-13 20:33:02 +00005465PyDoc_STRVAR(isupper__doc__,
Guido van Rossum77f6a652002-04-03 22:41:51 +00005466"S.isupper() -> bool\n\
Guido van Rossumd57fd912000-03-10 22:53:23 +00005467\n\
Martin v. Löwis6828e182003-10-18 09:55:08 +00005468Return True if all cased characters in S are uppercase and there is\n\
Martin v. Löwis14f8b4c2002-06-13 20:33:02 +00005469at least one cased character in S, False otherwise.");
Guido van Rossumd57fd912000-03-10 22:53:23 +00005470
5471static PyObject*
Martin v. Löwise3eb1f22001-08-16 13:15:00 +00005472unicode_isupper(PyUnicodeObject *self)
Guido van Rossumd57fd912000-03-10 22:53:23 +00005473{
5474 register const Py_UNICODE *p = PyUnicode_AS_UNICODE(self);
5475 register const Py_UNICODE *e;
5476 int cased;
5477
Guido van Rossumd57fd912000-03-10 22:53:23 +00005478 /* Shortcut for single character strings */
5479 if (PyUnicode_GET_SIZE(self) == 1)
Guido van Rossum77f6a652002-04-03 22:41:51 +00005480 return PyBool_FromLong(Py_UNICODE_ISUPPER(*p) != 0);
Guido van Rossumd57fd912000-03-10 22:53:23 +00005481
Marc-André Lemburg60bc8092000-06-14 09:18:32 +00005482 /* Special case for empty strings */
Martin v. Löwisdea59e52006-01-05 10:00:36 +00005483 if (PyUnicode_GET_SIZE(self) == 0)
Guido van Rossum77f6a652002-04-03 22:41:51 +00005484 return PyBool_FromLong(0);
Marc-André Lemburg60bc8092000-06-14 09:18:32 +00005485
Guido van Rossumd57fd912000-03-10 22:53:23 +00005486 e = p + PyUnicode_GET_SIZE(self);
5487 cased = 0;
5488 for (; p < e; p++) {
5489 register const Py_UNICODE ch = *p;
Tim Petersced69f82003-09-16 20:30:58 +00005490
Guido van Rossumd57fd912000-03-10 22:53:23 +00005491 if (Py_UNICODE_ISLOWER(ch) || Py_UNICODE_ISTITLE(ch))
Guido van Rossum77f6a652002-04-03 22:41:51 +00005492 return PyBool_FromLong(0);
Guido van Rossumd57fd912000-03-10 22:53:23 +00005493 else if (!cased && Py_UNICODE_ISUPPER(ch))
5494 cased = 1;
5495 }
Guido van Rossum77f6a652002-04-03 22:41:51 +00005496 return PyBool_FromLong(cased);
Guido van Rossumd57fd912000-03-10 22:53:23 +00005497}
5498
Martin v. Löwis14f8b4c2002-06-13 20:33:02 +00005499PyDoc_STRVAR(istitle__doc__,
Guido van Rossum77f6a652002-04-03 22:41:51 +00005500"S.istitle() -> bool\n\
Guido van Rossumd57fd912000-03-10 22:53:23 +00005501\n\
Martin v. Löwis6828e182003-10-18 09:55:08 +00005502Return True if S is a titlecased string and there is at least one\n\
5503character in S, i.e. upper- and titlecase characters may only\n\
5504follow uncased characters and lowercase characters only cased ones.\n\
5505Return False otherwise.");
Guido van Rossumd57fd912000-03-10 22:53:23 +00005506
5507static PyObject*
Martin v. Löwise3eb1f22001-08-16 13:15:00 +00005508unicode_istitle(PyUnicodeObject *self)
Guido van Rossumd57fd912000-03-10 22:53:23 +00005509{
5510 register const Py_UNICODE *p = PyUnicode_AS_UNICODE(self);
5511 register const Py_UNICODE *e;
5512 int cased, previous_is_cased;
5513
Guido van Rossumd57fd912000-03-10 22:53:23 +00005514 /* Shortcut for single character strings */
5515 if (PyUnicode_GET_SIZE(self) == 1)
Guido van Rossum77f6a652002-04-03 22:41:51 +00005516 return PyBool_FromLong((Py_UNICODE_ISTITLE(*p) != 0) ||
5517 (Py_UNICODE_ISUPPER(*p) != 0));
Guido van Rossumd57fd912000-03-10 22:53:23 +00005518
Marc-André Lemburg60bc8092000-06-14 09:18:32 +00005519 /* Special case for empty strings */
Martin v. Löwisdea59e52006-01-05 10:00:36 +00005520 if (PyUnicode_GET_SIZE(self) == 0)
Guido van Rossum77f6a652002-04-03 22:41:51 +00005521 return PyBool_FromLong(0);
Marc-André Lemburg60bc8092000-06-14 09:18:32 +00005522
Guido van Rossumd57fd912000-03-10 22:53:23 +00005523 e = p + PyUnicode_GET_SIZE(self);
5524 cased = 0;
5525 previous_is_cased = 0;
5526 for (; p < e; p++) {
5527 register const Py_UNICODE ch = *p;
Tim Petersced69f82003-09-16 20:30:58 +00005528
Guido van Rossumd57fd912000-03-10 22:53:23 +00005529 if (Py_UNICODE_ISUPPER(ch) || Py_UNICODE_ISTITLE(ch)) {
5530 if (previous_is_cased)
Guido van Rossum77f6a652002-04-03 22:41:51 +00005531 return PyBool_FromLong(0);
Guido van Rossumd57fd912000-03-10 22:53:23 +00005532 previous_is_cased = 1;
5533 cased = 1;
5534 }
5535 else if (Py_UNICODE_ISLOWER(ch)) {
5536 if (!previous_is_cased)
Guido van Rossum77f6a652002-04-03 22:41:51 +00005537 return PyBool_FromLong(0);
Guido van Rossumd57fd912000-03-10 22:53:23 +00005538 previous_is_cased = 1;
5539 cased = 1;
5540 }
5541 else
5542 previous_is_cased = 0;
5543 }
Guido van Rossum77f6a652002-04-03 22:41:51 +00005544 return PyBool_FromLong(cased);
Guido van Rossumd57fd912000-03-10 22:53:23 +00005545}
5546
Martin v. Löwis14f8b4c2002-06-13 20:33:02 +00005547PyDoc_STRVAR(isspace__doc__,
Guido van Rossum77f6a652002-04-03 22:41:51 +00005548"S.isspace() -> bool\n\
Guido van Rossumd57fd912000-03-10 22:53:23 +00005549\n\
Martin v. Löwis6828e182003-10-18 09:55:08 +00005550Return True if all characters in S are whitespace\n\
5551and there is at least one character in S, False otherwise.");
Guido van Rossumd57fd912000-03-10 22:53:23 +00005552
5553static PyObject*
Martin v. Löwise3eb1f22001-08-16 13:15:00 +00005554unicode_isspace(PyUnicodeObject *self)
Guido van Rossumd57fd912000-03-10 22:53:23 +00005555{
5556 register const Py_UNICODE *p = PyUnicode_AS_UNICODE(self);
5557 register const Py_UNICODE *e;
5558
Guido van Rossumd57fd912000-03-10 22:53:23 +00005559 /* Shortcut for single character strings */
5560 if (PyUnicode_GET_SIZE(self) == 1 &&
5561 Py_UNICODE_ISSPACE(*p))
Guido van Rossum77f6a652002-04-03 22:41:51 +00005562 return PyBool_FromLong(1);
Guido van Rossumd57fd912000-03-10 22:53:23 +00005563
Marc-André Lemburg60bc8092000-06-14 09:18:32 +00005564 /* Special case for empty strings */
Martin v. Löwisdea59e52006-01-05 10:00:36 +00005565 if (PyUnicode_GET_SIZE(self) == 0)
Guido van Rossum77f6a652002-04-03 22:41:51 +00005566 return PyBool_FromLong(0);
Marc-André Lemburg60bc8092000-06-14 09:18:32 +00005567
Guido van Rossumd57fd912000-03-10 22:53:23 +00005568 e = p + PyUnicode_GET_SIZE(self);
5569 for (; p < e; p++) {
5570 if (!Py_UNICODE_ISSPACE(*p))
Guido van Rossum77f6a652002-04-03 22:41:51 +00005571 return PyBool_FromLong(0);
Guido van Rossumd57fd912000-03-10 22:53:23 +00005572 }
Guido van Rossum77f6a652002-04-03 22:41:51 +00005573 return PyBool_FromLong(1);
Guido van Rossumd57fd912000-03-10 22:53:23 +00005574}
5575
Martin v. Löwis14f8b4c2002-06-13 20:33:02 +00005576PyDoc_STRVAR(isalpha__doc__,
Guido van Rossum77f6a652002-04-03 22:41:51 +00005577"S.isalpha() -> bool\n\
Marc-André Lemburga7acf422000-07-05 09:49:44 +00005578\n\
Martin v. Löwis6828e182003-10-18 09:55:08 +00005579Return True if all characters in S are alphabetic\n\
Martin v. Löwis14f8b4c2002-06-13 20:33:02 +00005580and there is at least one character in S, False otherwise.");
Marc-André Lemburga7acf422000-07-05 09:49:44 +00005581
5582static PyObject*
Martin v. Löwise3eb1f22001-08-16 13:15:00 +00005583unicode_isalpha(PyUnicodeObject *self)
Marc-André Lemburga7acf422000-07-05 09:49:44 +00005584{
5585 register const Py_UNICODE *p = PyUnicode_AS_UNICODE(self);
5586 register const Py_UNICODE *e;
5587
Marc-André Lemburga7acf422000-07-05 09:49:44 +00005588 /* Shortcut for single character strings */
5589 if (PyUnicode_GET_SIZE(self) == 1 &&
5590 Py_UNICODE_ISALPHA(*p))
Guido van Rossum77f6a652002-04-03 22:41:51 +00005591 return PyBool_FromLong(1);
Marc-André Lemburga7acf422000-07-05 09:49:44 +00005592
5593 /* Special case for empty strings */
Martin v. Löwisdea59e52006-01-05 10:00:36 +00005594 if (PyUnicode_GET_SIZE(self) == 0)
Guido van Rossum77f6a652002-04-03 22:41:51 +00005595 return PyBool_FromLong(0);
Marc-André Lemburga7acf422000-07-05 09:49:44 +00005596
5597 e = p + PyUnicode_GET_SIZE(self);
5598 for (; p < e; p++) {
5599 if (!Py_UNICODE_ISALPHA(*p))
Guido van Rossum77f6a652002-04-03 22:41:51 +00005600 return PyBool_FromLong(0);
Marc-André Lemburga7acf422000-07-05 09:49:44 +00005601 }
Guido van Rossum77f6a652002-04-03 22:41:51 +00005602 return PyBool_FromLong(1);
Marc-André Lemburga7acf422000-07-05 09:49:44 +00005603}
5604
Martin v. Löwis14f8b4c2002-06-13 20:33:02 +00005605PyDoc_STRVAR(isalnum__doc__,
Guido van Rossum77f6a652002-04-03 22:41:51 +00005606"S.isalnum() -> bool\n\
Marc-André Lemburga7acf422000-07-05 09:49:44 +00005607\n\
Martin v. Löwis6828e182003-10-18 09:55:08 +00005608Return True if all characters in S are alphanumeric\n\
Martin v. Löwis14f8b4c2002-06-13 20:33:02 +00005609and there is at least one character in S, False otherwise.");
Marc-André Lemburga7acf422000-07-05 09:49:44 +00005610
5611static PyObject*
Martin v. Löwise3eb1f22001-08-16 13:15:00 +00005612unicode_isalnum(PyUnicodeObject *self)
Marc-André Lemburga7acf422000-07-05 09:49:44 +00005613{
5614 register const Py_UNICODE *p = PyUnicode_AS_UNICODE(self);
5615 register const Py_UNICODE *e;
5616
Marc-André Lemburga7acf422000-07-05 09:49:44 +00005617 /* Shortcut for single character strings */
5618 if (PyUnicode_GET_SIZE(self) == 1 &&
5619 Py_UNICODE_ISALNUM(*p))
Guido van Rossum77f6a652002-04-03 22:41:51 +00005620 return PyBool_FromLong(1);
Marc-André Lemburga7acf422000-07-05 09:49:44 +00005621
5622 /* Special case for empty strings */
Martin v. Löwisdea59e52006-01-05 10:00:36 +00005623 if (PyUnicode_GET_SIZE(self) == 0)
Guido van Rossum77f6a652002-04-03 22:41:51 +00005624 return PyBool_FromLong(0);
Marc-André Lemburga7acf422000-07-05 09:49:44 +00005625
5626 e = p + PyUnicode_GET_SIZE(self);
5627 for (; p < e; p++) {
5628 if (!Py_UNICODE_ISALNUM(*p))
Guido van Rossum77f6a652002-04-03 22:41:51 +00005629 return PyBool_FromLong(0);
Marc-André Lemburga7acf422000-07-05 09:49:44 +00005630 }
Guido van Rossum77f6a652002-04-03 22:41:51 +00005631 return PyBool_FromLong(1);
Marc-André Lemburga7acf422000-07-05 09:49:44 +00005632}
5633
Martin v. Löwis14f8b4c2002-06-13 20:33:02 +00005634PyDoc_STRVAR(isdecimal__doc__,
Guido van Rossum77f6a652002-04-03 22:41:51 +00005635"S.isdecimal() -> bool\n\
Guido van Rossumd57fd912000-03-10 22:53:23 +00005636\n\
Guido van Rossum77f6a652002-04-03 22:41:51 +00005637Return True if there are only decimal characters in S,\n\
Martin v. Löwis14f8b4c2002-06-13 20:33:02 +00005638False otherwise.");
Guido van Rossumd57fd912000-03-10 22:53:23 +00005639
5640static PyObject*
Martin v. Löwise3eb1f22001-08-16 13:15:00 +00005641unicode_isdecimal(PyUnicodeObject *self)
Guido van Rossumd57fd912000-03-10 22:53:23 +00005642{
5643 register const Py_UNICODE *p = PyUnicode_AS_UNICODE(self);
5644 register const Py_UNICODE *e;
5645
Guido van Rossumd57fd912000-03-10 22:53:23 +00005646 /* Shortcut for single character strings */
5647 if (PyUnicode_GET_SIZE(self) == 1 &&
5648 Py_UNICODE_ISDECIMAL(*p))
Guido van Rossum77f6a652002-04-03 22:41:51 +00005649 return PyBool_FromLong(1);
Guido van Rossumd57fd912000-03-10 22:53:23 +00005650
Marc-André Lemburg60bc8092000-06-14 09:18:32 +00005651 /* Special case for empty strings */
Martin v. Löwisdea59e52006-01-05 10:00:36 +00005652 if (PyUnicode_GET_SIZE(self) == 0)
Guido van Rossum77f6a652002-04-03 22:41:51 +00005653 return PyBool_FromLong(0);
Marc-André Lemburg60bc8092000-06-14 09:18:32 +00005654
Guido van Rossumd57fd912000-03-10 22:53:23 +00005655 e = p + PyUnicode_GET_SIZE(self);
5656 for (; p < e; p++) {
5657 if (!Py_UNICODE_ISDECIMAL(*p))
Guido van Rossum77f6a652002-04-03 22:41:51 +00005658 return PyBool_FromLong(0);
Guido van Rossumd57fd912000-03-10 22:53:23 +00005659 }
Guido van Rossum77f6a652002-04-03 22:41:51 +00005660 return PyBool_FromLong(1);
Guido van Rossumd57fd912000-03-10 22:53:23 +00005661}
5662
Martin v. Löwis14f8b4c2002-06-13 20:33:02 +00005663PyDoc_STRVAR(isdigit__doc__,
Guido van Rossum77f6a652002-04-03 22:41:51 +00005664"S.isdigit() -> bool\n\
Guido van Rossumd57fd912000-03-10 22:53:23 +00005665\n\
Martin v. Löwis6828e182003-10-18 09:55:08 +00005666Return True if all characters in S are digits\n\
5667and there is at least one character in S, False otherwise.");
Guido van Rossumd57fd912000-03-10 22:53:23 +00005668
5669static PyObject*
Martin v. Löwise3eb1f22001-08-16 13:15:00 +00005670unicode_isdigit(PyUnicodeObject *self)
Guido van Rossumd57fd912000-03-10 22:53:23 +00005671{
5672 register const Py_UNICODE *p = PyUnicode_AS_UNICODE(self);
5673 register const Py_UNICODE *e;
5674
Guido van Rossumd57fd912000-03-10 22:53:23 +00005675 /* Shortcut for single character strings */
5676 if (PyUnicode_GET_SIZE(self) == 1 &&
5677 Py_UNICODE_ISDIGIT(*p))
Guido van Rossum77f6a652002-04-03 22:41:51 +00005678 return PyBool_FromLong(1);
Guido van Rossumd57fd912000-03-10 22:53:23 +00005679
Marc-André Lemburg60bc8092000-06-14 09:18:32 +00005680 /* Special case for empty strings */
Martin v. Löwisdea59e52006-01-05 10:00:36 +00005681 if (PyUnicode_GET_SIZE(self) == 0)
Guido van Rossum77f6a652002-04-03 22:41:51 +00005682 return PyBool_FromLong(0);
Marc-André Lemburg60bc8092000-06-14 09:18:32 +00005683
Guido van Rossumd57fd912000-03-10 22:53:23 +00005684 e = p + PyUnicode_GET_SIZE(self);
5685 for (; p < e; p++) {
5686 if (!Py_UNICODE_ISDIGIT(*p))
Guido van Rossum77f6a652002-04-03 22:41:51 +00005687 return PyBool_FromLong(0);
Guido van Rossumd57fd912000-03-10 22:53:23 +00005688 }
Guido van Rossum77f6a652002-04-03 22:41:51 +00005689 return PyBool_FromLong(1);
Guido van Rossumd57fd912000-03-10 22:53:23 +00005690}
5691
Martin v. Löwis14f8b4c2002-06-13 20:33:02 +00005692PyDoc_STRVAR(isnumeric__doc__,
Guido van Rossum77f6a652002-04-03 22:41:51 +00005693"S.isnumeric() -> bool\n\
Guido van Rossumd57fd912000-03-10 22:53:23 +00005694\n\
Guido van Rossum77f6a652002-04-03 22:41:51 +00005695Return True if there are only numeric characters in S,\n\
Martin v. Löwis14f8b4c2002-06-13 20:33:02 +00005696False otherwise.");
Guido van Rossumd57fd912000-03-10 22:53:23 +00005697
5698static PyObject*
Martin v. Löwise3eb1f22001-08-16 13:15:00 +00005699unicode_isnumeric(PyUnicodeObject *self)
Guido van Rossumd57fd912000-03-10 22:53:23 +00005700{
5701 register const Py_UNICODE *p = PyUnicode_AS_UNICODE(self);
5702 register const Py_UNICODE *e;
5703
Guido van Rossumd57fd912000-03-10 22:53:23 +00005704 /* Shortcut for single character strings */
5705 if (PyUnicode_GET_SIZE(self) == 1 &&
5706 Py_UNICODE_ISNUMERIC(*p))
Guido van Rossum77f6a652002-04-03 22:41:51 +00005707 return PyBool_FromLong(1);
Guido van Rossumd57fd912000-03-10 22:53:23 +00005708
Marc-André Lemburg60bc8092000-06-14 09:18:32 +00005709 /* Special case for empty strings */
Martin v. Löwisdea59e52006-01-05 10:00:36 +00005710 if (PyUnicode_GET_SIZE(self) == 0)
Guido van Rossum77f6a652002-04-03 22:41:51 +00005711 return PyBool_FromLong(0);
Marc-André Lemburg60bc8092000-06-14 09:18:32 +00005712
Guido van Rossumd57fd912000-03-10 22:53:23 +00005713 e = p + PyUnicode_GET_SIZE(self);
5714 for (; p < e; p++) {
5715 if (!Py_UNICODE_ISNUMERIC(*p))
Guido van Rossum77f6a652002-04-03 22:41:51 +00005716 return PyBool_FromLong(0);
Guido van Rossumd57fd912000-03-10 22:53:23 +00005717 }
Guido van Rossum77f6a652002-04-03 22:41:51 +00005718 return PyBool_FromLong(1);
Guido van Rossumd57fd912000-03-10 22:53:23 +00005719}
5720
Martin v. Löwis14f8b4c2002-06-13 20:33:02 +00005721PyDoc_STRVAR(join__doc__,
Guido van Rossumd57fd912000-03-10 22:53:23 +00005722"S.join(sequence) -> unicode\n\
5723\n\
5724Return a string which is the concatenation of the strings in the\n\
Martin v. Löwis14f8b4c2002-06-13 20:33:02 +00005725sequence. The separator between elements is S.");
Guido van Rossumd57fd912000-03-10 22:53:23 +00005726
5727static PyObject*
Martin v. Löwise3eb1f22001-08-16 13:15:00 +00005728unicode_join(PyObject *self, PyObject *data)
Guido van Rossumd57fd912000-03-10 22:53:23 +00005729{
Martin v. Löwise3eb1f22001-08-16 13:15:00 +00005730 return PyUnicode_Join(self, data);
Guido van Rossumd57fd912000-03-10 22:53:23 +00005731}
5732
Martin v. Löwis18e16552006-02-15 17:27:45 +00005733static Py_ssize_t
Guido van Rossumd57fd912000-03-10 22:53:23 +00005734unicode_length(PyUnicodeObject *self)
5735{
5736 return self->length;
5737}
5738
Martin v. Löwis14f8b4c2002-06-13 20:33:02 +00005739PyDoc_STRVAR(ljust__doc__,
Hye-Shik Chang974ed7c2004-06-02 16:49:17 +00005740"S.ljust(width[, fillchar]) -> int\n\
Guido van Rossumd57fd912000-03-10 22:53:23 +00005741\n\
5742Return S left justified in a Unicode string of length width. Padding is\n\
Raymond Hettinger4f8f9762003-11-26 08:21:35 +00005743done using the specified fill character (default is a space).");
Guido van Rossumd57fd912000-03-10 22:53:23 +00005744
5745static PyObject *
5746unicode_ljust(PyUnicodeObject *self, PyObject *args)
5747{
Martin v. Löwis412fb672006-04-13 06:34:32 +00005748 Py_ssize_t width;
Raymond Hettinger4f8f9762003-11-26 08:21:35 +00005749 Py_UNICODE fillchar = ' ';
5750
Martin v. Löwis412fb672006-04-13 06:34:32 +00005751 if (!PyArg_ParseTuple(args, "n|O&:ljust", &width, convert_uc, &fillchar))
Guido van Rossumd57fd912000-03-10 22:53:23 +00005752 return NULL;
5753
Tim Peters7a29bd52001-09-12 03:03:31 +00005754 if (self->length >= width && PyUnicode_CheckExact(self)) {
Guido van Rossumd57fd912000-03-10 22:53:23 +00005755 Py_INCREF(self);
5756 return (PyObject*) self;
5757 }
5758
Raymond Hettinger4f8f9762003-11-26 08:21:35 +00005759 return (PyObject*) pad(self, 0, width - self->length, fillchar);
Guido van Rossumd57fd912000-03-10 22:53:23 +00005760}
5761
Martin v. Löwis14f8b4c2002-06-13 20:33:02 +00005762PyDoc_STRVAR(lower__doc__,
Guido van Rossumd57fd912000-03-10 22:53:23 +00005763"S.lower() -> unicode\n\
5764\n\
Martin v. Löwis14f8b4c2002-06-13 20:33:02 +00005765Return a copy of the string S converted to lowercase.");
Guido van Rossumd57fd912000-03-10 22:53:23 +00005766
5767static PyObject*
Martin v. Löwise3eb1f22001-08-16 13:15:00 +00005768unicode_lower(PyUnicodeObject *self)
Guido van Rossumd57fd912000-03-10 22:53:23 +00005769{
Guido van Rossumd57fd912000-03-10 22:53:23 +00005770 return fixup(self, fixlower);
5771}
5772
Walter Dörwaldde02bcb2002-04-22 17:42:37 +00005773#define LEFTSTRIP 0
5774#define RIGHTSTRIP 1
5775#define BOTHSTRIP 2
5776
5777/* Arrays indexed by above */
5778static const char *stripformat[] = {"|O:lstrip", "|O:rstrip", "|O:strip"};
5779
5780#define STRIPNAME(i) (stripformat[i]+3)
5781
Walter Dörwaldde02bcb2002-04-22 17:42:37 +00005782/* externally visible for str.strip(unicode) */
5783PyObject *
5784_PyUnicode_XStrip(PyUnicodeObject *self, int striptype, PyObject *sepobj)
5785{
5786 Py_UNICODE *s = PyUnicode_AS_UNICODE(self);
Martin v. Löwis18e16552006-02-15 17:27:45 +00005787 Py_ssize_t len = PyUnicode_GET_SIZE(self);
Walter Dörwaldde02bcb2002-04-22 17:42:37 +00005788 Py_UNICODE *sep = PyUnicode_AS_UNICODE(sepobj);
Martin v. Löwis18e16552006-02-15 17:27:45 +00005789 Py_ssize_t seplen = PyUnicode_GET_SIZE(sepobj);
5790 Py_ssize_t i, j;
Walter Dörwaldde02bcb2002-04-22 17:42:37 +00005791
Fredrik Lundhb63588c2006-05-23 18:44:25 +00005792 BLOOM_MASK sepmask = make_bloom_mask(sep, seplen);
5793
Walter Dörwaldde02bcb2002-04-22 17:42:37 +00005794 i = 0;
5795 if (striptype != RIGHTSTRIP) {
Fredrik Lundhb63588c2006-05-23 18:44:25 +00005796 while (i < len && BLOOM_MEMBER(sepmask, s[i], sep, seplen)) {
5797 i++;
5798 }
Walter Dörwaldde02bcb2002-04-22 17:42:37 +00005799 }
5800
5801 j = len;
5802 if (striptype != LEFTSTRIP) {
Fredrik Lundhb63588c2006-05-23 18:44:25 +00005803 do {
5804 j--;
5805 } while (j >= i && BLOOM_MEMBER(sepmask, s[j], sep, seplen));
5806 j++;
Walter Dörwaldde02bcb2002-04-22 17:42:37 +00005807 }
5808
5809 if (i == 0 && j == len && PyUnicode_CheckExact(self)) {
Fredrik Lundhb63588c2006-05-23 18:44:25 +00005810 Py_INCREF(self);
5811 return (PyObject*)self;
Walter Dörwaldde02bcb2002-04-22 17:42:37 +00005812 }
5813 else
Fredrik Lundhb63588c2006-05-23 18:44:25 +00005814 return PyUnicode_FromUnicode(s+i, j-i);
Walter Dörwaldde02bcb2002-04-22 17:42:37 +00005815}
5816
Guido van Rossumd57fd912000-03-10 22:53:23 +00005817
5818static PyObject *
Walter Dörwaldde02bcb2002-04-22 17:42:37 +00005819do_strip(PyUnicodeObject *self, int striptype)
Guido van Rossumd57fd912000-03-10 22:53:23 +00005820{
Walter Dörwaldde02bcb2002-04-22 17:42:37 +00005821 Py_UNICODE *s = PyUnicode_AS_UNICODE(self);
Martin v. Löwis18e16552006-02-15 17:27:45 +00005822 Py_ssize_t len = PyUnicode_GET_SIZE(self), i, j;
Walter Dörwaldde02bcb2002-04-22 17:42:37 +00005823
5824 i = 0;
5825 if (striptype != RIGHTSTRIP) {
5826 while (i < len && Py_UNICODE_ISSPACE(s[i])) {
5827 i++;
5828 }
5829 }
5830
5831 j = len;
5832 if (striptype != LEFTSTRIP) {
5833 do {
5834 j--;
5835 } while (j >= i && Py_UNICODE_ISSPACE(s[j]));
5836 j++;
5837 }
5838
5839 if (i == 0 && j == len && PyUnicode_CheckExact(self)) {
5840 Py_INCREF(self);
5841 return (PyObject*)self;
5842 }
5843 else
5844 return PyUnicode_FromUnicode(s+i, j-i);
Guido van Rossumd57fd912000-03-10 22:53:23 +00005845}
5846
Walter Dörwaldde02bcb2002-04-22 17:42:37 +00005847
5848static PyObject *
5849do_argstrip(PyUnicodeObject *self, int striptype, PyObject *args)
5850{
5851 PyObject *sep = NULL;
5852
5853 if (!PyArg_ParseTuple(args, (char *)stripformat[striptype], &sep))
5854 return NULL;
5855
5856 if (sep != NULL && sep != Py_None) {
5857 if (PyUnicode_Check(sep))
5858 return _PyUnicode_XStrip(self, striptype, sep);
5859 else if (PyString_Check(sep)) {
5860 PyObject *res;
5861 sep = PyUnicode_FromObject(sep);
5862 if (sep==NULL)
5863 return NULL;
5864 res = _PyUnicode_XStrip(self, striptype, sep);
5865 Py_DECREF(sep);
5866 return res;
5867 }
5868 else {
5869 PyErr_Format(PyExc_TypeError,
5870 "%s arg must be None, unicode or str",
5871 STRIPNAME(striptype));
5872 return NULL;
5873 }
5874 }
5875
5876 return do_strip(self, striptype);
5877}
5878
5879
Martin v. Löwis14f8b4c2002-06-13 20:33:02 +00005880PyDoc_STRVAR(strip__doc__,
Neal Norwitzffe33b72003-04-10 22:35:32 +00005881"S.strip([chars]) -> unicode\n\
Walter Dörwaldde02bcb2002-04-22 17:42:37 +00005882\n\
5883Return a copy of the string S with leading and trailing\n\
5884whitespace removed.\n\
Neal Norwitzffe33b72003-04-10 22:35:32 +00005885If chars is given and not None, remove characters in chars instead.\n\
5886If chars is a str, it will be converted to unicode before stripping");
Walter Dörwaldde02bcb2002-04-22 17:42:37 +00005887
5888static PyObject *
5889unicode_strip(PyUnicodeObject *self, PyObject *args)
5890{
5891 if (PyTuple_GET_SIZE(args) == 0)
5892 return do_strip(self, BOTHSTRIP); /* Common case */
5893 else
5894 return do_argstrip(self, BOTHSTRIP, args);
5895}
5896
5897
Martin v. Löwis14f8b4c2002-06-13 20:33:02 +00005898PyDoc_STRVAR(lstrip__doc__,
Neal Norwitzffe33b72003-04-10 22:35:32 +00005899"S.lstrip([chars]) -> unicode\n\
Walter Dörwaldde02bcb2002-04-22 17:42:37 +00005900\n\
5901Return a copy of the string S with leading whitespace removed.\n\
Neal Norwitzffe33b72003-04-10 22:35:32 +00005902If chars is given and not None, remove characters in chars instead.\n\
5903If chars is a str, it will be converted to unicode before stripping");
Walter Dörwaldde02bcb2002-04-22 17:42:37 +00005904
5905static PyObject *
5906unicode_lstrip(PyUnicodeObject *self, PyObject *args)
5907{
5908 if (PyTuple_GET_SIZE(args) == 0)
5909 return do_strip(self, LEFTSTRIP); /* Common case */
5910 else
5911 return do_argstrip(self, LEFTSTRIP, args);
5912}
5913
5914
Martin v. Löwis14f8b4c2002-06-13 20:33:02 +00005915PyDoc_STRVAR(rstrip__doc__,
Neal Norwitzffe33b72003-04-10 22:35:32 +00005916"S.rstrip([chars]) -> unicode\n\
Walter Dörwaldde02bcb2002-04-22 17:42:37 +00005917\n\
5918Return a copy of the string S with trailing whitespace removed.\n\
Neal Norwitzffe33b72003-04-10 22:35:32 +00005919If chars is given and not None, remove characters in chars instead.\n\
5920If chars is a str, it will be converted to unicode before stripping");
Walter Dörwaldde02bcb2002-04-22 17:42:37 +00005921
5922static PyObject *
5923unicode_rstrip(PyUnicodeObject *self, PyObject *args)
5924{
5925 if (PyTuple_GET_SIZE(args) == 0)
5926 return do_strip(self, RIGHTSTRIP); /* Common case */
5927 else
5928 return do_argstrip(self, RIGHTSTRIP, args);
5929}
5930
5931
Guido van Rossumd57fd912000-03-10 22:53:23 +00005932static PyObject*
Martin v. Löwis18e16552006-02-15 17:27:45 +00005933unicode_repeat(PyUnicodeObject *str, Py_ssize_t len)
Guido van Rossumd57fd912000-03-10 22:53:23 +00005934{
5935 PyUnicodeObject *u;
5936 Py_UNICODE *p;
Martin v. Löwis18e16552006-02-15 17:27:45 +00005937 Py_ssize_t nchars;
Tim Peters8f422462000-09-09 06:13:41 +00005938 size_t nbytes;
Guido van Rossumd57fd912000-03-10 22:53:23 +00005939
5940 if (len < 0)
5941 len = 0;
5942
Tim Peters7a29bd52001-09-12 03:03:31 +00005943 if (len == 1 && PyUnicode_CheckExact(str)) {
Guido van Rossumd57fd912000-03-10 22:53:23 +00005944 /* no repeat, return original string */
5945 Py_INCREF(str);
5946 return (PyObject*) str;
5947 }
Tim Peters8f422462000-09-09 06:13:41 +00005948
5949 /* ensure # of chars needed doesn't overflow int and # of bytes
5950 * needed doesn't overflow size_t
5951 */
5952 nchars = len * str->length;
5953 if (len && nchars / len != str->length) {
5954 PyErr_SetString(PyExc_OverflowError,
5955 "repeated string is too long");
5956 return NULL;
5957 }
5958 nbytes = (nchars + 1) * sizeof(Py_UNICODE);
5959 if (nbytes / sizeof(Py_UNICODE) != (size_t)(nchars + 1)) {
5960 PyErr_SetString(PyExc_OverflowError,
5961 "repeated string is too long");
5962 return NULL;
5963 }
5964 u = _PyUnicode_New(nchars);
Guido van Rossumd57fd912000-03-10 22:53:23 +00005965 if (!u)
5966 return NULL;
5967
5968 p = u->str;
5969
Fredrik Lundhf1d60a52006-05-22 16:29:30 +00005970 if (str->length == 1 && len > 0) {
5971 Py_UNICODE_FILL(p, str->str[0], len);
Fredrik Lundh8a8e05a2006-05-22 17:12:58 +00005972 } else {
Tim Peters1bacc642006-05-23 05:47:16 +00005973 Py_ssize_t done = 0; /* number of characters copied this far */
Fredrik Lundh8a8e05a2006-05-22 17:12:58 +00005974 if (done < nchars) {
Fredrik Lundhf1d60a52006-05-22 16:29:30 +00005975 Py_UNICODE_COPY(p, str->str, str->length);
Fredrik Lundh8a8e05a2006-05-22 17:12:58 +00005976 done = str->length;
5977 }
5978 while (done < nchars) {
5979 int n = (done <= nchars-done) ? done : nchars-done;
5980 Py_UNICODE_COPY(p+done, p, n);
5981 done += n;
5982 }
5983 }
Guido van Rossumd57fd912000-03-10 22:53:23 +00005984
5985 return (PyObject*) u;
5986}
5987
5988PyObject *PyUnicode_Replace(PyObject *obj,
5989 PyObject *subobj,
5990 PyObject *replobj,
Martin v. Löwis18e16552006-02-15 17:27:45 +00005991 Py_ssize_t maxcount)
Guido van Rossumd57fd912000-03-10 22:53:23 +00005992{
5993 PyObject *self;
5994 PyObject *str1;
5995 PyObject *str2;
5996 PyObject *result;
5997
5998 self = PyUnicode_FromObject(obj);
5999 if (self == NULL)
6000 return NULL;
6001 str1 = PyUnicode_FromObject(subobj);
6002 if (str1 == NULL) {
6003 Py_DECREF(self);
6004 return NULL;
6005 }
6006 str2 = PyUnicode_FromObject(replobj);
6007 if (str2 == NULL) {
6008 Py_DECREF(self);
6009 Py_DECREF(str1);
6010 return NULL;
6011 }
Tim Petersced69f82003-09-16 20:30:58 +00006012 result = replace((PyUnicodeObject *)self,
6013 (PyUnicodeObject *)str1,
6014 (PyUnicodeObject *)str2,
Guido van Rossumd57fd912000-03-10 22:53:23 +00006015 maxcount);
6016 Py_DECREF(self);
6017 Py_DECREF(str1);
6018 Py_DECREF(str2);
6019 return result;
6020}
6021
Martin v. Löwis14f8b4c2002-06-13 20:33:02 +00006022PyDoc_STRVAR(replace__doc__,
Guido van Rossumd57fd912000-03-10 22:53:23 +00006023"S.replace (old, new[, maxsplit]) -> unicode\n\
6024\n\
6025Return a copy of S with all occurrences of substring\n\
6026old replaced by new. If the optional argument maxsplit is\n\
Martin v. Löwis14f8b4c2002-06-13 20:33:02 +00006027given, only the first maxsplit occurrences are replaced.");
Guido van Rossumd57fd912000-03-10 22:53:23 +00006028
6029static PyObject*
6030unicode_replace(PyUnicodeObject *self, PyObject *args)
6031{
6032 PyUnicodeObject *str1;
6033 PyUnicodeObject *str2;
Martin v. Löwis18e16552006-02-15 17:27:45 +00006034 Py_ssize_t maxcount = -1;
Guido van Rossumd57fd912000-03-10 22:53:23 +00006035 PyObject *result;
6036
Martin v. Löwis18e16552006-02-15 17:27:45 +00006037 if (!PyArg_ParseTuple(args, "OO|n:replace", &str1, &str2, &maxcount))
Guido van Rossumd57fd912000-03-10 22:53:23 +00006038 return NULL;
6039 str1 = (PyUnicodeObject *)PyUnicode_FromObject((PyObject *)str1);
6040 if (str1 == NULL)
6041 return NULL;
6042 str2 = (PyUnicodeObject *)PyUnicode_FromObject((PyObject *)str2);
Walter Dörwaldf6b56ae2003-02-09 23:42:56 +00006043 if (str2 == NULL) {
6044 Py_DECREF(str1);
Guido van Rossumd57fd912000-03-10 22:53:23 +00006045 return NULL;
Walter Dörwaldf6b56ae2003-02-09 23:42:56 +00006046 }
Guido van Rossumd57fd912000-03-10 22:53:23 +00006047
6048 result = replace(self, str1, str2, maxcount);
6049
6050 Py_DECREF(str1);
6051 Py_DECREF(str2);
6052 return result;
6053}
6054
6055static
6056PyObject *unicode_repr(PyObject *unicode)
6057{
6058 return unicodeescape_string(PyUnicode_AS_UNICODE(unicode),
6059 PyUnicode_GET_SIZE(unicode),
6060 1);
6061}
6062
Martin v. Löwis14f8b4c2002-06-13 20:33:02 +00006063PyDoc_STRVAR(rfind__doc__,
Guido van Rossumd57fd912000-03-10 22:53:23 +00006064"S.rfind(sub [,start [,end]]) -> int\n\
6065\n\
6066Return the highest index in S where substring sub is found,\n\
6067such that sub is contained within s[start,end]. Optional\n\
6068arguments start and end are interpreted as in slice notation.\n\
6069\n\
Martin v. Löwis14f8b4c2002-06-13 20:33:02 +00006070Return -1 on failure.");
Guido van Rossumd57fd912000-03-10 22:53:23 +00006071
6072static PyObject *
6073unicode_rfind(PyUnicodeObject *self, PyObject *args)
6074{
6075 PyUnicodeObject *substring;
Martin v. Löwis18e16552006-02-15 17:27:45 +00006076 Py_ssize_t start = 0;
Martin v. Löwis412fb672006-04-13 06:34:32 +00006077 Py_ssize_t end = PY_SSIZE_T_MAX;
Guido van Rossumd57fd912000-03-10 22:53:23 +00006078 PyObject *result;
6079
Guido van Rossumb8872e62000-05-09 14:14:27 +00006080 if (!PyArg_ParseTuple(args, "O|O&O&:rfind", &substring,
6081 _PyEval_SliceIndex, &start, _PyEval_SliceIndex, &end))
Guido van Rossumd57fd912000-03-10 22:53:23 +00006082 return NULL;
6083 substring = (PyUnicodeObject *)PyUnicode_FromObject(
6084 (PyObject *)substring);
6085 if (substring == NULL)
6086 return NULL;
6087
Martin v. Löwis18e16552006-02-15 17:27:45 +00006088 result = PyInt_FromSsize_t(findstring(self, substring, start, end, -1));
Guido van Rossumd57fd912000-03-10 22:53:23 +00006089
6090 Py_DECREF(substring);
6091 return result;
6092}
6093
Martin v. Löwis14f8b4c2002-06-13 20:33:02 +00006094PyDoc_STRVAR(rindex__doc__,
Guido van Rossumd57fd912000-03-10 22:53:23 +00006095"S.rindex(sub [,start [,end]]) -> int\n\
6096\n\
Martin v. Löwis14f8b4c2002-06-13 20:33:02 +00006097Like S.rfind() but raise ValueError when the substring is not found.");
Guido van Rossumd57fd912000-03-10 22:53:23 +00006098
6099static PyObject *
6100unicode_rindex(PyUnicodeObject *self, PyObject *args)
6101{
Martin v. Löwis18e16552006-02-15 17:27:45 +00006102 Py_ssize_t result;
Guido van Rossumd57fd912000-03-10 22:53:23 +00006103 PyUnicodeObject *substring;
Martin v. Löwis18e16552006-02-15 17:27:45 +00006104 Py_ssize_t start = 0;
Martin v. Löwis412fb672006-04-13 06:34:32 +00006105 Py_ssize_t end = PY_SSIZE_T_MAX;
Guido van Rossumd57fd912000-03-10 22:53:23 +00006106
Guido van Rossumb8872e62000-05-09 14:14:27 +00006107 if (!PyArg_ParseTuple(args, "O|O&O&:rindex", &substring,
6108 _PyEval_SliceIndex, &start, _PyEval_SliceIndex, &end))
Guido van Rossumd57fd912000-03-10 22:53:23 +00006109 return NULL;
6110 substring = (PyUnicodeObject *)PyUnicode_FromObject(
6111 (PyObject *)substring);
6112 if (substring == NULL)
6113 return NULL;
6114
6115 result = findstring(self, substring, start, end, -1);
6116
6117 Py_DECREF(substring);
6118 if (result < 0) {
6119 PyErr_SetString(PyExc_ValueError, "substring not found");
6120 return NULL;
6121 }
Martin v. Löwis18e16552006-02-15 17:27:45 +00006122 return PyInt_FromSsize_t(result);
Guido van Rossumd57fd912000-03-10 22:53:23 +00006123}
6124
Martin v. Löwis14f8b4c2002-06-13 20:33:02 +00006125PyDoc_STRVAR(rjust__doc__,
Raymond Hettinger4f8f9762003-11-26 08:21:35 +00006126"S.rjust(width[, fillchar]) -> unicode\n\
Guido van Rossumd57fd912000-03-10 22:53:23 +00006127\n\
6128Return S right justified in a Unicode string of length width. Padding is\n\
Raymond Hettinger4f8f9762003-11-26 08:21:35 +00006129done using the specified fill character (default is a space).");
Guido van Rossumd57fd912000-03-10 22:53:23 +00006130
6131static PyObject *
6132unicode_rjust(PyUnicodeObject *self, PyObject *args)
6133{
Martin v. Löwis412fb672006-04-13 06:34:32 +00006134 Py_ssize_t width;
Raymond Hettinger4f8f9762003-11-26 08:21:35 +00006135 Py_UNICODE fillchar = ' ';
6136
Martin v. Löwis412fb672006-04-13 06:34:32 +00006137 if (!PyArg_ParseTuple(args, "n|O&:rjust", &width, convert_uc, &fillchar))
Guido van Rossumd57fd912000-03-10 22:53:23 +00006138 return NULL;
6139
Tim Peters7a29bd52001-09-12 03:03:31 +00006140 if (self->length >= width && PyUnicode_CheckExact(self)) {
Guido van Rossumd57fd912000-03-10 22:53:23 +00006141 Py_INCREF(self);
6142 return (PyObject*) self;
6143 }
6144
Raymond Hettinger4f8f9762003-11-26 08:21:35 +00006145 return (PyObject*) pad(self, width - self->length, 0, fillchar);
Guido van Rossumd57fd912000-03-10 22:53:23 +00006146}
6147
Guido van Rossumd57fd912000-03-10 22:53:23 +00006148static PyObject*
Martin v. Löwis18e16552006-02-15 17:27:45 +00006149unicode_slice(PyUnicodeObject *self, Py_ssize_t start, Py_ssize_t end)
Guido van Rossumd57fd912000-03-10 22:53:23 +00006150{
6151 /* standard clamping */
6152 if (start < 0)
6153 start = 0;
6154 if (end < 0)
6155 end = 0;
6156 if (end > self->length)
6157 end = self->length;
Tim Peters7a29bd52001-09-12 03:03:31 +00006158 if (start == 0 && end == self->length && PyUnicode_CheckExact(self)) {
Guido van Rossumd57fd912000-03-10 22:53:23 +00006159 /* full slice, return original string */
6160 Py_INCREF(self);
6161 return (PyObject*) self;
6162 }
6163 if (start > end)
6164 start = end;
6165 /* copy slice */
6166 return (PyObject*) PyUnicode_FromUnicode(self->str + start,
6167 end - start);
6168}
6169
6170PyObject *PyUnicode_Split(PyObject *s,
6171 PyObject *sep,
Martin v. Löwis18e16552006-02-15 17:27:45 +00006172 Py_ssize_t maxsplit)
Guido van Rossumd57fd912000-03-10 22:53:23 +00006173{
6174 PyObject *result;
Tim Petersced69f82003-09-16 20:30:58 +00006175
Guido van Rossumd57fd912000-03-10 22:53:23 +00006176 s = PyUnicode_FromObject(s);
6177 if (s == NULL)
6178 return NULL;
6179 if (sep != NULL) {
6180 sep = PyUnicode_FromObject(sep);
6181 if (sep == NULL) {
6182 Py_DECREF(s);
6183 return NULL;
6184 }
6185 }
6186
6187 result = split((PyUnicodeObject *)s, (PyUnicodeObject *)sep, maxsplit);
6188
6189 Py_DECREF(s);
6190 Py_XDECREF(sep);
6191 return result;
6192}
6193
Martin v. Löwis14f8b4c2002-06-13 20:33:02 +00006194PyDoc_STRVAR(split__doc__,
Guido van Rossumd57fd912000-03-10 22:53:23 +00006195"S.split([sep [,maxsplit]]) -> list of strings\n\
6196\n\
6197Return a list of the words in S, using sep as the\n\
6198delimiter string. If maxsplit is given, at most maxsplit\n\
Thomas Hellerca0d2cb2004-09-15 11:41:32 +00006199splits are done. If sep is not specified or is None,\n\
Walter Dörwald782afc52004-09-14 09:40:45 +00006200any whitespace string is a separator.");
Guido van Rossumd57fd912000-03-10 22:53:23 +00006201
6202static PyObject*
6203unicode_split(PyUnicodeObject *self, PyObject *args)
6204{
6205 PyObject *substring = Py_None;
Martin v. Löwis18e16552006-02-15 17:27:45 +00006206 Py_ssize_t maxcount = -1;
Guido van Rossumd57fd912000-03-10 22:53:23 +00006207
Martin v. Löwis18e16552006-02-15 17:27:45 +00006208 if (!PyArg_ParseTuple(args, "|On:split", &substring, &maxcount))
Guido van Rossumd57fd912000-03-10 22:53:23 +00006209 return NULL;
6210
6211 if (substring == Py_None)
6212 return split(self, NULL, maxcount);
6213 else if (PyUnicode_Check(substring))
6214 return split(self, (PyUnicodeObject *)substring, maxcount);
6215 else
6216 return PyUnicode_Split((PyObject *)self, substring, maxcount);
6217}
6218
Fredrik Lundh06a69dd2006-05-26 08:54:28 +00006219PyObject *
6220PyUnicode_Partition(PyObject *str_in, PyObject *sep_in)
6221{
6222 PyObject* str_obj;
6223 PyObject* sep_obj;
Fredrik Lundh06a69dd2006-05-26 08:54:28 +00006224 PyObject* out;
Fredrik Lundhb9479482006-05-26 17:22:38 +00006225
Fredrik Lundh06a69dd2006-05-26 08:54:28 +00006226 str_obj = PyUnicode_FromObject(str_in);
6227 if (!str_obj)
6228 return NULL;
6229 sep_obj = PyUnicode_FromObject(sep_in);
Fredrik Lundhb9479482006-05-26 17:22:38 +00006230 if (!sep_obj) {
6231 Py_DECREF(str_obj);
6232 return NULL;
Fredrik Lundh06a69dd2006-05-26 08:54:28 +00006233 }
6234
Fredrik Lundh58b5e842006-05-26 19:24:53 +00006235 out = stringlib_partition(
Fredrik Lundhb9479482006-05-26 17:22:38 +00006236 str_obj, PyUnicode_AS_UNICODE(str_obj), PyUnicode_GET_SIZE(str_obj),
6237 sep_obj, PyUnicode_AS_UNICODE(sep_obj), PyUnicode_GET_SIZE(sep_obj)
6238 );
Fredrik Lundh06a69dd2006-05-26 08:54:28 +00006239
Fredrik Lundhb9479482006-05-26 17:22:38 +00006240 Py_DECREF(sep_obj);
6241 Py_DECREF(str_obj);
Fredrik Lundh06a69dd2006-05-26 08:54:28 +00006242
6243 return out;
Fredrik Lundh06a69dd2006-05-26 08:54:28 +00006244}
6245
Fredrik Lundhb3167cb2006-05-26 18:15:38 +00006246
6247PyObject *
6248PyUnicode_RPartition(PyObject *str_in, PyObject *sep_in)
6249{
6250 PyObject* str_obj;
6251 PyObject* sep_obj;
6252 PyObject* out;
6253
6254 str_obj = PyUnicode_FromObject(str_in);
6255 if (!str_obj)
6256 return NULL;
6257 sep_obj = PyUnicode_FromObject(sep_in);
6258 if (!sep_obj) {
6259 Py_DECREF(str_obj);
6260 return NULL;
6261 }
6262
Fredrik Lundh58b5e842006-05-26 19:24:53 +00006263 out = stringlib_rpartition(
Fredrik Lundhb3167cb2006-05-26 18:15:38 +00006264 str_obj, PyUnicode_AS_UNICODE(str_obj), PyUnicode_GET_SIZE(str_obj),
6265 sep_obj, PyUnicode_AS_UNICODE(sep_obj), PyUnicode_GET_SIZE(sep_obj)
6266 );
6267
6268 Py_DECREF(sep_obj);
6269 Py_DECREF(str_obj);
6270
6271 return out;
6272}
6273
Fredrik Lundh06a69dd2006-05-26 08:54:28 +00006274PyDoc_STRVAR(partition__doc__,
6275"S.partition(sep) -> (head, sep, tail)\n\
6276\n\
6277Searches for the separator sep in S, and returns the part before it,\n\
6278the separator itself, and the part after it. If the separator is not\n\
6279found, returns S and two empty strings.");
6280
6281static PyObject*
Fredrik Lundh450277f2006-05-26 09:46:59 +00006282unicode_partition(PyUnicodeObject *self, PyObject *separator)
Fredrik Lundh06a69dd2006-05-26 08:54:28 +00006283{
Fredrik Lundh06a69dd2006-05-26 08:54:28 +00006284 return PyUnicode_Partition((PyObject *)self, separator);
6285}
6286
Fredrik Lundhb3167cb2006-05-26 18:15:38 +00006287PyDoc_STRVAR(rpartition__doc__,
6288"S.rpartition(sep) -> (head, sep, tail)\n\
6289\n\
6290Searches for the separator sep in S, starting at the end of S, and returns\n\
6291the part before it, the separator itself, and the part after it. If the\n\
6292separator is not found, returns S and two empty strings.");
6293
6294static PyObject*
6295unicode_rpartition(PyUnicodeObject *self, PyObject *separator)
6296{
6297 return PyUnicode_RPartition((PyObject *)self, separator);
6298}
6299
Hye-Shik Chang3ae811b2003-12-15 18:49:53 +00006300PyObject *PyUnicode_RSplit(PyObject *s,
6301 PyObject *sep,
Martin v. Löwis18e16552006-02-15 17:27:45 +00006302 Py_ssize_t maxsplit)
Hye-Shik Chang3ae811b2003-12-15 18:49:53 +00006303{
6304 PyObject *result;
6305
6306 s = PyUnicode_FromObject(s);
6307 if (s == NULL)
6308 return NULL;
6309 if (sep != NULL) {
6310 sep = PyUnicode_FromObject(sep);
6311 if (sep == NULL) {
6312 Py_DECREF(s);
6313 return NULL;
6314 }
6315 }
6316
6317 result = rsplit((PyUnicodeObject *)s, (PyUnicodeObject *)sep, maxsplit);
6318
6319 Py_DECREF(s);
6320 Py_XDECREF(sep);
6321 return result;
6322}
6323
6324PyDoc_STRVAR(rsplit__doc__,
6325"S.rsplit([sep [,maxsplit]]) -> list of strings\n\
6326\n\
6327Return a list of the words in S, using sep as the\n\
6328delimiter string, starting at the end of the string and\n\
6329working to the front. If maxsplit is given, at most maxsplit\n\
6330splits are done. If sep is not specified, any whitespace string\n\
6331is a separator.");
6332
6333static PyObject*
6334unicode_rsplit(PyUnicodeObject *self, PyObject *args)
6335{
6336 PyObject *substring = Py_None;
Martin v. Löwis18e16552006-02-15 17:27:45 +00006337 Py_ssize_t maxcount = -1;
Hye-Shik Chang3ae811b2003-12-15 18:49:53 +00006338
Martin v. Löwis18e16552006-02-15 17:27:45 +00006339 if (!PyArg_ParseTuple(args, "|On:rsplit", &substring, &maxcount))
Hye-Shik Chang3ae811b2003-12-15 18:49:53 +00006340 return NULL;
6341
6342 if (substring == Py_None)
6343 return rsplit(self, NULL, maxcount);
6344 else if (PyUnicode_Check(substring))
6345 return rsplit(self, (PyUnicodeObject *)substring, maxcount);
6346 else
6347 return PyUnicode_RSplit((PyObject *)self, substring, maxcount);
6348}
6349
Martin v. Löwis14f8b4c2002-06-13 20:33:02 +00006350PyDoc_STRVAR(splitlines__doc__,
Guido van Rossum86662912000-04-11 15:38:46 +00006351"S.splitlines([keepends]]) -> list of strings\n\
Guido van Rossumd57fd912000-03-10 22:53:23 +00006352\n\
6353Return a list of the lines in S, breaking at line boundaries.\n\
Guido van Rossum86662912000-04-11 15:38:46 +00006354Line breaks are not included in the resulting list unless keepends\n\
Martin v. Löwis14f8b4c2002-06-13 20:33:02 +00006355is given and true.");
Guido van Rossumd57fd912000-03-10 22:53:23 +00006356
6357static PyObject*
6358unicode_splitlines(PyUnicodeObject *self, PyObject *args)
6359{
Guido van Rossum86662912000-04-11 15:38:46 +00006360 int keepends = 0;
Guido van Rossumd57fd912000-03-10 22:53:23 +00006361
Guido van Rossum86662912000-04-11 15:38:46 +00006362 if (!PyArg_ParseTuple(args, "|i:splitlines", &keepends))
Guido van Rossumd57fd912000-03-10 22:53:23 +00006363 return NULL;
6364
Guido van Rossum86662912000-04-11 15:38:46 +00006365 return PyUnicode_Splitlines((PyObject *)self, keepends);
Guido van Rossumd57fd912000-03-10 22:53:23 +00006366}
6367
6368static
6369PyObject *unicode_str(PyUnicodeObject *self)
6370{
Fred Drakee4315f52000-05-09 19:53:39 +00006371 return PyUnicode_AsEncodedString((PyObject *)self, NULL, NULL);
Guido van Rossumd57fd912000-03-10 22:53:23 +00006372}
6373
Martin v. Löwis14f8b4c2002-06-13 20:33:02 +00006374PyDoc_STRVAR(swapcase__doc__,
Guido van Rossumd57fd912000-03-10 22:53:23 +00006375"S.swapcase() -> unicode\n\
6376\n\
6377Return a copy of S with uppercase characters converted to lowercase\n\
Martin v. Löwis14f8b4c2002-06-13 20:33:02 +00006378and vice versa.");
Guido van Rossumd57fd912000-03-10 22:53:23 +00006379
6380static PyObject*
Martin v. Löwise3eb1f22001-08-16 13:15:00 +00006381unicode_swapcase(PyUnicodeObject *self)
Guido van Rossumd57fd912000-03-10 22:53:23 +00006382{
Guido van Rossumd57fd912000-03-10 22:53:23 +00006383 return fixup(self, fixswapcase);
6384}
6385
Martin v. Löwis14f8b4c2002-06-13 20:33:02 +00006386PyDoc_STRVAR(translate__doc__,
Guido van Rossumd57fd912000-03-10 22:53:23 +00006387"S.translate(table) -> unicode\n\
6388\n\
6389Return a copy of the string S, where all characters have been mapped\n\
6390through the given translation table, which must be a mapping of\n\
Walter Dörwald5c1ee172002-09-04 20:31:32 +00006391Unicode ordinals to Unicode ordinals, Unicode strings or None.\n\
6392Unmapped characters are left untouched. Characters mapped to None\n\
6393are deleted.");
Guido van Rossumd57fd912000-03-10 22:53:23 +00006394
6395static PyObject*
Martin v. Löwise3eb1f22001-08-16 13:15:00 +00006396unicode_translate(PyUnicodeObject *self, PyObject *table)
Guido van Rossumd57fd912000-03-10 22:53:23 +00006397{
Tim Petersced69f82003-09-16 20:30:58 +00006398 return PyUnicode_TranslateCharmap(self->str,
Guido van Rossumd57fd912000-03-10 22:53:23 +00006399 self->length,
Tim Petersced69f82003-09-16 20:30:58 +00006400 table,
Guido van Rossumd57fd912000-03-10 22:53:23 +00006401 "ignore");
6402}
6403
Martin v. Löwis14f8b4c2002-06-13 20:33:02 +00006404PyDoc_STRVAR(upper__doc__,
Guido van Rossumd57fd912000-03-10 22:53:23 +00006405"S.upper() -> unicode\n\
6406\n\
Martin v. Löwis14f8b4c2002-06-13 20:33:02 +00006407Return a copy of S converted to uppercase.");
Guido van Rossumd57fd912000-03-10 22:53:23 +00006408
6409static PyObject*
Martin v. Löwise3eb1f22001-08-16 13:15:00 +00006410unicode_upper(PyUnicodeObject *self)
Guido van Rossumd57fd912000-03-10 22:53:23 +00006411{
Guido van Rossumd57fd912000-03-10 22:53:23 +00006412 return fixup(self, fixupper);
6413}
6414
Martin v. Löwis14f8b4c2002-06-13 20:33:02 +00006415PyDoc_STRVAR(zfill__doc__,
Guido van Rossumd57fd912000-03-10 22:53:23 +00006416"S.zfill(width) -> unicode\n\
6417\n\
6418Pad a numeric string x with zeros on the left, to fill a field\n\
Martin v. Löwis14f8b4c2002-06-13 20:33:02 +00006419of the specified width. The string x is never truncated.");
Guido van Rossumd57fd912000-03-10 22:53:23 +00006420
6421static PyObject *
6422unicode_zfill(PyUnicodeObject *self, PyObject *args)
6423{
Martin v. Löwis18e16552006-02-15 17:27:45 +00006424 Py_ssize_t fill;
Guido van Rossumd57fd912000-03-10 22:53:23 +00006425 PyUnicodeObject *u;
6426
Martin v. Löwis18e16552006-02-15 17:27:45 +00006427 Py_ssize_t width;
6428 if (!PyArg_ParseTuple(args, "n:zfill", &width))
Guido van Rossumd57fd912000-03-10 22:53:23 +00006429 return NULL;
6430
6431 if (self->length >= width) {
Walter Dörwald0fe940c2002-04-15 18:42:15 +00006432 if (PyUnicode_CheckExact(self)) {
6433 Py_INCREF(self);
6434 return (PyObject*) self;
6435 }
6436 else
6437 return PyUnicode_FromUnicode(
6438 PyUnicode_AS_UNICODE(self),
6439 PyUnicode_GET_SIZE(self)
6440 );
Guido van Rossumd57fd912000-03-10 22:53:23 +00006441 }
6442
6443 fill = width - self->length;
6444
6445 u = pad(self, fill, 0, '0');
6446
Walter Dörwald068325e2002-04-15 13:36:47 +00006447 if (u == NULL)
6448 return NULL;
6449
Guido van Rossumd57fd912000-03-10 22:53:23 +00006450 if (u->str[fill] == '+' || u->str[fill] == '-') {
6451 /* move sign to beginning of string */
6452 u->str[0] = u->str[fill];
6453 u->str[fill] = '0';
6454 }
6455
6456 return (PyObject*) u;
6457}
Guido van Rossumd57fd912000-03-10 22:53:23 +00006458
6459#if 0
6460static PyObject*
Martin v. Löwise3eb1f22001-08-16 13:15:00 +00006461unicode_freelistsize(PyUnicodeObject *self)
Guido van Rossumd57fd912000-03-10 22:53:23 +00006462{
Guido van Rossumd57fd912000-03-10 22:53:23 +00006463 return PyInt_FromLong(unicode_freelist_size);
6464}
6465#endif
6466
Martin v. Löwis14f8b4c2002-06-13 20:33:02 +00006467PyDoc_STRVAR(startswith__doc__,
Guido van Rossum77f6a652002-04-03 22:41:51 +00006468"S.startswith(prefix[, start[, end]]) -> bool\n\
Guido van Rossumd57fd912000-03-10 22:53:23 +00006469\n\
Guido van Rossuma7132182003-04-09 19:32:45 +00006470Return True if S starts with the specified prefix, False otherwise.\n\
6471With optional start, test S beginning at that position.\n\
6472With optional end, stop comparing S at that position.");
Guido van Rossumd57fd912000-03-10 22:53:23 +00006473
6474static PyObject *
6475unicode_startswith(PyUnicodeObject *self,
6476 PyObject *args)
6477{
6478 PyUnicodeObject *substring;
Martin v. Löwis18e16552006-02-15 17:27:45 +00006479 Py_ssize_t start = 0;
Martin v. Löwis412fb672006-04-13 06:34:32 +00006480 Py_ssize_t end = PY_SSIZE_T_MAX;
Guido van Rossumd57fd912000-03-10 22:53:23 +00006481 PyObject *result;
6482
Guido van Rossumb8872e62000-05-09 14:14:27 +00006483 if (!PyArg_ParseTuple(args, "O|O&O&:startswith", &substring,
6484 _PyEval_SliceIndex, &start, _PyEval_SliceIndex, &end))
Guido van Rossumd57fd912000-03-10 22:53:23 +00006485 return NULL;
6486 substring = (PyUnicodeObject *)PyUnicode_FromObject(
6487 (PyObject *)substring);
6488 if (substring == NULL)
6489 return NULL;
6490
Guido van Rossum77f6a652002-04-03 22:41:51 +00006491 result = PyBool_FromLong(tailmatch(self, substring, start, end, -1));
Guido van Rossumd57fd912000-03-10 22:53:23 +00006492
6493 Py_DECREF(substring);
6494 return result;
6495}
6496
6497
Martin v. Löwis14f8b4c2002-06-13 20:33:02 +00006498PyDoc_STRVAR(endswith__doc__,
Guido van Rossum77f6a652002-04-03 22:41:51 +00006499"S.endswith(suffix[, start[, end]]) -> bool\n\
Guido van Rossumd57fd912000-03-10 22:53:23 +00006500\n\
Guido van Rossuma7132182003-04-09 19:32:45 +00006501Return True if S ends with the specified suffix, False otherwise.\n\
6502With optional start, test S beginning at that position.\n\
6503With optional end, stop comparing S at that position.");
Guido van Rossumd57fd912000-03-10 22:53:23 +00006504
6505static PyObject *
6506unicode_endswith(PyUnicodeObject *self,
6507 PyObject *args)
6508{
6509 PyUnicodeObject *substring;
Martin v. Löwis18e16552006-02-15 17:27:45 +00006510 Py_ssize_t start = 0;
Martin v. Löwis412fb672006-04-13 06:34:32 +00006511 Py_ssize_t end = PY_SSIZE_T_MAX;
Guido van Rossumd57fd912000-03-10 22:53:23 +00006512 PyObject *result;
6513
Guido van Rossumb8872e62000-05-09 14:14:27 +00006514 if (!PyArg_ParseTuple(args, "O|O&O&:endswith", &substring,
6515 _PyEval_SliceIndex, &start, _PyEval_SliceIndex, &end))
Guido van Rossumd57fd912000-03-10 22:53:23 +00006516 return NULL;
6517 substring = (PyUnicodeObject *)PyUnicode_FromObject(
6518 (PyObject *)substring);
6519 if (substring == NULL)
6520 return NULL;
6521
Guido van Rossum77f6a652002-04-03 22:41:51 +00006522 result = PyBool_FromLong(tailmatch(self, substring, start, end, +1));
Guido van Rossumd57fd912000-03-10 22:53:23 +00006523
6524 Py_DECREF(substring);
6525 return result;
6526}
6527
6528
Guido van Rossum5d9113d2003-01-29 17:58:45 +00006529
6530static PyObject *
6531unicode_getnewargs(PyUnicodeObject *v)
6532{
6533 return Py_BuildValue("(u#)", v->str, v->length);
6534}
6535
6536
Guido van Rossumd57fd912000-03-10 22:53:23 +00006537static PyMethodDef unicode_methods[] = {
6538
6539 /* Order is according to common usage: often used methods should
6540 appear first, since lookup is done sequentially. */
6541
Georg Brandlecdc0a92006-03-30 12:19:07 +00006542 {"encode", (PyCFunction) unicode_encode, METH_VARARGS, encode__doc__},
Martin v. Löwise3eb1f22001-08-16 13:15:00 +00006543 {"replace", (PyCFunction) unicode_replace, METH_VARARGS, replace__doc__},
6544 {"split", (PyCFunction) unicode_split, METH_VARARGS, split__doc__},
Hye-Shik Chang3ae811b2003-12-15 18:49:53 +00006545 {"rsplit", (PyCFunction) unicode_rsplit, METH_VARARGS, rsplit__doc__},
Martin v. Löwise3eb1f22001-08-16 13:15:00 +00006546 {"join", (PyCFunction) unicode_join, METH_O, join__doc__},
6547 {"capitalize", (PyCFunction) unicode_capitalize, METH_NOARGS, capitalize__doc__},
6548 {"title", (PyCFunction) unicode_title, METH_NOARGS, title__doc__},
6549 {"center", (PyCFunction) unicode_center, METH_VARARGS, center__doc__},
6550 {"count", (PyCFunction) unicode_count, METH_VARARGS, count__doc__},
6551 {"expandtabs", (PyCFunction) unicode_expandtabs, METH_VARARGS, expandtabs__doc__},
6552 {"find", (PyCFunction) unicode_find, METH_VARARGS, find__doc__},
Fredrik Lundh450277f2006-05-26 09:46:59 +00006553 {"partition", (PyCFunction) unicode_partition, METH_O, partition__doc__},
Martin v. Löwise3eb1f22001-08-16 13:15:00 +00006554 {"index", (PyCFunction) unicode_index, METH_VARARGS, index__doc__},
6555 {"ljust", (PyCFunction) unicode_ljust, METH_VARARGS, ljust__doc__},
6556 {"lower", (PyCFunction) unicode_lower, METH_NOARGS, lower__doc__},
Walter Dörwaldde02bcb2002-04-22 17:42:37 +00006557 {"lstrip", (PyCFunction) unicode_lstrip, METH_VARARGS, lstrip__doc__},
Marc-André Lemburgd2d45982004-07-08 17:57:32 +00006558 {"decode", (PyCFunction) unicode_decode, METH_VARARGS, decode__doc__},
Martin v. Löwise3eb1f22001-08-16 13:15:00 +00006559/* {"maketrans", (PyCFunction) unicode_maketrans, METH_VARARGS, maketrans__doc__}, */
6560 {"rfind", (PyCFunction) unicode_rfind, METH_VARARGS, rfind__doc__},
6561 {"rindex", (PyCFunction) unicode_rindex, METH_VARARGS, rindex__doc__},
6562 {"rjust", (PyCFunction) unicode_rjust, METH_VARARGS, rjust__doc__},
Walter Dörwaldde02bcb2002-04-22 17:42:37 +00006563 {"rstrip", (PyCFunction) unicode_rstrip, METH_VARARGS, rstrip__doc__},
Fredrik Lundhb3167cb2006-05-26 18:15:38 +00006564 {"rpartition", (PyCFunction) unicode_rpartition, METH_O, rpartition__doc__},
Martin v. Löwise3eb1f22001-08-16 13:15:00 +00006565 {"splitlines", (PyCFunction) unicode_splitlines, METH_VARARGS, splitlines__doc__},
Walter Dörwaldde02bcb2002-04-22 17:42:37 +00006566 {"strip", (PyCFunction) unicode_strip, METH_VARARGS, strip__doc__},
Martin v. Löwise3eb1f22001-08-16 13:15:00 +00006567 {"swapcase", (PyCFunction) unicode_swapcase, METH_NOARGS, swapcase__doc__},
6568 {"translate", (PyCFunction) unicode_translate, METH_O, translate__doc__},
6569 {"upper", (PyCFunction) unicode_upper, METH_NOARGS, upper__doc__},
6570 {"startswith", (PyCFunction) unicode_startswith, METH_VARARGS, startswith__doc__},
6571 {"endswith", (PyCFunction) unicode_endswith, METH_VARARGS, endswith__doc__},
6572 {"islower", (PyCFunction) unicode_islower, METH_NOARGS, islower__doc__},
6573 {"isupper", (PyCFunction) unicode_isupper, METH_NOARGS, isupper__doc__},
6574 {"istitle", (PyCFunction) unicode_istitle, METH_NOARGS, istitle__doc__},
6575 {"isspace", (PyCFunction) unicode_isspace, METH_NOARGS, isspace__doc__},
6576 {"isdecimal", (PyCFunction) unicode_isdecimal, METH_NOARGS, isdecimal__doc__},
6577 {"isdigit", (PyCFunction) unicode_isdigit, METH_NOARGS, isdigit__doc__},
6578 {"isnumeric", (PyCFunction) unicode_isnumeric, METH_NOARGS, isnumeric__doc__},
6579 {"isalpha", (PyCFunction) unicode_isalpha, METH_NOARGS, isalpha__doc__},
6580 {"isalnum", (PyCFunction) unicode_isalnum, METH_NOARGS, isalnum__doc__},
Martin v. Löwise3eb1f22001-08-16 13:15:00 +00006581 {"zfill", (PyCFunction) unicode_zfill, METH_VARARGS, zfill__doc__},
Walter Dörwald068325e2002-04-15 13:36:47 +00006582#if 0
Martin v. Löwise3eb1f22001-08-16 13:15:00 +00006583 {"capwords", (PyCFunction) unicode_capwords, METH_NOARGS, capwords__doc__},
Guido van Rossumd57fd912000-03-10 22:53:23 +00006584#endif
6585
6586#if 0
6587 /* This one is just used for debugging the implementation. */
Martin v. Löwise3eb1f22001-08-16 13:15:00 +00006588 {"freelistsize", (PyCFunction) unicode_freelistsize, METH_NOARGS},
Guido van Rossumd57fd912000-03-10 22:53:23 +00006589#endif
6590
Guido van Rossum5d9113d2003-01-29 17:58:45 +00006591 {"__getnewargs__", (PyCFunction)unicode_getnewargs, METH_NOARGS},
Guido van Rossumd57fd912000-03-10 22:53:23 +00006592 {NULL, NULL}
6593};
6594
Neil Schemenauerce30bc92002-11-18 16:10:18 +00006595static PyObject *
6596unicode_mod(PyObject *v, PyObject *w)
6597{
6598 if (!PyUnicode_Check(v)) {
6599 Py_INCREF(Py_NotImplemented);
6600 return Py_NotImplemented;
6601 }
6602 return PyUnicode_Format(v, w);
6603}
6604
6605static PyNumberMethods unicode_as_number = {
6606 0, /*nb_add*/
6607 0, /*nb_subtract*/
6608 0, /*nb_multiply*/
6609 0, /*nb_divide*/
6610 unicode_mod, /*nb_remainder*/
6611};
6612
Guido van Rossumd57fd912000-03-10 22:53:23 +00006613static PySequenceMethods unicode_as_sequence = {
Martin v. Löwis18e16552006-02-15 17:27:45 +00006614 (lenfunc) unicode_length, /* sq_length */
Georg Brandl347b3002006-03-30 11:57:00 +00006615 PyUnicode_Concat, /* sq_concat */
Martin v. Löwis18e16552006-02-15 17:27:45 +00006616 (ssizeargfunc) unicode_repeat, /* sq_repeat */
6617 (ssizeargfunc) unicode_getitem, /* sq_item */
6618 (ssizessizeargfunc) unicode_slice, /* sq_slice */
Guido van Rossumd57fd912000-03-10 22:53:23 +00006619 0, /* sq_ass_item */
6620 0, /* sq_ass_slice */
Georg Brandl347b3002006-03-30 11:57:00 +00006621 PyUnicode_Contains, /* sq_contains */
Guido van Rossumd57fd912000-03-10 22:53:23 +00006622};
6623
Guido van Rossum38fff8c2006-03-07 18:50:55 +00006624#define HASINDEX(o) PyType_HasFeature((o)->ob_type, Py_TPFLAGS_HAVE_INDEX)
6625
Michael W. Hudson5efaf7e2002-06-11 10:55:12 +00006626static PyObject*
6627unicode_subscript(PyUnicodeObject* self, PyObject* item)
6628{
Guido van Rossum38fff8c2006-03-07 18:50:55 +00006629 PyNumberMethods *nb = item->ob_type->tp_as_number;
6630 if (nb != NULL && HASINDEX(item) && nb->nb_index != NULL) {
6631 Py_ssize_t i = nb->nb_index(item);
Michael W. Hudson5efaf7e2002-06-11 10:55:12 +00006632 if (i == -1 && PyErr_Occurred())
6633 return NULL;
6634 if (i < 0)
Martin v. Löwisdea59e52006-01-05 10:00:36 +00006635 i += PyUnicode_GET_SIZE(self);
Michael W. Hudson5efaf7e2002-06-11 10:55:12 +00006636 return unicode_getitem(self, i);
6637 } else if (PySlice_Check(item)) {
Martin v. Löwis18e16552006-02-15 17:27:45 +00006638 Py_ssize_t start, stop, step, slicelength, cur, i;
Michael W. Hudson5efaf7e2002-06-11 10:55:12 +00006639 Py_UNICODE* source_buf;
6640 Py_UNICODE* result_buf;
6641 PyObject* result;
6642
Martin v. Löwisdea59e52006-01-05 10:00:36 +00006643 if (PySlice_GetIndicesEx((PySliceObject*)item, PyUnicode_GET_SIZE(self),
Michael W. Hudson5efaf7e2002-06-11 10:55:12 +00006644 &start, &stop, &step, &slicelength) < 0) {
6645 return NULL;
6646 }
6647
6648 if (slicelength <= 0) {
6649 return PyUnicode_FromUnicode(NULL, 0);
6650 } else {
6651 source_buf = PyUnicode_AS_UNICODE((PyObject*)self);
Anthony Baxtera6286212006-04-11 07:42:36 +00006652 result_buf = (Py_UNICODE *)PyMem_MALLOC(slicelength*
6653 sizeof(Py_UNICODE));
Martin v. Löwisdea59e52006-01-05 10:00:36 +00006654
6655 if (result_buf == NULL)
6656 return PyErr_NoMemory();
Michael W. Hudson5efaf7e2002-06-11 10:55:12 +00006657
6658 for (cur = start, i = 0; i < slicelength; cur += step, i++) {
6659 result_buf[i] = source_buf[cur];
6660 }
Tim Petersced69f82003-09-16 20:30:58 +00006661
Michael W. Hudson5efaf7e2002-06-11 10:55:12 +00006662 result = PyUnicode_FromUnicode(result_buf, slicelength);
6663 PyMem_FREE(result_buf);
6664 return result;
6665 }
6666 } else {
6667 PyErr_SetString(PyExc_TypeError, "string indices must be integers");
6668 return NULL;
6669 }
6670}
6671
6672static PyMappingMethods unicode_as_mapping = {
Martin v. Löwis18e16552006-02-15 17:27:45 +00006673 (lenfunc)unicode_length, /* mp_length */
Michael W. Hudson5efaf7e2002-06-11 10:55:12 +00006674 (binaryfunc)unicode_subscript, /* mp_subscript */
6675 (objobjargproc)0, /* mp_ass_subscript */
6676};
6677
Martin v. Löwis18e16552006-02-15 17:27:45 +00006678static Py_ssize_t
Guido van Rossumd57fd912000-03-10 22:53:23 +00006679unicode_buffer_getreadbuf(PyUnicodeObject *self,
Martin v. Löwis18e16552006-02-15 17:27:45 +00006680 Py_ssize_t index,
Guido van Rossumd57fd912000-03-10 22:53:23 +00006681 const void **ptr)
6682{
6683 if (index != 0) {
6684 PyErr_SetString(PyExc_SystemError,
6685 "accessing non-existent unicode segment");
6686 return -1;
6687 }
6688 *ptr = (void *) self->str;
6689 return PyUnicode_GET_DATA_SIZE(self);
6690}
6691
Martin v. Löwis18e16552006-02-15 17:27:45 +00006692static Py_ssize_t
6693unicode_buffer_getwritebuf(PyUnicodeObject *self, Py_ssize_t index,
Guido van Rossumd57fd912000-03-10 22:53:23 +00006694 const void **ptr)
6695{
6696 PyErr_SetString(PyExc_TypeError,
Neal Norwitz20e72132002-06-13 21:25:17 +00006697 "cannot use unicode as modifiable buffer");
Guido van Rossumd57fd912000-03-10 22:53:23 +00006698 return -1;
6699}
6700
6701static int
6702unicode_buffer_getsegcount(PyUnicodeObject *self,
Martin v. Löwis18e16552006-02-15 17:27:45 +00006703 Py_ssize_t *lenp)
Guido van Rossumd57fd912000-03-10 22:53:23 +00006704{
6705 if (lenp)
6706 *lenp = PyUnicode_GET_DATA_SIZE(self);
6707 return 1;
6708}
6709
Martin v. Löwiseb079f12006-02-16 14:32:27 +00006710static Py_ssize_t
Guido van Rossumd57fd912000-03-10 22:53:23 +00006711unicode_buffer_getcharbuf(PyUnicodeObject *self,
Martin v. Löwis18e16552006-02-15 17:27:45 +00006712 Py_ssize_t index,
Guido van Rossumd57fd912000-03-10 22:53:23 +00006713 const void **ptr)
6714{
6715 PyObject *str;
Tim Petersced69f82003-09-16 20:30:58 +00006716
Guido van Rossumd57fd912000-03-10 22:53:23 +00006717 if (index != 0) {
6718 PyErr_SetString(PyExc_SystemError,
6719 "accessing non-existent unicode segment");
6720 return -1;
6721 }
Marc-André Lemburgbff879c2000-08-03 18:46:08 +00006722 str = _PyUnicode_AsDefaultEncodedString((PyObject *)self, NULL);
Guido van Rossumd57fd912000-03-10 22:53:23 +00006723 if (str == NULL)
6724 return -1;
6725 *ptr = (void *) PyString_AS_STRING(str);
6726 return PyString_GET_SIZE(str);
6727}
6728
6729/* Helpers for PyUnicode_Format() */
6730
6731static PyObject *
Martin v. Löwis18e16552006-02-15 17:27:45 +00006732getnextarg(PyObject *args, Py_ssize_t arglen, Py_ssize_t *p_argidx)
Guido van Rossumd57fd912000-03-10 22:53:23 +00006733{
Martin v. Löwis18e16552006-02-15 17:27:45 +00006734 Py_ssize_t argidx = *p_argidx;
Guido van Rossumd57fd912000-03-10 22:53:23 +00006735 if (argidx < arglen) {
6736 (*p_argidx)++;
6737 if (arglen < 0)
6738 return args;
6739 else
6740 return PyTuple_GetItem(args, argidx);
6741 }
6742 PyErr_SetString(PyExc_TypeError,
6743 "not enough arguments for format string");
6744 return NULL;
6745}
6746
6747#define F_LJUST (1<<0)
6748#define F_SIGN (1<<1)
6749#define F_BLANK (1<<2)
6750#define F_ALT (1<<3)
6751#define F_ZERO (1<<4)
6752
Martin v. Löwis18e16552006-02-15 17:27:45 +00006753static Py_ssize_t
Neal Norwitzfc76d632006-01-10 06:03:13 +00006754strtounicode(Py_UNICODE *buffer, const char *charbuffer)
Guido van Rossumd57fd912000-03-10 22:53:23 +00006755{
Martin v. Löwis18e16552006-02-15 17:27:45 +00006756 register Py_ssize_t i;
6757 Py_ssize_t len = strlen(charbuffer);
Guido van Rossumd57fd912000-03-10 22:53:23 +00006758 for (i = len - 1; i >= 0; i--)
6759 buffer[i] = (Py_UNICODE) charbuffer[i];
6760
Guido van Rossumd57fd912000-03-10 22:53:23 +00006761 return len;
6762}
6763
Neal Norwitzfc76d632006-01-10 06:03:13 +00006764static int
6765doubletounicode(Py_UNICODE *buffer, size_t len, const char *format, double x)
6766{
Tim Peters15231542006-02-16 01:08:01 +00006767 Py_ssize_t result;
6768
Neal Norwitzfc76d632006-01-10 06:03:13 +00006769 PyOS_ascii_formatd((char *)buffer, len, format, x);
Tim Peters15231542006-02-16 01:08:01 +00006770 result = strtounicode(buffer, (char *)buffer);
6771 return Py_SAFE_DOWNCAST(result, Py_ssize_t, int);
Neal Norwitzfc76d632006-01-10 06:03:13 +00006772}
6773
6774static int
6775longtounicode(Py_UNICODE *buffer, size_t len, const char *format, long x)
6776{
Tim Peters15231542006-02-16 01:08:01 +00006777 Py_ssize_t result;
6778
Neal Norwitzfc76d632006-01-10 06:03:13 +00006779 PyOS_snprintf((char *)buffer, len, format, x);
Tim Peters15231542006-02-16 01:08:01 +00006780 result = strtounicode(buffer, (char *)buffer);
6781 return Py_SAFE_DOWNCAST(result, Py_ssize_t, int);
Neal Norwitzfc76d632006-01-10 06:03:13 +00006782}
6783
Guido van Rossum078151d2002-08-11 04:24:12 +00006784/* XXX To save some code duplication, formatfloat/long/int could have been
6785 shared with stringobject.c, converting from 8-bit to Unicode after the
6786 formatting is done. */
6787
Guido van Rossumd57fd912000-03-10 22:53:23 +00006788static int
6789formatfloat(Py_UNICODE *buf,
Marc-André Lemburgf28dd832000-06-30 10:29:57 +00006790 size_t buflen,
Guido van Rossumd57fd912000-03-10 22:53:23 +00006791 int flags,
6792 int prec,
6793 int type,
6794 PyObject *v)
6795{
Marc-André Lemburgf28dd832000-06-30 10:29:57 +00006796 /* fmt = '%#.' + `prec` + `type`
6797 worst case length = 3 + 10 (len of INT_MAX) + 1 = 14 (use 20)*/
Guido van Rossumd57fd912000-03-10 22:53:23 +00006798 char fmt[20];
6799 double x;
Tim Petersced69f82003-09-16 20:30:58 +00006800
Guido van Rossumd57fd912000-03-10 22:53:23 +00006801 x = PyFloat_AsDouble(v);
6802 if (x == -1.0 && PyErr_Occurred())
6803 return -1;
6804 if (prec < 0)
6805 prec = 6;
Guido van Rossumd57fd912000-03-10 22:53:23 +00006806 if (type == 'f' && (fabs(x) / 1e25) >= 1e25)
6807 type = 'g';
Marc-André Lemburg79f57832002-12-29 19:44:06 +00006808 /* Worst case length calc to ensure no buffer overrun:
6809
6810 'g' formats:
6811 fmt = %#.<prec>g
6812 buf = '-' + [0-9]*prec + '.' + 'e+' + (longest exp
6813 for any double rep.)
6814 len = 1 + prec + 1 + 2 + 5 = 9 + prec
6815
6816 'f' formats:
6817 buf = '-' + [0-9]*x + '.' + [0-9]*prec (with x < 50)
6818 len = 1 + 50 + 1 + prec = 52 + prec
6819
Marc-André Lemburgf28dd832000-06-30 10:29:57 +00006820 If prec=0 the effective precision is 1 (the leading digit is
Tim Petersced69f82003-09-16 20:30:58 +00006821 always given), therefore increase the length by one.
Marc-André Lemburg79f57832002-12-29 19:44:06 +00006822
6823 */
6824 if ((type == 'g' && buflen <= (size_t)10 + (size_t)prec) ||
6825 (type == 'f' && buflen <= (size_t)53 + (size_t)prec)) {
Marc-André Lemburgf28dd832000-06-30 10:29:57 +00006826 PyErr_SetString(PyExc_OverflowError,
Marc-André Lemburg79f57832002-12-29 19:44:06 +00006827 "formatted float is too long (precision too large?)");
Marc-André Lemburgf28dd832000-06-30 10:29:57 +00006828 return -1;
6829 }
Marc-André Lemburg79f57832002-12-29 19:44:06 +00006830 PyOS_snprintf(fmt, sizeof(fmt), "%%%s.%d%c",
6831 (flags&F_ALT) ? "#" : "",
6832 prec, type);
Neal Norwitzfc76d632006-01-10 06:03:13 +00006833 return doubletounicode(buf, buflen, fmt, x);
Guido van Rossumd57fd912000-03-10 22:53:23 +00006834}
6835
Tim Peters38fd5b62000-09-21 05:43:11 +00006836static PyObject*
6837formatlong(PyObject *val, int flags, int prec, int type)
6838{
6839 char *buf;
6840 int i, len;
6841 PyObject *str; /* temporary string object. */
6842 PyUnicodeObject *result;
6843
6844 str = _PyString_FormatLong(val, flags, prec, type, &buf, &len);
6845 if (!str)
6846 return NULL;
6847 result = _PyUnicode_New(len);
Hye-Shik Chang4af5c8c2006-03-07 15:39:21 +00006848 if (!result) {
6849 Py_DECREF(str);
6850 return NULL;
6851 }
Tim Peters38fd5b62000-09-21 05:43:11 +00006852 for (i = 0; i < len; i++)
6853 result->str[i] = buf[i];
6854 result->str[len] = 0;
6855 Py_DECREF(str);
6856 return (PyObject*)result;
6857}
6858
Guido van Rossumd57fd912000-03-10 22:53:23 +00006859static int
6860formatint(Py_UNICODE *buf,
Marc-André Lemburgf28dd832000-06-30 10:29:57 +00006861 size_t buflen,
Guido van Rossumd57fd912000-03-10 22:53:23 +00006862 int flags,
6863 int prec,
6864 int type,
6865 PyObject *v)
6866{
Marc-André Lemburgf28dd832000-06-30 10:29:57 +00006867 /* fmt = '%#.' + `prec` + 'l' + `type`
Andrew MacIntyre5e9c80d2002-02-28 11:38:24 +00006868 * worst case length = 3 + 19 (worst len of INT_MAX on 64-bit machine)
6869 * + 1 + 1
6870 * = 24
6871 */
Tim Peters38fd5b62000-09-21 05:43:11 +00006872 char fmt[64]; /* plenty big enough! */
Guido van Rossum6c9e1302003-11-29 23:52:13 +00006873 char *sign;
Guido van Rossumd57fd912000-03-10 22:53:23 +00006874 long x;
6875
6876 x = PyInt_AsLong(v);
6877 if (x == -1 && PyErr_Occurred())
Andrew MacIntyre5e9c80d2002-02-28 11:38:24 +00006878 return -1;
Guido van Rossum6c9e1302003-11-29 23:52:13 +00006879 if (x < 0 && type == 'u') {
6880 type = 'd';
Guido van Rossum078151d2002-08-11 04:24:12 +00006881 }
Guido van Rossum6c9e1302003-11-29 23:52:13 +00006882 if (x < 0 && (type == 'x' || type == 'X' || type == 'o'))
6883 sign = "-";
6884 else
6885 sign = "";
Guido van Rossumd57fd912000-03-10 22:53:23 +00006886 if (prec < 0)
Andrew MacIntyre5e9c80d2002-02-28 11:38:24 +00006887 prec = 1;
6888
Guido van Rossum6c9e1302003-11-29 23:52:13 +00006889 /* buf = '+'/'-'/'' + '0'/'0x'/'' + '[0-9]'*max(prec, len(x in octal))
6890 * worst case buf = '-0x' + [0-9]*prec, where prec >= 11
Andrew MacIntyre5e9c80d2002-02-28 11:38:24 +00006891 */
Guido van Rossum6c9e1302003-11-29 23:52:13 +00006892 if (buflen <= 14 || buflen <= (size_t)3 + (size_t)prec) {
Marc-André Lemburgf28dd832000-06-30 10:29:57 +00006893 PyErr_SetString(PyExc_OverflowError,
Andrew MacIntyre5e9c80d2002-02-28 11:38:24 +00006894 "formatted integer is too long (precision too large?)");
Marc-André Lemburgf28dd832000-06-30 10:29:57 +00006895 return -1;
6896 }
Andrew MacIntyre5e9c80d2002-02-28 11:38:24 +00006897
6898 if ((flags & F_ALT) &&
6899 (type == 'x' || type == 'X')) {
Tim Petersced69f82003-09-16 20:30:58 +00006900 /* When converting under %#x or %#X, there are a number
Andrew MacIntyre5e9c80d2002-02-28 11:38:24 +00006901 * of issues that cause pain:
6902 * - when 0 is being converted, the C standard leaves off
6903 * the '0x' or '0X', which is inconsistent with other
6904 * %#x/%#X conversions and inconsistent with Python's
6905 * hex() function
6906 * - there are platforms that violate the standard and
6907 * convert 0 with the '0x' or '0X'
6908 * (Metrowerks, Compaq Tru64)
6909 * - there are platforms that give '0x' when converting
Tim Petersced69f82003-09-16 20:30:58 +00006910 * under %#X, but convert 0 in accordance with the
Andrew MacIntyre5e9c80d2002-02-28 11:38:24 +00006911 * standard (OS/2 EMX)
Tim Petersced69f82003-09-16 20:30:58 +00006912 *
Andrew MacIntyre5e9c80d2002-02-28 11:38:24 +00006913 * We can achieve the desired consistency by inserting our
6914 * own '0x' or '0X' prefix, and substituting %x/%X in place
6915 * of %#x/%#X.
6916 *
6917 * Note that this is the same approach as used in
6918 * formatint() in stringobject.c
Andrew MacIntyrec4874392002-02-26 11:36:35 +00006919 */
Guido van Rossum6c9e1302003-11-29 23:52:13 +00006920 PyOS_snprintf(fmt, sizeof(fmt), "%s0%c%%.%dl%c",
6921 sign, type, prec, type);
Andrew MacIntyrec4874392002-02-26 11:36:35 +00006922 }
Andrew MacIntyre5e9c80d2002-02-28 11:38:24 +00006923 else {
Guido van Rossum6c9e1302003-11-29 23:52:13 +00006924 PyOS_snprintf(fmt, sizeof(fmt), "%s%%%s.%dl%c",
6925 sign, (flags&F_ALT) ? "#" : "",
Andrew MacIntyre5e9c80d2002-02-28 11:38:24 +00006926 prec, type);
Tim Petersb3d8d1f2001-04-28 05:38:26 +00006927 }
Guido van Rossum6c9e1302003-11-29 23:52:13 +00006928 if (sign[0])
Neal Norwitzfc76d632006-01-10 06:03:13 +00006929 return longtounicode(buf, buflen, fmt, -x);
Guido van Rossum6c9e1302003-11-29 23:52:13 +00006930 else
Neal Norwitzfc76d632006-01-10 06:03:13 +00006931 return longtounicode(buf, buflen, fmt, x);
Guido van Rossumd57fd912000-03-10 22:53:23 +00006932}
6933
6934static int
6935formatchar(Py_UNICODE *buf,
Marc-André Lemburgf28dd832000-06-30 10:29:57 +00006936 size_t buflen,
6937 PyObject *v)
Guido van Rossumd57fd912000-03-10 22:53:23 +00006938{
Marc-André Lemburgf28dd832000-06-30 10:29:57 +00006939 /* presume that the buffer is at least 2 characters long */
Marc-André Lemburgd4ab4a52000-06-08 17:54:00 +00006940 if (PyUnicode_Check(v)) {
6941 if (PyUnicode_GET_SIZE(v) != 1)
6942 goto onError;
Guido van Rossumd57fd912000-03-10 22:53:23 +00006943 buf[0] = PyUnicode_AS_UNICODE(v)[0];
Marc-André Lemburgd4ab4a52000-06-08 17:54:00 +00006944 }
Guido van Rossumd57fd912000-03-10 22:53:23 +00006945
Marc-André Lemburgd4ab4a52000-06-08 17:54:00 +00006946 else if (PyString_Check(v)) {
Tim Petersced69f82003-09-16 20:30:58 +00006947 if (PyString_GET_SIZE(v) != 1)
Marc-André Lemburgd4ab4a52000-06-08 17:54:00 +00006948 goto onError;
6949 buf[0] = (Py_UNICODE)PyString_AS_STRING(v)[0];
6950 }
Guido van Rossumd57fd912000-03-10 22:53:23 +00006951
6952 else {
6953 /* Integer input truncated to a character */
6954 long x;
6955 x = PyInt_AsLong(v);
6956 if (x == -1 && PyErr_Occurred())
Marc-André Lemburgd4ab4a52000-06-08 17:54:00 +00006957 goto onError;
Marc-André Lemburgcc8764c2002-08-11 12:23:04 +00006958#ifdef Py_UNICODE_WIDE
6959 if (x < 0 || x > 0x10ffff) {
Walter Dörwald44f527f2003-04-02 16:37:24 +00006960 PyErr_SetString(PyExc_OverflowError,
Marc-André Lemburgcc8764c2002-08-11 12:23:04 +00006961 "%c arg not in range(0x110000) "
6962 "(wide Python build)");
6963 return -1;
6964 }
6965#else
6966 if (x < 0 || x > 0xffff) {
Walter Dörwald44f527f2003-04-02 16:37:24 +00006967 PyErr_SetString(PyExc_OverflowError,
Marc-André Lemburgcc8764c2002-08-11 12:23:04 +00006968 "%c arg not in range(0x10000) "
6969 "(narrow Python build)");
6970 return -1;
6971 }
6972#endif
6973 buf[0] = (Py_UNICODE) x;
Guido van Rossumd57fd912000-03-10 22:53:23 +00006974 }
6975 buf[1] = '\0';
6976 return 1;
Marc-André Lemburgd4ab4a52000-06-08 17:54:00 +00006977
6978 onError:
6979 PyErr_SetString(PyExc_TypeError,
6980 "%c requires int or char");
6981 return -1;
Guido van Rossumd57fd912000-03-10 22:53:23 +00006982}
6983
Marc-André Lemburgf28dd832000-06-30 10:29:57 +00006984/* fmt%(v1,v2,...) is roughly equivalent to sprintf(fmt, v1, v2, ...)
6985
6986 FORMATBUFLEN is the length of the buffer in which the floats, ints, &
6987 chars are formatted. XXX This is a magic number. Each formatting
6988 routine does bounds checking to ensure no overflow, but a better
6989 solution may be to malloc a buffer of appropriate size for each
6990 format. For now, the current solution is sufficient.
6991*/
6992#define FORMATBUFLEN (size_t)120
6993
Guido van Rossumd57fd912000-03-10 22:53:23 +00006994PyObject *PyUnicode_Format(PyObject *format,
6995 PyObject *args)
6996{
6997 Py_UNICODE *fmt, *res;
Martin v. Löwis18e16552006-02-15 17:27:45 +00006998 Py_ssize_t fmtcnt, rescnt, reslen, arglen, argidx;
Guido van Rossumd57fd912000-03-10 22:53:23 +00006999 int args_owned = 0;
7000 PyUnicodeObject *result = NULL;
7001 PyObject *dict = NULL;
7002 PyObject *uformat;
Tim Petersced69f82003-09-16 20:30:58 +00007003
Guido van Rossumd57fd912000-03-10 22:53:23 +00007004 if (format == NULL || args == NULL) {
7005 PyErr_BadInternalCall();
7006 return NULL;
7007 }
7008 uformat = PyUnicode_FromObject(format);
Fred Drakee4315f52000-05-09 19:53:39 +00007009 if (uformat == NULL)
7010 return NULL;
Guido van Rossumd57fd912000-03-10 22:53:23 +00007011 fmt = PyUnicode_AS_UNICODE(uformat);
7012 fmtcnt = PyUnicode_GET_SIZE(uformat);
7013
7014 reslen = rescnt = fmtcnt + 100;
7015 result = _PyUnicode_New(reslen);
7016 if (result == NULL)
7017 goto onError;
7018 res = PyUnicode_AS_UNICODE(result);
7019
7020 if (PyTuple_Check(args)) {
7021 arglen = PyTuple_Size(args);
7022 argidx = 0;
7023 }
7024 else {
7025 arglen = -1;
7026 argidx = -2;
7027 }
Neal Norwitz80a1bf42002-11-12 23:01:12 +00007028 if (args->ob_type->tp_as_mapping && !PyTuple_Check(args) &&
7029 !PyObject_TypeCheck(args, &PyBaseString_Type))
Guido van Rossumd57fd912000-03-10 22:53:23 +00007030 dict = args;
7031
7032 while (--fmtcnt >= 0) {
7033 if (*fmt != '%') {
7034 if (--rescnt < 0) {
7035 rescnt = fmtcnt + 100;
7036 reslen += rescnt;
Marc-André Lemburg8155e0e2001-04-23 14:44:21 +00007037 if (_PyUnicode_Resize(&result, reslen) < 0)
Hye-Shik Chang4af5c8c2006-03-07 15:39:21 +00007038 goto onError;
Guido van Rossumd57fd912000-03-10 22:53:23 +00007039 res = PyUnicode_AS_UNICODE(result) + reslen - rescnt;
7040 --rescnt;
7041 }
7042 *res++ = *fmt++;
7043 }
7044 else {
7045 /* Got a format specifier */
7046 int flags = 0;
Martin v. Löwis18e16552006-02-15 17:27:45 +00007047 Py_ssize_t width = -1;
Guido van Rossumd57fd912000-03-10 22:53:23 +00007048 int prec = -1;
Guido van Rossumd57fd912000-03-10 22:53:23 +00007049 Py_UNICODE c = '\0';
7050 Py_UNICODE fill;
7051 PyObject *v = NULL;
7052 PyObject *temp = NULL;
Marc-André Lemburgf28dd832000-06-30 10:29:57 +00007053 Py_UNICODE *pbuf;
Guido van Rossumd57fd912000-03-10 22:53:23 +00007054 Py_UNICODE sign;
Martin v. Löwis18e16552006-02-15 17:27:45 +00007055 Py_ssize_t len;
Marc-André Lemburgf28dd832000-06-30 10:29:57 +00007056 Py_UNICODE formatbuf[FORMATBUFLEN]; /* For format{float,int,char}() */
Guido van Rossumd57fd912000-03-10 22:53:23 +00007057
7058 fmt++;
7059 if (*fmt == '(') {
7060 Py_UNICODE *keystart;
Martin v. Löwis18e16552006-02-15 17:27:45 +00007061 Py_ssize_t keylen;
Guido van Rossumd57fd912000-03-10 22:53:23 +00007062 PyObject *key;
7063 int pcount = 1;
7064
7065 if (dict == NULL) {
7066 PyErr_SetString(PyExc_TypeError,
Tim Petersced69f82003-09-16 20:30:58 +00007067 "format requires a mapping");
Guido van Rossumd57fd912000-03-10 22:53:23 +00007068 goto onError;
7069 }
7070 ++fmt;
7071 --fmtcnt;
7072 keystart = fmt;
7073 /* Skip over balanced parentheses */
7074 while (pcount > 0 && --fmtcnt >= 0) {
7075 if (*fmt == ')')
7076 --pcount;
7077 else if (*fmt == '(')
7078 ++pcount;
7079 fmt++;
7080 }
7081 keylen = fmt - keystart - 1;
7082 if (fmtcnt < 0 || pcount > 0) {
7083 PyErr_SetString(PyExc_ValueError,
7084 "incomplete format key");
7085 goto onError;
7086 }
Marc-André Lemburg72f82132001-11-20 15:18:49 +00007087#if 0
Fred Drakee4315f52000-05-09 19:53:39 +00007088 /* keys are converted to strings using UTF-8 and
Guido van Rossumd57fd912000-03-10 22:53:23 +00007089 then looked up since Python uses strings to hold
7090 variables names etc. in its namespaces and we
Fred Drakee4315f52000-05-09 19:53:39 +00007091 wouldn't want to break common idioms. */
Guido van Rossumd57fd912000-03-10 22:53:23 +00007092 key = PyUnicode_EncodeUTF8(keystart,
7093 keylen,
7094 NULL);
Marc-André Lemburg72f82132001-11-20 15:18:49 +00007095#else
7096 key = PyUnicode_FromUnicode(keystart, keylen);
7097#endif
Guido van Rossumd57fd912000-03-10 22:53:23 +00007098 if (key == NULL)
7099 goto onError;
7100 if (args_owned) {
7101 Py_DECREF(args);
7102 args_owned = 0;
7103 }
7104 args = PyObject_GetItem(dict, key);
7105 Py_DECREF(key);
7106 if (args == NULL) {
7107 goto onError;
7108 }
7109 args_owned = 1;
7110 arglen = -1;
7111 argidx = -2;
7112 }
7113 while (--fmtcnt >= 0) {
7114 switch (c = *fmt++) {
7115 case '-': flags |= F_LJUST; continue;
7116 case '+': flags |= F_SIGN; continue;
7117 case ' ': flags |= F_BLANK; continue;
7118 case '#': flags |= F_ALT; continue;
7119 case '0': flags |= F_ZERO; continue;
7120 }
7121 break;
7122 }
7123 if (c == '*') {
7124 v = getnextarg(args, arglen, &argidx);
7125 if (v == NULL)
7126 goto onError;
7127 if (!PyInt_Check(v)) {
7128 PyErr_SetString(PyExc_TypeError,
7129 "* wants int");
7130 goto onError;
7131 }
7132 width = PyInt_AsLong(v);
7133 if (width < 0) {
7134 flags |= F_LJUST;
7135 width = -width;
7136 }
7137 if (--fmtcnt >= 0)
7138 c = *fmt++;
7139 }
7140 else if (c >= '0' && c <= '9') {
7141 width = c - '0';
7142 while (--fmtcnt >= 0) {
7143 c = *fmt++;
7144 if (c < '0' || c > '9')
7145 break;
7146 if ((width*10) / 10 != width) {
7147 PyErr_SetString(PyExc_ValueError,
7148 "width too big");
7149 goto onError;
7150 }
7151 width = width*10 + (c - '0');
7152 }
7153 }
7154 if (c == '.') {
7155 prec = 0;
7156 if (--fmtcnt >= 0)
7157 c = *fmt++;
7158 if (c == '*') {
7159 v = getnextarg(args, arglen, &argidx);
7160 if (v == NULL)
7161 goto onError;
7162 if (!PyInt_Check(v)) {
7163 PyErr_SetString(PyExc_TypeError,
7164 "* wants int");
7165 goto onError;
7166 }
7167 prec = PyInt_AsLong(v);
7168 if (prec < 0)
7169 prec = 0;
7170 if (--fmtcnt >= 0)
7171 c = *fmt++;
7172 }
7173 else if (c >= '0' && c <= '9') {
7174 prec = c - '0';
7175 while (--fmtcnt >= 0) {
7176 c = Py_CHARMASK(*fmt++);
7177 if (c < '0' || c > '9')
7178 break;
7179 if ((prec*10) / 10 != prec) {
7180 PyErr_SetString(PyExc_ValueError,
7181 "prec too big");
7182 goto onError;
7183 }
7184 prec = prec*10 + (c - '0');
7185 }
7186 }
7187 } /* prec */
7188 if (fmtcnt >= 0) {
7189 if (c == 'h' || c == 'l' || c == 'L') {
Guido van Rossumd57fd912000-03-10 22:53:23 +00007190 if (--fmtcnt >= 0)
7191 c = *fmt++;
7192 }
7193 }
7194 if (fmtcnt < 0) {
7195 PyErr_SetString(PyExc_ValueError,
7196 "incomplete format");
7197 goto onError;
7198 }
7199 if (c != '%') {
7200 v = getnextarg(args, arglen, &argidx);
7201 if (v == NULL)
7202 goto onError;
7203 }
7204 sign = 0;
7205 fill = ' ';
7206 switch (c) {
7207
7208 case '%':
Marc-André Lemburgf28dd832000-06-30 10:29:57 +00007209 pbuf = formatbuf;
7210 /* presume that buffer length is at least 1 */
7211 pbuf[0] = '%';
Guido van Rossumd57fd912000-03-10 22:53:23 +00007212 len = 1;
7213 break;
7214
7215 case 's':
7216 case 'r':
7217 if (PyUnicode_Check(v) && c == 's') {
7218 temp = v;
7219 Py_INCREF(temp);
7220 }
7221 else {
7222 PyObject *unicode;
7223 if (c == 's')
Marc-André Lemburgd25c6502004-07-23 16:13:25 +00007224 temp = PyObject_Unicode(v);
Guido van Rossumd57fd912000-03-10 22:53:23 +00007225 else
7226 temp = PyObject_Repr(v);
7227 if (temp == NULL)
7228 goto onError;
Marc-André Lemburgd25c6502004-07-23 16:13:25 +00007229 if (PyUnicode_Check(temp))
7230 /* nothing to do */;
7231 else if (PyString_Check(temp)) {
7232 /* convert to string to Unicode */
Thomas Woutersa96affe2006-03-12 00:29:36 +00007233 unicode = PyUnicode_Decode(PyString_AS_STRING(temp),
Guido van Rossumd57fd912000-03-10 22:53:23 +00007234 PyString_GET_SIZE(temp),
Thomas Woutersa96affe2006-03-12 00:29:36 +00007235 NULL,
Guido van Rossumd57fd912000-03-10 22:53:23 +00007236 "strict");
Thomas Woutersa96affe2006-03-12 00:29:36 +00007237 Py_DECREF(temp);
7238 temp = unicode;
7239 if (temp == NULL)
7240 goto onError;
7241 }
Marc-André Lemburgd25c6502004-07-23 16:13:25 +00007242 else {
7243 Py_DECREF(temp);
7244 PyErr_SetString(PyExc_TypeError,
7245 "%s argument has non-string str()");
7246 goto onError;
7247 }
7248 }
Marc-André Lemburgf28dd832000-06-30 10:29:57 +00007249 pbuf = PyUnicode_AS_UNICODE(temp);
Guido van Rossumd57fd912000-03-10 22:53:23 +00007250 len = PyUnicode_GET_SIZE(temp);
7251 if (prec >= 0 && len > prec)
7252 len = prec;
7253 break;
7254
7255 case 'i':
7256 case 'd':
7257 case 'u':
7258 case 'o':
7259 case 'x':
7260 case 'X':
7261 if (c == 'i')
7262 c = 'd';
Tim Petersa3a3a032000-11-30 05:22:44 +00007263 if (PyLong_Check(v)) {
Tim Peters38fd5b62000-09-21 05:43:11 +00007264 temp = formatlong(v, flags, prec, c);
7265 if (!temp)
7266 goto onError;
7267 pbuf = PyUnicode_AS_UNICODE(temp);
7268 len = PyUnicode_GET_SIZE(temp);
Tim Peters38fd5b62000-09-21 05:43:11 +00007269 sign = 1;
Guido van Rossumd57fd912000-03-10 22:53:23 +00007270 }
Tim Peters38fd5b62000-09-21 05:43:11 +00007271 else {
7272 pbuf = formatbuf;
7273 len = formatint(pbuf, sizeof(formatbuf)/sizeof(Py_UNICODE),
7274 flags, prec, c, v);
7275 if (len < 0)
7276 goto onError;
Guido van Rossum6c9e1302003-11-29 23:52:13 +00007277 sign = 1;
Tim Peters38fd5b62000-09-21 05:43:11 +00007278 }
7279 if (flags & F_ZERO)
7280 fill = '0';
Guido van Rossumd57fd912000-03-10 22:53:23 +00007281 break;
7282
7283 case 'e':
7284 case 'E':
7285 case 'f':
Raymond Hettinger9bfe5332003-08-27 04:55:52 +00007286 case 'F':
Guido van Rossumd57fd912000-03-10 22:53:23 +00007287 case 'g':
7288 case 'G':
Raymond Hettinger9bfe5332003-08-27 04:55:52 +00007289 if (c == 'F')
7290 c = 'f';
Marc-André Lemburgf28dd832000-06-30 10:29:57 +00007291 pbuf = formatbuf;
7292 len = formatfloat(pbuf, sizeof(formatbuf)/sizeof(Py_UNICODE),
7293 flags, prec, c, v);
Guido van Rossumd57fd912000-03-10 22:53:23 +00007294 if (len < 0)
7295 goto onError;
7296 sign = 1;
Tim Peters38fd5b62000-09-21 05:43:11 +00007297 if (flags & F_ZERO)
Guido van Rossumd57fd912000-03-10 22:53:23 +00007298 fill = '0';
7299 break;
7300
7301 case 'c':
Marc-André Lemburgf28dd832000-06-30 10:29:57 +00007302 pbuf = formatbuf;
7303 len = formatchar(pbuf, sizeof(formatbuf)/sizeof(Py_UNICODE), v);
Guido van Rossumd57fd912000-03-10 22:53:23 +00007304 if (len < 0)
7305 goto onError;
7306 break;
7307
7308 default:
7309 PyErr_Format(PyExc_ValueError,
Andrew M. Kuchling6ca89172000-12-15 13:07:46 +00007310 "unsupported format character '%c' (0x%x) "
7311 "at index %i",
Tim Petersced69f82003-09-16 20:30:58 +00007312 (31<=c && c<=126) ? (char)c : '?',
Marc-André Lemburg24e53b62002-09-24 09:32:14 +00007313 (int)c,
Guido van Rossumefc11882002-09-12 14:43:41 +00007314 (int)(fmt -1 - PyUnicode_AS_UNICODE(uformat)));
Guido van Rossumd57fd912000-03-10 22:53:23 +00007315 goto onError;
7316 }
7317 if (sign) {
Marc-André Lemburgf28dd832000-06-30 10:29:57 +00007318 if (*pbuf == '-' || *pbuf == '+') {
7319 sign = *pbuf++;
Guido van Rossumd57fd912000-03-10 22:53:23 +00007320 len--;
7321 }
7322 else if (flags & F_SIGN)
7323 sign = '+';
7324 else if (flags & F_BLANK)
7325 sign = ' ';
7326 else
7327 sign = 0;
7328 }
7329 if (width < len)
7330 width = len;
Guido van Rossum049cd6b2002-10-11 00:43:48 +00007331 if (rescnt - (sign != 0) < width) {
Guido van Rossumd57fd912000-03-10 22:53:23 +00007332 reslen -= rescnt;
7333 rescnt = width + fmtcnt + 100;
7334 reslen += rescnt;
Guido van Rossum049cd6b2002-10-11 00:43:48 +00007335 if (reslen < 0) {
Hye-Shik Chang4af5c8c2006-03-07 15:39:21 +00007336 Py_XDECREF(temp);
Thomas Woutersa96affe2006-03-12 00:29:36 +00007337 PyErr_NoMemory();
7338 goto onError;
Guido van Rossum049cd6b2002-10-11 00:43:48 +00007339 }
Thomas Woutersa96affe2006-03-12 00:29:36 +00007340 if (_PyUnicode_Resize(&result, reslen) < 0) {
7341 Py_XDECREF(temp);
7342 goto onError;
7343 }
Guido van Rossumd57fd912000-03-10 22:53:23 +00007344 res = PyUnicode_AS_UNICODE(result)
7345 + reslen - rescnt;
7346 }
7347 if (sign) {
7348 if (fill != ' ')
7349 *res++ = sign;
7350 rescnt--;
7351 if (width > len)
7352 width--;
7353 }
Tim Peters38fd5b62000-09-21 05:43:11 +00007354 if ((flags & F_ALT) && (c == 'x' || c == 'X')) {
7355 assert(pbuf[0] == '0');
Tim Petersfff53252001-04-12 18:38:48 +00007356 assert(pbuf[1] == c);
7357 if (fill != ' ') {
7358 *res++ = *pbuf++;
7359 *res++ = *pbuf++;
Tim Peters38fd5b62000-09-21 05:43:11 +00007360 }
Tim Petersfff53252001-04-12 18:38:48 +00007361 rescnt -= 2;
7362 width -= 2;
7363 if (width < 0)
7364 width = 0;
7365 len -= 2;
Tim Peters38fd5b62000-09-21 05:43:11 +00007366 }
Guido van Rossumd57fd912000-03-10 22:53:23 +00007367 if (width > len && !(flags & F_LJUST)) {
7368 do {
7369 --rescnt;
7370 *res++ = fill;
7371 } while (--width > len);
7372 }
Tim Peters38fd5b62000-09-21 05:43:11 +00007373 if (fill == ' ') {
7374 if (sign)
7375 *res++ = sign;
Tim Petersfff53252001-04-12 18:38:48 +00007376 if ((flags & F_ALT) && (c == 'x' || c == 'X')) {
Tim Peters38fd5b62000-09-21 05:43:11 +00007377 assert(pbuf[0] == '0');
Tim Petersfff53252001-04-12 18:38:48 +00007378 assert(pbuf[1] == c);
Tim Peters38fd5b62000-09-21 05:43:11 +00007379 *res++ = *pbuf++;
7380 *res++ = *pbuf++;
7381 }
7382 }
Marc-André Lemburg8155e0e2001-04-23 14:44:21 +00007383 Py_UNICODE_COPY(res, pbuf, len);
Guido van Rossumd57fd912000-03-10 22:53:23 +00007384 res += len;
7385 rescnt -= len;
7386 while (--width >= len) {
7387 --rescnt;
7388 *res++ = ' ';
7389 }
7390 if (dict && (argidx < arglen) && c != '%') {
7391 PyErr_SetString(PyExc_TypeError,
Raymond Hettinger0ebac972002-05-21 15:14:57 +00007392 "not all arguments converted during string formatting");
Thomas Woutersa96affe2006-03-12 00:29:36 +00007393 Py_XDECREF(temp);
Guido van Rossumd57fd912000-03-10 22:53:23 +00007394 goto onError;
7395 }
7396 Py_XDECREF(temp);
7397 } /* '%' */
7398 } /* until end */
7399 if (argidx < arglen && !dict) {
7400 PyErr_SetString(PyExc_TypeError,
Raymond Hettinger0ebac972002-05-21 15:14:57 +00007401 "not all arguments converted during string formatting");
Guido van Rossumd57fd912000-03-10 22:53:23 +00007402 goto onError;
7403 }
7404
Thomas Woutersa96affe2006-03-12 00:29:36 +00007405 if (_PyUnicode_Resize(&result, reslen - rescnt) < 0)
7406 goto onError;
Guido van Rossumd57fd912000-03-10 22:53:23 +00007407 if (args_owned) {
7408 Py_DECREF(args);
7409 }
7410 Py_DECREF(uformat);
Guido van Rossumd57fd912000-03-10 22:53:23 +00007411 return (PyObject *)result;
7412
7413 onError:
7414 Py_XDECREF(result);
7415 Py_DECREF(uformat);
7416 if (args_owned) {
7417 Py_DECREF(args);
7418 }
7419 return NULL;
7420}
7421
7422static PyBufferProcs unicode_as_buffer = {
Martin v. Löwis18e16552006-02-15 17:27:45 +00007423 (readbufferproc) unicode_buffer_getreadbuf,
7424 (writebufferproc) unicode_buffer_getwritebuf,
7425 (segcountproc) unicode_buffer_getsegcount,
7426 (charbufferproc) unicode_buffer_getcharbuf,
Guido van Rossumd57fd912000-03-10 22:53:23 +00007427};
7428
Jeremy Hylton938ace62002-07-17 16:30:39 +00007429static PyObject *
Guido van Rossume023fe02001-08-30 03:12:59 +00007430unicode_subtype_new(PyTypeObject *type, PyObject *args, PyObject *kwds);
7431
Tim Peters6d6c1a32001-08-02 04:15:00 +00007432static PyObject *
7433unicode_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
7434{
7435 PyObject *x = NULL;
Martin v. Löwis15e62742006-02-27 16:46:16 +00007436 static char *kwlist[] = {"string", "encoding", "errors", 0};
Tim Peters6d6c1a32001-08-02 04:15:00 +00007437 char *encoding = NULL;
7438 char *errors = NULL;
7439
Guido van Rossume023fe02001-08-30 03:12:59 +00007440 if (type != &PyUnicode_Type)
7441 return unicode_subtype_new(type, args, kwds);
Tim Peters6d6c1a32001-08-02 04:15:00 +00007442 if (!PyArg_ParseTupleAndKeywords(args, kwds, "|Oss:unicode",
7443 kwlist, &x, &encoding, &errors))
7444 return NULL;
7445 if (x == NULL)
7446 return (PyObject *)_PyUnicode_New(0);
Guido van Rossumb8c65bc2001-10-19 02:01:31 +00007447 if (encoding == NULL && errors == NULL)
7448 return PyObject_Unicode(x);
7449 else
Tim Peters6d6c1a32001-08-02 04:15:00 +00007450 return PyUnicode_FromEncodedObject(x, encoding, errors);
7451}
7452
Guido van Rossume023fe02001-08-30 03:12:59 +00007453static PyObject *
7454unicode_subtype_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
7455{
Tim Petersaf90b3e2001-09-12 05:18:58 +00007456 PyUnicodeObject *tmp, *pnew;
Martin v. Löwis18e16552006-02-15 17:27:45 +00007457 Py_ssize_t n;
Guido van Rossume023fe02001-08-30 03:12:59 +00007458
7459 assert(PyType_IsSubtype(type, &PyUnicode_Type));
7460 tmp = (PyUnicodeObject *)unicode_new(&PyUnicode_Type, args, kwds);
7461 if (tmp == NULL)
7462 return NULL;
7463 assert(PyUnicode_Check(tmp));
Tim Petersaf90b3e2001-09-12 05:18:58 +00007464 pnew = (PyUnicodeObject *) type->tp_alloc(type, n = tmp->length);
Raymond Hettingerf4667932003-06-28 20:04:25 +00007465 if (pnew == NULL) {
7466 Py_DECREF(tmp);
Guido van Rossume023fe02001-08-30 03:12:59 +00007467 return NULL;
Raymond Hettingerf4667932003-06-28 20:04:25 +00007468 }
Tim Petersaf90b3e2001-09-12 05:18:58 +00007469 pnew->str = PyMem_NEW(Py_UNICODE, n+1);
7470 if (pnew->str == NULL) {
7471 _Py_ForgetReference((PyObject *)pnew);
Neil Schemenauer58aa8612002-04-12 03:07:20 +00007472 PyObject_Del(pnew);
Raymond Hettingerf4667932003-06-28 20:04:25 +00007473 Py_DECREF(tmp);
Neal Norwitzec74f2f2003-02-11 23:05:40 +00007474 return PyErr_NoMemory();
Guido van Rossume023fe02001-08-30 03:12:59 +00007475 }
Tim Petersaf90b3e2001-09-12 05:18:58 +00007476 Py_UNICODE_COPY(pnew->str, tmp->str, n+1);
7477 pnew->length = n;
7478 pnew->hash = tmp->hash;
Guido van Rossume023fe02001-08-30 03:12:59 +00007479 Py_DECREF(tmp);
Tim Petersaf90b3e2001-09-12 05:18:58 +00007480 return (PyObject *)pnew;
Guido van Rossume023fe02001-08-30 03:12:59 +00007481}
7482
Martin v. Löwis14f8b4c2002-06-13 20:33:02 +00007483PyDoc_STRVAR(unicode_doc,
Tim Peters6d6c1a32001-08-02 04:15:00 +00007484"unicode(string [, encoding[, errors]]) -> object\n\
7485\n\
7486Create a new Unicode object from the given encoded string.\n\
Skip Montanaro35b37a52002-07-26 16:22:46 +00007487encoding defaults to the current default string encoding.\n\
7488errors can be 'strict', 'replace' or 'ignore' and defaults to 'strict'.");
Tim Peters6d6c1a32001-08-02 04:15:00 +00007489
Guido van Rossumd57fd912000-03-10 22:53:23 +00007490PyTypeObject PyUnicode_Type = {
7491 PyObject_HEAD_INIT(&PyType_Type)
7492 0, /* ob_size */
7493 "unicode", /* tp_name */
7494 sizeof(PyUnicodeObject), /* tp_size */
7495 0, /* tp_itemsize */
7496 /* Slots */
Guido van Rossum9475a232001-10-05 20:51:39 +00007497 (destructor)unicode_dealloc, /* tp_dealloc */
Guido van Rossumd57fd912000-03-10 22:53:23 +00007498 0, /* tp_print */
Tim Peters6d6c1a32001-08-02 04:15:00 +00007499 0, /* tp_getattr */
Guido van Rossumd57fd912000-03-10 22:53:23 +00007500 0, /* tp_setattr */
7501 (cmpfunc) unicode_compare, /* tp_compare */
Georg Brandl347b3002006-03-30 11:57:00 +00007502 unicode_repr, /* tp_repr */
Neil Schemenauerce30bc92002-11-18 16:10:18 +00007503 &unicode_as_number, /* tp_as_number */
Guido van Rossumd57fd912000-03-10 22:53:23 +00007504 &unicode_as_sequence, /* tp_as_sequence */
Michael W. Hudson5efaf7e2002-06-11 10:55:12 +00007505 &unicode_as_mapping, /* tp_as_mapping */
Guido van Rossumd57fd912000-03-10 22:53:23 +00007506 (hashfunc) unicode_hash, /* tp_hash*/
7507 0, /* tp_call*/
7508 (reprfunc) unicode_str, /* tp_str */
Tim Peters6d6c1a32001-08-02 04:15:00 +00007509 PyObject_GenericGetAttr, /* tp_getattro */
7510 0, /* tp_setattro */
Guido van Rossumd57fd912000-03-10 22:53:23 +00007511 &unicode_as_buffer, /* tp_as_buffer */
Neil Schemenauerce30bc92002-11-18 16:10:18 +00007512 Py_TPFLAGS_DEFAULT | Py_TPFLAGS_CHECKTYPES |
7513 Py_TPFLAGS_BASETYPE, /* tp_flags */
Tim Peters6d6c1a32001-08-02 04:15:00 +00007514 unicode_doc, /* tp_doc */
7515 0, /* tp_traverse */
7516 0, /* tp_clear */
7517 0, /* tp_richcompare */
7518 0, /* tp_weaklistoffset */
7519 0, /* tp_iter */
7520 0, /* tp_iternext */
7521 unicode_methods, /* tp_methods */
7522 0, /* tp_members */
7523 0, /* tp_getset */
Guido van Rossumcacfc072002-05-24 19:01:59 +00007524 &PyBaseString_Type, /* tp_base */
Tim Peters6d6c1a32001-08-02 04:15:00 +00007525 0, /* tp_dict */
7526 0, /* tp_descr_get */
7527 0, /* tp_descr_set */
7528 0, /* tp_dictoffset */
7529 0, /* tp_init */
7530 0, /* tp_alloc */
7531 unicode_new, /* tp_new */
Neil Schemenauer58aa8612002-04-12 03:07:20 +00007532 PyObject_Del, /* tp_free */
Guido van Rossumd57fd912000-03-10 22:53:23 +00007533};
7534
7535/* Initialize the Unicode implementation */
7536
Thomas Wouters78890102000-07-22 19:25:51 +00007537void _PyUnicode_Init(void)
Guido van Rossumd57fd912000-03-10 22:53:23 +00007538{
Marc-André Lemburg8155e0e2001-04-23 14:44:21 +00007539 int i;
7540
Fredrik Lundhb63588c2006-05-23 18:44:25 +00007541 /* XXX - move this array to unicodectype.c ? */
7542 Py_UNICODE linebreak[] = {
7543 0x000A, /* LINE FEED */
7544 0x000D, /* CARRIAGE RETURN */
7545 0x001C, /* FILE SEPARATOR */
7546 0x001D, /* GROUP SEPARATOR */
7547 0x001E, /* RECORD SEPARATOR */
7548 0x0085, /* NEXT LINE */
7549 0x2028, /* LINE SEPARATOR */
7550 0x2029, /* PARAGRAPH SEPARATOR */
7551 };
7552
Fred Drakee4315f52000-05-09 19:53:39 +00007553 /* Init the implementation */
Marc-André Lemburgd4ab4a52000-06-08 17:54:00 +00007554 unicode_freelist = NULL;
7555 unicode_freelist_size = 0;
Guido van Rossumd57fd912000-03-10 22:53:23 +00007556 unicode_empty = _PyUnicode_New(0);
Marc-André Lemburg90e81472000-06-07 09:13:21 +00007557 strcpy(unicode_default_encoding, "ascii");
Marc-André Lemburg8155e0e2001-04-23 14:44:21 +00007558 for (i = 0; i < 256; i++)
7559 unicode_latin1[i] = NULL;
Guido van Rossumcacfc072002-05-24 19:01:59 +00007560 if (PyType_Ready(&PyUnicode_Type) < 0)
7561 Py_FatalError("Can't initialize 'unicode'");
Fredrik Lundhb63588c2006-05-23 18:44:25 +00007562
7563 /* initialize the linebreak bloom filter */
7564 bloom_linebreak = make_bloom_mask(
7565 linebreak, sizeof(linebreak) / sizeof(linebreak[0])
7566 );
Guido van Rossumd57fd912000-03-10 22:53:23 +00007567}
7568
7569/* Finalize the Unicode implementation */
7570
7571void
Thomas Wouters78890102000-07-22 19:25:51 +00007572_PyUnicode_Fini(void)
Guido van Rossumd57fd912000-03-10 22:53:23 +00007573{
Barry Warsaw5b4c2282000-10-03 20:45:26 +00007574 PyUnicodeObject *u;
Marc-André Lemburg8155e0e2001-04-23 14:44:21 +00007575 int i;
Guido van Rossumd57fd912000-03-10 22:53:23 +00007576
Guido van Rossum4ae8ef82000-10-03 18:09:04 +00007577 Py_XDECREF(unicode_empty);
7578 unicode_empty = NULL;
Barry Warsaw5b4c2282000-10-03 20:45:26 +00007579
Marc-André Lemburg8155e0e2001-04-23 14:44:21 +00007580 for (i = 0; i < 256; i++) {
7581 if (unicode_latin1[i]) {
7582 Py_DECREF(unicode_latin1[i]);
7583 unicode_latin1[i] = NULL;
7584 }
7585 }
7586
Barry Warsaw5b4c2282000-10-03 20:45:26 +00007587 for (u = unicode_freelist; u != NULL;) {
Guido van Rossumd57fd912000-03-10 22:53:23 +00007588 PyUnicodeObject *v = u;
7589 u = *(PyUnicodeObject **)u;
Guido van Rossumfd4b9572000-04-10 13:51:10 +00007590 if (v->str)
Guido van Rossumb18618d2000-05-03 23:44:39 +00007591 PyMem_DEL(v->str);
Marc-André Lemburgbff879c2000-08-03 18:46:08 +00007592 Py_XDECREF(v->defenc);
Neil Schemenauer58aa8612002-04-12 03:07:20 +00007593 PyObject_Del(v);
Guido van Rossumd57fd912000-03-10 22:53:23 +00007594 }
Marc-André Lemburgd4ab4a52000-06-08 17:54:00 +00007595 unicode_freelist = NULL;
7596 unicode_freelist_size = 0;
Guido van Rossumd57fd912000-03-10 22:53:23 +00007597}
Martin v. Löwis9a3a9f72003-05-18 12:31:09 +00007598
Anthony Baxterac6bd462006-04-13 02:06:09 +00007599#ifdef __cplusplus
7600}
7601#endif
7602
7603
Martin v. Löwis9a3a9f72003-05-18 12:31:09 +00007604/*
7605Local variables:
7606c-basic-offset: 4
7607indent-tabs-mode: nil
7608End:
7609*/