Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 1 | #ifndef Py_UNICODEOBJECT_H |
| 2 | #define Py_UNICODEOBJECT_H |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 3 | |
Christian Heimes | af98da1 | 2008-01-27 15:18:18 +0000 | [diff] [blame] | 4 | #include <stdarg.h> |
| 5 | |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 6 | /* |
| 7 | |
| 8 | Unicode implementation based on original code by Fredrik Lundh, |
| 9 | modified by Marc-Andre Lemburg (mal@lemburg.com) according to the |
Alexander Belopolsky | 83283c2 | 2010-11-16 14:29:01 +0000 | [diff] [blame] | 10 | Unicode Integration Proposal. (See |
| 11 | http://www.egenix.com/files/python/unicode-proposal.txt). |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 12 | |
Guido van Rossum | 16b1ad9 | 2000-08-03 16:24:25 +0000 | [diff] [blame] | 13 | Copyright (c) Corporation for National Research Initiatives. |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 14 | |
| 15 | |
| 16 | Original header: |
| 17 | -------------------------------------------------------------------- |
| 18 | |
| 19 | * Yet another Unicode string type for Python. This type supports the |
| 20 | * 16-bit Basic Multilingual Plane (BMP) only. |
| 21 | * |
| 22 | * Written by Fredrik Lundh, January 1999. |
| 23 | * |
| 24 | * Copyright (c) 1999 by Secret Labs AB. |
| 25 | * Copyright (c) 1999 by Fredrik Lundh. |
| 26 | * |
| 27 | * fredrik@pythonware.com |
| 28 | * http://www.pythonware.com |
| 29 | * |
| 30 | * -------------------------------------------------------------------- |
| 31 | * This Unicode String Type is |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 32 | * |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 33 | * Copyright (c) 1999 by Secret Labs AB |
| 34 | * Copyright (c) 1999 by Fredrik Lundh |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 35 | * |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 36 | * By obtaining, using, and/or copying this software and/or its |
| 37 | * associated documentation, you agree that you have read, understood, |
| 38 | * and will comply with the following terms and conditions: |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 39 | * |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 40 | * Permission to use, copy, modify, and distribute this software and its |
| 41 | * associated documentation for any purpose and without fee is hereby |
| 42 | * granted, provided that the above copyright notice appears in all |
| 43 | * copies, and that both that copyright notice and this permission notice |
| 44 | * appear in supporting documentation, and that the name of Secret Labs |
| 45 | * AB or the author not be used in advertising or publicity pertaining to |
| 46 | * distribution of the software without specific, written prior |
| 47 | * permission. |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 48 | * |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 49 | * SECRET LABS AB AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO |
| 50 | * THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND |
| 51 | * FITNESS. IN NO EVENT SHALL SECRET LABS AB OR THE AUTHOR BE LIABLE FOR |
| 52 | * ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES |
| 53 | * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN |
| 54 | * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT |
| 55 | * OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. |
| 56 | * -------------------------------------------------------------------- */ |
| 57 | |
Marc-André Lemburg | 5e6007c | 2001-09-19 11:21:03 +0000 | [diff] [blame] | 58 | #include <ctype.h> |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 59 | |
| 60 | /* === Internal API ======================================================= */ |
| 61 | |
| 62 | /* --- Internal Unicode Format -------------------------------------------- */ |
| 63 | |
Christian Heimes | 0625e89 | 2008-01-07 21:04:21 +0000 | [diff] [blame] | 64 | /* Python 3.x requires unicode */ |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 65 | #define Py_USING_UNICODE |
Christian Heimes | 0625e89 | 2008-01-07 21:04:21 +0000 | [diff] [blame] | 66 | |
Martin v. Löwis | d63a3b8 | 2011-09-28 07:41:54 +0200 | [diff] [blame] | 67 | #ifndef SIZEOF_WCHAR_T |
| 68 | #error Must define SIZEOF_WCHAR_T |
Fredrik Lundh | 9b14ab3 | 2001-06-26 22:59:49 +0000 | [diff] [blame] | 69 | #endif |
| 70 | |
Martin v. Löwis | d63a3b8 | 2011-09-28 07:41:54 +0200 | [diff] [blame] | 71 | #define Py_UNICODE_SIZE SIZEOF_WCHAR_T |
| 72 | |
| 73 | /* If wchar_t can be used for UCS-4 storage, set Py_UNICODE_WIDE. |
| 74 | Otherwise, Unicode strings are stored as UCS-2 (with limited support |
| 75 | for UTF-16) */ |
Fredrik Lundh | 8f45585 | 2001-06-27 18:59:43 +0000 | [diff] [blame] | 76 | |
| 77 | #if Py_UNICODE_SIZE >= 4 |
| 78 | #define Py_UNICODE_WIDE |
Martin v. Löwis | 0ba70cc | 2001-06-26 22:22:37 +0000 | [diff] [blame] | 79 | #endif |
Fredrik Lundh | 1294ad0 | 2001-06-26 17:17:07 +0000 | [diff] [blame] | 80 | |
Amaury Forgeot d'Arc | feb7307 | 2010-09-12 22:42:57 +0000 | [diff] [blame] | 81 | /* Set these flags if the platform has "wchar.h" and the |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 82 | wchar_t type is a 16-bit unsigned type */ |
| 83 | /* #define HAVE_WCHAR_H */ |
| 84 | /* #define HAVE_USABLE_WCHAR_T */ |
| 85 | |
Martin v. Löwis | d63a3b8 | 2011-09-28 07:41:54 +0200 | [diff] [blame] | 86 | /* Py_UNICODE was the native Unicode storage format (code unit) used by |
| 87 | Python and represents a single Unicode element in the Unicode type. |
Georg Brandl | c6bc4c6 | 2011-10-05 16:23:09 +0200 | [diff] [blame] | 88 | With PEP 393, Py_UNICODE is deprecated and replaced with a |
Martin v. Löwis | d63a3b8 | 2011-09-28 07:41:54 +0200 | [diff] [blame] | 89 | typedef to wchar_t. */ |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 90 | |
Martin v. Löwis | d63a3b8 | 2011-09-28 07:41:54 +0200 | [diff] [blame] | 91 | #ifndef Py_LIMITED_API |
| 92 | #define PY_UNICODE_TYPE wchar_t |
| 93 | typedef wchar_t Py_UNICODE; |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 94 | #endif |
| 95 | |
| 96 | /* If the compiler provides a wchar_t type we try to support it |
Victor Stinner | 137c34c | 2010-09-29 10:25:54 +0000 | [diff] [blame] | 97 | through the interface functions PyUnicode_FromWideChar(), |
| 98 | PyUnicode_AsWideChar() and PyUnicode_AsWideCharString(). */ |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 99 | |
| 100 | #ifdef HAVE_USABLE_WCHAR_T |
Marc-André Lemburg | 1a731c6 | 2000-08-11 11:43:10 +0000 | [diff] [blame] | 101 | # ifndef HAVE_WCHAR_H |
| 102 | # define HAVE_WCHAR_H |
| 103 | # endif |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 104 | #endif |
| 105 | |
Martin v. Löwis | d63a3b8 | 2011-09-28 07:41:54 +0200 | [diff] [blame] | 106 | #if defined(MS_WINDOWS) |
Victor Stinner | 99b9538 | 2011-07-04 14:23:54 +0200 | [diff] [blame] | 107 | # define HAVE_MBCS |
| 108 | #endif |
| 109 | |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 110 | #ifdef HAVE_WCHAR_H |
Guido van Rossum | 24bdb04 | 2000-03-28 20:29:59 +0000 | [diff] [blame] | 111 | /* Work around a cosmetic bug in BSDI 4.x wchar.h; thanks to Thomas Wouters */ |
| 112 | # ifdef _HAVE_BSDI |
| 113 | # include <time.h> |
| 114 | # endif |
Marc-André Lemburg | 5e6007c | 2001-09-19 11:21:03 +0000 | [diff] [blame] | 115 | # include <wchar.h> |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 116 | #endif |
| 117 | |
Georg Brandl | c6bc4c6 | 2011-10-05 16:23:09 +0200 | [diff] [blame] | 118 | /* Py_UCS4 and Py_UCS2 are typedefs for the respective |
Martin v. Löwis | d63a3b8 | 2011-09-28 07:41:54 +0200 | [diff] [blame] | 119 | unicode representations. */ |
Victor Stinner | 7c8bbbb | 2011-11-20 18:28:29 +0100 | [diff] [blame] | 120 | #if SIZEOF_INT == 4 |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 121 | typedef unsigned int Py_UCS4; |
Victor Stinner | 7c8bbbb | 2011-11-20 18:28:29 +0100 | [diff] [blame] | 122 | #elif SIZEOF_LONG == 4 |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 123 | typedef unsigned long Py_UCS4; |
Marc-André Lemburg | b5ac6f6 | 2001-07-31 14:30:16 +0000 | [diff] [blame] | 124 | #else |
Martin v. Löwis | d63a3b8 | 2011-09-28 07:41:54 +0200 | [diff] [blame] | 125 | #error "Could not find a proper typedef for Py_UCS4" |
Marc-André Lemburg | b5ac6f6 | 2001-07-31 14:30:16 +0000 | [diff] [blame] | 126 | #endif |
| 127 | |
Victor Stinner | 7c8bbbb | 2011-11-20 18:28:29 +0100 | [diff] [blame] | 128 | #if SIZEOF_SHORT == 2 |
Martin v. Löwis | d63a3b8 | 2011-09-28 07:41:54 +0200 | [diff] [blame] | 129 | typedef unsigned short Py_UCS2; |
Victor Stinner | 7c8bbbb | 2011-11-20 18:28:29 +0100 | [diff] [blame] | 130 | #else |
| 131 | #error "Could not find a proper typedef for Py_UCS2" |
| 132 | #endif |
| 133 | |
Martin v. Löwis | d63a3b8 | 2011-09-28 07:41:54 +0200 | [diff] [blame] | 134 | typedef unsigned char Py_UCS1; |
| 135 | |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 136 | /* --- Internal Unicode Operations ---------------------------------------- */ |
| 137 | |
Benjamin Peterson | 960cf0f | 2009-01-09 04:11:44 +0000 | [diff] [blame] | 138 | /* Since splitting on whitespace is an important use case, and |
| 139 | whitespace in most situations is solely ASCII whitespace, we |
| 140 | optimize for the common case by using a quick look-up table |
| 141 | _Py_ascii_whitespace (see below) with an inlined check. |
Christian Heimes | 190d79e | 2008-01-30 11:58:22 +0000 | [diff] [blame] | 142 | |
Benjamin Peterson | 960cf0f | 2009-01-09 04:11:44 +0000 | [diff] [blame] | 143 | */ |
Martin v. Löwis | 4d0d471 | 2010-12-03 20:14:31 +0000 | [diff] [blame] | 144 | #ifndef Py_LIMITED_API |
Christian Heimes | 190d79e | 2008-01-30 11:58:22 +0000 | [diff] [blame] | 145 | #define Py_UNICODE_ISSPACE(ch) \ |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 146 | ((ch) < 128U ? _Py_ascii_whitespace[(ch)] : _PyUnicode_IsWhitespace(ch)) |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 147 | |
| 148 | #define Py_UNICODE_ISLOWER(ch) _PyUnicode_IsLowercase(ch) |
| 149 | #define Py_UNICODE_ISUPPER(ch) _PyUnicode_IsUppercase(ch) |
| 150 | #define Py_UNICODE_ISTITLE(ch) _PyUnicode_IsTitlecase(ch) |
| 151 | #define Py_UNICODE_ISLINEBREAK(ch) _PyUnicode_IsLinebreak(ch) |
| 152 | |
| 153 | #define Py_UNICODE_TOLOWER(ch) _PyUnicode_ToLowercase(ch) |
| 154 | #define Py_UNICODE_TOUPPER(ch) _PyUnicode_ToUppercase(ch) |
| 155 | #define Py_UNICODE_TOTITLE(ch) _PyUnicode_ToTitlecase(ch) |
| 156 | |
| 157 | #define Py_UNICODE_ISDECIMAL(ch) _PyUnicode_IsDecimalDigit(ch) |
| 158 | #define Py_UNICODE_ISDIGIT(ch) _PyUnicode_IsDigit(ch) |
| 159 | #define Py_UNICODE_ISNUMERIC(ch) _PyUnicode_IsNumeric(ch) |
Georg Brandl | 559e5d7 | 2008-06-11 18:37:52 +0000 | [diff] [blame] | 160 | #define Py_UNICODE_ISPRINTABLE(ch) _PyUnicode_IsPrintable(ch) |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 161 | |
| 162 | #define Py_UNICODE_TODECIMAL(ch) _PyUnicode_ToDecimalDigit(ch) |
| 163 | #define Py_UNICODE_TODIGIT(ch) _PyUnicode_ToDigit(ch) |
| 164 | #define Py_UNICODE_TONUMERIC(ch) _PyUnicode_ToNumeric(ch) |
| 165 | |
Marc-André Lemburg | f03e741 | 2000-07-05 09:45:59 +0000 | [diff] [blame] | 166 | #define Py_UNICODE_ISALPHA(ch) _PyUnicode_IsAlpha(ch) |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 167 | |
Marc-André Lemburg | a9c103b | 2000-07-03 10:52:13 +0000 | [diff] [blame] | 168 | #define Py_UNICODE_ISALNUM(ch) \ |
| 169 | (Py_UNICODE_ISALPHA(ch) || \ |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 170 | Py_UNICODE_ISDECIMAL(ch) || \ |
| 171 | Py_UNICODE_ISDIGIT(ch) || \ |
| 172 | Py_UNICODE_ISNUMERIC(ch)) |
Marc-André Lemburg | a9c103b | 2000-07-03 10:52:13 +0000 | [diff] [blame] | 173 | |
Martin v. Löwis | d63a3b8 | 2011-09-28 07:41:54 +0200 | [diff] [blame] | 174 | #define Py_UNICODE_COPY(target, source, length) \ |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 175 | Py_MEMCPY((target), (source), (length)*sizeof(Py_UNICODE)) |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 176 | |
Benjamin Peterson | 960cf0f | 2009-01-09 04:11:44 +0000 | [diff] [blame] | 177 | #define Py_UNICODE_FILL(target, value, length) \ |
| 178 | do {Py_ssize_t i_; Py_UNICODE *t_ = (target); Py_UNICODE v_ = (value);\ |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 179 | for (i_ = 0; i_ < (length); i_++) t_[i_] = v_;\ |
Thomas Wouters | 477c8d5 | 2006-05-27 19:21:47 +0000 | [diff] [blame] | 180 | } while (0) |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 181 | |
Ezio Melotti | 8c9375b | 2011-08-22 20:03:25 +0300 | [diff] [blame] | 182 | /* macros to work with surrogates */ |
Victor Stinner | 76df43d | 2012-10-30 01:42:39 +0100 | [diff] [blame] | 183 | #define Py_UNICODE_IS_SURROGATE(ch) (0xD800 <= (ch) && (ch) <= 0xDFFF) |
| 184 | #define Py_UNICODE_IS_HIGH_SURROGATE(ch) (0xD800 <= (ch) && (ch) <= 0xDBFF) |
| 185 | #define Py_UNICODE_IS_LOW_SURROGATE(ch) (0xDC00 <= (ch) && (ch) <= 0xDFFF) |
Ezio Melotti | 8c9375b | 2011-08-22 20:03:25 +0300 | [diff] [blame] | 186 | /* Join two surrogate characters and return a single Py_UCS4 value. */ |
| 187 | #define Py_UNICODE_JOIN_SURROGATES(high, low) \ |
| 188 | (((((Py_UCS4)(high) & 0x03FF) << 10) | \ |
| 189 | ((Py_UCS4)(low) & 0x03FF)) + 0x10000) |
Victor Stinner | 551ac95 | 2011-11-29 22:58:13 +0100 | [diff] [blame] | 190 | /* high surrogate = top 10 bits added to D800 */ |
Antoine Pitrou | 27f6a3b | 2012-06-15 22:15:23 +0200 | [diff] [blame] | 191 | #define Py_UNICODE_HIGH_SURROGATE(ch) (0xD800 - (0x10000 >> 10) + ((ch) >> 10)) |
Victor Stinner | 551ac95 | 2011-11-29 22:58:13 +0100 | [diff] [blame] | 192 | /* low surrogate = bottom 10 bits added to DC00 */ |
Antoine Pitrou | 27f6a3b | 2012-06-15 22:15:23 +0200 | [diff] [blame] | 193 | #define Py_UNICODE_LOW_SURROGATE(ch) (0xDC00 + ((ch) & 0x3FF)) |
Ezio Melotti | 8c9375b | 2011-08-22 20:03:25 +0300 | [diff] [blame] | 194 | |
Alexander Belopolsky | 83283c2 | 2010-11-16 14:29:01 +0000 | [diff] [blame] | 195 | /* Check if substring matches at given offset. The offset must be |
| 196 | valid, and the substring must not be empty. */ |
Benjamin Peterson | 960cf0f | 2009-01-09 04:11:44 +0000 | [diff] [blame] | 197 | |
Thomas Wouters | 477c8d5 | 2006-05-27 19:21:47 +0000 | [diff] [blame] | 198 | #define Py_UNICODE_MATCH(string, offset, substring) \ |
Martin v. Löwis | d63a3b8 | 2011-09-28 07:41:54 +0200 | [diff] [blame] | 199 | ((*((string)->wstr + (offset)) == *((substring)->wstr)) && \ |
| 200 | ((*((string)->wstr + (offset) + (substring)->wstr_length-1) == *((substring)->wstr + (substring)->wstr_length-1))) && \ |
| 201 | !memcmp((string)->wstr + (offset), (substring)->wstr, (substring)->wstr_length*sizeof(Py_UNICODE))) |
| 202 | |
Martin v. Löwis | 4d0d471 | 2010-12-03 20:14:31 +0000 | [diff] [blame] | 203 | #endif /* Py_LIMITED_API */ |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 204 | |
Barry Warsaw | 51ac580 | 2000-03-20 16:36:48 +0000 | [diff] [blame] | 205 | #ifdef __cplusplus |
| 206 | extern "C" { |
| 207 | #endif |
| 208 | |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 209 | /* --- Unicode Type ------------------------------------------------------- */ |
| 210 | |
Martin v. Löwis | 4d0d471 | 2010-12-03 20:14:31 +0000 | [diff] [blame] | 211 | #ifndef Py_LIMITED_API |
Martin v. Löwis | d63a3b8 | 2011-09-28 07:41:54 +0200 | [diff] [blame] | 212 | |
| 213 | /* ASCII-only strings created through PyUnicode_New use the PyASCIIObject |
| 214 | structure. state.ascii and state.compact are set, and the data |
| 215 | immediately follow the structure. utf8_length and wstr_length can be found |
| 216 | in the length field; the utf8 pointer is equal to the data pointer. */ |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 217 | typedef struct { |
Éric Araujo | 80a348c | 2011-10-05 01:11:12 +0200 | [diff] [blame] | 218 | /* There are 4 forms of Unicode strings: |
Victor Stinner | 910337b | 2011-10-03 03:20:16 +0200 | [diff] [blame] | 219 | |
| 220 | - compact ascii: |
| 221 | |
| 222 | * structure = PyASCIIObject |
Victor Stinner | 7a9105a | 2011-12-12 00:13:42 +0100 | [diff] [blame] | 223 | * test: PyUnicode_IS_COMPACT_ASCII(op) |
Victor Stinner | 910337b | 2011-10-03 03:20:16 +0200 | [diff] [blame] | 224 | * kind = PyUnicode_1BYTE_KIND |
| 225 | * compact = 1 |
| 226 | * ascii = 1 |
| 227 | * ready = 1 |
Victor Stinner | 30134f5 | 2011-10-04 01:32:45 +0200 | [diff] [blame] | 228 | * (length is the length of the utf8 and wstr strings) |
| 229 | * (data starts just after the structure) |
| 230 | * (since ASCII is decoded from UTF-8, the utf8 string are the data) |
Victor Stinner | 910337b | 2011-10-03 03:20:16 +0200 | [diff] [blame] | 231 | |
| 232 | - compact: |
| 233 | |
| 234 | * structure = PyCompactUnicodeObject |
Victor Stinner | 80bc72d | 2011-12-22 03:23:10 +0100 | [diff] [blame] | 235 | * test: PyUnicode_IS_COMPACT(op) && !PyUnicode_IS_ASCII(op) |
Victor Stinner | 910337b | 2011-10-03 03:20:16 +0200 | [diff] [blame] | 236 | * kind = PyUnicode_1BYTE_KIND, PyUnicode_2BYTE_KIND or |
| 237 | PyUnicode_4BYTE_KIND |
| 238 | * compact = 1 |
| 239 | * ready = 1 |
Victor Stinner | a3b334d | 2011-10-03 13:53:37 +0200 | [diff] [blame] | 240 | * ascii = 0 |
Victor Stinner | 30134f5 | 2011-10-04 01:32:45 +0200 | [diff] [blame] | 241 | * utf8 is not shared with data |
Victor Stinner | a41463c | 2011-10-04 01:05:08 +0200 | [diff] [blame] | 242 | * utf8_length = 0 if utf8 is NULL |
| 243 | * wstr is shared with data and wstr_length=length |
| 244 | if kind=PyUnicode_2BYTE_KIND and sizeof(wchar_t)=2 |
Victor Stinner | e30c0a1 | 2011-11-04 20:54:05 +0100 | [diff] [blame] | 245 | or if kind=PyUnicode_4BYTE_KIND and sizeof(wchar_t)=4 |
Victor Stinner | a41463c | 2011-10-04 01:05:08 +0200 | [diff] [blame] | 246 | * wstr_length = 0 if wstr is NULL |
Victor Stinner | 30134f5 | 2011-10-04 01:32:45 +0200 | [diff] [blame] | 247 | * (data starts just after the structure) |
Victor Stinner | 910337b | 2011-10-03 03:20:16 +0200 | [diff] [blame] | 248 | |
Victor Stinner | 8cfcbed | 2011-10-03 23:19:21 +0200 | [diff] [blame] | 249 | - legacy string, not ready: |
Victor Stinner | 910337b | 2011-10-03 03:20:16 +0200 | [diff] [blame] | 250 | |
| 251 | * structure = PyUnicodeObject |
Victor Stinner | 7a9105a | 2011-12-12 00:13:42 +0100 | [diff] [blame] | 252 | * test: kind == PyUnicode_WCHAR_KIND |
Victor Stinner | e30c0a1 | 2011-11-04 20:54:05 +0100 | [diff] [blame] | 253 | * length = 0 (use wstr_length) |
| 254 | * hash = -1 |
Victor Stinner | 910337b | 2011-10-03 03:20:16 +0200 | [diff] [blame] | 255 | * kind = PyUnicode_WCHAR_KIND |
| 256 | * compact = 0 |
Victor Stinner | 30134f5 | 2011-10-04 01:32:45 +0200 | [diff] [blame] | 257 | * ascii = 0 |
Victor Stinner | 910337b | 2011-10-03 03:20:16 +0200 | [diff] [blame] | 258 | * ready = 0 |
Victor Stinner | e30c0a1 | 2011-11-04 20:54:05 +0100 | [diff] [blame] | 259 | * interned = SSTATE_NOT_INTERNED |
Victor Stinner | 910337b | 2011-10-03 03:20:16 +0200 | [diff] [blame] | 260 | * wstr is not NULL |
| 261 | * data.any is NULL |
| 262 | * utf8 is NULL |
Victor Stinner | a41463c | 2011-10-04 01:05:08 +0200 | [diff] [blame] | 263 | * utf8_length = 0 |
Victor Stinner | 910337b | 2011-10-03 03:20:16 +0200 | [diff] [blame] | 264 | |
Victor Stinner | 8cfcbed | 2011-10-03 23:19:21 +0200 | [diff] [blame] | 265 | - legacy string, ready: |
Victor Stinner | 910337b | 2011-10-03 03:20:16 +0200 | [diff] [blame] | 266 | |
| 267 | * structure = PyUnicodeObject structure |
Victor Stinner | 7a9105a | 2011-12-12 00:13:42 +0100 | [diff] [blame] | 268 | * test: !PyUnicode_IS_COMPACT(op) && kind != PyUnicode_WCHAR_KIND |
Victor Stinner | 910337b | 2011-10-03 03:20:16 +0200 | [diff] [blame] | 269 | * kind = PyUnicode_1BYTE_KIND, PyUnicode_2BYTE_KIND or |
| 270 | PyUnicode_4BYTE_KIND |
| 271 | * compact = 0 |
| 272 | * ready = 1 |
| 273 | * data.any is not NULL |
Victor Stinner | a41463c | 2011-10-04 01:05:08 +0200 | [diff] [blame] | 274 | * utf8 is shared and utf8_length = length with data.any if ascii = 1 |
| 275 | * utf8_length = 0 if utf8 is NULL |
Victor Stinner | e30c0a1 | 2011-11-04 20:54:05 +0100 | [diff] [blame] | 276 | * wstr is shared with data.any and wstr_length = length |
Victor Stinner | a41463c | 2011-10-04 01:05:08 +0200 | [diff] [blame] | 277 | if kind=PyUnicode_2BYTE_KIND and sizeof(wchar_t)=2 |
| 278 | or if kind=PyUnicode_4BYTE_KIND and sizeof(wchar_4)=4 |
| 279 | * wstr_length = 0 if wstr is NULL |
Victor Stinner | 910337b | 2011-10-03 03:20:16 +0200 | [diff] [blame] | 280 | |
Victor Stinner | 8cfcbed | 2011-10-03 23:19:21 +0200 | [diff] [blame] | 281 | Compact strings use only one memory block (structure + characters), |
| 282 | whereas legacy strings use one block for the structure and one block |
| 283 | for characters. |
Victor Stinner | 910337b | 2011-10-03 03:20:16 +0200 | [diff] [blame] | 284 | |
Victor Stinner | 8cfcbed | 2011-10-03 23:19:21 +0200 | [diff] [blame] | 285 | Legacy strings are created by PyUnicode_FromUnicode() and |
| 286 | PyUnicode_FromStringAndSize(NULL, size) functions. They become ready |
| 287 | when PyUnicode_READY() is called. |
| 288 | |
| 289 | See also _PyUnicode_CheckConsistency(). |
| 290 | */ |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 291 | PyObject_HEAD |
Martin v. Löwis | d63a3b8 | 2011-09-28 07:41:54 +0200 | [diff] [blame] | 292 | Py_ssize_t length; /* Number of code points in the string */ |
Benjamin Peterson | 8f67d08 | 2010-10-17 20:54:53 +0000 | [diff] [blame] | 293 | Py_hash_t hash; /* Hash value; -1 if not set */ |
Martin v. Löwis | d63a3b8 | 2011-09-28 07:41:54 +0200 | [diff] [blame] | 294 | struct { |
| 295 | /* |
| 296 | SSTATE_NOT_INTERNED (0) |
| 297 | SSTATE_INTERNED_MORTAL (1) |
| 298 | SSTATE_INTERNED_IMMORTAL (2) |
| 299 | |
| 300 | If interned != SSTATE_NOT_INTERNED, the two references from the |
| 301 | dictionary to this object are *not* counted in ob_refcnt. |
| 302 | */ |
| 303 | unsigned int interned:2; |
| 304 | /* Character size: |
| 305 | |
Victor Stinner | 4d0d54b | 2011-10-05 01:31:05 +0200 | [diff] [blame] | 306 | - PyUnicode_WCHAR_KIND (0): |
| 307 | |
| 308 | * character type = wchar_t (16 or 32 bits, depending on the |
| 309 | platform) |
| 310 | |
| 311 | - PyUnicode_1BYTE_KIND (1): |
| 312 | |
| 313 | * character type = Py_UCS1 (8 bits, unsigned) |
Victor Stinner | 77faf69 | 2011-11-20 18:56:05 +0100 | [diff] [blame] | 314 | * all characters are in the range U+0000-U+00FF (latin1) |
| 315 | * if ascii is set, all characters are in the range U+0000-U+007F |
| 316 | (ASCII), otherwise at least one character is in the range |
Victor Stinner | 1d4b35f | 2011-10-06 01:51:19 +0200 | [diff] [blame] | 317 | U+0080-U+00FF |
Victor Stinner | 4d0d54b | 2011-10-05 01:31:05 +0200 | [diff] [blame] | 318 | |
| 319 | - PyUnicode_2BYTE_KIND (2): |
| 320 | |
| 321 | * character type = Py_UCS2 (16 bits, unsigned) |
Victor Stinner | 77faf69 | 2011-11-20 18:56:05 +0100 | [diff] [blame] | 322 | * all characters are in the range U+0000-U+FFFF (BMP) |
| 323 | * at least one character is in the range U+0100-U+FFFF |
Victor Stinner | 4d0d54b | 2011-10-05 01:31:05 +0200 | [diff] [blame] | 324 | |
Martin v. Löwis | c47adb0 | 2011-10-07 20:55:35 +0200 | [diff] [blame] | 325 | - PyUnicode_4BYTE_KIND (4): |
Victor Stinner | 4d0d54b | 2011-10-05 01:31:05 +0200 | [diff] [blame] | 326 | |
| 327 | * character type = Py_UCS4 (32 bits, unsigned) |
Victor Stinner | 77faf69 | 2011-11-20 18:56:05 +0100 | [diff] [blame] | 328 | * all characters are in the range U+0000-U+10FFFF |
| 329 | * at least one character is in the range U+10000-U+10FFFF |
Martin v. Löwis | d63a3b8 | 2011-09-28 07:41:54 +0200 | [diff] [blame] | 330 | */ |
Martin v. Löwis | c47adb0 | 2011-10-07 20:55:35 +0200 | [diff] [blame] | 331 | unsigned int kind:3; |
Martin v. Löwis | d63a3b8 | 2011-09-28 07:41:54 +0200 | [diff] [blame] | 332 | /* Compact is with respect to the allocation scheme. Compact unicode |
| 333 | objects only require one memory block while non-compact objects use |
| 334 | one block for the PyUnicodeObject struct and another for its data |
| 335 | buffer. */ |
| 336 | unsigned int compact:1; |
Victor Stinner | 77faf69 | 2011-11-20 18:56:05 +0100 | [diff] [blame] | 337 | /* The string only contains characters in the range U+0000-U+007F (ASCII) |
Victor Stinner | 1d4b35f | 2011-10-06 01:51:19 +0200 | [diff] [blame] | 338 | and the kind is PyUnicode_1BYTE_KIND. If ascii is set and compact is |
| 339 | set, use the PyASCIIObject structure. */ |
Martin v. Löwis | d63a3b8 | 2011-09-28 07:41:54 +0200 | [diff] [blame] | 340 | unsigned int ascii:1; |
| 341 | /* The ready flag indicates whether the object layout is initialized |
| 342 | completely. This means that this is either a compact object, or |
| 343 | the data pointer is filled out. The bit is redundant, and helps |
| 344 | to minimize the test in PyUnicode_IS_READY(). */ |
| 345 | unsigned int ready:1; |
Antoine Pitrou | 8c6f8dc | 2014-03-23 22:55:03 +0100 | [diff] [blame] | 346 | /* Padding to ensure that PyUnicode_DATA() is always aligned to |
| 347 | 4 bytes (see issue #19537 on m68k). */ |
| 348 | unsigned int :24; |
Martin v. Löwis | d63a3b8 | 2011-09-28 07:41:54 +0200 | [diff] [blame] | 349 | } state; |
| 350 | wchar_t *wstr; /* wchar_t representation (null-terminated) */ |
| 351 | } PyASCIIObject; |
| 352 | |
| 353 | /* Non-ASCII strings allocated through PyUnicode_New use the |
Georg Brandl | c6bc4c6 | 2011-10-05 16:23:09 +0200 | [diff] [blame] | 354 | PyCompactUnicodeObject structure. state.compact is set, and the data |
Martin v. Löwis | d63a3b8 | 2011-09-28 07:41:54 +0200 | [diff] [blame] | 355 | immediately follow the structure. */ |
| 356 | typedef struct { |
| 357 | PyASCIIObject _base; |
| 358 | Py_ssize_t utf8_length; /* Number of bytes in utf8, excluding the |
| 359 | * terminating \0. */ |
| 360 | char *utf8; /* UTF-8 representation (null-terminated) */ |
| 361 | Py_ssize_t wstr_length; /* Number of code points in wstr, possible |
| 362 | * surrogates count as two code points. */ |
| 363 | } PyCompactUnicodeObject; |
| 364 | |
| 365 | /* Strings allocated through PyUnicode_FromUnicode(NULL, len) use the |
| 366 | PyUnicodeObject structure. The actual string data is initially in the wstr |
Victor Stinner | a3b334d | 2011-10-03 13:53:37 +0200 | [diff] [blame] | 367 | block, and copied into the data block using _PyUnicode_Ready. */ |
Martin v. Löwis | d63a3b8 | 2011-09-28 07:41:54 +0200 | [diff] [blame] | 368 | typedef struct { |
| 369 | PyCompactUnicodeObject _base; |
| 370 | union { |
| 371 | void *any; |
| 372 | Py_UCS1 *latin1; |
| 373 | Py_UCS2 *ucs2; |
| 374 | Py_UCS4 *ucs4; |
| 375 | } data; /* Canonical, smallest-form Unicode buffer */ |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 376 | } PyUnicodeObject; |
Martin v. Löwis | 4d0d471 | 2010-12-03 20:14:31 +0000 | [diff] [blame] | 377 | #endif |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 378 | |
Mark Hammond | 91a681d | 2002-08-12 07:21:58 +0000 | [diff] [blame] | 379 | PyAPI_DATA(PyTypeObject) PyUnicode_Type; |
Christian Heimes | a22e8bd | 2007-11-29 22:35:39 +0000 | [diff] [blame] | 380 | PyAPI_DATA(PyTypeObject) PyUnicodeIter_Type; |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 381 | |
Thomas Wouters | 27d517b | 2007-02-25 20:39:11 +0000 | [diff] [blame] | 382 | #define PyUnicode_Check(op) \ |
Christian Heimes | 90aa764 | 2007-12-19 02:45:37 +0000 | [diff] [blame] | 383 | PyType_FastSubclass(Py_TYPE(op), Py_TPFLAGS_UNICODE_SUBCLASS) |
| 384 | #define PyUnicode_CheckExact(op) (Py_TYPE(op) == &PyUnicode_Type) |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 385 | |
| 386 | /* Fast access macros */ |
Martin v. Löwis | 4d0d471 | 2010-12-03 20:14:31 +0000 | [diff] [blame] | 387 | #ifndef Py_LIMITED_API |
Martin v. Löwis | d63a3b8 | 2011-09-28 07:41:54 +0200 | [diff] [blame] | 388 | |
| 389 | #define PyUnicode_WSTR_LENGTH(op) \ |
Victor Stinner | a3b334d | 2011-10-03 13:53:37 +0200 | [diff] [blame] | 390 | (PyUnicode_IS_COMPACT_ASCII(op) ? \ |
Martin v. Löwis | d63a3b8 | 2011-09-28 07:41:54 +0200 | [diff] [blame] | 391 | ((PyASCIIObject*)op)->length : \ |
| 392 | ((PyCompactUnicodeObject*)op)->wstr_length) |
| 393 | |
| 394 | /* Returns the deprecated Py_UNICODE representation's size in code units |
| 395 | (this includes surrogate pairs as 2 units). |
| 396 | If the Py_UNICODE representation is not available, it will be computed |
| 397 | on request. Use PyUnicode_GET_LENGTH() for the length in code points. */ |
| 398 | |
Victor Stinner | f3ae620 | 2011-11-21 02:24:49 +0100 | [diff] [blame] | 399 | #define PyUnicode_GET_SIZE(op) \ |
| 400 | (assert(PyUnicode_Check(op)), \ |
| 401 | (((PyASCIIObject *)(op))->wstr) ? \ |
| 402 | PyUnicode_WSTR_LENGTH(op) : \ |
| 403 | ((void)PyUnicode_AsUnicode((PyObject *)(op)), \ |
| 404 | assert(((PyASCIIObject *)(op))->wstr), \ |
| 405 | PyUnicode_WSTR_LENGTH(op))) |
Martin v. Löwis | d63a3b8 | 2011-09-28 07:41:54 +0200 | [diff] [blame] | 406 | |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 407 | #define PyUnicode_GET_DATA_SIZE(op) \ |
Martin v. Löwis | d63a3b8 | 2011-09-28 07:41:54 +0200 | [diff] [blame] | 408 | (PyUnicode_GET_SIZE(op) * Py_UNICODE_SIZE) |
| 409 | |
| 410 | /* Alias for PyUnicode_AsUnicode(). This will create a wchar_t/Py_UNICODE |
| 411 | representation on demand. Using this macro is very inefficient now, |
| 412 | try to port your code to use the new PyUnicode_*BYTE_DATA() macros or |
| 413 | use PyUnicode_WRITE() and PyUnicode_READ(). */ |
| 414 | |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 415 | #define PyUnicode_AS_UNICODE(op) \ |
Martin v. Löwis | d63a3b8 | 2011-09-28 07:41:54 +0200 | [diff] [blame] | 416 | (assert(PyUnicode_Check(op)), \ |
| 417 | (((PyASCIIObject *)(op))->wstr) ? (((PyASCIIObject *)(op))->wstr) : \ |
| 418 | PyUnicode_AsUnicode((PyObject *)(op))) |
| 419 | |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 420 | #define PyUnicode_AS_DATA(op) \ |
Martin v. Löwis | d63a3b8 | 2011-09-28 07:41:54 +0200 | [diff] [blame] | 421 | ((const char *)(PyUnicode_AS_UNICODE(op))) |
| 422 | |
| 423 | |
Georg Brandl | c6bc4c6 | 2011-10-05 16:23:09 +0200 | [diff] [blame] | 424 | /* --- Flexible String Representation Helper Macros (PEP 393) -------------- */ |
Martin v. Löwis | d63a3b8 | 2011-09-28 07:41:54 +0200 | [diff] [blame] | 425 | |
Victor Stinner | 6f9568b | 2011-11-17 00:12:44 +0100 | [diff] [blame] | 426 | /* Values for PyASCIIObject.state: */ |
Martin v. Löwis | d63a3b8 | 2011-09-28 07:41:54 +0200 | [diff] [blame] | 427 | |
| 428 | /* Interning state. */ |
| 429 | #define SSTATE_NOT_INTERNED 0 |
| 430 | #define SSTATE_INTERNED_MORTAL 1 |
| 431 | #define SSTATE_INTERNED_IMMORTAL 2 |
| 432 | |
Victor Stinner | a3b334d | 2011-10-03 13:53:37 +0200 | [diff] [blame] | 433 | /* Return true if the string contains only ASCII characters, or 0 if not. The |
Victor Stinner | 24c74be | 2011-12-12 01:24:20 +0100 | [diff] [blame] | 434 | string may be compact (PyUnicode_IS_COMPACT_ASCII) or not, but must be |
| 435 | ready. */ |
| 436 | #define PyUnicode_IS_ASCII(op) \ |
| 437 | (assert(PyUnicode_Check(op)), \ |
| 438 | assert(PyUnicode_IS_READY(op)), \ |
| 439 | ((PyASCIIObject*)op)->state.ascii) |
Victor Stinner | a3b334d | 2011-10-03 13:53:37 +0200 | [diff] [blame] | 440 | |
| 441 | /* Return true if the string is compact or 0 if not. |
| 442 | No type checks or Ready calls are performed. */ |
| 443 | #define PyUnicode_IS_COMPACT(op) \ |
| 444 | (((PyASCIIObject*)(op))->state.compact) |
| 445 | |
| 446 | /* Return true if the string is a compact ASCII string (use PyASCIIObject |
| 447 | structure), or 0 if not. No type checks or Ready calls are performed. */ |
| 448 | #define PyUnicode_IS_COMPACT_ASCII(op) \ |
Victor Stinner | 24c74be | 2011-12-12 01:24:20 +0100 | [diff] [blame] | 449 | (((PyASCIIObject*)op)->state.ascii && PyUnicode_IS_COMPACT(op)) |
Martin v. Löwis | d63a3b8 | 2011-09-28 07:41:54 +0200 | [diff] [blame] | 450 | |
Victor Stinner | 52e2cc8 | 2011-12-19 22:14:45 +0100 | [diff] [blame] | 451 | enum PyUnicode_Kind { |
Martin v. Löwis | d63a3b8 | 2011-09-28 07:41:54 +0200 | [diff] [blame] | 452 | /* String contains only wstr byte characters. This is only possible |
Victor Stinner | a3b334d | 2011-10-03 13:53:37 +0200 | [diff] [blame] | 453 | when the string was created with a legacy API and _PyUnicode_Ready() |
Martin v. Löwis | d63a3b8 | 2011-09-28 07:41:54 +0200 | [diff] [blame] | 454 | has not been called yet. */ |
Victor Stinner | 52e2cc8 | 2011-12-19 22:14:45 +0100 | [diff] [blame] | 455 | PyUnicode_WCHAR_KIND = 0, |
| 456 | /* Return values of the PyUnicode_KIND() macro: */ |
Martin v. Löwis | c47adb0 | 2011-10-07 20:55:35 +0200 | [diff] [blame] | 457 | PyUnicode_1BYTE_KIND = 1, |
| 458 | PyUnicode_2BYTE_KIND = 2, |
| 459 | PyUnicode_4BYTE_KIND = 4 |
| 460 | }; |
Martin v. Löwis | d63a3b8 | 2011-09-28 07:41:54 +0200 | [diff] [blame] | 461 | |
Georg Brandl | 4975a9b | 2011-10-05 16:12:21 +0200 | [diff] [blame] | 462 | /* Return pointers to the canonical representation cast to unsigned char, |
Martin v. Löwis | d63a3b8 | 2011-09-28 07:41:54 +0200 | [diff] [blame] | 463 | Py_UCS2, or Py_UCS4 for direct character access. |
Martin v. Löwis | c47adb0 | 2011-10-07 20:55:35 +0200 | [diff] [blame] | 464 | No checks are performed, use PyUnicode_KIND() before to ensure |
| 465 | these will work correctly. */ |
Martin v. Löwis | d63a3b8 | 2011-09-28 07:41:54 +0200 | [diff] [blame] | 466 | |
| 467 | #define PyUnicode_1BYTE_DATA(op) ((Py_UCS1*)PyUnicode_DATA(op)) |
| 468 | #define PyUnicode_2BYTE_DATA(op) ((Py_UCS2*)PyUnicode_DATA(op)) |
| 469 | #define PyUnicode_4BYTE_DATA(op) ((Py_UCS4*)PyUnicode_DATA(op)) |
| 470 | |
Victor Stinner | 157f83f | 2011-09-28 21:41:31 +0200 | [diff] [blame] | 471 | /* Return one of the PyUnicode_*_KIND values defined above. */ |
Martin v. Löwis | d63a3b8 | 2011-09-28 07:41:54 +0200 | [diff] [blame] | 472 | #define PyUnicode_KIND(op) \ |
| 473 | (assert(PyUnicode_Check(op)), \ |
| 474 | assert(PyUnicode_IS_READY(op)), \ |
| 475 | ((PyASCIIObject *)(op))->state.kind) |
| 476 | |
Victor Stinner | 157f83f | 2011-09-28 21:41:31 +0200 | [diff] [blame] | 477 | /* Return a void pointer to the raw unicode buffer. */ |
Martin v. Löwis | d63a3b8 | 2011-09-28 07:41:54 +0200 | [diff] [blame] | 478 | #define _PyUnicode_COMPACT_DATA(op) \ |
Victor Stinner | 55c7e00 | 2011-10-18 23:32:53 +0200 | [diff] [blame] | 479 | (PyUnicode_IS_ASCII(op) ? \ |
Martin v. Löwis | d63a3b8 | 2011-09-28 07:41:54 +0200 | [diff] [blame] | 480 | ((void*)((PyASCIIObject*)(op) + 1)) : \ |
| 481 | ((void*)((PyCompactUnicodeObject*)(op) + 1))) |
| 482 | |
| 483 | #define _PyUnicode_NONCOMPACT_DATA(op) \ |
| 484 | (assert(((PyUnicodeObject*)(op))->data.any), \ |
| 485 | ((((PyUnicodeObject *)(op))->data.any))) |
| 486 | |
| 487 | #define PyUnicode_DATA(op) \ |
| 488 | (assert(PyUnicode_Check(op)), \ |
| 489 | PyUnicode_IS_COMPACT(op) ? _PyUnicode_COMPACT_DATA(op) : \ |
| 490 | _PyUnicode_NONCOMPACT_DATA(op)) |
| 491 | |
Martin v. Löwis | d63a3b8 | 2011-09-28 07:41:54 +0200 | [diff] [blame] | 492 | /* In the access macros below, "kind" may be evaluated more than once. |
| 493 | All other macro parameters are evaluated exactly once, so it is safe |
| 494 | to put side effects into them (such as increasing the index). */ |
| 495 | |
| 496 | /* Write into the canonical representation, this macro does not do any sanity |
| 497 | checks and is intended for usage in loops. The caller should cache the |
Georg Brandl | 07de325 | 2011-10-05 16:47:38 +0200 | [diff] [blame] | 498 | kind and data pointers obtained from other macro calls. |
Martin v. Löwis | d63a3b8 | 2011-09-28 07:41:54 +0200 | [diff] [blame] | 499 | index is the index in the string (starts at 0) and value is the new |
Georg Brandl | c6bc4c6 | 2011-10-05 16:23:09 +0200 | [diff] [blame] | 500 | code point value which should be written to that location. */ |
Martin v. Löwis | d63a3b8 | 2011-09-28 07:41:54 +0200 | [diff] [blame] | 501 | #define PyUnicode_WRITE(kind, data, index, value) \ |
| 502 | do { \ |
| 503 | switch ((kind)) { \ |
| 504 | case PyUnicode_1BYTE_KIND: { \ |
| 505 | ((Py_UCS1 *)(data))[(index)] = (Py_UCS1)(value); \ |
| 506 | break; \ |
| 507 | } \ |
| 508 | case PyUnicode_2BYTE_KIND: { \ |
| 509 | ((Py_UCS2 *)(data))[(index)] = (Py_UCS2)(value); \ |
| 510 | break; \ |
| 511 | } \ |
| 512 | default: { \ |
| 513 | assert((kind) == PyUnicode_4BYTE_KIND); \ |
| 514 | ((Py_UCS4 *)(data))[(index)] = (Py_UCS4)(value); \ |
| 515 | } \ |
| 516 | } \ |
| 517 | } while (0) |
| 518 | |
Georg Brandl | 07de325 | 2011-10-05 16:47:38 +0200 | [diff] [blame] | 519 | /* Read a code point from the string's canonical representation. No checks |
Martin v. Löwis | d63a3b8 | 2011-09-28 07:41:54 +0200 | [diff] [blame] | 520 | or ready calls are performed. */ |
| 521 | #define PyUnicode_READ(kind, data, index) \ |
| 522 | ((Py_UCS4) \ |
| 523 | ((kind) == PyUnicode_1BYTE_KIND ? \ |
Victor Stinner | 7a48ff7 | 2011-10-02 00:55:25 +0200 | [diff] [blame] | 524 | ((const Py_UCS1 *)(data))[(index)] : \ |
Martin v. Löwis | d63a3b8 | 2011-09-28 07:41:54 +0200 | [diff] [blame] | 525 | ((kind) == PyUnicode_2BYTE_KIND ? \ |
| 526 | ((const Py_UCS2 *)(data))[(index)] : \ |
| 527 | ((const Py_UCS4 *)(data))[(index)] \ |
| 528 | ) \ |
| 529 | )) |
| 530 | |
| 531 | /* PyUnicode_READ_CHAR() is less efficient than PyUnicode_READ() because it |
| 532 | calls PyUnicode_KIND() and might call it twice. For single reads, use |
| 533 | PyUnicode_READ_CHAR, for multiple consecutive reads callers should |
| 534 | cache kind and use PyUnicode_READ instead. */ |
| 535 | #define PyUnicode_READ_CHAR(unicode, index) \ |
Victor Stinner | 3794376 | 2011-10-02 20:33:18 +0200 | [diff] [blame] | 536 | (assert(PyUnicode_Check(unicode)), \ |
| 537 | assert(PyUnicode_IS_READY(unicode)), \ |
| 538 | (Py_UCS4) \ |
| 539 | (PyUnicode_KIND((unicode)) == PyUnicode_1BYTE_KIND ? \ |
| 540 | ((const Py_UCS1 *)(PyUnicode_DATA((unicode))))[(index)] : \ |
| 541 | (PyUnicode_KIND((unicode)) == PyUnicode_2BYTE_KIND ? \ |
| 542 | ((const Py_UCS2 *)(PyUnicode_DATA((unicode))))[(index)] : \ |
| 543 | ((const Py_UCS4 *)(PyUnicode_DATA((unicode))))[(index)] \ |
| 544 | ) \ |
| 545 | )) |
Martin v. Löwis | d63a3b8 | 2011-09-28 07:41:54 +0200 | [diff] [blame] | 546 | |
| 547 | /* Returns the length of the unicode string. The caller has to make sure that |
| 548 | the string has it's canonical representation set before calling |
| 549 | this macro. Call PyUnicode_(FAST_)Ready to ensure that. */ |
| 550 | #define PyUnicode_GET_LENGTH(op) \ |
| 551 | (assert(PyUnicode_Check(op)), \ |
| 552 | assert(PyUnicode_IS_READY(op)), \ |
| 553 | ((PyASCIIObject *)(op))->length) |
| 554 | |
| 555 | |
| 556 | /* Fast check to determine whether an object is ready. Equivalent to |
| 557 | PyUnicode_IS_COMPACT(op) || ((PyUnicodeObject*)(op))->data.any) */ |
| 558 | |
| 559 | #define PyUnicode_IS_READY(op) (((PyASCIIObject*)op)->state.ready) |
| 560 | |
Victor Stinner | a3b334d | 2011-10-03 13:53:37 +0200 | [diff] [blame] | 561 | /* PyUnicode_READY() does less work than _PyUnicode_Ready() in the best |
Martin v. Löwis | d63a3b8 | 2011-09-28 07:41:54 +0200 | [diff] [blame] | 562 | case. If the canonical representation is not yet set, it will still call |
Victor Stinner | a3b334d | 2011-10-03 13:53:37 +0200 | [diff] [blame] | 563 | _PyUnicode_Ready(). |
Martin v. Löwis | d63a3b8 | 2011-09-28 07:41:54 +0200 | [diff] [blame] | 564 | Returns 0 on success and -1 on errors. */ |
| 565 | #define PyUnicode_READY(op) \ |
| 566 | (assert(PyUnicode_Check(op)), \ |
| 567 | (PyUnicode_IS_READY(op) ? \ |
Victor Stinner | d8f6510 | 2011-09-29 19:43:17 +0200 | [diff] [blame] | 568 | 0 : _PyUnicode_Ready((PyObject *)(op)))) |
Martin v. Löwis | d63a3b8 | 2011-09-28 07:41:54 +0200 | [diff] [blame] | 569 | |
Martin v. Löwis | d63a3b8 | 2011-09-28 07:41:54 +0200 | [diff] [blame] | 570 | /* Return a maximum character value which is suitable for creating another |
| 571 | string based on op. This is always an approximation but more efficient |
Georg Brandl | c6bc4c6 | 2011-10-05 16:23:09 +0200 | [diff] [blame] | 572 | than iterating over the string. */ |
Martin v. Löwis | d63a3b8 | 2011-09-28 07:41:54 +0200 | [diff] [blame] | 573 | #define PyUnicode_MAX_CHAR_VALUE(op) \ |
| 574 | (assert(PyUnicode_IS_READY(op)), \ |
Victor Stinner | 8813104 | 2011-10-13 01:12:01 +0200 | [diff] [blame] | 575 | (PyUnicode_IS_ASCII(op) ? \ |
| 576 | (0x7f) : \ |
Martin v. Löwis | d63a3b8 | 2011-09-28 07:41:54 +0200 | [diff] [blame] | 577 | (PyUnicode_KIND(op) == PyUnicode_1BYTE_KIND ? \ |
Victor Stinner | 8813104 | 2011-10-13 01:12:01 +0200 | [diff] [blame] | 578 | (0xffU) : \ |
Martin v. Löwis | d63a3b8 | 2011-09-28 07:41:54 +0200 | [diff] [blame] | 579 | (PyUnicode_KIND(op) == PyUnicode_2BYTE_KIND ? \ |
Victor Stinner | 8813104 | 2011-10-13 01:12:01 +0200 | [diff] [blame] | 580 | (0xffffU) : \ |
| 581 | (0x10ffffU))))) |
Martin v. Löwis | d63a3b8 | 2011-09-28 07:41:54 +0200 | [diff] [blame] | 582 | |
Martin v. Löwis | 4d0d471 | 2010-12-03 20:14:31 +0000 | [diff] [blame] | 583 | #endif |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 584 | |
| 585 | /* --- Constants ---------------------------------------------------------- */ |
| 586 | |
| 587 | /* This Unicode character will be used as replacement character during |
| 588 | decoding if the errors argument is set to "replace". Note: the |
| 589 | Unicode character U+FFFD is the official REPLACEMENT CHARACTER in |
| 590 | Unicode 3.0. */ |
| 591 | |
Victor Stinner | 5ce1b0d | 2011-09-28 20:29:27 +0200 | [diff] [blame] | 592 | #define Py_UNICODE_REPLACEMENT_CHARACTER ((Py_UCS4) 0xFFFD) |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 593 | |
| 594 | /* === Public API ========================================================= */ |
| 595 | |
| 596 | /* --- Plain Py_UNICODE --------------------------------------------------- */ |
| 597 | |
Martin v. Löwis | d63a3b8 | 2011-09-28 07:41:54 +0200 | [diff] [blame] | 598 | /* With PEP 393, this is the recommended way to allocate a new unicode object. |
| 599 | This function will allocate the object and its buffer in a single memory |
| 600 | block. Objects created using this function are not resizable. */ |
| 601 | #ifndef Py_LIMITED_API |
| 602 | PyAPI_FUNC(PyObject*) PyUnicode_New( |
| 603 | Py_ssize_t size, /* Number of code points in the new string */ |
| 604 | Py_UCS4 maxchar /* maximum code point value in the string */ |
| 605 | ); |
| 606 | #endif |
| 607 | |
Victor Stinner | d8f6510 | 2011-09-29 19:43:17 +0200 | [diff] [blame] | 608 | /* Initializes the canonical string representation from a the deprecated |
| 609 | wstr/Py_UNICODE representation. This function is used to convert Unicode |
| 610 | objects which were created using the old API to the new flexible format |
| 611 | introduced with PEP 393. |
| 612 | |
| 613 | Don't call this function directly, use the public PyUnicode_READY() macro |
| 614 | instead. */ |
Martin v. Löwis | d63a3b8 | 2011-09-28 07:41:54 +0200 | [diff] [blame] | 615 | #ifndef Py_LIMITED_API |
| 616 | PyAPI_FUNC(int) _PyUnicode_Ready( |
Victor Stinner | d8f6510 | 2011-09-29 19:43:17 +0200 | [diff] [blame] | 617 | PyObject *unicode /* Unicode object */ |
Martin v. Löwis | d63a3b8 | 2011-09-28 07:41:54 +0200 | [diff] [blame] | 618 | ); |
| 619 | #endif |
| 620 | |
Victor Stinner | 034f6cf | 2011-09-30 02:26:44 +0200 | [diff] [blame] | 621 | /* Get a copy of a Unicode string. */ |
Victor Stinner | bf6e560 | 2011-12-12 01:53:47 +0100 | [diff] [blame] | 622 | #ifndef Py_LIMITED_API |
| 623 | PyAPI_FUNC(PyObject*) _PyUnicode_Copy( |
Victor Stinner | 034f6cf | 2011-09-30 02:26:44 +0200 | [diff] [blame] | 624 | PyObject *unicode |
| 625 | ); |
Victor Stinner | bf6e560 | 2011-12-12 01:53:47 +0100 | [diff] [blame] | 626 | #endif |
Victor Stinner | 034f6cf | 2011-09-30 02:26:44 +0200 | [diff] [blame] | 627 | |
Martin v. Löwis | d63a3b8 | 2011-09-28 07:41:54 +0200 | [diff] [blame] | 628 | /* Copy character from one unicode object into another, this function performs |
Victor Stinner | 3fe5531 | 2012-01-04 00:33:50 +0100 | [diff] [blame] | 629 | character conversion when necessary and falls back to memcpy() if possible. |
Victor Stinner | be78eaf | 2011-09-28 21:37:03 +0200 | [diff] [blame] | 630 | |
Victor Stinner | 3fe5531 | 2012-01-04 00:33:50 +0100 | [diff] [blame] | 631 | Fail if to is too small (smaller than *how_many* or smaller than |
Victor Stinner | a0702ab | 2011-09-29 14:14:38 +0200 | [diff] [blame] | 632 | len(from)-from_start), or if kind(from[from_start:from_start+how_many]) > |
Victor Stinner | 3fe5531 | 2012-01-04 00:33:50 +0100 | [diff] [blame] | 633 | kind(to), or if *to* has more than 1 reference. |
Victor Stinner | be78eaf | 2011-09-28 21:37:03 +0200 | [diff] [blame] | 634 | |
| 635 | Return the number of written character, or return -1 and raise an exception |
| 636 | on error. |
| 637 | |
| 638 | Pseudo-code: |
| 639 | |
| 640 | how_many = min(how_many, len(from) - from_start) |
| 641 | to[to_start:to_start+how_many] = from[from_start:from_start+how_many] |
| 642 | return how_many |
Victor Stinner | a0702ab | 2011-09-29 14:14:38 +0200 | [diff] [blame] | 643 | |
| 644 | Note: The function doesn't write a terminating null character. |
Victor Stinner | be78eaf | 2011-09-28 21:37:03 +0200 | [diff] [blame] | 645 | */ |
Martin v. Löwis | d63a3b8 | 2011-09-28 07:41:54 +0200 | [diff] [blame] | 646 | #ifndef Py_LIMITED_API |
Victor Stinner | be78eaf | 2011-09-28 21:37:03 +0200 | [diff] [blame] | 647 | PyAPI_FUNC(Py_ssize_t) PyUnicode_CopyCharacters( |
Martin v. Löwis | d63a3b8 | 2011-09-28 07:41:54 +0200 | [diff] [blame] | 648 | PyObject *to, |
| 649 | Py_ssize_t to_start, |
| 650 | PyObject *from, |
| 651 | Py_ssize_t from_start, |
| 652 | Py_ssize_t how_many |
| 653 | ); |
Victor Stinner | d3f0882 | 2012-05-29 12:57:52 +0200 | [diff] [blame] | 654 | |
| 655 | /* Unsafe version of PyUnicode_CopyCharacters(): don't check arguments and so |
| 656 | may crash if parameters are invalid (e.g. if the output string |
| 657 | is too short). */ |
| 658 | PyAPI_FUNC(void) _PyUnicode_FastCopyCharacters( |
| 659 | PyObject *to, |
| 660 | Py_ssize_t to_start, |
| 661 | PyObject *from, |
| 662 | Py_ssize_t from_start, |
| 663 | Py_ssize_t how_many |
| 664 | ); |
Martin v. Löwis | d63a3b8 | 2011-09-28 07:41:54 +0200 | [diff] [blame] | 665 | #endif |
| 666 | |
Victor Stinner | d3f0882 | 2012-05-29 12:57:52 +0200 | [diff] [blame] | 667 | #ifndef Py_LIMITED_API |
Victor Stinner | 3fe5531 | 2012-01-04 00:33:50 +0100 | [diff] [blame] | 668 | /* Fill a string with a character: write fill_char into |
| 669 | unicode[start:start+length]. |
| 670 | |
| 671 | Fail if fill_char is bigger than the string maximum character, or if the |
| 672 | string has more than 1 reference. |
| 673 | |
| 674 | Return the number of written character, or return -1 and raise an exception |
| 675 | on error. */ |
Victor Stinner | 3fe5531 | 2012-01-04 00:33:50 +0100 | [diff] [blame] | 676 | PyAPI_FUNC(Py_ssize_t) PyUnicode_Fill( |
| 677 | PyObject *unicode, |
| 678 | Py_ssize_t start, |
| 679 | Py_ssize_t length, |
| 680 | Py_UCS4 fill_char |
| 681 | ); |
Victor Stinner | d3f0882 | 2012-05-29 12:57:52 +0200 | [diff] [blame] | 682 | |
| 683 | /* Unsafe version of PyUnicode_Fill(): don't check arguments and so may crash |
| 684 | if parameters are invalid (e.g. if length is longer than the string). */ |
| 685 | PyAPI_FUNC(void) _PyUnicode_FastFill( |
| 686 | PyObject *unicode, |
| 687 | Py_ssize_t start, |
| 688 | Py_ssize_t length, |
| 689 | Py_UCS4 fill_char |
| 690 | ); |
Victor Stinner | 3fe5531 | 2012-01-04 00:33:50 +0100 | [diff] [blame] | 691 | #endif |
| 692 | |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 693 | /* Create a Unicode Object from the Py_UNICODE buffer u of the given |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 694 | size. |
Marc-André Lemburg | 8155e0e | 2001-04-23 14:44:21 +0000 | [diff] [blame] | 695 | |
| 696 | u may be NULL which causes the contents to be undefined. It is the |
| 697 | user's responsibility to fill in the needed data afterwards. Note |
| 698 | that modifying the Unicode object contents after construction is |
| 699 | only allowed if u was set to NULL. |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 700 | |
| 701 | The buffer is copied into the new object. */ |
| 702 | |
Martin v. Löwis | 4d0d471 | 2010-12-03 20:14:31 +0000 | [diff] [blame] | 703 | #ifndef Py_LIMITED_API |
Mark Hammond | 91a681d | 2002-08-12 07:21:58 +0000 | [diff] [blame] | 704 | PyAPI_FUNC(PyObject*) PyUnicode_FromUnicode( |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 705 | const Py_UNICODE *u, /* Unicode buffer */ |
Martin v. Löwis | 18e1655 | 2006-02-15 17:27:45 +0000 | [diff] [blame] | 706 | Py_ssize_t size /* size of buffer */ |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 707 | ); |
Martin v. Löwis | 4d0d471 | 2010-12-03 20:14:31 +0000 | [diff] [blame] | 708 | #endif |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 709 | |
Georg Brandl | 952867a | 2010-06-27 10:17:12 +0000 | [diff] [blame] | 710 | /* Similar to PyUnicode_FromUnicode(), but u points to UTF-8 encoded bytes */ |
Walter Dörwald | d203431 | 2007-05-18 16:29:38 +0000 | [diff] [blame] | 711 | PyAPI_FUNC(PyObject*) PyUnicode_FromStringAndSize( |
Victor Stinner | 0d71116 | 2010-12-27 02:39:20 +0000 | [diff] [blame] | 712 | const char *u, /* UTF-8 encoded string */ |
Victor Stinner | dc2081f | 2010-12-27 01:49:29 +0000 | [diff] [blame] | 713 | Py_ssize_t size /* size of buffer */ |
Walter Dörwald | d203431 | 2007-05-18 16:29:38 +0000 | [diff] [blame] | 714 | ); |
| 715 | |
Walter Dörwald | acaa5a1 | 2007-05-05 12:00:46 +0000 | [diff] [blame] | 716 | /* Similar to PyUnicode_FromUnicode(), but u points to null-terminated |
Martin v. Löwis | d63a3b8 | 2011-09-28 07:41:54 +0200 | [diff] [blame] | 717 | UTF-8 encoded bytes. The size is determined with strlen(). */ |
Walter Dörwald | acaa5a1 | 2007-05-05 12:00:46 +0000 | [diff] [blame] | 718 | PyAPI_FUNC(PyObject*) PyUnicode_FromString( |
Victor Stinner | dc2081f | 2010-12-27 01:49:29 +0000 | [diff] [blame] | 719 | const char *u /* UTF-8 encoded string */ |
Walter Dörwald | acaa5a1 | 2007-05-05 12:00:46 +0000 | [diff] [blame] | 720 | ); |
| 721 | |
Victor Stinner | d3f0882 | 2012-05-29 12:57:52 +0200 | [diff] [blame] | 722 | #ifndef Py_LIMITED_API |
Victor Stinner | b9275c1 | 2011-10-05 14:01:42 +0200 | [diff] [blame] | 723 | /* Create a new string from a buffer of Py_UCS1, Py_UCS2 or Py_UCS4 characters. |
| 724 | Scan the string to find the maximum character. */ |
Martin v. Löwis | d63a3b8 | 2011-09-28 07:41:54 +0200 | [diff] [blame] | 725 | PyAPI_FUNC(PyObject*) PyUnicode_FromKindAndData( |
| 726 | int kind, |
| 727 | const void *buffer, |
| 728 | Py_ssize_t size); |
Victor Stinner | d3f0882 | 2012-05-29 12:57:52 +0200 | [diff] [blame] | 729 | |
| 730 | /* Create a new string from a buffer of ASCII characters. |
| 731 | WARNING: Don't check if the string contains any non-ASCII character. */ |
| 732 | PyAPI_FUNC(PyObject*) _PyUnicode_FromASCII( |
| 733 | const char *buffer, |
| 734 | Py_ssize_t size); |
Martin v. Löwis | d63a3b8 | 2011-09-28 07:41:54 +0200 | [diff] [blame] | 735 | #endif |
| 736 | |
| 737 | PyAPI_FUNC(PyObject*) PyUnicode_Substring( |
| 738 | PyObject *str, |
| 739 | Py_ssize_t start, |
| 740 | Py_ssize_t end); |
| 741 | |
Victor Stinner | ece58de | 2012-04-23 23:36:38 +0200 | [diff] [blame] | 742 | #ifndef Py_LIMITED_API |
| 743 | /* Compute the maximum character of the substring unicode[start:end]. |
| 744 | Return 127 for an empty string. */ |
| 745 | PyAPI_FUNC(Py_UCS4) _PyUnicode_FindMaxChar ( |
| 746 | PyObject *unicode, |
| 747 | Py_ssize_t start, |
| 748 | Py_ssize_t end); |
| 749 | #endif |
| 750 | |
Georg Brandl | db6c7f5 | 2011-10-07 11:19:11 +0200 | [diff] [blame] | 751 | /* Copy the string into a UCS4 buffer including the null character if copy_null |
Martin v. Löwis | d63a3b8 | 2011-09-28 07:41:54 +0200 | [diff] [blame] | 752 | is set. Return NULL and raise an exception on error. Raise a ValueError if |
| 753 | the buffer is smaller than the string. Return buffer on success. |
| 754 | |
| 755 | buflen is the length of the buffer in (Py_UCS4) characters. */ |
| 756 | PyAPI_FUNC(Py_UCS4*) PyUnicode_AsUCS4( |
| 757 | PyObject *unicode, |
| 758 | Py_UCS4* buffer, |
| 759 | Py_ssize_t buflen, |
| 760 | int copy_null); |
| 761 | |
| 762 | /* Copy the string into a UCS4 buffer. A new buffer is allocated using |
| 763 | * PyMem_Malloc; if this fails, NULL is returned with a memory error |
| 764 | exception set. */ |
| 765 | PyAPI_FUNC(Py_UCS4*) PyUnicode_AsUCS4Copy(PyObject *unicode); |
| 766 | |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 767 | /* Return a read-only pointer to the Unicode object's internal |
Martin v. Löwis | d63a3b8 | 2011-09-28 07:41:54 +0200 | [diff] [blame] | 768 | Py_UNICODE buffer. |
| 769 | If the wchar_t/Py_UNICODE representation is not yet available, this |
| 770 | function will calculate it. */ |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 771 | |
Martin v. Löwis | 4d0d471 | 2010-12-03 20:14:31 +0000 | [diff] [blame] | 772 | #ifndef Py_LIMITED_API |
Mark Hammond | 91a681d | 2002-08-12 07:21:58 +0000 | [diff] [blame] | 773 | PyAPI_FUNC(Py_UNICODE *) PyUnicode_AsUnicode( |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 774 | PyObject *unicode /* Unicode object */ |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 775 | ); |
Martin v. Löwis | 4d0d471 | 2010-12-03 20:14:31 +0000 | [diff] [blame] | 776 | #endif |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 777 | |
Martin v. Löwis | d63a3b8 | 2011-09-28 07:41:54 +0200 | [diff] [blame] | 778 | /* Return a read-only pointer to the Unicode object's internal |
| 779 | Py_UNICODE buffer and save the length at size. |
| 780 | If the wchar_t/Py_UNICODE representation is not yet available, this |
| 781 | function will calculate it. */ |
| 782 | |
| 783 | #ifndef Py_LIMITED_API |
| 784 | PyAPI_FUNC(Py_UNICODE *) PyUnicode_AsUnicodeAndSize( |
| 785 | PyObject *unicode, /* Unicode object */ |
| 786 | Py_ssize_t *size /* location where to save the length */ |
| 787 | ); |
| 788 | #endif |
| 789 | |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 790 | /* Get the length of the Unicode object. */ |
| 791 | |
Martin v. Löwis | d63a3b8 | 2011-09-28 07:41:54 +0200 | [diff] [blame] | 792 | PyAPI_FUNC(Py_ssize_t) PyUnicode_GetLength( |
| 793 | PyObject *unicode |
| 794 | ); |
| 795 | |
Victor Stinner | 157f83f | 2011-09-28 21:41:31 +0200 | [diff] [blame] | 796 | /* Get the number of Py_UNICODE units in the |
Martin v. Löwis | d63a3b8 | 2011-09-28 07:41:54 +0200 | [diff] [blame] | 797 | string representation. */ |
| 798 | |
Martin v. Löwis | 18e1655 | 2006-02-15 17:27:45 +0000 | [diff] [blame] | 799 | PyAPI_FUNC(Py_ssize_t) PyUnicode_GetSize( |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 800 | PyObject *unicode /* Unicode object */ |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 801 | ); |
| 802 | |
Martin v. Löwis | d63a3b8 | 2011-09-28 07:41:54 +0200 | [diff] [blame] | 803 | /* Read a character from the string. */ |
| 804 | |
| 805 | PyAPI_FUNC(Py_UCS4) PyUnicode_ReadChar( |
| 806 | PyObject *unicode, |
| 807 | Py_ssize_t index |
| 808 | ); |
| 809 | |
| 810 | /* Write a character to the string. The string must have been created through |
Victor Stinner | cd9950f | 2011-10-02 00:34:53 +0200 | [diff] [blame] | 811 | PyUnicode_New, must not be shared, and must not have been hashed yet. |
| 812 | |
| 813 | Return 0 on success, -1 on error. */ |
Martin v. Löwis | d63a3b8 | 2011-09-28 07:41:54 +0200 | [diff] [blame] | 814 | |
| 815 | PyAPI_FUNC(int) PyUnicode_WriteChar( |
| 816 | PyObject *unicode, |
| 817 | Py_ssize_t index, |
| 818 | Py_UCS4 character |
| 819 | ); |
| 820 | |
Martin v. Löwis | 4d0d471 | 2010-12-03 20:14:31 +0000 | [diff] [blame] | 821 | #ifndef Py_LIMITED_API |
Martin v. Löwis | ce9b5a5 | 2001-06-27 06:28:56 +0000 | [diff] [blame] | 822 | /* Get the maximum ordinal for a Unicode character. */ |
Mark Hammond | 91a681d | 2002-08-12 07:21:58 +0000 | [diff] [blame] | 823 | PyAPI_FUNC(Py_UNICODE) PyUnicode_GetMax(void); |
Martin v. Löwis | 4d0d471 | 2010-12-03 20:14:31 +0000 | [diff] [blame] | 824 | #endif |
Martin v. Löwis | ce9b5a5 | 2001-06-27 06:28:56 +0000 | [diff] [blame] | 825 | |
Victor Stinner | b0a82a6 | 2011-12-12 13:08:33 +0100 | [diff] [blame] | 826 | /* Resize an Unicode object. The length is the number of characters, except |
| 827 | if the kind of the string is PyUnicode_WCHAR_KIND: in this case, the length |
| 828 | is the number of Py_UNICODE characters. |
Guido van Rossum | 52c2359 | 2000-04-10 13:41:41 +0000 | [diff] [blame] | 829 | |
| 830 | *unicode is modified to point to the new (resized) object and 0 |
| 831 | returned on success. |
| 832 | |
Victor Stinner | b0a82a6 | 2011-12-12 13:08:33 +0100 | [diff] [blame] | 833 | Try to resize the string in place (which is usually faster than allocating |
| 834 | a new string and copy characters), or create a new string. |
Guido van Rossum | 52c2359 | 2000-04-10 13:41:41 +0000 | [diff] [blame] | 835 | |
| 836 | Error handling is implemented as follows: an exception is set, -1 |
Victor Stinner | 16e6a80 | 2011-12-12 13:24:15 +0100 | [diff] [blame] | 837 | is returned and *unicode left untouched. |
| 838 | |
| 839 | WARNING: The function doesn't check string content, the result may not be a |
| 840 | string in canonical representation. */ |
Guido van Rossum | 52c2359 | 2000-04-10 13:41:41 +0000 | [diff] [blame] | 841 | |
Mark Hammond | 91a681d | 2002-08-12 07:21:58 +0000 | [diff] [blame] | 842 | PyAPI_FUNC(int) PyUnicode_Resize( |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 843 | PyObject **unicode, /* Pointer to the Unicode object */ |
| 844 | Py_ssize_t length /* New length */ |
Guido van Rossum | 52c2359 | 2000-04-10 13:41:41 +0000 | [diff] [blame] | 845 | ); |
| 846 | |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 847 | /* Coerce obj to an Unicode object and return a reference with |
| 848 | *incremented* refcount. |
| 849 | |
| 850 | Coercion is done in the following way: |
| 851 | |
Serhiy Storchaka | b757c83 | 2014-12-05 22:25:22 +0200 | [diff] [blame] | 852 | 1. bytes, bytearray and other bytes-like objects are decoded |
Alexander Belopolsky | 83283c2 | 2010-11-16 14:29:01 +0000 | [diff] [blame] | 853 | under the assumptions that they contain data using the UTF-8 |
| 854 | encoding. Decoding is done in "strict" mode. |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 855 | |
Guido van Rossum | b8c65bc | 2001-10-19 02:01:31 +0000 | [diff] [blame] | 856 | 2. All other objects (including Unicode objects) raise an |
| 857 | exception. |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 858 | |
| 859 | The API returns NULL in case of an error. The caller is responsible |
| 860 | for decref'ing the returned objects. |
| 861 | |
| 862 | */ |
| 863 | |
Mark Hammond | 91a681d | 2002-08-12 07:21:58 +0000 | [diff] [blame] | 864 | PyAPI_FUNC(PyObject*) PyUnicode_FromEncodedObject( |
Antoine Pitrou | 9ed5f27 | 2013-08-13 20:18:52 +0200 | [diff] [blame] | 865 | PyObject *obj, /* Object */ |
Marc-André Lemburg | 5a5c81a | 2000-07-07 13:46:42 +0000 | [diff] [blame] | 866 | const char *encoding, /* encoding */ |
| 867 | const char *errors /* error handling */ |
| 868 | ); |
| 869 | |
Guido van Rossum | b8c65bc | 2001-10-19 02:01:31 +0000 | [diff] [blame] | 870 | /* Coerce obj to an Unicode object and return a reference with |
Marc-André Lemburg | 5a5c81a | 2000-07-07 13:46:42 +0000 | [diff] [blame] | 871 | *incremented* refcount. |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 872 | |
Guido van Rossum | b8c65bc | 2001-10-19 02:01:31 +0000 | [diff] [blame] | 873 | Unicode objects are passed back as-is (subclasses are converted to |
| 874 | true Unicode objects), all other objects are delegated to |
| 875 | PyUnicode_FromEncodedObject(obj, NULL, "strict") which results in |
Georg Brandl | 952867a | 2010-06-27 10:17:12 +0000 | [diff] [blame] | 876 | using UTF-8 encoding as basis for decoding the object. |
Marc-André Lemburg | 5a5c81a | 2000-07-07 13:46:42 +0000 | [diff] [blame] | 877 | |
| 878 | The API returns NULL in case of an error. The caller is responsible |
| 879 | for decref'ing the returned objects. |
| 880 | |
| 881 | */ |
| 882 | |
Mark Hammond | 91a681d | 2002-08-12 07:21:58 +0000 | [diff] [blame] | 883 | PyAPI_FUNC(PyObject*) PyUnicode_FromObject( |
Antoine Pitrou | 9ed5f27 | 2013-08-13 20:18:52 +0200 | [diff] [blame] | 884 | PyObject *obj /* Object */ |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 885 | ); |
| 886 | |
Victor Stinner | 1205f27 | 2010-09-11 00:54:47 +0000 | [diff] [blame] | 887 | PyAPI_FUNC(PyObject *) PyUnicode_FromFormatV( |
| 888 | const char *format, /* ASCII-encoded string */ |
| 889 | va_list vargs |
| 890 | ); |
| 891 | PyAPI_FUNC(PyObject *) PyUnicode_FromFormat( |
| 892 | const char *format, /* ASCII-encoded string */ |
| 893 | ... |
| 894 | ); |
Walter Dörwald | d203431 | 2007-05-18 16:29:38 +0000 | [diff] [blame] | 895 | |
Martin v. Löwis | 4d0d471 | 2010-12-03 20:14:31 +0000 | [diff] [blame] | 896 | #ifndef Py_LIMITED_API |
Victor Stinner | d3f0882 | 2012-05-29 12:57:52 +0200 | [diff] [blame] | 897 | typedef struct { |
| 898 | PyObject *buffer; |
| 899 | void *data; |
| 900 | enum PyUnicode_Kind kind; |
| 901 | Py_UCS4 maxchar; |
| 902 | Py_ssize_t size; |
| 903 | Py_ssize_t pos; |
Victor Stinner | 8f674cc | 2013-04-17 23:02:17 +0200 | [diff] [blame] | 904 | |
| 905 | /* minimum number of allocated characters (default: 0) */ |
Victor Stinner | d3f0882 | 2012-05-29 12:57:52 +0200 | [diff] [blame] | 906 | Py_ssize_t min_length; |
Victor Stinner | 8f674cc | 2013-04-17 23:02:17 +0200 | [diff] [blame] | 907 | |
| 908 | /* minimum character (default: 127, ASCII) */ |
| 909 | Py_UCS4 min_char; |
| 910 | |
| 911 | /* If non-zero, overallocate the buffer by 25% (default: 0). */ |
Victor Stinner | d7b7c74 | 2012-06-04 22:52:12 +0200 | [diff] [blame] | 912 | unsigned char overallocate; |
Victor Stinner | 8f674cc | 2013-04-17 23:02:17 +0200 | [diff] [blame] | 913 | |
Victor Stinner | d7b7c74 | 2012-06-04 22:52:12 +0200 | [diff] [blame] | 914 | /* If readonly is 1, buffer is a shared string (cannot be modified) |
| 915 | and size is set to 0. */ |
| 916 | unsigned char readonly; |
Victor Stinner | d3f0882 | 2012-05-29 12:57:52 +0200 | [diff] [blame] | 917 | } _PyUnicodeWriter ; |
| 918 | |
| 919 | /* Initialize a Unicode writer. |
Victor Stinner | 8f674cc | 2013-04-17 23:02:17 +0200 | [diff] [blame] | 920 | * |
| 921 | * By default, the minimum buffer size is 0 character and overallocation is |
| 922 | * disabled. Set min_length, min_char and overallocate attributes to control |
| 923 | * the allocation of the buffer. */ |
Victor Stinner | d3f0882 | 2012-05-29 12:57:52 +0200 | [diff] [blame] | 924 | PyAPI_FUNC(void) |
Victor Stinner | 8f674cc | 2013-04-17 23:02:17 +0200 | [diff] [blame] | 925 | _PyUnicodeWriter_Init(_PyUnicodeWriter *writer); |
Victor Stinner | d3f0882 | 2012-05-29 12:57:52 +0200 | [diff] [blame] | 926 | |
| 927 | /* Prepare the buffer to write 'length' characters |
| 928 | with the specified maximum character. |
| 929 | |
| 930 | Return 0 on success, raise an exception and return -1 on error. */ |
| 931 | #define _PyUnicodeWriter_Prepare(WRITER, LENGTH, MAXCHAR) \ |
| 932 | (((MAXCHAR) <= (WRITER)->maxchar \ |
| 933 | && (LENGTH) <= (WRITER)->size - (WRITER)->pos) \ |
| 934 | ? 0 \ |
| 935 | : (((LENGTH) == 0) \ |
| 936 | ? 0 \ |
| 937 | : _PyUnicodeWriter_PrepareInternal((WRITER), (LENGTH), (MAXCHAR)))) |
| 938 | |
| 939 | /* Don't call this function directly, use the _PyUnicodeWriter_Prepare() macro |
| 940 | instead. */ |
| 941 | PyAPI_FUNC(int) |
| 942 | _PyUnicodeWriter_PrepareInternal(_PyUnicodeWriter *writer, |
| 943 | Py_ssize_t length, Py_UCS4 maxchar); |
| 944 | |
Victor Stinner | a0dd021 | 2013-04-11 22:09:04 +0200 | [diff] [blame] | 945 | /* Append a Unicode character. |
| 946 | Return 0 on success, raise an exception and return -1 on error. */ |
| 947 | PyAPI_FUNC(int) |
| 948 | _PyUnicodeWriter_WriteChar(_PyUnicodeWriter *writer, |
| 949 | Py_UCS4 ch |
| 950 | ); |
| 951 | |
Victor Stinner | e215d96 | 2012-10-06 23:03:36 +0200 | [diff] [blame] | 952 | /* Append a Unicode string. |
| 953 | Return 0 on success, raise an exception and return -1 on error. */ |
Victor Stinner | d3f0882 | 2012-05-29 12:57:52 +0200 | [diff] [blame] | 954 | PyAPI_FUNC(int) |
Victor Stinner | e215d96 | 2012-10-06 23:03:36 +0200 | [diff] [blame] | 955 | _PyUnicodeWriter_WriteStr(_PyUnicodeWriter *writer, |
| 956 | PyObject *str /* Unicode string */ |
| 957 | ); |
Victor Stinner | d3f0882 | 2012-05-29 12:57:52 +0200 | [diff] [blame] | 958 | |
Victor Stinner | cfc4c13 | 2013-04-03 01:48:39 +0200 | [diff] [blame] | 959 | /* Append a substring of a Unicode string. |
| 960 | Return 0 on success, raise an exception and return -1 on error. */ |
| 961 | PyAPI_FUNC(int) |
| 962 | _PyUnicodeWriter_WriteSubstring(_PyUnicodeWriter *writer, |
| 963 | PyObject *str, /* Unicode string */ |
| 964 | Py_ssize_t start, |
| 965 | Py_ssize_t end |
| 966 | ); |
| 967 | |
Victor Stinner | 4a58707 | 2013-11-19 12:54:53 +0100 | [diff] [blame] | 968 | /* Append a ASCII-encoded byte string. |
| 969 | Return 0 on success, raise an exception and return -1 on error. */ |
| 970 | PyAPI_FUNC(int) |
| 971 | _PyUnicodeWriter_WriteASCIIString(_PyUnicodeWriter *writer, |
| 972 | const char *str, /* ASCII-encoded byte string */ |
| 973 | Py_ssize_t len /* number of bytes, or -1 if unknown */ |
| 974 | ); |
| 975 | |
Victor Stinner | e215d96 | 2012-10-06 23:03:36 +0200 | [diff] [blame] | 976 | /* Append a latin1-encoded byte string. |
| 977 | Return 0 on success, raise an exception and return -1 on error. */ |
| 978 | PyAPI_FUNC(int) |
Victor Stinner | 4a58707 | 2013-11-19 12:54:53 +0100 | [diff] [blame] | 979 | _PyUnicodeWriter_WriteLatin1String(_PyUnicodeWriter *writer, |
| 980 | const char *str, /* latin1-encoded byte string */ |
| 981 | Py_ssize_t len /* length in bytes */ |
Victor Stinner | e215d96 | 2012-10-06 23:03:36 +0200 | [diff] [blame] | 982 | ); |
| 983 | |
Victor Stinner | f476405 | 2013-04-18 23:21:19 +0200 | [diff] [blame] | 984 | /* Get the value of the writer as an Unicode string. Clear the |
Victor Stinner | e215d96 | 2012-10-06 23:03:36 +0200 | [diff] [blame] | 985 | buffer of the writer. Raise an exception and return NULL |
| 986 | on error. */ |
Victor Stinner | d3f0882 | 2012-05-29 12:57:52 +0200 | [diff] [blame] | 987 | PyAPI_FUNC(PyObject *) |
| 988 | _PyUnicodeWriter_Finish(_PyUnicodeWriter *writer); |
| 989 | |
Victor Stinner | e215d96 | 2012-10-06 23:03:36 +0200 | [diff] [blame] | 990 | /* Deallocate memory of a writer (clear its internal buffer). */ |
Victor Stinner | d3f0882 | 2012-05-29 12:57:52 +0200 | [diff] [blame] | 991 | PyAPI_FUNC(void) |
| 992 | _PyUnicodeWriter_Dealloc(_PyUnicodeWriter *writer); |
| 993 | #endif |
| 994 | |
| 995 | #ifndef Py_LIMITED_API |
Eric Smith | 4a7d76d | 2008-05-30 18:10:19 +0000 | [diff] [blame] | 996 | /* Format the object based on the format_spec, as defined in PEP 3101 |
| 997 | (Advanced String Formatting). */ |
Victor Stinner | d3f0882 | 2012-05-29 12:57:52 +0200 | [diff] [blame] | 998 | PyAPI_FUNC(int) _PyUnicode_FormatAdvancedWriter( |
| 999 | _PyUnicodeWriter *writer, |
| 1000 | PyObject *obj, |
| 1001 | PyObject *format_spec, |
| 1002 | Py_ssize_t start, |
| 1003 | Py_ssize_t end); |
Martin v. Löwis | 4d0d471 | 2010-12-03 20:14:31 +0000 | [diff] [blame] | 1004 | #endif |
Eric Smith | 4a7d76d | 2008-05-30 18:10:19 +0000 | [diff] [blame] | 1005 | |
Walter Dörwald | 1680713 | 2007-05-25 13:52:07 +0000 | [diff] [blame] | 1006 | PyAPI_FUNC(void) PyUnicode_InternInPlace(PyObject **); |
| 1007 | PyAPI_FUNC(void) PyUnicode_InternImmortal(PyObject **); |
Victor Stinner | dc2081f | 2010-12-27 01:49:29 +0000 | [diff] [blame] | 1008 | PyAPI_FUNC(PyObject *) PyUnicode_InternFromString( |
| 1009 | const char *u /* UTF-8 encoded string */ |
| 1010 | ); |
Martin v. Löwis | 4d0d471 | 2010-12-03 20:14:31 +0000 | [diff] [blame] | 1011 | #ifndef Py_LIMITED_API |
Walter Dörwald | 1680713 | 2007-05-25 13:52:07 +0000 | [diff] [blame] | 1012 | PyAPI_FUNC(void) _Py_ReleaseInternedUnicodeStrings(void); |
Martin v. Löwis | 4d0d471 | 2010-12-03 20:14:31 +0000 | [diff] [blame] | 1013 | #endif |
Walter Dörwald | 1680713 | 2007-05-25 13:52:07 +0000 | [diff] [blame] | 1014 | |
| 1015 | /* Use only if you know it's a string */ |
Martin v. Löwis | d63a3b8 | 2011-09-28 07:41:54 +0200 | [diff] [blame] | 1016 | #define PyUnicode_CHECK_INTERNED(op) \ |
| 1017 | (((PyASCIIObject *)(op))->state.interned) |
Walter Dörwald | 1680713 | 2007-05-25 13:52:07 +0000 | [diff] [blame] | 1018 | |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 1019 | /* --- wchar_t support for platforms which support it --------------------- */ |
| 1020 | |
| 1021 | #ifdef HAVE_WCHAR_H |
| 1022 | |
Georg Brandl | 952867a | 2010-06-27 10:17:12 +0000 | [diff] [blame] | 1023 | /* Create a Unicode Object from the wchar_t buffer w of the given |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 1024 | size. |
| 1025 | |
| 1026 | The buffer is copied into the new object. */ |
| 1027 | |
Mark Hammond | 91a681d | 2002-08-12 07:21:58 +0000 | [diff] [blame] | 1028 | PyAPI_FUNC(PyObject*) PyUnicode_FromWideChar( |
Antoine Pitrou | 9ed5f27 | 2013-08-13 20:18:52 +0200 | [diff] [blame] | 1029 | const wchar_t *w, /* wchar_t buffer */ |
Martin v. Löwis | 18e1655 | 2006-02-15 17:27:45 +0000 | [diff] [blame] | 1030 | Py_ssize_t size /* size of buffer */ |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 1031 | ); |
| 1032 | |
Marc-André Lemburg | a9cadcd | 2004-11-22 13:02:31 +0000 | [diff] [blame] | 1033 | /* Copies the Unicode Object contents into the wchar_t buffer w. At |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 1034 | most size wchar_t characters are copied. |
| 1035 | |
Marc-André Lemburg | a9cadcd | 2004-11-22 13:02:31 +0000 | [diff] [blame] | 1036 | Note that the resulting wchar_t string may or may not be |
| 1037 | 0-terminated. It is the responsibility of the caller to make sure |
| 1038 | that the wchar_t string is 0-terminated in case this is required by |
| 1039 | the application. |
| 1040 | |
| 1041 | Returns the number of wchar_t characters copied (excluding a |
| 1042 | possibly trailing 0-termination character) or -1 in case of an |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 1043 | error. */ |
| 1044 | |
Martin v. Löwis | 18e1655 | 2006-02-15 17:27:45 +0000 | [diff] [blame] | 1045 | PyAPI_FUNC(Py_ssize_t) PyUnicode_AsWideChar( |
Martin v. Löwis | 4d0d471 | 2010-12-03 20:14:31 +0000 | [diff] [blame] | 1046 | PyObject *unicode, /* Unicode object */ |
Antoine Pitrou | 9ed5f27 | 2013-08-13 20:18:52 +0200 | [diff] [blame] | 1047 | wchar_t *w, /* wchar_t buffer */ |
Martin v. Löwis | 18e1655 | 2006-02-15 17:27:45 +0000 | [diff] [blame] | 1048 | Py_ssize_t size /* size of buffer */ |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 1049 | ); |
| 1050 | |
Victor Stinner | 137c34c | 2010-09-29 10:25:54 +0000 | [diff] [blame] | 1051 | /* Convert the Unicode object to a wide character string. The output string |
| 1052 | always ends with a nul character. If size is not NULL, write the number of |
Victor Stinner | d88d983 | 2011-09-06 02:00:05 +0200 | [diff] [blame] | 1053 | wide characters (excluding the null character) into *size. |
Victor Stinner | 137c34c | 2010-09-29 10:25:54 +0000 | [diff] [blame] | 1054 | |
| 1055 | Returns a buffer allocated by PyMem_Alloc() (use PyMem_Free() to free it) |
| 1056 | on success. On error, returns NULL, *size is undefined and raises a |
| 1057 | MemoryError. */ |
| 1058 | |
| 1059 | PyAPI_FUNC(wchar_t*) PyUnicode_AsWideCharString( |
Victor Stinner | beb4135b | 2010-10-07 01:02:42 +0000 | [diff] [blame] | 1060 | PyObject *unicode, /* Unicode object */ |
Victor Stinner | 137c34c | 2010-09-29 10:25:54 +0000 | [diff] [blame] | 1061 | Py_ssize_t *size /* number of characters of the result */ |
| 1062 | ); |
| 1063 | |
Victor Stinner | 9f789e7 | 2011-10-01 03:57:28 +0200 | [diff] [blame] | 1064 | #ifndef Py_LIMITED_API |
Martin v. Löwis | d63a3b8 | 2011-09-28 07:41:54 +0200 | [diff] [blame] | 1065 | PyAPI_FUNC(void*) _PyUnicode_AsKind(PyObject *s, unsigned int kind); |
Victor Stinner | 9f789e7 | 2011-10-01 03:57:28 +0200 | [diff] [blame] | 1066 | #endif |
Martin v. Löwis | d63a3b8 | 2011-09-28 07:41:54 +0200 | [diff] [blame] | 1067 | |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 1068 | #endif |
| 1069 | |
Marc-André Lemburg | cc8764c | 2002-08-11 12:23:04 +0000 | [diff] [blame] | 1070 | /* --- Unicode ordinals --------------------------------------------------- */ |
| 1071 | |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 1072 | /* Create a Unicode Object from the given Unicode code point ordinal. |
| 1073 | |
Ezio Melotti | e7f9037 | 2012-10-05 03:33:31 +0300 | [diff] [blame] | 1074 | The ordinal must be in range(0x110000). A ValueError is |
Marc-André Lemburg | cc8764c | 2002-08-11 12:23:04 +0000 | [diff] [blame] | 1075 | raised in case it is not. |
| 1076 | |
| 1077 | */ |
| 1078 | |
Marc-André Lemburg | 9c329de | 2002-08-12 08:19:10 +0000 | [diff] [blame] | 1079 | PyAPI_FUNC(PyObject*) PyUnicode_FromOrdinal(int ordinal); |
Marc-André Lemburg | cc8764c | 2002-08-11 12:23:04 +0000 | [diff] [blame] | 1080 | |
Benjamin Peterson | 960cf0f | 2009-01-09 04:11:44 +0000 | [diff] [blame] | 1081 | /* --- Free-list management ----------------------------------------------- */ |
| 1082 | |
| 1083 | /* Clear the free list used by the Unicode implementation. |
| 1084 | |
| 1085 | This can be used to release memory used for objects on the free |
| 1086 | list back to the Python memory allocator. |
| 1087 | |
| 1088 | */ |
| 1089 | |
| 1090 | PyAPI_FUNC(int) PyUnicode_ClearFreeList(void); |
| 1091 | |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 1092 | /* === Builtin Codecs ===================================================== |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 1093 | |
| 1094 | Many of these APIs take two arguments encoding and errors. These |
| 1095 | parameters encoding and errors have the same semantics as the ones |
Alexander Belopolsky | 83283c2 | 2010-11-16 14:29:01 +0000 | [diff] [blame] | 1096 | of the builtin str() API. |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 1097 | |
Georg Brandl | 952867a | 2010-06-27 10:17:12 +0000 | [diff] [blame] | 1098 | Setting encoding to NULL causes the default encoding (UTF-8) to be used. |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 1099 | |
| 1100 | Error handling is set by errors which may also be set to NULL |
| 1101 | meaning to use the default handling defined for the codec. Default |
| 1102 | error handling for all builtin codecs is "strict" (ValueErrors are |
| 1103 | raised). |
| 1104 | |
| 1105 | The codecs all use a similar interface. Only deviation from the |
| 1106 | generic ones are documented. |
| 1107 | |
| 1108 | */ |
| 1109 | |
Fred Drake | cb093fe | 2000-05-09 19:51:53 +0000 | [diff] [blame] | 1110 | /* --- Manage the default encoding ---------------------------------------- */ |
| 1111 | |
Alexander Belopolsky | 83283c2 | 2010-11-16 14:29:01 +0000 | [diff] [blame] | 1112 | /* Returns a pointer to the default encoding (UTF-8) of the |
Marc-André Lemburg | 9155aa7 | 2008-04-29 11:14:08 +0000 | [diff] [blame] | 1113 | Unicode object unicode and the size of the encoded representation |
| 1114 | in bytes stored in *size. |
Christian Heimes | 5894ba7 | 2007-11-04 11:43:14 +0000 | [diff] [blame] | 1115 | |
Marc-André Lemburg | 9155aa7 | 2008-04-29 11:14:08 +0000 | [diff] [blame] | 1116 | In case of an error, no *size is set. |
Guido van Rossum | 7d1df6c | 2007-08-29 13:53:23 +0000 | [diff] [blame] | 1117 | |
Georg Brandl | c6bc4c6 | 2011-10-05 16:23:09 +0200 | [diff] [blame] | 1118 | This function caches the UTF-8 encoded string in the unicodeobject |
| 1119 | and subsequent calls will return the same string. The memory is released |
Martin v. Löwis | d63a3b8 | 2011-09-28 07:41:54 +0200 | [diff] [blame] | 1120 | when the unicodeobject is deallocated. |
| 1121 | |
| 1122 | _PyUnicode_AsStringAndSize is a #define for PyUnicode_AsUTF8AndSize to |
| 1123 | support the previous internal function with the same behaviour. |
| 1124 | |
Marc-André Lemburg | 4cc0f24 | 2008-08-07 18:54:33 +0000 | [diff] [blame] | 1125 | *** This API is for interpreter INTERNAL USE ONLY and will likely |
Alexander Belopolsky | 83283c2 | 2010-11-16 14:29:01 +0000 | [diff] [blame] | 1126 | *** be removed or changed in the future. |
Marc-André Lemburg | 4cc0f24 | 2008-08-07 18:54:33 +0000 | [diff] [blame] | 1127 | |
| 1128 | *** If you need to access the Unicode object as UTF-8 bytes string, |
| 1129 | *** please use PyUnicode_AsUTF8String() instead. |
Martin v. Löwis | 5b22213 | 2007-06-10 09:51:05 +0000 | [diff] [blame] | 1130 | */ |
| 1131 | |
Martin v. Löwis | 4d0d471 | 2010-12-03 20:14:31 +0000 | [diff] [blame] | 1132 | #ifndef Py_LIMITED_API |
Martin v. Löwis | d63a3b8 | 2011-09-28 07:41:54 +0200 | [diff] [blame] | 1133 | PyAPI_FUNC(char *) PyUnicode_AsUTF8AndSize( |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 1134 | PyObject *unicode, |
Marc-André Lemburg | 9155aa7 | 2008-04-29 11:14:08 +0000 | [diff] [blame] | 1135 | Py_ssize_t *size); |
Martin v. Löwis | d63a3b8 | 2011-09-28 07:41:54 +0200 | [diff] [blame] | 1136 | #define _PyUnicode_AsStringAndSize PyUnicode_AsUTF8AndSize |
Martin v. Löwis | 4d0d471 | 2010-12-03 20:14:31 +0000 | [diff] [blame] | 1137 | #endif |
Guido van Rossum | 7d1df6c | 2007-08-29 13:53:23 +0000 | [diff] [blame] | 1138 | |
Alexander Belopolsky | 83283c2 | 2010-11-16 14:29:01 +0000 | [diff] [blame] | 1139 | /* Returns a pointer to the default encoding (UTF-8) of the |
Marc-André Lemburg | 9155aa7 | 2008-04-29 11:14:08 +0000 | [diff] [blame] | 1140 | Unicode object unicode. |
Guido van Rossum | 7d1df6c | 2007-08-29 13:53:23 +0000 | [diff] [blame] | 1141 | |
Martin v. Löwis | d63a3b8 | 2011-09-28 07:41:54 +0200 | [diff] [blame] | 1142 | Like PyUnicode_AsUTF8AndSize(), this also caches the UTF-8 representation |
| 1143 | in the unicodeobject. |
| 1144 | |
| 1145 | _PyUnicode_AsString is a #define for PyUnicode_AsUTF8 to |
| 1146 | support the previous internal function with the same behaviour. |
| 1147 | |
Marc-André Lemburg | 9155aa7 | 2008-04-29 11:14:08 +0000 | [diff] [blame] | 1148 | Use of this API is DEPRECATED since no size information can be |
Marc-André Lemburg | 4cc0f24 | 2008-08-07 18:54:33 +0000 | [diff] [blame] | 1149 | extracted from the returned data. |
| 1150 | |
| 1151 | *** This API is for interpreter INTERNAL USE ONLY and will likely |
| 1152 | *** be removed or changed for Python 3.1. |
| 1153 | |
| 1154 | *** If you need to access the Unicode object as UTF-8 bytes string, |
| 1155 | *** please use PyUnicode_AsUTF8String() instead. |
Guido van Rossum | 7d1df6c | 2007-08-29 13:53:23 +0000 | [diff] [blame] | 1156 | |
Marc-André Lemburg | 9155aa7 | 2008-04-29 11:14:08 +0000 | [diff] [blame] | 1157 | */ |
Martin v. Löwis | 5b22213 | 2007-06-10 09:51:05 +0000 | [diff] [blame] | 1158 | |
Martin v. Löwis | 4d0d471 | 2010-12-03 20:14:31 +0000 | [diff] [blame] | 1159 | #ifndef Py_LIMITED_API |
Martin v. Löwis | d63a3b8 | 2011-09-28 07:41:54 +0200 | [diff] [blame] | 1160 | PyAPI_FUNC(char *) PyUnicode_AsUTF8(PyObject *unicode); |
| 1161 | #define _PyUnicode_AsString PyUnicode_AsUTF8 |
Martin v. Löwis | 4d0d471 | 2010-12-03 20:14:31 +0000 | [diff] [blame] | 1162 | #endif |
Martin v. Löwis | 5b22213 | 2007-06-10 09:51:05 +0000 | [diff] [blame] | 1163 | |
Alexander Belopolsky | 83283c2 | 2010-11-16 14:29:01 +0000 | [diff] [blame] | 1164 | /* Returns "utf-8". */ |
Fred Drake | cb093fe | 2000-05-09 19:51:53 +0000 | [diff] [blame] | 1165 | |
Mark Hammond | 91a681d | 2002-08-12 07:21:58 +0000 | [diff] [blame] | 1166 | PyAPI_FUNC(const char*) PyUnicode_GetDefaultEncoding(void); |
Fred Drake | cb093fe | 2000-05-09 19:51:53 +0000 | [diff] [blame] | 1167 | |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 1168 | /* --- Generic Codecs ----------------------------------------------------- */ |
| 1169 | |
| 1170 | /* Create a Unicode object by decoding the encoded string s of the |
| 1171 | given size. */ |
| 1172 | |
Mark Hammond | 91a681d | 2002-08-12 07:21:58 +0000 | [diff] [blame] | 1173 | PyAPI_FUNC(PyObject*) PyUnicode_Decode( |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 1174 | const char *s, /* encoded string */ |
Martin v. Löwis | 18e1655 | 2006-02-15 17:27:45 +0000 | [diff] [blame] | 1175 | Py_ssize_t size, /* size of buffer */ |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 1176 | const char *encoding, /* encoding */ |
| 1177 | const char *errors /* error handling */ |
| 1178 | ); |
| 1179 | |
Marc-André Lemburg | b2750b5 | 2008-06-06 12:18:17 +0000 | [diff] [blame] | 1180 | /* Decode a Unicode object unicode and return the result as Python |
| 1181 | object. */ |
| 1182 | |
| 1183 | PyAPI_FUNC(PyObject*) PyUnicode_AsDecodedObject( |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 1184 | PyObject *unicode, /* Unicode object */ |
| 1185 | const char *encoding, /* encoding */ |
| 1186 | const char *errors /* error handling */ |
Marc-André Lemburg | b2750b5 | 2008-06-06 12:18:17 +0000 | [diff] [blame] | 1187 | ); |
| 1188 | |
| 1189 | /* Decode a Unicode object unicode and return the result as Unicode |
| 1190 | object. */ |
| 1191 | |
| 1192 | PyAPI_FUNC(PyObject*) PyUnicode_AsDecodedUnicode( |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 1193 | PyObject *unicode, /* Unicode object */ |
| 1194 | const char *encoding, /* encoding */ |
| 1195 | const char *errors /* error handling */ |
Marc-André Lemburg | b2750b5 | 2008-06-06 12:18:17 +0000 | [diff] [blame] | 1196 | ); |
| 1197 | |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 1198 | /* Encodes a Py_UNICODE buffer of the given size and returns a |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 1199 | Python string object. */ |
| 1200 | |
Martin v. Löwis | 4d0d471 | 2010-12-03 20:14:31 +0000 | [diff] [blame] | 1201 | #ifndef Py_LIMITED_API |
Mark Hammond | 91a681d | 2002-08-12 07:21:58 +0000 | [diff] [blame] | 1202 | PyAPI_FUNC(PyObject*) PyUnicode_Encode( |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 1203 | const Py_UNICODE *s, /* Unicode char buffer */ |
Martin v. Löwis | 18e1655 | 2006-02-15 17:27:45 +0000 | [diff] [blame] | 1204 | Py_ssize_t size, /* number of Py_UNICODE chars to encode */ |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 1205 | const char *encoding, /* encoding */ |
| 1206 | const char *errors /* error handling */ |
| 1207 | ); |
Martin v. Löwis | 4d0d471 | 2010-12-03 20:14:31 +0000 | [diff] [blame] | 1208 | #endif |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 1209 | |
Marc-André Lemburg | d2d4598 | 2004-07-08 17:57:32 +0000 | [diff] [blame] | 1210 | /* Encodes a Unicode object and returns the result as Python |
| 1211 | object. */ |
| 1212 | |
| 1213 | PyAPI_FUNC(PyObject*) PyUnicode_AsEncodedObject( |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 1214 | PyObject *unicode, /* Unicode object */ |
| 1215 | const char *encoding, /* encoding */ |
| 1216 | const char *errors /* error handling */ |
Marc-André Lemburg | d2d4598 | 2004-07-08 17:57:32 +0000 | [diff] [blame] | 1217 | ); |
| 1218 | |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 1219 | /* Encodes a Unicode object and returns the result as Python string |
| 1220 | object. */ |
| 1221 | |
Mark Hammond | 91a681d | 2002-08-12 07:21:58 +0000 | [diff] [blame] | 1222 | PyAPI_FUNC(PyObject*) PyUnicode_AsEncodedString( |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 1223 | PyObject *unicode, /* Unicode object */ |
| 1224 | const char *encoding, /* encoding */ |
| 1225 | const char *errors /* error handling */ |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 1226 | ); |
| 1227 | |
Marc-André Lemburg | b2750b5 | 2008-06-06 12:18:17 +0000 | [diff] [blame] | 1228 | /* Encodes a Unicode object and returns the result as Unicode |
| 1229 | object. */ |
| 1230 | |
| 1231 | PyAPI_FUNC(PyObject*) PyUnicode_AsEncodedUnicode( |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 1232 | PyObject *unicode, /* Unicode object */ |
| 1233 | const char *encoding, /* encoding */ |
| 1234 | const char *errors /* error handling */ |
Marc-André Lemburg | b2750b5 | 2008-06-06 12:18:17 +0000 | [diff] [blame] | 1235 | ); |
| 1236 | |
| 1237 | /* Build an encoding map. */ |
| 1238 | |
Thomas Wouters | 73e5a5b | 2006-06-08 15:35:45 +0000 | [diff] [blame] | 1239 | PyAPI_FUNC(PyObject*) PyUnicode_BuildEncodingMap( |
| 1240 | PyObject* string /* 256 character map */ |
| 1241 | ); |
| 1242 | |
Marc-André Lemburg | c60e6f7 | 2001-09-20 10:35:46 +0000 | [diff] [blame] | 1243 | /* --- UTF-7 Codecs ------------------------------------------------------- */ |
| 1244 | |
Mark Hammond | 91a681d | 2002-08-12 07:21:58 +0000 | [diff] [blame] | 1245 | PyAPI_FUNC(PyObject*) PyUnicode_DecodeUTF7( |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 1246 | const char *string, /* UTF-7 encoded string */ |
| 1247 | Py_ssize_t length, /* size of string */ |
| 1248 | const char *errors /* error handling */ |
Marc-André Lemburg | c60e6f7 | 2001-09-20 10:35:46 +0000 | [diff] [blame] | 1249 | ); |
| 1250 | |
Christian Heimes | 5d14c2b | 2007-11-20 23:38:09 +0000 | [diff] [blame] | 1251 | PyAPI_FUNC(PyObject*) PyUnicode_DecodeUTF7Stateful( |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 1252 | const char *string, /* UTF-7 encoded string */ |
| 1253 | Py_ssize_t length, /* size of string */ |
| 1254 | const char *errors, /* error handling */ |
| 1255 | Py_ssize_t *consumed /* bytes consumed */ |
Christian Heimes | 5d14c2b | 2007-11-20 23:38:09 +0000 | [diff] [blame] | 1256 | ); |
| 1257 | |
Martin v. Löwis | 4d0d471 | 2010-12-03 20:14:31 +0000 | [diff] [blame] | 1258 | #ifndef Py_LIMITED_API |
Mark Hammond | 91a681d | 2002-08-12 07:21:58 +0000 | [diff] [blame] | 1259 | PyAPI_FUNC(PyObject*) PyUnicode_EncodeUTF7( |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 1260 | const Py_UNICODE *data, /* Unicode char buffer */ |
| 1261 | Py_ssize_t length, /* number of Py_UNICODE chars to encode */ |
| 1262 | int base64SetO, /* Encode RFC2152 Set O characters in base64 */ |
| 1263 | int base64WhiteSpace, /* Encode whitespace (sp, ht, nl, cr) in base64 */ |
| 1264 | const char *errors /* error handling */ |
Marc-André Lemburg | c60e6f7 | 2001-09-20 10:35:46 +0000 | [diff] [blame] | 1265 | ); |
Martin v. Löwis | 1db7c13 | 2011-11-10 18:24:32 +0100 | [diff] [blame] | 1266 | PyAPI_FUNC(PyObject*) _PyUnicode_EncodeUTF7( |
| 1267 | PyObject *unicode, /* Unicode object */ |
| 1268 | int base64SetO, /* Encode RFC2152 Set O characters in base64 */ |
| 1269 | int base64WhiteSpace, /* Encode whitespace (sp, ht, nl, cr) in base64 */ |
| 1270 | const char *errors /* error handling */ |
| 1271 | ); |
Martin v. Löwis | 4d0d471 | 2010-12-03 20:14:31 +0000 | [diff] [blame] | 1272 | #endif |
Marc-André Lemburg | c60e6f7 | 2001-09-20 10:35:46 +0000 | [diff] [blame] | 1273 | |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 1274 | /* --- UTF-8 Codecs ------------------------------------------------------- */ |
| 1275 | |
Mark Hammond | 91a681d | 2002-08-12 07:21:58 +0000 | [diff] [blame] | 1276 | PyAPI_FUNC(PyObject*) PyUnicode_DecodeUTF8( |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 1277 | const char *string, /* UTF-8 encoded string */ |
| 1278 | Py_ssize_t length, /* size of string */ |
| 1279 | const char *errors /* error handling */ |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 1280 | ); |
| 1281 | |
Walter Dörwald | 6965203 | 2004-09-07 20:24:22 +0000 | [diff] [blame] | 1282 | PyAPI_FUNC(PyObject*) PyUnicode_DecodeUTF8Stateful( |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 1283 | const char *string, /* UTF-8 encoded string */ |
| 1284 | Py_ssize_t length, /* size of string */ |
| 1285 | const char *errors, /* error handling */ |
| 1286 | Py_ssize_t *consumed /* bytes consumed */ |
Walter Dörwald | 6965203 | 2004-09-07 20:24:22 +0000 | [diff] [blame] | 1287 | ); |
| 1288 | |
Mark Hammond | 91a681d | 2002-08-12 07:21:58 +0000 | [diff] [blame] | 1289 | PyAPI_FUNC(PyObject*) PyUnicode_AsUTF8String( |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 1290 | PyObject *unicode /* Unicode object */ |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 1291 | ); |
| 1292 | |
Martin v. Löwis | 4d0d471 | 2010-12-03 20:14:31 +0000 | [diff] [blame] | 1293 | #ifndef Py_LIMITED_API |
Martin v. Löwis | d63a3b8 | 2011-09-28 07:41:54 +0200 | [diff] [blame] | 1294 | PyAPI_FUNC(PyObject*) _PyUnicode_AsUTF8String( |
| 1295 | PyObject *unicode, |
| 1296 | const char *errors); |
| 1297 | |
Mark Hammond | 91a681d | 2002-08-12 07:21:58 +0000 | [diff] [blame] | 1298 | PyAPI_FUNC(PyObject*) PyUnicode_EncodeUTF8( |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 1299 | const Py_UNICODE *data, /* Unicode char buffer */ |
| 1300 | Py_ssize_t length, /* number of Py_UNICODE chars to encode */ |
| 1301 | const char *errors /* error handling */ |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 1302 | ); |
Martin v. Löwis | 4d0d471 | 2010-12-03 20:14:31 +0000 | [diff] [blame] | 1303 | #endif |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 1304 | |
Walter Dörwald | 41980ca | 2007-08-16 21:55:45 +0000 | [diff] [blame] | 1305 | /* --- UTF-32 Codecs ------------------------------------------------------ */ |
| 1306 | |
| 1307 | /* Decodes length bytes from a UTF-32 encoded buffer string and returns |
| 1308 | the corresponding Unicode object. |
| 1309 | |
| 1310 | errors (if non-NULL) defines the error handling. It defaults |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 1311 | to "strict". |
Walter Dörwald | 41980ca | 2007-08-16 21:55:45 +0000 | [diff] [blame] | 1312 | |
| 1313 | If byteorder is non-NULL, the decoder starts decoding using the |
| 1314 | given byte order: |
| 1315 | |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 1316 | *byteorder == -1: little endian |
| 1317 | *byteorder == 0: native order |
| 1318 | *byteorder == 1: big endian |
Walter Dörwald | 41980ca | 2007-08-16 21:55:45 +0000 | [diff] [blame] | 1319 | |
| 1320 | In native mode, the first four bytes of the stream are checked for a |
| 1321 | BOM mark. If found, the BOM mark is analysed, the byte order |
| 1322 | adjusted and the BOM skipped. In the other modes, no BOM mark |
| 1323 | interpretation is done. After completion, *byteorder is set to the |
| 1324 | current byte order at the end of input data. |
| 1325 | |
| 1326 | If byteorder is NULL, the codec starts in native order mode. |
| 1327 | |
| 1328 | */ |
| 1329 | |
| 1330 | PyAPI_FUNC(PyObject*) PyUnicode_DecodeUTF32( |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 1331 | const char *string, /* UTF-32 encoded string */ |
| 1332 | Py_ssize_t length, /* size of string */ |
| 1333 | const char *errors, /* error handling */ |
| 1334 | int *byteorder /* pointer to byteorder to use |
| 1335 | 0=native;-1=LE,1=BE; updated on |
| 1336 | exit */ |
Walter Dörwald | 41980ca | 2007-08-16 21:55:45 +0000 | [diff] [blame] | 1337 | ); |
| 1338 | |
| 1339 | PyAPI_FUNC(PyObject*) PyUnicode_DecodeUTF32Stateful( |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 1340 | const char *string, /* UTF-32 encoded string */ |
| 1341 | Py_ssize_t length, /* size of string */ |
| 1342 | const char *errors, /* error handling */ |
| 1343 | int *byteorder, /* pointer to byteorder to use |
| 1344 | 0=native;-1=LE,1=BE; updated on |
| 1345 | exit */ |
| 1346 | Py_ssize_t *consumed /* bytes consumed */ |
Walter Dörwald | 41980ca | 2007-08-16 21:55:45 +0000 | [diff] [blame] | 1347 | ); |
| 1348 | |
| 1349 | /* Returns a Python string using the UTF-32 encoding in native byte |
| 1350 | order. The string always starts with a BOM mark. */ |
| 1351 | |
| 1352 | PyAPI_FUNC(PyObject*) PyUnicode_AsUTF32String( |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 1353 | PyObject *unicode /* Unicode object */ |
Walter Dörwald | 41980ca | 2007-08-16 21:55:45 +0000 | [diff] [blame] | 1354 | ); |
| 1355 | |
| 1356 | /* Returns a Python string object holding the UTF-32 encoded value of |
| 1357 | the Unicode data. |
| 1358 | |
| 1359 | If byteorder is not 0, output is written according to the following |
| 1360 | byte order: |
| 1361 | |
| 1362 | byteorder == -1: little endian |
| 1363 | byteorder == 0: native byte order (writes a BOM mark) |
| 1364 | byteorder == 1: big endian |
| 1365 | |
| 1366 | If byteorder is 0, the output string will always start with the |
| 1367 | Unicode BOM mark (U+FEFF). In the other two modes, no BOM mark is |
| 1368 | prepended. |
| 1369 | |
| 1370 | */ |
| 1371 | |
Martin v. Löwis | 4d0d471 | 2010-12-03 20:14:31 +0000 | [diff] [blame] | 1372 | #ifndef Py_LIMITED_API |
Walter Dörwald | 41980ca | 2007-08-16 21:55:45 +0000 | [diff] [blame] | 1373 | PyAPI_FUNC(PyObject*) PyUnicode_EncodeUTF32( |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 1374 | const Py_UNICODE *data, /* Unicode char buffer */ |
| 1375 | Py_ssize_t length, /* number of Py_UNICODE chars to encode */ |
| 1376 | const char *errors, /* error handling */ |
| 1377 | int byteorder /* byteorder to use 0=BOM+native;-1=LE,1=BE */ |
Walter Dörwald | 41980ca | 2007-08-16 21:55:45 +0000 | [diff] [blame] | 1378 | ); |
Martin v. Löwis | 1db7c13 | 2011-11-10 18:24:32 +0100 | [diff] [blame] | 1379 | PyAPI_FUNC(PyObject*) _PyUnicode_EncodeUTF32( |
| 1380 | PyObject *object, /* Unicode object */ |
| 1381 | const char *errors, /* error handling */ |
| 1382 | int byteorder /* byteorder to use 0=BOM+native;-1=LE,1=BE */ |
| 1383 | ); |
Martin v. Löwis | 4d0d471 | 2010-12-03 20:14:31 +0000 | [diff] [blame] | 1384 | #endif |
Walter Dörwald | 41980ca | 2007-08-16 21:55:45 +0000 | [diff] [blame] | 1385 | |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 1386 | /* --- UTF-16 Codecs ------------------------------------------------------ */ |
| 1387 | |
Guido van Rossum | 9e896b3 | 2000-04-05 20:11:21 +0000 | [diff] [blame] | 1388 | /* Decodes length bytes from a UTF-16 encoded buffer string and returns |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 1389 | the corresponding Unicode object. |
| 1390 | |
| 1391 | errors (if non-NULL) defines the error handling. It defaults |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 1392 | to "strict". |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 1393 | |
| 1394 | If byteorder is non-NULL, the decoder starts decoding using the |
| 1395 | given byte order: |
| 1396 | |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 1397 | *byteorder == -1: little endian |
| 1398 | *byteorder == 0: native order |
| 1399 | *byteorder == 1: big endian |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 1400 | |
Marc-André Lemburg | 489b56e | 2001-05-21 20:30:15 +0000 | [diff] [blame] | 1401 | In native mode, the first two bytes of the stream are checked for a |
| 1402 | BOM mark. If found, the BOM mark is analysed, the byte order |
| 1403 | adjusted and the BOM skipped. In the other modes, no BOM mark |
| 1404 | interpretation is done. After completion, *byteorder is set to the |
| 1405 | current byte order at the end of input data. |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 1406 | |
| 1407 | If byteorder is NULL, the codec starts in native order mode. |
| 1408 | |
| 1409 | */ |
| 1410 | |
Mark Hammond | 91a681d | 2002-08-12 07:21:58 +0000 | [diff] [blame] | 1411 | PyAPI_FUNC(PyObject*) PyUnicode_DecodeUTF16( |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 1412 | const char *string, /* UTF-16 encoded string */ |
| 1413 | Py_ssize_t length, /* size of string */ |
| 1414 | const char *errors, /* error handling */ |
| 1415 | int *byteorder /* pointer to byteorder to use |
| 1416 | 0=native;-1=LE,1=BE; updated on |
| 1417 | exit */ |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 1418 | ); |
| 1419 | |
Walter Dörwald | 6965203 | 2004-09-07 20:24:22 +0000 | [diff] [blame] | 1420 | PyAPI_FUNC(PyObject*) PyUnicode_DecodeUTF16Stateful( |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 1421 | const char *string, /* UTF-16 encoded string */ |
| 1422 | Py_ssize_t length, /* size of string */ |
| 1423 | const char *errors, /* error handling */ |
| 1424 | int *byteorder, /* pointer to byteorder to use |
| 1425 | 0=native;-1=LE,1=BE; updated on |
| 1426 | exit */ |
| 1427 | Py_ssize_t *consumed /* bytes consumed */ |
Walter Dörwald | 6965203 | 2004-09-07 20:24:22 +0000 | [diff] [blame] | 1428 | ); |
| 1429 | |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 1430 | /* Returns a Python string using the UTF-16 encoding in native byte |
| 1431 | order. The string always starts with a BOM mark. */ |
| 1432 | |
Mark Hammond | 91a681d | 2002-08-12 07:21:58 +0000 | [diff] [blame] | 1433 | PyAPI_FUNC(PyObject*) PyUnicode_AsUTF16String( |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 1434 | PyObject *unicode /* Unicode object */ |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 1435 | ); |
| 1436 | |
| 1437 | /* Returns a Python string object holding the UTF-16 encoded value of |
Guido van Rossum | 9e896b3 | 2000-04-05 20:11:21 +0000 | [diff] [blame] | 1438 | the Unicode data. |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 1439 | |
| 1440 | If byteorder is not 0, output is written according to the following |
| 1441 | byte order: |
| 1442 | |
| 1443 | byteorder == -1: little endian |
| 1444 | byteorder == 0: native byte order (writes a BOM mark) |
| 1445 | byteorder == 1: big endian |
| 1446 | |
| 1447 | If byteorder is 0, the output string will always start with the |
| 1448 | Unicode BOM mark (U+FEFF). In the other two modes, no BOM mark is |
| 1449 | prepended. |
| 1450 | |
| 1451 | Note that Py_UNICODE data is being interpreted as UTF-16 reduced to |
| 1452 | UCS-2. This trick makes it possible to add full UTF-16 capabilities |
Thomas Wouters | 7e47402 | 2000-07-16 12:04:32 +0000 | [diff] [blame] | 1453 | at a later point without compromising the APIs. |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 1454 | |
| 1455 | */ |
| 1456 | |
Martin v. Löwis | 4d0d471 | 2010-12-03 20:14:31 +0000 | [diff] [blame] | 1457 | #ifndef Py_LIMITED_API |
Mark Hammond | 91a681d | 2002-08-12 07:21:58 +0000 | [diff] [blame] | 1458 | PyAPI_FUNC(PyObject*) PyUnicode_EncodeUTF16( |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 1459 | const Py_UNICODE *data, /* Unicode char buffer */ |
| 1460 | Py_ssize_t length, /* number of Py_UNICODE chars to encode */ |
| 1461 | const char *errors, /* error handling */ |
| 1462 | int byteorder /* byteorder to use 0=BOM+native;-1=LE,1=BE */ |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 1463 | ); |
Martin v. Löwis | 1db7c13 | 2011-11-10 18:24:32 +0100 | [diff] [blame] | 1464 | PyAPI_FUNC(PyObject*) _PyUnicode_EncodeUTF16( |
| 1465 | PyObject* unicode, /* Unicode object */ |
| 1466 | const char *errors, /* error handling */ |
| 1467 | int byteorder /* byteorder to use 0=BOM+native;-1=LE,1=BE */ |
| 1468 | ); |
Martin v. Löwis | 4d0d471 | 2010-12-03 20:14:31 +0000 | [diff] [blame] | 1469 | #endif |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 1470 | |
| 1471 | /* --- Unicode-Escape Codecs ---------------------------------------------- */ |
| 1472 | |
Mark Hammond | 91a681d | 2002-08-12 07:21:58 +0000 | [diff] [blame] | 1473 | PyAPI_FUNC(PyObject*) PyUnicode_DecodeUnicodeEscape( |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 1474 | const char *string, /* Unicode-Escape encoded string */ |
| 1475 | Py_ssize_t length, /* size of string */ |
| 1476 | const char *errors /* error handling */ |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 1477 | ); |
| 1478 | |
Mark Hammond | 91a681d | 2002-08-12 07:21:58 +0000 | [diff] [blame] | 1479 | PyAPI_FUNC(PyObject*) PyUnicode_AsUnicodeEscapeString( |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 1480 | PyObject *unicode /* Unicode object */ |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 1481 | ); |
| 1482 | |
Martin v. Löwis | 4d0d471 | 2010-12-03 20:14:31 +0000 | [diff] [blame] | 1483 | #ifndef Py_LIMITED_API |
Mark Hammond | 91a681d | 2002-08-12 07:21:58 +0000 | [diff] [blame] | 1484 | PyAPI_FUNC(PyObject*) PyUnicode_EncodeUnicodeEscape( |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 1485 | const Py_UNICODE *data, /* Unicode char buffer */ |
| 1486 | Py_ssize_t length /* Number of Py_UNICODE chars to encode */ |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 1487 | ); |
Martin v. Löwis | 4d0d471 | 2010-12-03 20:14:31 +0000 | [diff] [blame] | 1488 | #endif |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 1489 | |
| 1490 | /* --- Raw-Unicode-Escape Codecs ------------------------------------------ */ |
| 1491 | |
Mark Hammond | 91a681d | 2002-08-12 07:21:58 +0000 | [diff] [blame] | 1492 | PyAPI_FUNC(PyObject*) PyUnicode_DecodeRawUnicodeEscape( |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 1493 | const char *string, /* Raw-Unicode-Escape encoded string */ |
| 1494 | Py_ssize_t length, /* size of string */ |
| 1495 | const char *errors /* error handling */ |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 1496 | ); |
| 1497 | |
Mark Hammond | 91a681d | 2002-08-12 07:21:58 +0000 | [diff] [blame] | 1498 | PyAPI_FUNC(PyObject*) PyUnicode_AsRawUnicodeEscapeString( |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 1499 | PyObject *unicode /* Unicode object */ |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 1500 | ); |
| 1501 | |
Martin v. Löwis | 4d0d471 | 2010-12-03 20:14:31 +0000 | [diff] [blame] | 1502 | #ifndef Py_LIMITED_API |
Mark Hammond | 91a681d | 2002-08-12 07:21:58 +0000 | [diff] [blame] | 1503 | PyAPI_FUNC(PyObject*) PyUnicode_EncodeRawUnicodeEscape( |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 1504 | const Py_UNICODE *data, /* Unicode char buffer */ |
| 1505 | Py_ssize_t length /* Number of Py_UNICODE chars to encode */ |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 1506 | ); |
Martin v. Löwis | 4d0d471 | 2010-12-03 20:14:31 +0000 | [diff] [blame] | 1507 | #endif |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 1508 | |
Walter Dörwald | a47d1c0 | 2005-08-30 10:23:14 +0000 | [diff] [blame] | 1509 | /* --- Unicode Internal Codec --------------------------------------------- |
| 1510 | |
| 1511 | Only for internal use in _codecsmodule.c */ |
| 1512 | |
Martin v. Löwis | 4d0d471 | 2010-12-03 20:14:31 +0000 | [diff] [blame] | 1513 | #ifndef Py_LIMITED_API |
Walter Dörwald | a47d1c0 | 2005-08-30 10:23:14 +0000 | [diff] [blame] | 1514 | PyObject *_PyUnicode_DecodeUnicodeInternal( |
| 1515 | const char *string, |
Martin v. Löwis | 18e1655 | 2006-02-15 17:27:45 +0000 | [diff] [blame] | 1516 | Py_ssize_t length, |
Walter Dörwald | a47d1c0 | 2005-08-30 10:23:14 +0000 | [diff] [blame] | 1517 | const char *errors |
| 1518 | ); |
Martin v. Löwis | 4d0d471 | 2010-12-03 20:14:31 +0000 | [diff] [blame] | 1519 | #endif |
Walter Dörwald | a47d1c0 | 2005-08-30 10:23:14 +0000 | [diff] [blame] | 1520 | |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 1521 | /* --- Latin-1 Codecs ----------------------------------------------------- |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 1522 | |
| 1523 | Note: Latin-1 corresponds to the first 256 Unicode ordinals. |
| 1524 | |
| 1525 | */ |
| 1526 | |
Mark Hammond | 91a681d | 2002-08-12 07:21:58 +0000 | [diff] [blame] | 1527 | PyAPI_FUNC(PyObject*) PyUnicode_DecodeLatin1( |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 1528 | const char *string, /* Latin-1 encoded string */ |
| 1529 | Py_ssize_t length, /* size of string */ |
| 1530 | const char *errors /* error handling */ |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 1531 | ); |
| 1532 | |
Mark Hammond | 91a681d | 2002-08-12 07:21:58 +0000 | [diff] [blame] | 1533 | PyAPI_FUNC(PyObject*) PyUnicode_AsLatin1String( |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 1534 | PyObject *unicode /* Unicode object */ |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 1535 | ); |
| 1536 | |
Martin v. Löwis | 4d0d471 | 2010-12-03 20:14:31 +0000 | [diff] [blame] | 1537 | #ifndef Py_LIMITED_API |
Martin v. Löwis | d63a3b8 | 2011-09-28 07:41:54 +0200 | [diff] [blame] | 1538 | PyAPI_FUNC(PyObject*) _PyUnicode_AsLatin1String( |
| 1539 | PyObject* unicode, |
| 1540 | const char* errors); |
| 1541 | |
Mark Hammond | 91a681d | 2002-08-12 07:21:58 +0000 | [diff] [blame] | 1542 | PyAPI_FUNC(PyObject*) PyUnicode_EncodeLatin1( |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 1543 | const Py_UNICODE *data, /* Unicode char buffer */ |
| 1544 | Py_ssize_t length, /* Number of Py_UNICODE chars to encode */ |
| 1545 | const char *errors /* error handling */ |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 1546 | ); |
Martin v. Löwis | 4d0d471 | 2010-12-03 20:14:31 +0000 | [diff] [blame] | 1547 | #endif |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 1548 | |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 1549 | /* --- ASCII Codecs ------------------------------------------------------- |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 1550 | |
| 1551 | Only 7-bit ASCII data is excepted. All other codes generate errors. |
| 1552 | |
| 1553 | */ |
| 1554 | |
Mark Hammond | 91a681d | 2002-08-12 07:21:58 +0000 | [diff] [blame] | 1555 | PyAPI_FUNC(PyObject*) PyUnicode_DecodeASCII( |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 1556 | const char *string, /* ASCII encoded string */ |
| 1557 | Py_ssize_t length, /* size of string */ |
| 1558 | const char *errors /* error handling */ |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 1559 | ); |
| 1560 | |
Mark Hammond | 91a681d | 2002-08-12 07:21:58 +0000 | [diff] [blame] | 1561 | PyAPI_FUNC(PyObject*) PyUnicode_AsASCIIString( |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 1562 | PyObject *unicode /* Unicode object */ |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 1563 | ); |
| 1564 | |
Martin v. Löwis | 4d0d471 | 2010-12-03 20:14:31 +0000 | [diff] [blame] | 1565 | #ifndef Py_LIMITED_API |
Martin v. Löwis | d63a3b8 | 2011-09-28 07:41:54 +0200 | [diff] [blame] | 1566 | PyAPI_FUNC(PyObject*) _PyUnicode_AsASCIIString( |
| 1567 | PyObject* unicode, |
| 1568 | const char* errors); |
| 1569 | |
Mark Hammond | 91a681d | 2002-08-12 07:21:58 +0000 | [diff] [blame] | 1570 | PyAPI_FUNC(PyObject*) PyUnicode_EncodeASCII( |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 1571 | const Py_UNICODE *data, /* Unicode char buffer */ |
| 1572 | Py_ssize_t length, /* Number of Py_UNICODE chars to encode */ |
| 1573 | const char *errors /* error handling */ |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 1574 | ); |
Martin v. Löwis | 4d0d471 | 2010-12-03 20:14:31 +0000 | [diff] [blame] | 1575 | #endif |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 1576 | |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 1577 | /* --- Character Map Codecs ----------------------------------------------- |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 1578 | |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 1579 | This codec uses mappings to encode and decode characters. |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 1580 | |
| 1581 | Decoding mappings must map single string characters to single |
| 1582 | Unicode characters, integers (which are then interpreted as Unicode |
| 1583 | ordinals) or None (meaning "undefined mapping" and causing an |
| 1584 | error). |
| 1585 | |
| 1586 | Encoding mappings must map single Unicode characters to single |
| 1587 | string characters, integers (which are then interpreted as Latin-1 |
| 1588 | ordinals) or None (meaning "undefined mapping" and causing an |
| 1589 | error). |
| 1590 | |
| 1591 | If a character lookup fails with a LookupError, the character is |
| 1592 | copied as-is meaning that its ordinal value will be interpreted as |
| 1593 | Unicode or Latin-1 ordinal resp. Because of this mappings only need |
| 1594 | to contain those mappings which map characters to different code |
| 1595 | points. |
| 1596 | |
| 1597 | */ |
| 1598 | |
Mark Hammond | 91a681d | 2002-08-12 07:21:58 +0000 | [diff] [blame] | 1599 | PyAPI_FUNC(PyObject*) PyUnicode_DecodeCharmap( |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 1600 | const char *string, /* Encoded string */ |
| 1601 | Py_ssize_t length, /* size of string */ |
| 1602 | PyObject *mapping, /* character mapping |
| 1603 | (char ordinal -> unicode ordinal) */ |
| 1604 | const char *errors /* error handling */ |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 1605 | ); |
| 1606 | |
Mark Hammond | 91a681d | 2002-08-12 07:21:58 +0000 | [diff] [blame] | 1607 | PyAPI_FUNC(PyObject*) PyUnicode_AsCharmapString( |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 1608 | PyObject *unicode, /* Unicode object */ |
| 1609 | PyObject *mapping /* character mapping |
| 1610 | (unicode ordinal -> char ordinal) */ |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 1611 | ); |
| 1612 | |
Martin v. Löwis | 4d0d471 | 2010-12-03 20:14:31 +0000 | [diff] [blame] | 1613 | #ifndef Py_LIMITED_API |
Mark Hammond | 91a681d | 2002-08-12 07:21:58 +0000 | [diff] [blame] | 1614 | PyAPI_FUNC(PyObject*) PyUnicode_EncodeCharmap( |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 1615 | const Py_UNICODE *data, /* Unicode char buffer */ |
| 1616 | Py_ssize_t length, /* Number of Py_UNICODE chars to encode */ |
| 1617 | PyObject *mapping, /* character mapping |
| 1618 | (unicode ordinal -> char ordinal) */ |
| 1619 | const char *errors /* error handling */ |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 1620 | ); |
Martin v. Löwis | 23e275b | 2011-11-02 18:02:51 +0100 | [diff] [blame] | 1621 | PyAPI_FUNC(PyObject*) _PyUnicode_EncodeCharmap( |
| 1622 | PyObject *unicode, /* Unicode object */ |
| 1623 | PyObject *mapping, /* character mapping |
| 1624 | (unicode ordinal -> char ordinal) */ |
| 1625 | const char *errors /* error handling */ |
| 1626 | ); |
Martin v. Löwis | 4d0d471 | 2010-12-03 20:14:31 +0000 | [diff] [blame] | 1627 | #endif |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 1628 | |
| 1629 | /* Translate a Py_UNICODE buffer of the given length by applying a |
| 1630 | character mapping table to it and return the resulting Unicode |
| 1631 | object. |
| 1632 | |
| 1633 | The mapping table must map Unicode ordinal integers to Unicode |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 1634 | ordinal integers or None (causing deletion of the character). |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 1635 | |
| 1636 | Mapping tables may be dictionaries or sequences. Unmapped character |
| 1637 | ordinals (ones which cause a LookupError) are left untouched and |
| 1638 | are copied as-is. |
| 1639 | |
| 1640 | */ |
| 1641 | |
Martin v. Löwis | 4d0d471 | 2010-12-03 20:14:31 +0000 | [diff] [blame] | 1642 | #ifndef Py_LIMITED_API |
Mark Hammond | 91a681d | 2002-08-12 07:21:58 +0000 | [diff] [blame] | 1643 | PyAPI_FUNC(PyObject *) PyUnicode_TranslateCharmap( |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 1644 | const Py_UNICODE *data, /* Unicode char buffer */ |
| 1645 | Py_ssize_t length, /* Number of Py_UNICODE chars to encode */ |
| 1646 | PyObject *table, /* Translate table */ |
| 1647 | const char *errors /* error handling */ |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 1648 | ); |
Martin v. Löwis | 4d0d471 | 2010-12-03 20:14:31 +0000 | [diff] [blame] | 1649 | #endif |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 1650 | |
Victor Stinner | 99b9538 | 2011-07-04 14:23:54 +0200 | [diff] [blame] | 1651 | #ifdef HAVE_MBCS |
Guido van Rossum | 24bdb04 | 2000-03-28 20:29:59 +0000 | [diff] [blame] | 1652 | |
Guido van Rossum | efec115 | 2000-03-28 02:01:15 +0000 | [diff] [blame] | 1653 | /* --- MBCS codecs for Windows -------------------------------------------- */ |
Guido van Rossum | 24bdb04 | 2000-03-28 20:29:59 +0000 | [diff] [blame] | 1654 | |
Mark Hammond | 91a681d | 2002-08-12 07:21:58 +0000 | [diff] [blame] | 1655 | PyAPI_FUNC(PyObject*) PyUnicode_DecodeMBCS( |
Guido van Rossum | efec115 | 2000-03-28 02:01:15 +0000 | [diff] [blame] | 1656 | const char *string, /* MBCS encoded string */ |
Martin v. Löwis | 18e1655 | 2006-02-15 17:27:45 +0000 | [diff] [blame] | 1657 | Py_ssize_t length, /* size of string */ |
Guido van Rossum | efec115 | 2000-03-28 02:01:15 +0000 | [diff] [blame] | 1658 | const char *errors /* error handling */ |
| 1659 | ); |
| 1660 | |
Thomas Wouters | 0e3f591 | 2006-08-11 14:57:12 +0000 | [diff] [blame] | 1661 | PyAPI_FUNC(PyObject*) PyUnicode_DecodeMBCSStateful( |
| 1662 | const char *string, /* MBCS encoded string */ |
| 1663 | Py_ssize_t length, /* size of string */ |
| 1664 | const char *errors, /* error handling */ |
| 1665 | Py_ssize_t *consumed /* bytes consumed */ |
| 1666 | ); |
| 1667 | |
Victor Stinner | 3a50e70 | 2011-10-18 21:21:00 +0200 | [diff] [blame] | 1668 | PyAPI_FUNC(PyObject*) PyUnicode_DecodeCodePageStateful( |
| 1669 | int code_page, /* code page number */ |
| 1670 | const char *string, /* encoded string */ |
| 1671 | Py_ssize_t length, /* size of string */ |
| 1672 | const char *errors, /* error handling */ |
| 1673 | Py_ssize_t *consumed /* bytes consumed */ |
| 1674 | ); |
| 1675 | |
Mark Hammond | 91a681d | 2002-08-12 07:21:58 +0000 | [diff] [blame] | 1676 | PyAPI_FUNC(PyObject*) PyUnicode_AsMBCSString( |
Guido van Rossum | efec115 | 2000-03-28 02:01:15 +0000 | [diff] [blame] | 1677 | PyObject *unicode /* Unicode object */ |
| 1678 | ); |
| 1679 | |
Martin v. Löwis | 4d0d471 | 2010-12-03 20:14:31 +0000 | [diff] [blame] | 1680 | #ifndef Py_LIMITED_API |
Mark Hammond | 91a681d | 2002-08-12 07:21:58 +0000 | [diff] [blame] | 1681 | PyAPI_FUNC(PyObject*) PyUnicode_EncodeMBCS( |
Guido van Rossum | efec115 | 2000-03-28 02:01:15 +0000 | [diff] [blame] | 1682 | const Py_UNICODE *data, /* Unicode char buffer */ |
Victor Stinner | 3a50e70 | 2011-10-18 21:21:00 +0200 | [diff] [blame] | 1683 | Py_ssize_t length, /* number of Py_UNICODE chars to encode */ |
Guido van Rossum | efec115 | 2000-03-28 02:01:15 +0000 | [diff] [blame] | 1684 | const char *errors /* error handling */ |
| 1685 | ); |
Martin v. Löwis | 4d0d471 | 2010-12-03 20:14:31 +0000 | [diff] [blame] | 1686 | #endif |
Guido van Rossum | efec115 | 2000-03-28 02:01:15 +0000 | [diff] [blame] | 1687 | |
Victor Stinner | 3a50e70 | 2011-10-18 21:21:00 +0200 | [diff] [blame] | 1688 | PyAPI_FUNC(PyObject*) PyUnicode_EncodeCodePage( |
| 1689 | int code_page, /* code page number */ |
| 1690 | PyObject *unicode, /* Unicode object */ |
| 1691 | const char *errors /* error handling */ |
| 1692 | ); |
| 1693 | |
Victor Stinner | 99b9538 | 2011-07-04 14:23:54 +0200 | [diff] [blame] | 1694 | #endif /* HAVE_MBCS */ |
Guido van Rossum | 24bdb04 | 2000-03-28 20:29:59 +0000 | [diff] [blame] | 1695 | |
Guido van Rossum | 9e896b3 | 2000-04-05 20:11:21 +0000 | [diff] [blame] | 1696 | /* --- Decimal Encoder ---------------------------------------------------- */ |
| 1697 | |
| 1698 | /* Takes a Unicode string holding a decimal value and writes it into |
| 1699 | an output buffer using standard ASCII digit codes. |
| 1700 | |
| 1701 | The output buffer has to provide at least length+1 bytes of storage |
| 1702 | area. The output string is 0-terminated. |
| 1703 | |
| 1704 | The encoder converts whitespace to ' ', decimal characters to their |
| 1705 | corresponding ASCII digit and all other Latin-1 characters except |
| 1706 | \0 as-is. Characters outside this range (Unicode ordinals 1-256) |
| 1707 | are treated as errors. This includes embedded NULL bytes. |
| 1708 | |
| 1709 | Error handling is defined by the errors argument: |
| 1710 | |
| 1711 | NULL or "strict": raise a ValueError |
| 1712 | "ignore": ignore the wrong characters (these are not copied to the |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 1713 | output buffer) |
Guido van Rossum | 9e896b3 | 2000-04-05 20:11:21 +0000 | [diff] [blame] | 1714 | "replace": replaces illegal characters with '?' |
| 1715 | |
| 1716 | Returns 0 on success, -1 on failure. |
| 1717 | |
| 1718 | */ |
| 1719 | |
Martin v. Löwis | 4d0d471 | 2010-12-03 20:14:31 +0000 | [diff] [blame] | 1720 | #ifndef Py_LIMITED_API |
Mark Hammond | 91a681d | 2002-08-12 07:21:58 +0000 | [diff] [blame] | 1721 | PyAPI_FUNC(int) PyUnicode_EncodeDecimal( |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 1722 | Py_UNICODE *s, /* Unicode buffer */ |
| 1723 | Py_ssize_t length, /* Number of Py_UNICODE chars to encode */ |
| 1724 | char *output, /* Output buffer; must have size >= length */ |
| 1725 | const char *errors /* error handling */ |
Guido van Rossum | 9e896b3 | 2000-04-05 20:11:21 +0000 | [diff] [blame] | 1726 | ); |
Martin v. Löwis | 4d0d471 | 2010-12-03 20:14:31 +0000 | [diff] [blame] | 1727 | #endif |
Guido van Rossum | 9e896b3 | 2000-04-05 20:11:21 +0000 | [diff] [blame] | 1728 | |
Alexander Belopolsky | 942af5a | 2010-12-04 03:38:46 +0000 | [diff] [blame] | 1729 | /* Transforms code points that have decimal digit property to the |
| 1730 | corresponding ASCII digit code points. |
| 1731 | |
| 1732 | Returns a new Unicode string on success, NULL on failure. |
| 1733 | */ |
| 1734 | |
Georg Brandl | b550308 | 2010-12-05 11:40:48 +0000 | [diff] [blame] | 1735 | #ifndef Py_LIMITED_API |
Alexander Belopolsky | 942af5a | 2010-12-04 03:38:46 +0000 | [diff] [blame] | 1736 | PyAPI_FUNC(PyObject*) PyUnicode_TransformDecimalToASCII( |
| 1737 | Py_UNICODE *s, /* Unicode buffer */ |
| 1738 | Py_ssize_t length /* Number of Py_UNICODE chars to transform */ |
| 1739 | ); |
Georg Brandl | b550308 | 2010-12-05 11:40:48 +0000 | [diff] [blame] | 1740 | #endif |
Alexander Belopolsky | 942af5a | 2010-12-04 03:38:46 +0000 | [diff] [blame] | 1741 | |
Victor Stinner | 6f9568b | 2011-11-17 00:12:44 +0100 | [diff] [blame] | 1742 | /* Similar to PyUnicode_TransformDecimalToASCII(), but takes a PyObject |
Martin v. Löwis | d63a3b8 | 2011-09-28 07:41:54 +0200 | [diff] [blame] | 1743 | as argument instead of a raw buffer and length. This function additionally |
| 1744 | transforms spaces to ASCII because this is what the callers in longobject, |
| 1745 | floatobject, and complexobject did anyways. */ |
| 1746 | |
| 1747 | #ifndef Py_LIMITED_API |
| 1748 | PyAPI_FUNC(PyObject*) _PyUnicode_TransformDecimalAndSpaceToASCII( |
| 1749 | PyObject *unicode /* Unicode object */ |
| 1750 | ); |
| 1751 | #endif |
| 1752 | |
Victor Stinner | af02e1c | 2011-12-16 23:56:01 +0100 | [diff] [blame] | 1753 | /* --- Locale encoding --------------------------------------------------- */ |
| 1754 | |
| 1755 | /* Decode a string from the current locale encoding. The decoder is strict if |
| 1756 | *surrogateescape* is equal to zero, otherwise it uses the 'surrogateescape' |
| 1757 | error handler (PEP 383) to escape undecodable bytes. If a byte sequence can |
| 1758 | be decoded as a surrogate character and *surrogateescape* is not equal to |
| 1759 | zero, the byte sequence is escaped using the 'surrogateescape' error handler |
| 1760 | instead of being decoded. *str* must end with a null character but cannot |
Victor Stinner | f2ea71f | 2011-12-17 04:13:41 +0100 | [diff] [blame] | 1761 | contain embedded null characters. */ |
Victor Stinner | af02e1c | 2011-12-16 23:56:01 +0100 | [diff] [blame] | 1762 | |
| 1763 | PyAPI_FUNC(PyObject*) PyUnicode_DecodeLocaleAndSize( |
| 1764 | const char *str, |
| 1765 | Py_ssize_t len, |
Victor Stinner | 1b57967 | 2011-12-17 05:47:23 +0100 | [diff] [blame] | 1766 | const char *errors); |
Victor Stinner | af02e1c | 2011-12-16 23:56:01 +0100 | [diff] [blame] | 1767 | |
| 1768 | /* Similar to PyUnicode_DecodeLocaleAndSize(), but compute the string |
| 1769 | length using strlen(). */ |
| 1770 | |
| 1771 | PyAPI_FUNC(PyObject*) PyUnicode_DecodeLocale( |
| 1772 | const char *str, |
Victor Stinner | 1b57967 | 2011-12-17 05:47:23 +0100 | [diff] [blame] | 1773 | const char *errors); |
Victor Stinner | af02e1c | 2011-12-16 23:56:01 +0100 | [diff] [blame] | 1774 | |
Victor Stinner | f2ea71f | 2011-12-17 04:13:41 +0100 | [diff] [blame] | 1775 | /* Encode a Unicode object to the current locale encoding. The encoder is |
| 1776 | strict is *surrogateescape* is equal to zero, otherwise the |
| 1777 | "surrogateescape" error handler is used. Return a bytes object. The string |
Victor Stinner | d45c7f8 | 2012-12-04 01:34:47 +0100 | [diff] [blame] | 1778 | cannot contain embedded null characters. */ |
Victor Stinner | f2ea71f | 2011-12-17 04:13:41 +0100 | [diff] [blame] | 1779 | |
| 1780 | PyAPI_FUNC(PyObject*) PyUnicode_EncodeLocale( |
| 1781 | PyObject *unicode, |
Victor Stinner | 1b57967 | 2011-12-17 05:47:23 +0100 | [diff] [blame] | 1782 | const char *errors |
Victor Stinner | f2ea71f | 2011-12-17 04:13:41 +0100 | [diff] [blame] | 1783 | ); |
| 1784 | |
Martin v. Löwis | 011e842 | 2009-05-05 04:43:17 +0000 | [diff] [blame] | 1785 | /* --- File system encoding ---------------------------------------------- */ |
| 1786 | |
Victor Stinner | 47fcb5b | 2010-08-13 23:59:58 +0000 | [diff] [blame] | 1787 | /* ParseTuple converter: encode str objects to bytes using |
| 1788 | PyUnicode_EncodeFSDefault(); bytes objects are output as-is. */ |
Martin v. Löwis | 011e842 | 2009-05-05 04:43:17 +0000 | [diff] [blame] | 1789 | |
| 1790 | PyAPI_FUNC(int) PyUnicode_FSConverter(PyObject*, void*); |
| 1791 | |
Victor Stinner | 47fcb5b | 2010-08-13 23:59:58 +0000 | [diff] [blame] | 1792 | /* ParseTuple converter: decode bytes objects to unicode using |
| 1793 | PyUnicode_DecodeFSDefaultAndSize(); str objects are output as-is. */ |
| 1794 | |
| 1795 | PyAPI_FUNC(int) PyUnicode_FSDecoder(PyObject*, void*); |
| 1796 | |
Victor Stinner | 77c3862 | 2010-05-14 15:58:55 +0000 | [diff] [blame] | 1797 | /* Decode a null-terminated string using Py_FileSystemDefaultEncoding |
| 1798 | and the "surrogateescape" error handler. |
Martin v. Löwis | 011e842 | 2009-05-05 04:43:17 +0000 | [diff] [blame] | 1799 | |
Victor Stinner | f3170cc | 2010-10-15 12:04:23 +0000 | [diff] [blame] | 1800 | If Py_FileSystemDefaultEncoding is not set, fall back to the locale |
| 1801 | encoding. |
Martin v. Löwis | 011e842 | 2009-05-05 04:43:17 +0000 | [diff] [blame] | 1802 | |
Benjamin Peterson | ccbd694 | 2010-05-15 17:43:18 +0000 | [diff] [blame] | 1803 | Use PyUnicode_DecodeFSDefaultAndSize() if the string length is known. |
Martin v. Löwis | 011e842 | 2009-05-05 04:43:17 +0000 | [diff] [blame] | 1804 | */ |
| 1805 | |
| 1806 | PyAPI_FUNC(PyObject*) PyUnicode_DecodeFSDefault( |
| 1807 | const char *s /* encoded string */ |
| 1808 | ); |
| 1809 | |
Victor Stinner | 77c3862 | 2010-05-14 15:58:55 +0000 | [diff] [blame] | 1810 | /* Decode a string using Py_FileSystemDefaultEncoding |
| 1811 | and the "surrogateescape" error handler. |
| 1812 | |
Victor Stinner | f3170cc | 2010-10-15 12:04:23 +0000 | [diff] [blame] | 1813 | If Py_FileSystemDefaultEncoding is not set, fall back to the locale |
| 1814 | encoding. |
Victor Stinner | 77c3862 | 2010-05-14 15:58:55 +0000 | [diff] [blame] | 1815 | */ |
| 1816 | |
Martin v. Löwis | 011e842 | 2009-05-05 04:43:17 +0000 | [diff] [blame] | 1817 | PyAPI_FUNC(PyObject*) PyUnicode_DecodeFSDefaultAndSize( |
| 1818 | const char *s, /* encoded string */ |
| 1819 | Py_ssize_t size /* size */ |
| 1820 | ); |
| 1821 | |
Victor Stinner | ae6265f | 2010-05-15 16:27:27 +0000 | [diff] [blame] | 1822 | /* Encode a Unicode object to Py_FileSystemDefaultEncoding with the |
Benjamin Peterson | ccbd694 | 2010-05-15 17:43:18 +0000 | [diff] [blame] | 1823 | "surrogateescape" error handler, and return bytes. |
Victor Stinner | ae6265f | 2010-05-15 16:27:27 +0000 | [diff] [blame] | 1824 | |
Victor Stinner | f3170cc | 2010-10-15 12:04:23 +0000 | [diff] [blame] | 1825 | If Py_FileSystemDefaultEncoding is not set, fall back to the locale |
| 1826 | encoding. |
Victor Stinner | ae6265f | 2010-05-15 16:27:27 +0000 | [diff] [blame] | 1827 | */ |
| 1828 | |
| 1829 | PyAPI_FUNC(PyObject*) PyUnicode_EncodeFSDefault( |
| 1830 | PyObject *unicode |
| 1831 | ); |
| 1832 | |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 1833 | /* --- Methods & Slots ---------------------------------------------------- |
| 1834 | |
| 1835 | These are capable of handling Unicode objects and strings on input |
| 1836 | (we refer to them as strings in the descriptions) and return |
Georg Brandl | c6bc4c6 | 2011-10-05 16:23:09 +0200 | [diff] [blame] | 1837 | Unicode objects or integers as appropriate. */ |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 1838 | |
| 1839 | /* Concat two strings giving a new Unicode string. */ |
| 1840 | |
Mark Hammond | 91a681d | 2002-08-12 07:21:58 +0000 | [diff] [blame] | 1841 | PyAPI_FUNC(PyObject*) PyUnicode_Concat( |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 1842 | PyObject *left, /* Left string */ |
| 1843 | PyObject *right /* Right string */ |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 1844 | ); |
| 1845 | |
Walter Dörwald | 1ab8330 | 2007-05-18 17:15:44 +0000 | [diff] [blame] | 1846 | /* Concat two strings and put the result in *pleft |
| 1847 | (sets *pleft to NULL on error) */ |
| 1848 | |
| 1849 | PyAPI_FUNC(void) PyUnicode_Append( |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 1850 | PyObject **pleft, /* Pointer to left string */ |
| 1851 | PyObject *right /* Right string */ |
Walter Dörwald | 1ab8330 | 2007-05-18 17:15:44 +0000 | [diff] [blame] | 1852 | ); |
| 1853 | |
| 1854 | /* Concat two strings, put the result in *pleft and drop the right object |
| 1855 | (sets *pleft to NULL on error) */ |
| 1856 | |
| 1857 | PyAPI_FUNC(void) PyUnicode_AppendAndDel( |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 1858 | PyObject **pleft, /* Pointer to left string */ |
| 1859 | PyObject *right /* Right string */ |
Walter Dörwald | 1ab8330 | 2007-05-18 17:15:44 +0000 | [diff] [blame] | 1860 | ); |
| 1861 | |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 1862 | /* Split a string giving a list of Unicode strings. |
| 1863 | |
| 1864 | If sep is NULL, splitting will be done at all whitespace |
| 1865 | substrings. Otherwise, splits occur at the given separator. |
| 1866 | |
| 1867 | At most maxsplit splits will be done. If negative, no limit is set. |
| 1868 | |
| 1869 | Separators are not included in the resulting list. |
| 1870 | |
| 1871 | */ |
| 1872 | |
Mark Hammond | 91a681d | 2002-08-12 07:21:58 +0000 | [diff] [blame] | 1873 | PyAPI_FUNC(PyObject*) PyUnicode_Split( |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 1874 | PyObject *s, /* String to split */ |
| 1875 | PyObject *sep, /* String separator */ |
| 1876 | Py_ssize_t maxsplit /* Maxsplit count */ |
| 1877 | ); |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 1878 | |
| 1879 | /* Dito, but split at line breaks. |
| 1880 | |
| 1881 | CRLF is considered to be one line break. Line breaks are not |
| 1882 | included in the resulting list. */ |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 1883 | |
Mark Hammond | 91a681d | 2002-08-12 07:21:58 +0000 | [diff] [blame] | 1884 | PyAPI_FUNC(PyObject*) PyUnicode_Splitlines( |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 1885 | PyObject *s, /* String to split */ |
| 1886 | int keepends /* If true, line end markers are included */ |
| 1887 | ); |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 1888 | |
Thomas Wouters | 477c8d5 | 2006-05-27 19:21:47 +0000 | [diff] [blame] | 1889 | /* Partition a string using a given separator. */ |
| 1890 | |
| 1891 | PyAPI_FUNC(PyObject*) PyUnicode_Partition( |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 1892 | PyObject *s, /* String to partition */ |
| 1893 | PyObject *sep /* String separator */ |
| 1894 | ); |
Thomas Wouters | 477c8d5 | 2006-05-27 19:21:47 +0000 | [diff] [blame] | 1895 | |
| 1896 | /* Partition a string using a given separator, searching from the end of the |
| 1897 | string. */ |
| 1898 | |
| 1899 | PyAPI_FUNC(PyObject*) PyUnicode_RPartition( |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 1900 | PyObject *s, /* String to partition */ |
| 1901 | PyObject *sep /* String separator */ |
| 1902 | ); |
Thomas Wouters | 477c8d5 | 2006-05-27 19:21:47 +0000 | [diff] [blame] | 1903 | |
Hye-Shik Chang | 3ae811b | 2003-12-15 18:49:53 +0000 | [diff] [blame] | 1904 | /* Split a string giving a list of Unicode strings. |
| 1905 | |
| 1906 | If sep is NULL, splitting will be done at all whitespace |
| 1907 | substrings. Otherwise, splits occur at the given separator. |
| 1908 | |
| 1909 | At most maxsplit splits will be done. But unlike PyUnicode_Split |
| 1910 | PyUnicode_RSplit splits from the end of the string. If negative, |
| 1911 | no limit is set. |
| 1912 | |
| 1913 | Separators are not included in the resulting list. |
| 1914 | |
| 1915 | */ |
| 1916 | |
| 1917 | PyAPI_FUNC(PyObject*) PyUnicode_RSplit( |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 1918 | PyObject *s, /* String to split */ |
| 1919 | PyObject *sep, /* String separator */ |
| 1920 | Py_ssize_t maxsplit /* Maxsplit count */ |
| 1921 | ); |
Hye-Shik Chang | 3ae811b | 2003-12-15 18:49:53 +0000 | [diff] [blame] | 1922 | |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 1923 | /* Translate a string by applying a character mapping table to it and |
| 1924 | return the resulting Unicode object. |
| 1925 | |
| 1926 | The mapping table must map Unicode ordinal integers to Unicode |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 1927 | ordinal integers or None (causing deletion of the character). |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 1928 | |
| 1929 | Mapping tables may be dictionaries or sequences. Unmapped character |
| 1930 | ordinals (ones which cause a LookupError) are left untouched and |
| 1931 | are copied as-is. |
| 1932 | |
| 1933 | */ |
| 1934 | |
Mark Hammond | 91a681d | 2002-08-12 07:21:58 +0000 | [diff] [blame] | 1935 | PyAPI_FUNC(PyObject *) PyUnicode_Translate( |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 1936 | PyObject *str, /* String */ |
| 1937 | PyObject *table, /* Translate table */ |
| 1938 | const char *errors /* error handling */ |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 1939 | ); |
| 1940 | |
| 1941 | /* Join a sequence of strings using the given separator and return |
| 1942 | the resulting Unicode string. */ |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 1943 | |
Mark Hammond | 91a681d | 2002-08-12 07:21:58 +0000 | [diff] [blame] | 1944 | PyAPI_FUNC(PyObject*) PyUnicode_Join( |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 1945 | PyObject *separator, /* Separator string */ |
| 1946 | PyObject *seq /* Sequence object */ |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 1947 | ); |
| 1948 | |
| 1949 | /* Return 1 if substr matches str[start:end] at the given tail end, 0 |
| 1950 | otherwise. */ |
| 1951 | |
Martin v. Löwis | 18e1655 | 2006-02-15 17:27:45 +0000 | [diff] [blame] | 1952 | PyAPI_FUNC(Py_ssize_t) PyUnicode_Tailmatch( |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 1953 | PyObject *str, /* String */ |
| 1954 | PyObject *substr, /* Prefix or Suffix string */ |
| 1955 | Py_ssize_t start, /* Start index */ |
| 1956 | Py_ssize_t end, /* Stop index */ |
| 1957 | int direction /* Tail end: -1 prefix, +1 suffix */ |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 1958 | ); |
| 1959 | |
| 1960 | /* Return the first position of substr in str[start:end] using the |
Marc-André Lemburg | 4da6fd6 | 2002-05-29 11:33:13 +0000 | [diff] [blame] | 1961 | given search direction or -1 if not found. -2 is returned in case |
| 1962 | an error occurred and an exception is set. */ |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 1963 | |
Martin v. Löwis | 18e1655 | 2006-02-15 17:27:45 +0000 | [diff] [blame] | 1964 | PyAPI_FUNC(Py_ssize_t) PyUnicode_Find( |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 1965 | PyObject *str, /* String */ |
| 1966 | PyObject *substr, /* Substring to find */ |
| 1967 | Py_ssize_t start, /* Start index */ |
| 1968 | Py_ssize_t end, /* Stop index */ |
| 1969 | int direction /* Find direction: +1 forward, -1 backward */ |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 1970 | ); |
| 1971 | |
Martin v. Löwis | d63a3b8 | 2011-09-28 07:41:54 +0200 | [diff] [blame] | 1972 | /* Like PyUnicode_Find, but search for single character only. */ |
| 1973 | PyAPI_FUNC(Py_ssize_t) PyUnicode_FindChar( |
| 1974 | PyObject *str, |
| 1975 | Py_UCS4 ch, |
| 1976 | Py_ssize_t start, |
| 1977 | Py_ssize_t end, |
| 1978 | int direction |
| 1979 | ); |
| 1980 | |
Barry Warsaw | 51ac580 | 2000-03-20 16:36:48 +0000 | [diff] [blame] | 1981 | /* Count the number of occurrences of substr in str[start:end]. */ |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 1982 | |
Martin v. Löwis | 18e1655 | 2006-02-15 17:27:45 +0000 | [diff] [blame] | 1983 | PyAPI_FUNC(Py_ssize_t) PyUnicode_Count( |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 1984 | PyObject *str, /* String */ |
| 1985 | PyObject *substr, /* Substring to count */ |
| 1986 | Py_ssize_t start, /* Start index */ |
| 1987 | Py_ssize_t end /* Stop index */ |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 1988 | ); |
| 1989 | |
Barry Warsaw | 51ac580 | 2000-03-20 16:36:48 +0000 | [diff] [blame] | 1990 | /* Replace at most maxcount occurrences of substr in str with replstr |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 1991 | and return the resulting Unicode object. */ |
| 1992 | |
Mark Hammond | 91a681d | 2002-08-12 07:21:58 +0000 | [diff] [blame] | 1993 | PyAPI_FUNC(PyObject *) PyUnicode_Replace( |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 1994 | PyObject *str, /* String */ |
| 1995 | PyObject *substr, /* Substring to find */ |
| 1996 | PyObject *replstr, /* Substring to replace */ |
| 1997 | Py_ssize_t maxcount /* Max. number of replacements to apply; |
| 1998 | -1 = all */ |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 1999 | ); |
| 2000 | |
| 2001 | /* Compare two strings and return -1, 0, 1 for less than, equal, |
Victor Stinner | 90db9c4 | 2012-10-04 21:53:50 +0200 | [diff] [blame] | 2002 | greater than resp. |
| 2003 | Raise an exception and return -1 on error. */ |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 2004 | |
Mark Hammond | 91a681d | 2002-08-12 07:21:58 +0000 | [diff] [blame] | 2005 | PyAPI_FUNC(int) PyUnicode_Compare( |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 2006 | PyObject *left, /* Left string */ |
| 2007 | PyObject *right /* Right string */ |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 2008 | ); |
| 2009 | |
Martin v. Löwis | 1c0689c | 2014-01-03 21:36:49 +0100 | [diff] [blame] | 2010 | #ifndef Py_LIMITED_API |
Victor Stinner | ad14ccd | 2013-11-07 00:46:04 +0100 | [diff] [blame] | 2011 | PyAPI_FUNC(int) _PyUnicode_CompareWithId( |
| 2012 | PyObject *left, /* Left string */ |
| 2013 | _Py_Identifier *right /* Right identifier */ |
| 2014 | ); |
Martin v. Löwis | 1c0689c | 2014-01-03 21:36:49 +0100 | [diff] [blame] | 2015 | #endif |
Victor Stinner | ad14ccd | 2013-11-07 00:46:04 +0100 | [diff] [blame] | 2016 | |
Martin v. Löwis | 5b22213 | 2007-06-10 09:51:05 +0000 | [diff] [blame] | 2017 | PyAPI_FUNC(int) PyUnicode_CompareWithASCIIString( |
| 2018 | PyObject *left, |
Victor Stinner | dc2081f | 2010-12-27 01:49:29 +0000 | [diff] [blame] | 2019 | const char *right /* ASCII-encoded string */ |
Martin v. Löwis | 5b22213 | 2007-06-10 09:51:05 +0000 | [diff] [blame] | 2020 | ); |
| 2021 | |
Thomas Wouters | 00ee7ba | 2006-08-21 19:07:27 +0000 | [diff] [blame] | 2022 | /* Rich compare two strings and return one of the following: |
| 2023 | |
| 2024 | - NULL in case an exception was raised |
Georg Brandl | c6bc4c6 | 2011-10-05 16:23:09 +0200 | [diff] [blame] | 2025 | - Py_True or Py_False for successfully comparisons |
Thomas Wouters | 00ee7ba | 2006-08-21 19:07:27 +0000 | [diff] [blame] | 2026 | - Py_NotImplemented in case the type combination is unknown |
| 2027 | |
| 2028 | Note that Py_EQ and Py_NE comparisons can cause a UnicodeWarning in |
| 2029 | case the conversion of the arguments to Unicode fails with a |
| 2030 | UnicodeDecodeError. |
| 2031 | |
| 2032 | Possible values for op: |
| 2033 | |
| 2034 | Py_GT, Py_GE, Py_EQ, Py_NE, Py_LT, Py_LE |
| 2035 | |
| 2036 | */ |
| 2037 | |
| 2038 | PyAPI_FUNC(PyObject *) PyUnicode_RichCompare( |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 2039 | PyObject *left, /* Left string */ |
| 2040 | PyObject *right, /* Right string */ |
| 2041 | int op /* Operation: Py_EQ, Py_NE, Py_GT, etc. */ |
Thomas Wouters | 00ee7ba | 2006-08-21 19:07:27 +0000 | [diff] [blame] | 2042 | ); |
| 2043 | |
Thomas Wouters | 7e47402 | 2000-07-16 12:04:32 +0000 | [diff] [blame] | 2044 | /* Apply a argument tuple or dictionary to a format string and return |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 2045 | the resulting Unicode string. */ |
| 2046 | |
Mark Hammond | 91a681d | 2002-08-12 07:21:58 +0000 | [diff] [blame] | 2047 | PyAPI_FUNC(PyObject *) PyUnicode_Format( |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 2048 | PyObject *format, /* Format string */ |
| 2049 | PyObject *args /* Argument tuple or dictionary */ |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 2050 | ); |
| 2051 | |
Guido van Rossum | d0d366b | 2000-03-13 23:22:24 +0000 | [diff] [blame] | 2052 | /* Checks whether element is contained in container and return 1/0 |
| 2053 | accordingly. |
| 2054 | |
| 2055 | element has to coerce to an one element Unicode string. -1 is |
| 2056 | returned in case of an error. */ |
| 2057 | |
Mark Hammond | 91a681d | 2002-08-12 07:21:58 +0000 | [diff] [blame] | 2058 | PyAPI_FUNC(int) PyUnicode_Contains( |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 2059 | PyObject *container, /* Container string */ |
| 2060 | PyObject *element /* Element string */ |
Guido van Rossum | d0d366b | 2000-03-13 23:22:24 +0000 | [diff] [blame] | 2061 | ); |
| 2062 | |
Antoine Pitrou | 1334884 | 2012-01-29 18:36:34 +0100 | [diff] [blame] | 2063 | /* Checks whether the string contains any NUL characters. */ |
| 2064 | |
| 2065 | #ifndef Py_LIMITED_API |
| 2066 | PyAPI_FUNC(int) _PyUnicode_HasNULChars(PyObject *); |
| 2067 | #endif |
| 2068 | |
Martin v. Löwis | 4738340 | 2007-08-15 07:32:56 +0000 | [diff] [blame] | 2069 | /* Checks whether argument is a valid identifier. */ |
| 2070 | |
| 2071 | PyAPI_FUNC(int) PyUnicode_IsIdentifier(PyObject *s); |
| 2072 | |
Martin v. Löwis | 4d0d471 | 2010-12-03 20:14:31 +0000 | [diff] [blame] | 2073 | #ifndef Py_LIMITED_API |
Walter Dörwald | de02bcb | 2002-04-22 17:42:37 +0000 | [diff] [blame] | 2074 | /* Externally visible for str.strip(unicode) */ |
Mark Hammond | 91a681d | 2002-08-12 07:21:58 +0000 | [diff] [blame] | 2075 | PyAPI_FUNC(PyObject *) _PyUnicode_XStrip( |
Victor Stinner | 9db1a8b | 2011-10-23 20:04:37 +0200 | [diff] [blame] | 2076 | PyObject *self, |
Walter Dörwald | de02bcb | 2002-04-22 17:42:37 +0000 | [diff] [blame] | 2077 | int striptype, |
| 2078 | PyObject *sepobj |
| 2079 | ); |
Martin v. Löwis | 4d0d471 | 2010-12-03 20:14:31 +0000 | [diff] [blame] | 2080 | #endif |
Walter Dörwald | de02bcb | 2002-04-22 17:42:37 +0000 | [diff] [blame] | 2081 | |
Eric Smith | a3b1ac8 | 2009-04-03 14:45:06 +0000 | [diff] [blame] | 2082 | /* Using explicit passed-in values, insert the thousands grouping |
| 2083 | into the string pointed to by buffer. For the argument descriptions, |
| 2084 | see Objects/stringlib/localeutil.h */ |
Martin v. Löwis | 4d0d471 | 2010-12-03 20:14:31 +0000 | [diff] [blame] | 2085 | #ifndef Py_LIMITED_API |
Martin v. Löwis | d63a3b8 | 2011-09-28 07:41:54 +0200 | [diff] [blame] | 2086 | PyAPI_FUNC(Py_ssize_t) _PyUnicode_InsertThousandsGrouping( |
Victor Stinner | c3cec78 | 2011-10-05 21:24:08 +0200 | [diff] [blame] | 2087 | PyObject *unicode, |
Victor Stinner | 41a863c | 2012-02-24 00:37:51 +0100 | [diff] [blame] | 2088 | Py_ssize_t index, |
Martin v. Löwis | d63a3b8 | 2011-09-28 07:41:54 +0200 | [diff] [blame] | 2089 | Py_ssize_t n_buffer, |
| 2090 | void *digits, |
| 2091 | Py_ssize_t n_digits, |
| 2092 | Py_ssize_t min_width, |
| 2093 | const char *grouping, |
Victor Stinner | 41a863c | 2012-02-24 00:37:51 +0100 | [diff] [blame] | 2094 | PyObject *thousands_sep, |
| 2095 | Py_UCS4 *maxchar); |
Martin v. Löwis | 4d0d471 | 2010-12-03 20:14:31 +0000 | [diff] [blame] | 2096 | #endif |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 2097 | /* === Characters Type APIs =============================================== */ |
| 2098 | |
Benjamin Peterson | 960cf0f | 2009-01-09 04:11:44 +0000 | [diff] [blame] | 2099 | /* Helper array used by Py_UNICODE_ISSPACE(). */ |
| 2100 | |
Martin v. Löwis | 4d0d471 | 2010-12-03 20:14:31 +0000 | [diff] [blame] | 2101 | #ifndef Py_LIMITED_API |
Benjamin Peterson | 960cf0f | 2009-01-09 04:11:44 +0000 | [diff] [blame] | 2102 | PyAPI_DATA(const unsigned char) _Py_ascii_whitespace[]; |
| 2103 | |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 2104 | /* These should not be used directly. Use the Py_UNICODE_IS* and |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 2105 | Py_UNICODE_TO* macros instead. |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 2106 | |
| 2107 | These APIs are implemented in Objects/unicodectype.c. |
| 2108 | |
| 2109 | */ |
| 2110 | |
Mark Hammond | 91a681d | 2002-08-12 07:21:58 +0000 | [diff] [blame] | 2111 | PyAPI_FUNC(int) _PyUnicode_IsLowercase( |
Amaury Forgeot d'Arc | 324ac65 | 2010-08-18 20:44:58 +0000 | [diff] [blame] | 2112 | Py_UCS4 ch /* Unicode character */ |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 2113 | ); |
| 2114 | |
Mark Hammond | 91a681d | 2002-08-12 07:21:58 +0000 | [diff] [blame] | 2115 | PyAPI_FUNC(int) _PyUnicode_IsUppercase( |
Amaury Forgeot d'Arc | 324ac65 | 2010-08-18 20:44:58 +0000 | [diff] [blame] | 2116 | Py_UCS4 ch /* Unicode character */ |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 2117 | ); |
| 2118 | |
Mark Hammond | 91a681d | 2002-08-12 07:21:58 +0000 | [diff] [blame] | 2119 | PyAPI_FUNC(int) _PyUnicode_IsTitlecase( |
Amaury Forgeot d'Arc | 324ac65 | 2010-08-18 20:44:58 +0000 | [diff] [blame] | 2120 | Py_UCS4 ch /* Unicode character */ |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 2121 | ); |
| 2122 | |
Martin v. Löwis | 13c3e38 | 2007-08-14 22:37:03 +0000 | [diff] [blame] | 2123 | PyAPI_FUNC(int) _PyUnicode_IsXidStart( |
Amaury Forgeot d'Arc | 324ac65 | 2010-08-18 20:44:58 +0000 | [diff] [blame] | 2124 | Py_UCS4 ch /* Unicode character */ |
Martin v. Löwis | 13c3e38 | 2007-08-14 22:37:03 +0000 | [diff] [blame] | 2125 | ); |
| 2126 | |
| 2127 | PyAPI_FUNC(int) _PyUnicode_IsXidContinue( |
Amaury Forgeot d'Arc | 324ac65 | 2010-08-18 20:44:58 +0000 | [diff] [blame] | 2128 | Py_UCS4 ch /* Unicode character */ |
Martin v. Löwis | 13c3e38 | 2007-08-14 22:37:03 +0000 | [diff] [blame] | 2129 | ); |
| 2130 | |
Mark Hammond | 91a681d | 2002-08-12 07:21:58 +0000 | [diff] [blame] | 2131 | PyAPI_FUNC(int) _PyUnicode_IsWhitespace( |
Amaury Forgeot d'Arc | 324ac65 | 2010-08-18 20:44:58 +0000 | [diff] [blame] | 2132 | const Py_UCS4 ch /* Unicode character */ |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 2133 | ); |
| 2134 | |
Mark Hammond | 91a681d | 2002-08-12 07:21:58 +0000 | [diff] [blame] | 2135 | PyAPI_FUNC(int) _PyUnicode_IsLinebreak( |
Amaury Forgeot d'Arc | 324ac65 | 2010-08-18 20:44:58 +0000 | [diff] [blame] | 2136 | const Py_UCS4 ch /* Unicode character */ |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 2137 | ); |
| 2138 | |
Amaury Forgeot d'Arc | 324ac65 | 2010-08-18 20:44:58 +0000 | [diff] [blame] | 2139 | PyAPI_FUNC(Py_UCS4) _PyUnicode_ToLowercase( |
| 2140 | Py_UCS4 ch /* Unicode character */ |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 2141 | ); |
| 2142 | |
Amaury Forgeot d'Arc | 324ac65 | 2010-08-18 20:44:58 +0000 | [diff] [blame] | 2143 | PyAPI_FUNC(Py_UCS4) _PyUnicode_ToUppercase( |
| 2144 | Py_UCS4 ch /* Unicode character */ |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 2145 | ); |
| 2146 | |
Amaury Forgeot d'Arc | 324ac65 | 2010-08-18 20:44:58 +0000 | [diff] [blame] | 2147 | PyAPI_FUNC(Py_UCS4) _PyUnicode_ToTitlecase( |
| 2148 | Py_UCS4 ch /* Unicode character */ |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 2149 | ); |
| 2150 | |
Benjamin Peterson | b2bf01d | 2012-01-11 18:17:06 -0500 | [diff] [blame] | 2151 | PyAPI_FUNC(int) _PyUnicode_ToLowerFull( |
| 2152 | Py_UCS4 ch, /* Unicode character */ |
| 2153 | Py_UCS4 *res |
| 2154 | ); |
| 2155 | |
| 2156 | PyAPI_FUNC(int) _PyUnicode_ToTitleFull( |
| 2157 | Py_UCS4 ch, /* Unicode character */ |
| 2158 | Py_UCS4 *res |
| 2159 | ); |
| 2160 | |
| 2161 | PyAPI_FUNC(int) _PyUnicode_ToUpperFull( |
| 2162 | Py_UCS4 ch, /* Unicode character */ |
| 2163 | Py_UCS4 *res |
| 2164 | ); |
| 2165 | |
Benjamin Peterson | d5890c8 | 2012-01-14 13:23:30 -0500 | [diff] [blame] | 2166 | PyAPI_FUNC(int) _PyUnicode_ToFoldedFull( |
| 2167 | Py_UCS4 ch, /* Unicode character */ |
| 2168 | Py_UCS4 *res |
| 2169 | ); |
| 2170 | |
Benjamin Peterson | b2bf01d | 2012-01-11 18:17:06 -0500 | [diff] [blame] | 2171 | PyAPI_FUNC(int) _PyUnicode_IsCaseIgnorable( |
Amaury Forgeot d'Arc | 77b1ecf | 2012-01-13 22:12:37 +0100 | [diff] [blame] | 2172 | Py_UCS4 ch /* Unicode character */ |
Benjamin Peterson | b2bf01d | 2012-01-11 18:17:06 -0500 | [diff] [blame] | 2173 | ); |
| 2174 | |
| 2175 | PyAPI_FUNC(int) _PyUnicode_IsCased( |
Amaury Forgeot d'Arc | 77b1ecf | 2012-01-13 22:12:37 +0100 | [diff] [blame] | 2176 | Py_UCS4 ch /* Unicode character */ |
Benjamin Peterson | b2bf01d | 2012-01-11 18:17:06 -0500 | [diff] [blame] | 2177 | ); |
| 2178 | |
Mark Hammond | 91a681d | 2002-08-12 07:21:58 +0000 | [diff] [blame] | 2179 | PyAPI_FUNC(int) _PyUnicode_ToDecimalDigit( |
Amaury Forgeot d'Arc | 324ac65 | 2010-08-18 20:44:58 +0000 | [diff] [blame] | 2180 | Py_UCS4 ch /* Unicode character */ |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 2181 | ); |
| 2182 | |
Mark Hammond | 91a681d | 2002-08-12 07:21:58 +0000 | [diff] [blame] | 2183 | PyAPI_FUNC(int) _PyUnicode_ToDigit( |
Amaury Forgeot d'Arc | 324ac65 | 2010-08-18 20:44:58 +0000 | [diff] [blame] | 2184 | Py_UCS4 ch /* Unicode character */ |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 2185 | ); |
| 2186 | |
Mark Hammond | 91a681d | 2002-08-12 07:21:58 +0000 | [diff] [blame] | 2187 | PyAPI_FUNC(double) _PyUnicode_ToNumeric( |
Amaury Forgeot d'Arc | 324ac65 | 2010-08-18 20:44:58 +0000 | [diff] [blame] | 2188 | Py_UCS4 ch /* Unicode character */ |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 2189 | ); |
| 2190 | |
Mark Hammond | 91a681d | 2002-08-12 07:21:58 +0000 | [diff] [blame] | 2191 | PyAPI_FUNC(int) _PyUnicode_IsDecimalDigit( |
Amaury Forgeot d'Arc | 324ac65 | 2010-08-18 20:44:58 +0000 | [diff] [blame] | 2192 | Py_UCS4 ch /* Unicode character */ |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 2193 | ); |
| 2194 | |
Mark Hammond | 91a681d | 2002-08-12 07:21:58 +0000 | [diff] [blame] | 2195 | PyAPI_FUNC(int) _PyUnicode_IsDigit( |
Amaury Forgeot d'Arc | 324ac65 | 2010-08-18 20:44:58 +0000 | [diff] [blame] | 2196 | Py_UCS4 ch /* Unicode character */ |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 2197 | ); |
| 2198 | |
Mark Hammond | 91a681d | 2002-08-12 07:21:58 +0000 | [diff] [blame] | 2199 | PyAPI_FUNC(int) _PyUnicode_IsNumeric( |
Amaury Forgeot d'Arc | 324ac65 | 2010-08-18 20:44:58 +0000 | [diff] [blame] | 2200 | Py_UCS4 ch /* Unicode character */ |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 2201 | ); |
| 2202 | |
Georg Brandl | 559e5d7 | 2008-06-11 18:37:52 +0000 | [diff] [blame] | 2203 | PyAPI_FUNC(int) _PyUnicode_IsPrintable( |
Amaury Forgeot d'Arc | 324ac65 | 2010-08-18 20:44:58 +0000 | [diff] [blame] | 2204 | Py_UCS4 ch /* Unicode character */ |
Georg Brandl | 559e5d7 | 2008-06-11 18:37:52 +0000 | [diff] [blame] | 2205 | ); |
| 2206 | |
Mark Hammond | 91a681d | 2002-08-12 07:21:58 +0000 | [diff] [blame] | 2207 | PyAPI_FUNC(int) _PyUnicode_IsAlpha( |
Amaury Forgeot d'Arc | 324ac65 | 2010-08-18 20:44:58 +0000 | [diff] [blame] | 2208 | Py_UCS4 ch /* Unicode character */ |
Marc-André Lemburg | f03e741 | 2000-07-05 09:45:59 +0000 | [diff] [blame] | 2209 | ); |
| 2210 | |
Victor Stinner | ef8d95c | 2010-08-16 22:03:11 +0000 | [diff] [blame] | 2211 | PyAPI_FUNC(size_t) Py_UNICODE_strlen( |
| 2212 | const Py_UNICODE *u |
| 2213 | ); |
Martin v. Löwis | 5b22213 | 2007-06-10 09:51:05 +0000 | [diff] [blame] | 2214 | |
| 2215 | PyAPI_FUNC(Py_UNICODE*) Py_UNICODE_strcpy( |
Victor Stinner | ef8d95c | 2010-08-16 22:03:11 +0000 | [diff] [blame] | 2216 | Py_UNICODE *s1, |
| 2217 | const Py_UNICODE *s2); |
Martin v. Löwis | 5b22213 | 2007-06-10 09:51:05 +0000 | [diff] [blame] | 2218 | |
Victor Stinner | c4eb765 | 2010-09-01 23:43:50 +0000 | [diff] [blame] | 2219 | PyAPI_FUNC(Py_UNICODE*) Py_UNICODE_strcat( |
| 2220 | Py_UNICODE *s1, const Py_UNICODE *s2); |
| 2221 | |
Martin v. Löwis | 5b22213 | 2007-06-10 09:51:05 +0000 | [diff] [blame] | 2222 | PyAPI_FUNC(Py_UNICODE*) Py_UNICODE_strncpy( |
Victor Stinner | ef8d95c | 2010-08-16 22:03:11 +0000 | [diff] [blame] | 2223 | Py_UNICODE *s1, |
| 2224 | const Py_UNICODE *s2, |
| 2225 | size_t n); |
Martin v. Löwis | 5b22213 | 2007-06-10 09:51:05 +0000 | [diff] [blame] | 2226 | |
| 2227 | PyAPI_FUNC(int) Py_UNICODE_strcmp( |
Victor Stinner | ef8d95c | 2010-08-16 22:03:11 +0000 | [diff] [blame] | 2228 | const Py_UNICODE *s1, |
| 2229 | const Py_UNICODE *s2 |
| 2230 | ); |
| 2231 | |
| 2232 | PyAPI_FUNC(int) Py_UNICODE_strncmp( |
| 2233 | const Py_UNICODE *s1, |
| 2234 | const Py_UNICODE *s2, |
| 2235 | size_t n |
| 2236 | ); |
Martin v. Löwis | 5b22213 | 2007-06-10 09:51:05 +0000 | [diff] [blame] | 2237 | |
| 2238 | PyAPI_FUNC(Py_UNICODE*) Py_UNICODE_strchr( |
Victor Stinner | ef8d95c | 2010-08-16 22:03:11 +0000 | [diff] [blame] | 2239 | const Py_UNICODE *s, |
| 2240 | Py_UNICODE c |
Martin v. Löwis | 5b22213 | 2007-06-10 09:51:05 +0000 | [diff] [blame] | 2241 | ); |
| 2242 | |
Victor Stinner | 331ea92 | 2010-08-10 16:37:20 +0000 | [diff] [blame] | 2243 | PyAPI_FUNC(Py_UNICODE*) Py_UNICODE_strrchr( |
Victor Stinner | ef8d95c | 2010-08-16 22:03:11 +0000 | [diff] [blame] | 2244 | const Py_UNICODE *s, |
| 2245 | Py_UNICODE c |
Victor Stinner | 331ea92 | 2010-08-10 16:37:20 +0000 | [diff] [blame] | 2246 | ); |
| 2247 | |
Victor Stinner | 71133ff | 2010-09-01 23:43:53 +0000 | [diff] [blame] | 2248 | /* Create a copy of a unicode string ending with a nul character. Return NULL |
| 2249 | and raise a MemoryError exception on memory allocation failure, otherwise |
| 2250 | return a new allocated buffer (use PyMem_Free() to free the buffer). */ |
| 2251 | |
Victor Stinner | 4640860 | 2010-09-03 16:18:00 +0000 | [diff] [blame] | 2252 | PyAPI_FUNC(Py_UNICODE*) PyUnicode_AsUnicodeCopy( |
Victor Stinner | 71133ff | 2010-09-01 23:43:53 +0000 | [diff] [blame] | 2253 | PyObject *unicode |
| 2254 | ); |
Martin v. Löwis | 4d0d471 | 2010-12-03 20:14:31 +0000 | [diff] [blame] | 2255 | #endif /* Py_LIMITED_API */ |
Victor Stinner | 71133ff | 2010-09-01 23:43:53 +0000 | [diff] [blame] | 2256 | |
Victor Stinner | fb9ea8c | 2011-10-06 01:45:57 +0200 | [diff] [blame] | 2257 | #if defined(Py_DEBUG) && !defined(Py_LIMITED_API) |
Victor Stinner | fb9ea8c | 2011-10-06 01:45:57 +0200 | [diff] [blame] | 2258 | PyAPI_FUNC(int) _PyUnicode_CheckConsistency( |
Victor Stinner | 7931d9a | 2011-11-04 00:22:48 +0100 | [diff] [blame] | 2259 | PyObject *op, |
Victor Stinner | fb9ea8c | 2011-10-06 01:45:57 +0200 | [diff] [blame] | 2260 | int check_content); |
| 2261 | #endif |
| 2262 | |
Martin v. Löwis | afe55bb | 2011-10-09 10:38:36 +0200 | [diff] [blame] | 2263 | /* Return an interned Unicode object for an Identifier; may fail if there is no memory.*/ |
| 2264 | PyAPI_FUNC(PyObject*) _PyUnicode_FromId(_Py_Identifier*); |
| 2265 | /* Clear all static strings. */ |
| 2266 | PyAPI_FUNC(void) _PyUnicode_ClearStaticStrings(void); |
| 2267 | |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 2268 | #ifdef __cplusplus |
| 2269 | } |
| 2270 | #endif |
Guido van Rossum | d822518 | 2000-03-10 22:33:05 +0000 | [diff] [blame] | 2271 | #endif /* !Py_UNICODEOBJECT_H */ |