blob: 2926c35ba11348bade4f7dddd4ad397db67be7f5 [file] [log] [blame]
Guido van Rossum2a70a3a2000-03-10 23:10:21 +00001/* ------------------------------------------------------------------------
2
Ezio Melotti4c5475d2010-03-22 23:16:42 +00003 unicodedata -- Provides access to the Unicode 5.2 data base.
Guido van Rossum2a70a3a2000-03-10 23:10:21 +00004
Ezio Melotti4c5475d2010-03-22 23:16:42 +00005 Data was extracted from the Unicode 5.2 UnicodeData.txt file.
Guido van Rossum2a70a3a2000-03-10 23:10:21 +00006
Fredrik Lundhcfcea492000-09-25 08:07:06 +00007 Written by Marc-Andre Lemburg (mal@lemburg.com).
8 Modified for Python 2.0 by Fredrik Lundh (fredrik@pythonware.com)
Florent Xiclunac934f322010-09-03 23:47:32 +00009 Modified by Martin v. Löwis (martin@v.loewis.de)
Guido van Rossum2a70a3a2000-03-10 23:10:21 +000010
Fredrik Lundhcfcea492000-09-25 08:07:06 +000011 Copyright (c) Corporation for National Research Initiatives.
Guido van Rossum2a70a3a2000-03-10 23:10:21 +000012
13 ------------------------------------------------------------------------ */
14
15#include "Python.h"
Fredrik Lundh06d12682001-01-24 07:59:11 +000016#include "ucnhash.h"
Martin v. Löwis480f1bb2006-03-09 23:38:20 +000017#include "structmember.h"
Fredrik Lundh06d12682001-01-24 07:59:11 +000018
19/* character properties */
Guido van Rossum2a70a3a2000-03-10 23:10:21 +000020
Fredrik Lundh7b7dd102001-01-21 22:41:08 +000021typedef struct {
Antoine Pitrouf95a1b32010-05-09 15:52:27 +000022 const unsigned char category; /* index into
23 _PyUnicode_CategoryNames */
24 const unsigned char combining; /* combining class value 0 - 255 */
25 const unsigned char bidirectional; /* index into
26 _PyUnicode_BidirectionalNames */
27 const unsigned char mirrored; /* true if mirrored in bidir mode */
28 const unsigned char east_asian_width; /* index into
29 _PyUnicode_EastAsianWidth */
Antoine Pitrou7a0fedf2009-04-27 22:31:40 +000030 const unsigned char normalization_quick_check; /* see is_normalized() */
Fredrik Lundh7b7dd102001-01-21 22:41:08 +000031} _PyUnicode_DatabaseRecord;
32
Martin v. Löwis480f1bb2006-03-09 23:38:20 +000033typedef struct change_record {
34 /* sequence of fields should be the same as in merge_old_version */
35 const unsigned char bidir_changed;
36 const unsigned char category_changed;
37 const unsigned char decimal_changed;
Martin v. Löwis93cbca32008-09-10 14:08:48 +000038 const unsigned char mirrored_changed;
Amaury Forgeot d'Arc7d520792009-10-06 21:03:20 +000039 const double numeric_changed;
Martin v. Löwis480f1bb2006-03-09 23:38:20 +000040} change_record;
41
Fredrik Lundh7b7dd102001-01-21 22:41:08 +000042/* data file generated by Tools/unicode/makeunicodedata.py */
43#include "unicodedata_db.h"
44
45static const _PyUnicode_DatabaseRecord*
Martin v. Löwis677bde22002-11-23 22:08:15 +000046_getrecord_ex(Py_UCS4 code)
Fredrik Lundh7b7dd102001-01-21 22:41:08 +000047{
Fredrik Lundh7b7dd102001-01-21 22:41:08 +000048 int index;
Neal Norwitze9c571f2003-02-28 03:14:37 +000049 if (code >= 0x110000)
Fredrik Lundh7b7dd102001-01-21 22:41:08 +000050 index = 0;
51 else {
52 index = index1[(code>>SHIFT)];
53 index = index2[(index<<SHIFT)+(code&((1<<SHIFT)-1))];
54 }
55
56 return &_PyUnicode_Database_Records[index];
57}
58
Martin v. Löwis480f1bb2006-03-09 23:38:20 +000059/* ------------- Previous-version API ------------------------------------- */
60typedef struct previous_version {
61 PyObject_HEAD
62 const char *name;
63 const change_record* (*getrecord)(Py_UCS4);
64 Py_UCS4 (*normalization)(Py_UCS4);
65} PreviousDBVersion;
66
67#define get_old_record(self, v) ((((PreviousDBVersion*)self)->getrecord)(v))
68
Martin v. Löwis480f1bb2006-03-09 23:38:20 +000069static PyMemberDef DB_members[] = {
Antoine Pitrouf95a1b32010-05-09 15:52:27 +000070 {"unidata_version", T_STRING, offsetof(PreviousDBVersion, name), READONLY},
Martin v. Löwis480f1bb2006-03-09 23:38:20 +000071 {NULL}
72};
73
Thomas Wouters89f507f2006-12-13 04:49:30 +000074/* forward declaration */
Martin v. Löwis5bd7c022006-03-10 11:20:04 +000075static PyTypeObject UCD_Type;
Martin v. Löwis1a214512008-06-11 05:26:20 +000076#define UCD_Check(o) (Py_TYPE(o)==&UCD_Type)
Martin v. Löwis480f1bb2006-03-09 23:38:20 +000077
78static PyObject*
79new_previous_version(const char*name, const change_record* (*getrecord)(Py_UCS4),
80 Py_UCS4 (*normalization)(Py_UCS4))
81{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +000082 PreviousDBVersion *self;
83 self = PyObject_New(PreviousDBVersion, &UCD_Type);
84 if (self == NULL)
85 return NULL;
86 self->name = name;
87 self->getrecord = getrecord;
Martin v. Löwis480f1bb2006-03-09 23:38:20 +000088 self->normalization = normalization;
Antoine Pitrouf95a1b32010-05-09 15:52:27 +000089 return (PyObject*)self;
Martin v. Löwis480f1bb2006-03-09 23:38:20 +000090}
91
Walter Dörwaldf342bfc2008-06-03 11:45:02 +000092
93static Py_UCS4 getuchar(PyUnicodeObject *obj)
94{
95 Py_UNICODE *v = PyUnicode_AS_UNICODE(obj);
96
97 if (PyUnicode_GET_SIZE(obj) == 1)
Antoine Pitrouf95a1b32010-05-09 15:52:27 +000098 return *v;
Walter Dörwaldf342bfc2008-06-03 11:45:02 +000099#ifndef Py_UNICODE_WIDE
100 else if ((PyUnicode_GET_SIZE(obj) == 2) &&
101 (0xD800 <= v[0] && v[0] <= 0xDBFF) &&
102 (0xDC00 <= v[1] && v[1] <= 0xDFFF))
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000103 return (((v[0] & 0x3FF)<<10) | (v[1] & 0x3FF)) + 0x10000;
Walter Dörwaldf342bfc2008-06-03 11:45:02 +0000104#endif
105 PyErr_SetString(PyExc_TypeError,
106 "need a single Unicode character as parameter");
107 return (Py_UCS4)-1;
108}
109
Guido van Rossum2a70a3a2000-03-10 23:10:21 +0000110/* --- Module API --------------------------------------------------------- */
111
Hye-Shik Changcf18a5d2005-04-04 16:32:07 +0000112PyDoc_STRVAR(unicodedata_decimal__doc__,
113"decimal(unichr[, default])\n\
114\n\
115Returns the decimal value assigned to the Unicode character unichr\n\
116as integer. If no such value is defined, default is returned, or, if\n\
117not given, ValueError is raised.");
118
Guido van Rossum2a70a3a2000-03-10 23:10:21 +0000119static PyObject *
Fredrik Lundh7b7dd102001-01-21 22:41:08 +0000120unicodedata_decimal(PyObject *self, PyObject *args)
Guido van Rossum2a70a3a2000-03-10 23:10:21 +0000121{
122 PyUnicodeObject *v;
123 PyObject *defobj = NULL;
Martin v. Löwis480f1bb2006-03-09 23:38:20 +0000124 int have_old = 0;
Guido van Rossum2a70a3a2000-03-10 23:10:21 +0000125 long rc;
Walter Dörwaldf342bfc2008-06-03 11:45:02 +0000126 Py_UCS4 c;
Guido van Rossum2a70a3a2000-03-10 23:10:21 +0000127
Fredrik Lundh06d12682001-01-24 07:59:11 +0000128 if (!PyArg_ParseTuple(args, "O!|O:decimal", &PyUnicode_Type, &v, &defobj))
Fredrik Lundh7b7dd102001-01-21 22:41:08 +0000129 return NULL;
Walter Dörwaldf342bfc2008-06-03 11:45:02 +0000130 c = getuchar(v);
131 if (c == (Py_UCS4)-1)
Fredrik Lundh7b7dd102001-01-21 22:41:08 +0000132 return NULL;
Martin v. Löwis480f1bb2006-03-09 23:38:20 +0000133
Martin v. Löwis1a214512008-06-11 05:26:20 +0000134 if (self && UCD_Check(self)) {
Walter Dörwaldf342bfc2008-06-03 11:45:02 +0000135 const change_record *old = get_old_record(self, c);
Martin v. Löwis480f1bb2006-03-09 23:38:20 +0000136 if (old->category_changed == 0) {
137 /* unassigned */
138 have_old = 1;
139 rc = -1;
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000140 }
Martin v. Löwis480f1bb2006-03-09 23:38:20 +0000141 else if (old->decimal_changed != 0xFF) {
142 have_old = 1;
143 rc = old->decimal_changed;
144 }
145 }
146
147 if (!have_old)
Walter Dörwaldf342bfc2008-06-03 11:45:02 +0000148 rc = Py_UNICODE_TODECIMAL(c);
Guido van Rossum2a70a3a2000-03-10 23:10:21 +0000149 if (rc < 0) {
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000150 if (defobj == NULL) {
151 PyErr_SetString(PyExc_ValueError,
152 "not a decimal");
Fredrik Lundh7b7dd102001-01-21 22:41:08 +0000153 return NULL;
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000154 }
155 else {
156 Py_INCREF(defobj);
157 return defobj;
158 }
Guido van Rossum2a70a3a2000-03-10 23:10:21 +0000159 }
Christian Heimes217cfd12007-12-02 14:31:20 +0000160 return PyLong_FromLong(rc);
Guido van Rossum2a70a3a2000-03-10 23:10:21 +0000161}
162
Hye-Shik Changcf18a5d2005-04-04 16:32:07 +0000163PyDoc_STRVAR(unicodedata_digit__doc__,
164"digit(unichr[, default])\n\
165\n\
166Returns the digit value assigned to the Unicode character unichr as\n\
167integer. If no such value is defined, default is returned, or, if\n\
168not given, ValueError is raised.");
169
Guido van Rossum2a70a3a2000-03-10 23:10:21 +0000170static PyObject *
Fredrik Lundh7b7dd102001-01-21 22:41:08 +0000171unicodedata_digit(PyObject *self, PyObject *args)
Guido van Rossum2a70a3a2000-03-10 23:10:21 +0000172{
173 PyUnicodeObject *v;
174 PyObject *defobj = NULL;
175 long rc;
Walter Dörwaldf342bfc2008-06-03 11:45:02 +0000176 Py_UCS4 c;
Guido van Rossum2a70a3a2000-03-10 23:10:21 +0000177
Fredrik Lundh06d12682001-01-24 07:59:11 +0000178 if (!PyArg_ParseTuple(args, "O!|O:digit", &PyUnicode_Type, &v, &defobj))
Fredrik Lundh7b7dd102001-01-21 22:41:08 +0000179 return NULL;
Walter Dörwaldf342bfc2008-06-03 11:45:02 +0000180 c = getuchar(v);
181 if (c == (Py_UCS4)-1)
Fredrik Lundh7b7dd102001-01-21 22:41:08 +0000182 return NULL;
Walter Dörwaldf342bfc2008-06-03 11:45:02 +0000183 rc = Py_UNICODE_TODIGIT(c);
Guido van Rossum2a70a3a2000-03-10 23:10:21 +0000184 if (rc < 0) {
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000185 if (defobj == NULL) {
186 PyErr_SetString(PyExc_ValueError, "not a digit");
Fredrik Lundh7b7dd102001-01-21 22:41:08 +0000187 return NULL;
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000188 }
189 else {
190 Py_INCREF(defobj);
191 return defobj;
192 }
Guido van Rossum2a70a3a2000-03-10 23:10:21 +0000193 }
Christian Heimes217cfd12007-12-02 14:31:20 +0000194 return PyLong_FromLong(rc);
Guido van Rossum2a70a3a2000-03-10 23:10:21 +0000195}
196
Hye-Shik Changcf18a5d2005-04-04 16:32:07 +0000197PyDoc_STRVAR(unicodedata_numeric__doc__,
198"numeric(unichr[, default])\n\
199\n\
200Returns the numeric value assigned to the Unicode character unichr\n\
201as float. If no such value is defined, default is returned, or, if\n\
202not given, ValueError is raised.");
203
Guido van Rossum2a70a3a2000-03-10 23:10:21 +0000204static PyObject *
Fredrik Lundh7b7dd102001-01-21 22:41:08 +0000205unicodedata_numeric(PyObject *self, PyObject *args)
Guido van Rossum2a70a3a2000-03-10 23:10:21 +0000206{
207 PyUnicodeObject *v;
208 PyObject *defobj = NULL;
Martin v. Löwis480f1bb2006-03-09 23:38:20 +0000209 int have_old = 0;
Guido van Rossum2a70a3a2000-03-10 23:10:21 +0000210 double rc;
Walter Dörwaldf342bfc2008-06-03 11:45:02 +0000211 Py_UCS4 c;
Guido van Rossum2a70a3a2000-03-10 23:10:21 +0000212
Fredrik Lundh06d12682001-01-24 07:59:11 +0000213 if (!PyArg_ParseTuple(args, "O!|O:numeric", &PyUnicode_Type, &v, &defobj))
Fredrik Lundh7b7dd102001-01-21 22:41:08 +0000214 return NULL;
Walter Dörwaldf342bfc2008-06-03 11:45:02 +0000215 c = getuchar(v);
216 if (c == (Py_UCS4)-1)
217 return NULL;
Martin v. Löwis480f1bb2006-03-09 23:38:20 +0000218
Martin v. Löwis1a214512008-06-11 05:26:20 +0000219 if (self && UCD_Check(self)) {
Walter Dörwaldf342bfc2008-06-03 11:45:02 +0000220 const change_record *old = get_old_record(self, c);
Martin v. Löwis480f1bb2006-03-09 23:38:20 +0000221 if (old->category_changed == 0) {
222 /* unassigned */
223 have_old = 1;
Thomas Wouters477c8d52006-05-27 19:21:47 +0000224 rc = -1.0;
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000225 }
Martin v. Löwis480f1bb2006-03-09 23:38:20 +0000226 else if (old->decimal_changed != 0xFF) {
227 have_old = 1;
228 rc = old->decimal_changed;
229 }
230 }
231
232 if (!have_old)
Walter Dörwaldf342bfc2008-06-03 11:45:02 +0000233 rc = Py_UNICODE_TONUMERIC(c);
Thomas Wouters477c8d52006-05-27 19:21:47 +0000234 if (rc == -1.0) {
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000235 if (defobj == NULL) {
236 PyErr_SetString(PyExc_ValueError, "not a numeric character");
237 return NULL;
238 }
239 else {
240 Py_INCREF(defobj);
241 return defobj;
242 }
Guido van Rossum2a70a3a2000-03-10 23:10:21 +0000243 }
244 return PyFloat_FromDouble(rc);
Guido van Rossum2a70a3a2000-03-10 23:10:21 +0000245}
246
Hye-Shik Changcf18a5d2005-04-04 16:32:07 +0000247PyDoc_STRVAR(unicodedata_category__doc__,
248"category(unichr)\n\
249\n\
250Returns the general category assigned to the Unicode character\n\
251unichr as string.");
252
Guido van Rossum2a70a3a2000-03-10 23:10:21 +0000253static PyObject *
Fredrik Lundh7b7dd102001-01-21 22:41:08 +0000254unicodedata_category(PyObject *self, PyObject *args)
Guido van Rossum2a70a3a2000-03-10 23:10:21 +0000255{
256 PyUnicodeObject *v;
257 int index;
Walter Dörwaldf342bfc2008-06-03 11:45:02 +0000258 Py_UCS4 c;
Guido van Rossum2a70a3a2000-03-10 23:10:21 +0000259
260 if (!PyArg_ParseTuple(args, "O!:category",
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000261 &PyUnicode_Type, &v))
262 return NULL;
Walter Dörwaldf342bfc2008-06-03 11:45:02 +0000263 c = getuchar(v);
264 if (c == (Py_UCS4)-1)
265 return NULL;
266 index = (int) _getrecord_ex(c)->category;
Martin v. Löwis1a214512008-06-11 05:26:20 +0000267 if (self && UCD_Check(self)) {
Walter Dörwaldf342bfc2008-06-03 11:45:02 +0000268 const change_record *old = get_old_record(self, c);
Martin v. Löwis480f1bb2006-03-09 23:38:20 +0000269 if (old->category_changed != 0xFF)
270 index = old->category_changed;
271 }
Walter Dörwald4254e762007-06-05 16:04:09 +0000272 return PyUnicode_FromString(_PyUnicode_CategoryNames[index]);
Guido van Rossum2a70a3a2000-03-10 23:10:21 +0000273}
274
Hye-Shik Changcf18a5d2005-04-04 16:32:07 +0000275PyDoc_STRVAR(unicodedata_bidirectional__doc__,
276"bidirectional(unichr)\n\
277\n\
278Returns the bidirectional category assigned to the Unicode character\n\
279unichr as string. If no such value is defined, an empty string is\n\
280returned.");
281
Guido van Rossum2a70a3a2000-03-10 23:10:21 +0000282static PyObject *
Fredrik Lundh7b7dd102001-01-21 22:41:08 +0000283unicodedata_bidirectional(PyObject *self, PyObject *args)
Guido van Rossum2a70a3a2000-03-10 23:10:21 +0000284{
285 PyUnicodeObject *v;
286 int index;
Walter Dörwaldf342bfc2008-06-03 11:45:02 +0000287 Py_UCS4 c;
Guido van Rossum2a70a3a2000-03-10 23:10:21 +0000288
289 if (!PyArg_ParseTuple(args, "O!:bidirectional",
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000290 &PyUnicode_Type, &v))
291 return NULL;
Walter Dörwaldf342bfc2008-06-03 11:45:02 +0000292 c = getuchar(v);
293 if (c == (Py_UCS4)-1)
294 return NULL;
295 index = (int) _getrecord_ex(c)->bidirectional;
Martin v. Löwis1a214512008-06-11 05:26:20 +0000296 if (self && UCD_Check(self)) {
Walter Dörwaldf342bfc2008-06-03 11:45:02 +0000297 const change_record *old = get_old_record(self, c);
Martin v. Löwis480f1bb2006-03-09 23:38:20 +0000298 if (old->category_changed == 0)
299 index = 0; /* unassigned */
300 else if (old->bidir_changed != 0xFF)
301 index = old->bidir_changed;
302 }
Walter Dörwald4254e762007-06-05 16:04:09 +0000303 return PyUnicode_FromString(_PyUnicode_BidirectionalNames[index]);
Guido van Rossum2a70a3a2000-03-10 23:10:21 +0000304}
305
Hye-Shik Changcf18a5d2005-04-04 16:32:07 +0000306PyDoc_STRVAR(unicodedata_combining__doc__,
307"combining(unichr)\n\
308\n\
309Returns the canonical combining class assigned to the Unicode\n\
310character unichr as integer. Returns 0 if no combining class is\n\
311defined.");
312
Guido van Rossum2a70a3a2000-03-10 23:10:21 +0000313static PyObject *
Fredrik Lundh7b7dd102001-01-21 22:41:08 +0000314unicodedata_combining(PyObject *self, PyObject *args)
Guido van Rossum2a70a3a2000-03-10 23:10:21 +0000315{
316 PyUnicodeObject *v;
Martin v. Löwis480f1bb2006-03-09 23:38:20 +0000317 int index;
Walter Dörwaldf342bfc2008-06-03 11:45:02 +0000318 Py_UCS4 c;
Guido van Rossum2a70a3a2000-03-10 23:10:21 +0000319
320 if (!PyArg_ParseTuple(args, "O!:combining",
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000321 &PyUnicode_Type, &v))
322 return NULL;
Walter Dörwaldf342bfc2008-06-03 11:45:02 +0000323 c = getuchar(v);
324 if (c == (Py_UCS4)-1)
325 return NULL;
326 index = (int) _getrecord_ex(c)->combining;
Martin v. Löwis1a214512008-06-11 05:26:20 +0000327 if (self && UCD_Check(self)) {
Walter Dörwaldf342bfc2008-06-03 11:45:02 +0000328 const change_record *old = get_old_record(self, c);
Martin v. Löwis480f1bb2006-03-09 23:38:20 +0000329 if (old->category_changed == 0)
330 index = 0; /* unassigned */
331 }
Christian Heimes217cfd12007-12-02 14:31:20 +0000332 return PyLong_FromLong(index);
Guido van Rossum2a70a3a2000-03-10 23:10:21 +0000333}
334
Hye-Shik Changcf18a5d2005-04-04 16:32:07 +0000335PyDoc_STRVAR(unicodedata_mirrored__doc__,
336"mirrored(unichr)\n\
337\n\
338Returns the mirrored property assigned to the Unicode character\n\
339unichr as integer. Returns 1 if the character has been identified as\n\
340a \"mirrored\" character in bidirectional text, 0 otherwise.");
341
Guido van Rossum2a70a3a2000-03-10 23:10:21 +0000342static PyObject *
Fredrik Lundh7b7dd102001-01-21 22:41:08 +0000343unicodedata_mirrored(PyObject *self, PyObject *args)
Guido van Rossum2a70a3a2000-03-10 23:10:21 +0000344{
345 PyUnicodeObject *v;
Martin v. Löwis480f1bb2006-03-09 23:38:20 +0000346 int index;
Walter Dörwaldf342bfc2008-06-03 11:45:02 +0000347 Py_UCS4 c;
Guido van Rossum2a70a3a2000-03-10 23:10:21 +0000348
349 if (!PyArg_ParseTuple(args, "O!:mirrored",
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000350 &PyUnicode_Type, &v))
351 return NULL;
Walter Dörwaldf342bfc2008-06-03 11:45:02 +0000352 c = getuchar(v);
353 if (c == (Py_UCS4)-1)
354 return NULL;
355 index = (int) _getrecord_ex(c)->mirrored;
Martin v. Löwis1a214512008-06-11 05:26:20 +0000356 if (self && UCD_Check(self)) {
Walter Dörwaldf342bfc2008-06-03 11:45:02 +0000357 const change_record *old = get_old_record(self, c);
Martin v. Löwis480f1bb2006-03-09 23:38:20 +0000358 if (old->category_changed == 0)
359 index = 0; /* unassigned */
Martin v. Löwis93cbca32008-09-10 14:08:48 +0000360 else if (old->mirrored_changed != 0xFF)
361 index = old->mirrored_changed;
Martin v. Löwis480f1bb2006-03-09 23:38:20 +0000362 }
Christian Heimes217cfd12007-12-02 14:31:20 +0000363 return PyLong_FromLong(index);
Guido van Rossum2a70a3a2000-03-10 23:10:21 +0000364}
365
Hye-Shik Changcf18a5d2005-04-04 16:32:07 +0000366PyDoc_STRVAR(unicodedata_east_asian_width__doc__,
367"east_asian_width(unichr)\n\
368\n\
369Returns the east asian width assigned to the Unicode character\n\
370unichr as string.");
371
Guido van Rossum2a70a3a2000-03-10 23:10:21 +0000372static PyObject *
Hye-Shik Change9ddfbb2004-08-04 07:38:35 +0000373unicodedata_east_asian_width(PyObject *self, PyObject *args)
374{
375 PyUnicodeObject *v;
376 int index;
Walter Dörwaldf342bfc2008-06-03 11:45:02 +0000377 Py_UCS4 c;
Hye-Shik Change9ddfbb2004-08-04 07:38:35 +0000378
379 if (!PyArg_ParseTuple(args, "O!:east_asian_width",
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000380 &PyUnicode_Type, &v))
381 return NULL;
Walter Dörwaldf342bfc2008-06-03 11:45:02 +0000382 c = getuchar(v);
383 if (c == (Py_UCS4)-1)
384 return NULL;
385 index = (int) _getrecord_ex(c)->east_asian_width;
Martin v. Löwis1a214512008-06-11 05:26:20 +0000386 if (self && UCD_Check(self)) {
Walter Dörwaldf342bfc2008-06-03 11:45:02 +0000387 const change_record *old = get_old_record(self, c);
Martin v. Löwis480f1bb2006-03-09 23:38:20 +0000388 if (old->category_changed == 0)
389 index = 0; /* unassigned */
390 }
Walter Dörwald4254e762007-06-05 16:04:09 +0000391 return PyUnicode_FromString(_PyUnicode_EastAsianWidthNames[index]);
Hye-Shik Change9ddfbb2004-08-04 07:38:35 +0000392}
393
Hye-Shik Changcf18a5d2005-04-04 16:32:07 +0000394PyDoc_STRVAR(unicodedata_decomposition__doc__,
395"decomposition(unichr)\n\
396\n\
397Returns the character decomposition mapping assigned to the Unicode\n\
398character unichr as string. An empty string is returned in case no\n\
399such mapping is defined.");
400
Hye-Shik Change9ddfbb2004-08-04 07:38:35 +0000401static PyObject *
Fredrik Lundh7b7dd102001-01-21 22:41:08 +0000402unicodedata_decomposition(PyObject *self, PyObject *args)
Guido van Rossum2a70a3a2000-03-10 23:10:21 +0000403{
404 PyUnicodeObject *v;
Fredrik Lundh7b7dd102001-01-21 22:41:08 +0000405 char decomp[256];
406 int code, index, count, i;
Thomas Wouters0e3f5912006-08-11 14:57:12 +0000407 unsigned int prefix_index;
Walter Dörwaldf342bfc2008-06-03 11:45:02 +0000408 Py_UCS4 c;
Guido van Rossum2a70a3a2000-03-10 23:10:21 +0000409
410 if (!PyArg_ParseTuple(args, "O!:decomposition",
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000411 &PyUnicode_Type, &v))
412 return NULL;
Walter Dörwaldf342bfc2008-06-03 11:45:02 +0000413 c = getuchar(v);
414 if (c == (Py_UCS4)-1)
415 return NULL;
Fredrik Lundh7b7dd102001-01-21 22:41:08 +0000416
Walter Dörwaldf342bfc2008-06-03 11:45:02 +0000417 code = (int)c;
Fredrik Lundh7b7dd102001-01-21 22:41:08 +0000418
Martin v. Löwis1a214512008-06-11 05:26:20 +0000419 if (self && UCD_Check(self)) {
Walter Dörwaldf342bfc2008-06-03 11:45:02 +0000420 const change_record *old = get_old_record(self, c);
Martin v. Löwis480f1bb2006-03-09 23:38:20 +0000421 if (old->category_changed == 0)
Walter Dörwald4254e762007-06-05 16:04:09 +0000422 return PyUnicode_FromString(""); /* unassigned */
Martin v. Löwis480f1bb2006-03-09 23:38:20 +0000423 }
424
Martin v. Löwis9def6a32002-10-18 16:11:54 +0000425 if (code < 0 || code >= 0x110000)
Fredrik Lundh7b7dd102001-01-21 22:41:08 +0000426 index = 0;
427 else {
428 index = decomp_index1[(code>>DECOMP_SHIFT)];
429 index = decomp_index2[(index<<DECOMP_SHIFT)+
430 (code&((1<<DECOMP_SHIFT)-1))];
431 }
432
Tim Peters69b83b12001-11-30 07:23:05 +0000433 /* high byte is number of hex bytes (usually one or two), low byte
Fredrik Lundh7b7dd102001-01-21 22:41:08 +0000434 is prefix code (from*/
435 count = decomp_data[index] >> 8;
436
437 /* XXX: could allocate the PyString up front instead
438 (strlen(prefix) + 5 * count + 1 bytes) */
439
Thomas Wouters0e3f5912006-08-11 14:57:12 +0000440 /* Based on how index is calculated above and decomp_data is generated
441 from Tools/unicode/makeunicodedata.py, it should not be possible
442 to overflow decomp_prefix. */
443 prefix_index = decomp_data[index] & 255;
444 assert(prefix_index < (sizeof(decomp_prefix)/sizeof(*decomp_prefix)));
445
Fredrik Lundh7b7dd102001-01-21 22:41:08 +0000446 /* copy prefix */
Thomas Wouters0e3f5912006-08-11 14:57:12 +0000447 i = strlen(decomp_prefix[prefix_index]);
448 memcpy(decomp, decomp_prefix[prefix_index], i);
Fredrik Lundh7b7dd102001-01-21 22:41:08 +0000449
450 while (count-- > 0) {
451 if (i)
452 decomp[i++] = ' ';
Tim Peters69b83b12001-11-30 07:23:05 +0000453 assert((size_t)i < sizeof(decomp));
454 PyOS_snprintf(decomp + i, sizeof(decomp) - i, "%04X",
455 decomp_data[++index]);
Fredrik Lundh7b7dd102001-01-21 22:41:08 +0000456 i += strlen(decomp + i);
457 }
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000458
Fredrik Lundh7b7dd102001-01-21 22:41:08 +0000459 decomp[i] = '\0';
460
Walter Dörwald4254e762007-06-05 16:04:09 +0000461 return PyUnicode_FromString(decomp);
Guido van Rossum2a70a3a2000-03-10 23:10:21 +0000462}
463
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000464static void
Martin v. Löwis480f1bb2006-03-09 23:38:20 +0000465get_decomp_record(PyObject *self, Py_UCS4 code, int *index, int *prefix, int *count)
Martin v. Löwis677bde22002-11-23 22:08:15 +0000466{
Neal Norwitze9c571f2003-02-28 03:14:37 +0000467 if (code >= 0x110000) {
Martin v. Löwis677bde22002-11-23 22:08:15 +0000468 *index = 0;
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000469 } else if (self && UCD_Check(self) &&
Martin v. Löwis1a214512008-06-11 05:26:20 +0000470 get_old_record(self, code)->category_changed==0) {
Martin v. Löwis480f1bb2006-03-09 23:38:20 +0000471 /* unassigned in old version */
472 *index = 0;
473 }
Martin v. Löwis677bde22002-11-23 22:08:15 +0000474 else {
475 *index = decomp_index1[(code>>DECOMP_SHIFT)];
476 *index = decomp_index2[(*index<<DECOMP_SHIFT)+
477 (code&((1<<DECOMP_SHIFT)-1))];
478 }
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000479
Martin v. Löwis677bde22002-11-23 22:08:15 +0000480 /* high byte is number of hex bytes (usually one or two), low byte
481 is prefix code (from*/
482 *count = decomp_data[*index] >> 8;
483 *prefix = decomp_data[*index] & 255;
484
485 (*index)++;
486}
487
488#define SBase 0xAC00
489#define LBase 0x1100
490#define VBase 0x1161
491#define TBase 0x11A7
492#define LCount 19
493#define VCount 21
494#define TCount 28
495#define NCount (VCount*TCount)
496#define SCount (LCount*NCount)
497
498static PyObject*
Martin v. Löwis480f1bb2006-03-09 23:38:20 +0000499nfd_nfkd(PyObject *self, PyObject *input, int k)
Martin v. Löwis677bde22002-11-23 22:08:15 +0000500{
501 PyObject *result;
502 Py_UNICODE *i, *end, *o;
503 /* Longest decomposition in Unicode 3.2: U+FDFA */
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000504 Py_UNICODE stack[20];
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000505 Py_ssize_t space, isize;
506 int index, prefix, count, stackptr;
Martin v. Löwis677bde22002-11-23 22:08:15 +0000507 unsigned char prev, cur;
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000508
Martin v. Löwis677bde22002-11-23 22:08:15 +0000509 stackptr = 0;
510 isize = PyUnicode_GET_SIZE(input);
511 /* Overallocate atmost 10 characters. */
512 space = (isize > 10 ? 10 : isize) + isize;
513 result = PyUnicode_FromUnicode(NULL, space);
514 if (!result)
515 return NULL;
516 i = PyUnicode_AS_UNICODE(input);
517 end = i + isize;
518 o = PyUnicode_AS_UNICODE(result);
519
520 while (i < end) {
521 stack[stackptr++] = *i++;
522 while(stackptr) {
523 Py_UNICODE code = stack[--stackptr];
Martin v. Löwisd2171d22003-11-06 20:47:57 +0000524 /* Hangul Decomposition adds three characters in
525 a single step, so we need atleast that much room. */
526 if (space < 3) {
Martin v. Löwis5b222132007-06-10 09:51:05 +0000527 Py_ssize_t newsize = PyUnicode_GET_SIZE(result) + 10;
Martin v. Löwisd2171d22003-11-06 20:47:57 +0000528 space += 10;
529 if (PyUnicode_Resize(&result, newsize) == -1)
Martin v. Löwis677bde22002-11-23 22:08:15 +0000530 return NULL;
Martin v. Löwisd2171d22003-11-06 20:47:57 +0000531 o = PyUnicode_AS_UNICODE(result) + newsize - space;
Martin v. Löwis677bde22002-11-23 22:08:15 +0000532 }
533 /* Hangul Decomposition. */
534 if (SBase <= code && code < (SBase+SCount)) {
535 int SIndex = code - SBase;
536 int L = LBase + SIndex / NCount;
537 int V = VBase + (SIndex % NCount) / TCount;
538 int T = TBase + SIndex % TCount;
539 *o++ = L;
540 *o++ = V;
541 space -= 2;
542 if (T != TBase) {
543 *o++ = T;
544 space --;
545 }
546 continue;
547 }
Martin v. Löwis480f1bb2006-03-09 23:38:20 +0000548 /* normalization changes */
Martin v. Löwis1a214512008-06-11 05:26:20 +0000549 if (self && UCD_Check(self)) {
Martin v. Löwis480f1bb2006-03-09 23:38:20 +0000550 Py_UCS4 value = ((PreviousDBVersion*)self)->normalization(code);
551 if (value != 0) {
552 stack[stackptr++] = value;
553 continue;
554 }
555 }
556
557 /* Other decompositions. */
558 get_decomp_record(self, code, &index, &prefix, &count);
Martin v. Löwis677bde22002-11-23 22:08:15 +0000559
560 /* Copy character if it is not decomposable, or has a
561 compatibility decomposition, but we do NFD. */
562 if (!count || (prefix && !k)) {
563 *o++ = code;
564 space--;
565 continue;
566 }
567 /* Copy decomposition onto the stack, in reverse
568 order. */
569 while(count) {
570 code = decomp_data[index + (--count)];
571 stack[stackptr++] = code;
572 }
573 }
574 }
575
576 /* Drop overallocation. Cannot fail. */
577 PyUnicode_Resize(&result, PyUnicode_GET_SIZE(result) - space);
578
579 /* Sort canonically. */
580 i = PyUnicode_AS_UNICODE(result);
581 prev = _getrecord_ex(*i)->combining;
582 end = i + PyUnicode_GET_SIZE(result);
583 for (i++; i < end; i++) {
584 cur = _getrecord_ex(*i)->combining;
585 if (prev == 0 || cur == 0 || prev <= cur) {
586 prev = cur;
587 continue;
588 }
589 /* Non-canonical order. Need to switch *i with previous. */
590 o = i - 1;
591 while (1) {
592 Py_UNICODE tmp = o[1];
593 o[1] = o[0];
594 o[0] = tmp;
595 o--;
596 if (o < PyUnicode_AS_UNICODE(result))
597 break;
598 prev = _getrecord_ex(*o)->combining;
599 if (prev == 0 || prev <= cur)
600 break;
601 }
602 prev = _getrecord_ex(*i)->combining;
603 }
604 return result;
605}
606
607static int
Martin v. Löwis480f1bb2006-03-09 23:38:20 +0000608find_nfc_index(PyObject *self, struct reindex* nfc, Py_UNICODE code)
Martin v. Löwis677bde22002-11-23 22:08:15 +0000609{
610 int index;
611 for (index = 0; nfc[index].start; index++) {
612 int start = nfc[index].start;
613 if (code < start)
614 return -1;
615 if (code <= start + nfc[index].count) {
616 int delta = code - start;
617 return nfc[index].index + delta;
618 }
619 }
620 return -1;
621}
622
623static PyObject*
Martin v. Löwis480f1bb2006-03-09 23:38:20 +0000624nfc_nfkc(PyObject *self, PyObject *input, int k)
Martin v. Löwis677bde22002-11-23 22:08:15 +0000625{
626 PyObject *result;
627 Py_UNICODE *i, *i1, *o, *end;
628 int f,l,index,index1,comb;
629 Py_UNICODE code;
630 Py_UNICODE *skipped[20];
631 int cskipped = 0;
632
Martin v. Löwis480f1bb2006-03-09 23:38:20 +0000633 result = nfd_nfkd(self, input, k);
Martin v. Löwis677bde22002-11-23 22:08:15 +0000634 if (!result)
635 return NULL;
636
637 /* We are going to modify result in-place.
638 If nfd_nfkd is changed to sometimes return the input,
639 this code needs to be reviewed. */
640 assert(result != input);
641
642 i = PyUnicode_AS_UNICODE(result);
643 end = i + PyUnicode_GET_SIZE(result);
644 o = PyUnicode_AS_UNICODE(result);
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000645
Martin v. Löwis677bde22002-11-23 22:08:15 +0000646 again:
647 while (i < end) {
648 for (index = 0; index < cskipped; index++) {
649 if (skipped[index] == i) {
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000650 /* *i character is skipped.
Martin v. Löwis677bde22002-11-23 22:08:15 +0000651 Remove from list. */
652 skipped[index] = skipped[cskipped-1];
653 cskipped--;
654 i++;
Martin v. Löwis2fb661f2002-12-07 14:56:36 +0000655 goto again; /* continue while */
Martin v. Löwis677bde22002-11-23 22:08:15 +0000656 }
657 }
658 /* Hangul Composition. We don't need to check for <LV,T>
659 pairs, since we always have decomposed data. */
660 if (LBase <= *i && *i < (LBase+LCount) &&
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000661 i + 1 < end &&
Martin v. Löwis677bde22002-11-23 22:08:15 +0000662 VBase <= i[1] && i[1] <= (VBase+VCount)) {
663 int LIndex, VIndex;
664 LIndex = i[0] - LBase;
665 VIndex = i[1] - VBase;
666 code = SBase + (LIndex*VCount+VIndex)*TCount;
667 i+=2;
668 if (i < end &&
669 TBase <= *i && *i <= (TBase+TCount)) {
670 code += *i-TBase;
671 i++;
672 }
673 *o++ = code;
674 continue;
675 }
676
Martin v. Löwis480f1bb2006-03-09 23:38:20 +0000677 f = find_nfc_index(self, nfc_first, *i);
Martin v. Löwis677bde22002-11-23 22:08:15 +0000678 if (f == -1) {
679 *o++ = *i++;
680 continue;
681 }
682 /* Find next unblocked character. */
683 i1 = i+1;
684 comb = 0;
685 while (i1 < end) {
686 int comb1 = _getrecord_ex(*i1)->combining;
Victor Stinner7ed9c4c2010-03-04 12:14:57 +0000687 if (comb && (comb1 == 0 || comb == comb1)) {
Martin v. Löwis677bde22002-11-23 22:08:15 +0000688 /* Character is blocked. */
689 i1++;
690 continue;
691 }
Martin v. Löwis480f1bb2006-03-09 23:38:20 +0000692 l = find_nfc_index(self, nfc_last, *i1);
Martin v. Löwis677bde22002-11-23 22:08:15 +0000693 /* *i1 cannot be combined with *i. If *i1
694 is a starter, we don't need to look further.
695 Otherwise, record the combining class. */
696 if (l == -1) {
697 not_combinable:
698 if (comb1 == 0)
699 break;
700 comb = comb1;
701 i1++;
702 continue;
703 }
704 index = f*TOTAL_LAST + l;
705 index1 = comp_index[index >> COMP_SHIFT];
706 code = comp_data[(index1<<COMP_SHIFT)+
707 (index&((1<<COMP_SHIFT)-1))];
708 if (code == 0)
709 goto not_combinable;
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000710
Martin v. Löwis677bde22002-11-23 22:08:15 +0000711 /* Replace the original character. */
712 *i = code;
713 /* Mark the second character unused. */
714 skipped[cskipped++] = i1;
715 i1++;
Martin v. Löwis480f1bb2006-03-09 23:38:20 +0000716 f = find_nfc_index(self, nfc_first, *i);
Martin v. Löwis677bde22002-11-23 22:08:15 +0000717 if (f == -1)
718 break;
719 }
720 *o++ = *i++;
721 }
722 if (o != end)
723 PyUnicode_Resize(&result, o - PyUnicode_AS_UNICODE(result));
724 return result;
725}
Antoine Pitrou7a0fedf2009-04-27 22:31:40 +0000726
727/* Return 1 if the input is certainly normalized, 0 if it might not be. */
728static int
729is_normalized(PyObject *self, PyObject *input, int nfc, int k)
730{
731 Py_UNICODE *i, *end;
732 unsigned char prev_combining = 0, quickcheck_mask;
733
734 /* An older version of the database is requested, quickchecks must be
735 disabled. */
736 if (self && UCD_Check(self))
737 return 0;
738
739 /* The two quickcheck bits at this shift mean 0=Yes, 1=Maybe, 2=No,
740 as described in http://unicode.org/reports/tr15/#Annex8. */
741 quickcheck_mask = 3 << ((nfc ? 4 : 0) + (k ? 2 : 0));
742
743 i = PyUnicode_AS_UNICODE(input);
744 end = i + PyUnicode_GET_SIZE(input);
745 while (i < end) {
746 const _PyUnicode_DatabaseRecord *record = _getrecord_ex(*i++);
747 unsigned char combining = record->combining;
748 unsigned char quickcheck = record->normalization_quick_check;
749
750 if (quickcheck & quickcheck_mask)
751 return 0; /* this string might need normalization */
752 if (combining && prev_combining > combining)
753 return 0; /* non-canonical sort order, not normalized */
754 prev_combining = combining;
755 }
756 return 1; /* certainly normalized */
757}
758
Hye-Shik Changcf18a5d2005-04-04 16:32:07 +0000759PyDoc_STRVAR(unicodedata_normalize__doc__,
760"normalize(form, unistr)\n\
761\n\
762Return the normal form 'form' for the Unicode string unistr. Valid\n\
763values for form are 'NFC', 'NFKC', 'NFD', and 'NFKD'.");
764
Martin v. Löwis677bde22002-11-23 22:08:15 +0000765static PyObject*
766unicodedata_normalize(PyObject *self, PyObject *args)
767{
768 char *form;
769 PyObject *input;
770
Hye-Shik Chang69dc1c82004-07-15 04:30:25 +0000771 if(!PyArg_ParseTuple(args, "sO!:normalize",
Martin v. Löwis677bde22002-11-23 22:08:15 +0000772 &form, &PyUnicode_Type, &input))
773 return NULL;
774
Martin v. Löwis61e40bd2004-04-17 19:36:48 +0000775 if (PyUnicode_GetSize(input) == 0) {
776 /* Special case empty input strings, since resizing
777 them later would cause internal errors. */
778 Py_INCREF(input);
779 return input;
780 }
781
Antoine Pitrou7a0fedf2009-04-27 22:31:40 +0000782 if (strcmp(form, "NFC") == 0) {
783 if (is_normalized(self, input, 1, 0)) {
784 Py_INCREF(input);
785 return input;
786 }
Martin v. Löwis480f1bb2006-03-09 23:38:20 +0000787 return nfc_nfkc(self, input, 0);
Antoine Pitrou7a0fedf2009-04-27 22:31:40 +0000788 }
789 if (strcmp(form, "NFKC") == 0) {
790 if (is_normalized(self, input, 1, 1)) {
791 Py_INCREF(input);
792 return input;
793 }
Martin v. Löwis480f1bb2006-03-09 23:38:20 +0000794 return nfc_nfkc(self, input, 1);
Antoine Pitrou7a0fedf2009-04-27 22:31:40 +0000795 }
796 if (strcmp(form, "NFD") == 0) {
797 if (is_normalized(self, input, 0, 0)) {
798 Py_INCREF(input);
799 return input;
800 }
Martin v. Löwis480f1bb2006-03-09 23:38:20 +0000801 return nfd_nfkd(self, input, 0);
Antoine Pitrou7a0fedf2009-04-27 22:31:40 +0000802 }
803 if (strcmp(form, "NFKD") == 0) {
804 if (is_normalized(self, input, 0, 1)) {
805 Py_INCREF(input);
806 return input;
807 }
Martin v. Löwis480f1bb2006-03-09 23:38:20 +0000808 return nfd_nfkd(self, input, 1);
Antoine Pitrou7a0fedf2009-04-27 22:31:40 +0000809 }
Martin v. Löwis677bde22002-11-23 22:08:15 +0000810 PyErr_SetString(PyExc_ValueError, "invalid normalization form");
811 return NULL;
812}
813
Fredrik Lundh06d12682001-01-24 07:59:11 +0000814/* -------------------------------------------------------------------- */
815/* unicode character name tables */
816
817/* data file generated by Tools/unicode/makeunicodedata.py */
818#include "unicodename_db.h"
819
820/* -------------------------------------------------------------------- */
821/* database code (cut and pasted from the unidb package) */
822
823static unsigned long
Fredrik Lundhb95896b2001-02-18 22:06:17 +0000824_gethash(const char *s, int len, int scale)
Fredrik Lundh06d12682001-01-24 07:59:11 +0000825{
826 int i;
827 unsigned long h = 0;
828 unsigned long ix;
829 for (i = 0; i < len; i++) {
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000830 h = (h * scale) + (unsigned char) toupper(Py_CHARMASK(s[i]));
Fredrik Lundh06d12682001-01-24 07:59:11 +0000831 ix = h & 0xff000000;
832 if (ix)
833 h = (h ^ ((ix>>24) & 0xff)) & 0x00ffffff;
834 }
835 return h;
836}
837
Martin v. Löwis7d41e292002-11-23 12:22:32 +0000838static char *hangul_syllables[][3] = {
839 { "G", "A", "" },
840 { "GG", "AE", "G" },
841 { "N", "YA", "GG" },
842 { "D", "YAE", "GS" },
843 { "DD", "EO", "N", },
844 { "R", "E", "NJ" },
845 { "M", "YEO", "NH" },
846 { "B", "YE", "D" },
847 { "BB", "O", "L" },
848 { "S", "WA", "LG" },
849 { "SS", "WAE", "LM" },
850 { "", "OE", "LB" },
851 { "J", "YO", "LS" },
852 { "JJ", "U", "LT" },
853 { "C", "WEO", "LP" },
854 { "K", "WE", "LH" },
855 { "T", "WI", "M" },
856 { "P", "YU", "B" },
857 { "H", "EU", "BS" },
858 { 0, "YI", "S" },
859 { 0, "I", "SS" },
860 { 0, 0, "NG" },
861 { 0, 0, "J" },
862 { 0, 0, "C" },
863 { 0, 0, "K" },
864 { 0, 0, "T" },
865 { 0, 0, "P" },
866 { 0, 0, "H" }
867};
868
Fredrik Lundh06d12682001-01-24 07:59:11 +0000869static int
Martin v. Löwis8d93ca12002-11-23 22:10:29 +0000870is_unified_ideograph(Py_UCS4 code)
871{
872 return (
873 (0x3400 <= code && code <= 0x4DB5) || /* CJK Ideograph Extension A */
Martin v. Löwisc3509122006-03-11 12:16:23 +0000874 (0x4E00 <= code && code <= 0x9FBB) || /* CJK Ideograph */
Martin v. Löwis8d93ca12002-11-23 22:10:29 +0000875 (0x20000 <= code && code <= 0x2A6D6));/* CJK Ideograph Extension B */
876}
877
878static int
Martin v. Löwis480f1bb2006-03-09 23:38:20 +0000879_getucname(PyObject *self, Py_UCS4 code, char* buffer, int buflen)
Fredrik Lundh06d12682001-01-24 07:59:11 +0000880{
881 int offset;
882 int i;
883 int word;
884 unsigned char* w;
885
Martin v. Löwisc3509122006-03-11 12:16:23 +0000886 if (code >= 0x110000)
887 return 0;
888
Martin v. Löwis1a214512008-06-11 05:26:20 +0000889 if (self && UCD_Check(self)) {
Martin v. Löwisc3509122006-03-11 12:16:23 +0000890 const change_record *old = get_old_record(self, code);
891 if (old->category_changed == 0) {
892 /* unassigned */
893 return 0;
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000894 }
Martin v. Löwisc3509122006-03-11 12:16:23 +0000895 }
896
Martin v. Löwis2f4be4e2002-11-23 17:11:06 +0000897 if (SBase <= code && code < SBase+SCount) {
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000898 /* Hangul syllable. */
899 int SIndex = code - SBase;
900 int L = SIndex / NCount;
901 int V = (SIndex % NCount) / TCount;
902 int T = SIndex % TCount;
Martin v. Löwis7d41e292002-11-23 12:22:32 +0000903
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000904 if (buflen < 27)
905 /* Worst case: HANGUL SYLLABLE <10chars>. */
906 return 0;
907 strcpy(buffer, "HANGUL SYLLABLE ");
908 buffer += 16;
909 strcpy(buffer, hangul_syllables[L][0]);
910 buffer += strlen(hangul_syllables[L][0]);
911 strcpy(buffer, hangul_syllables[V][1]);
912 buffer += strlen(hangul_syllables[V][1]);
913 strcpy(buffer, hangul_syllables[T][2]);
914 buffer += strlen(hangul_syllables[T][2]);
915 *buffer = '\0';
916 return 1;
Martin v. Löwis7d41e292002-11-23 12:22:32 +0000917 }
918
Martin v. Löwis8d93ca12002-11-23 22:10:29 +0000919 if (is_unified_ideograph(code)) {
Martin v. Löwisef7fe2e2002-11-23 18:01:32 +0000920 if (buflen < 28)
921 /* Worst case: CJK UNIFIED IDEOGRAPH-20000 */
922 return 0;
923 sprintf(buffer, "CJK UNIFIED IDEOGRAPH-%X", code);
924 return 1;
925 }
926
Fredrik Lundh06d12682001-01-24 07:59:11 +0000927 /* get offset into phrasebook */
928 offset = phrasebook_offset1[(code>>phrasebook_shift)];
929 offset = phrasebook_offset2[(offset<<phrasebook_shift) +
930 (code&((1<<phrasebook_shift)-1))];
931 if (!offset)
932 return 0;
933
934 i = 0;
935
936 for (;;) {
937 /* get word index */
938 word = phrasebook[offset] - phrasebook_short;
939 if (word >= 0) {
940 word = (word << 8) + phrasebook[offset+1];
941 offset += 2;
942 } else
943 word = phrasebook[offset++];
944 if (i) {
945 if (i > buflen)
946 return 0; /* buffer overflow */
947 buffer[i++] = ' ';
948 }
949 /* copy word string from lexicon. the last character in the
950 word has bit 7 set. the last word in a string ends with
951 0x80 */
952 w = lexicon + lexicon_offset[word];
953 while (*w < 128) {
954 if (i >= buflen)
955 return 0; /* buffer overflow */
956 buffer[i++] = *w++;
957 }
958 if (i >= buflen)
959 return 0; /* buffer overflow */
960 buffer[i++] = *w & 127;
961 if (*w == 128)
962 break; /* end of word */
963 }
964
965 return 1;
966}
967
968static int
Martin v. Löwis480f1bb2006-03-09 23:38:20 +0000969_cmpname(PyObject *self, int code, const char* name, int namelen)
Fredrik Lundh06d12682001-01-24 07:59:11 +0000970{
971 /* check if code corresponds to the given name */
972 int i;
973 char buffer[NAME_MAXLEN];
Martin v. Löwis480f1bb2006-03-09 23:38:20 +0000974 if (!_getucname(self, code, buffer, sizeof(buffer)))
Fredrik Lundh06d12682001-01-24 07:59:11 +0000975 return 0;
976 for (i = 0; i < namelen; i++) {
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000977 if (toupper(Py_CHARMASK(name[i])) != buffer[i])
Fredrik Lundh06d12682001-01-24 07:59:11 +0000978 return 0;
979 }
980 return buffer[namelen] == '\0';
981}
982
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000983static void
Martin v. Löwis7d41e292002-11-23 12:22:32 +0000984find_syllable(const char *str, int *len, int *pos, int count, int column)
985{
986 int i, len1;
987 *len = -1;
988 for (i = 0; i < count; i++) {
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000989 char *s = hangul_syllables[i][column];
990 len1 = strlen(s);
991 if (len1 <= *len)
992 continue;
993 if (strncmp(str, s, len1) == 0) {
994 *len = len1;
995 *pos = i;
996 }
Martin v. Löwis7d41e292002-11-23 12:22:32 +0000997 }
998 if (*len == -1) {
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000999 *len = 0;
Martin v. Löwis7d41e292002-11-23 12:22:32 +00001000 }
1001}
1002
Fredrik Lundh06d12682001-01-24 07:59:11 +00001003static int
Martin v. Löwis480f1bb2006-03-09 23:38:20 +00001004_getcode(PyObject* self, const char* name, int namelen, Py_UCS4* code)
Fredrik Lundh06d12682001-01-24 07:59:11 +00001005{
1006 unsigned int h, v;
1007 unsigned int mask = code_size-1;
1008 unsigned int i, incr;
1009
Martin v. Löwis7d41e292002-11-23 12:22:32 +00001010 /* Check for hangul syllables. */
1011 if (strncmp(name, "HANGUL SYLLABLE ", 16) == 0) {
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001012 int len, L = -1, V = -1, T = -1;
1013 const char *pos = name + 16;
1014 find_syllable(pos, &len, &L, LCount, 0);
1015 pos += len;
1016 find_syllable(pos, &len, &V, VCount, 1);
1017 pos += len;
1018 find_syllable(pos, &len, &T, TCount, 2);
1019 pos += len;
1020 if (L != -1 && V != -1 && T != -1 && pos-name == namelen) {
1021 *code = SBase + (L*VCount+V)*TCount + T;
1022 return 1;
1023 }
Martin v. Löwisef7fe2e2002-11-23 18:01:32 +00001024 /* Otherwise, it's an illegal syllable name. */
1025 return 0;
1026 }
1027
1028 /* Check for unified ideographs. */
1029 if (strncmp(name, "CJK UNIFIED IDEOGRAPH-", 22) == 0) {
1030 /* Four or five hexdigits must follow. */
1031 v = 0;
1032 name += 22;
1033 namelen -= 22;
1034 if (namelen != 4 && namelen != 5)
1035 return 0;
1036 while (namelen--) {
1037 v *= 16;
1038 if (*name >= '0' && *name <= '9')
1039 v += *name - '0';
1040 else if (*name >= 'A' && *name <= 'F')
1041 v += *name - 'A' + 10;
1042 else
1043 return 0;
1044 name++;
1045 }
Martin v. Löwis8d93ca12002-11-23 22:10:29 +00001046 if (!is_unified_ideograph(v))
1047 return 0;
Martin v. Löwisef7fe2e2002-11-23 18:01:32 +00001048 *code = v;
1049 return 1;
Martin v. Löwis7d41e292002-11-23 12:22:32 +00001050 }
1051
Fredrik Lundh06d12682001-01-24 07:59:11 +00001052 /* the following is the same as python's dictionary lookup, with
1053 only minor changes. see the makeunicodedata script for more
1054 details */
1055
Fredrik Lundhb95896b2001-02-18 22:06:17 +00001056 h = (unsigned int) _gethash(name, namelen, code_magic);
Fredrik Lundh06d12682001-01-24 07:59:11 +00001057 i = (~h) & mask;
1058 v = code_hash[i];
1059 if (!v)
1060 return 0;
Martin v. Löwis480f1bb2006-03-09 23:38:20 +00001061 if (_cmpname(self, v, name, namelen)) {
Fredrik Lundh06d12682001-01-24 07:59:11 +00001062 *code = v;
1063 return 1;
1064 }
1065 incr = (h ^ (h >> 3)) & mask;
1066 if (!incr)
1067 incr = mask;
1068 for (;;) {
1069 i = (i + incr) & mask;
1070 v = code_hash[i];
1071 if (!v)
Fredrik Lundhae763672001-02-18 11:41:49 +00001072 return 0;
Martin v. Löwis480f1bb2006-03-09 23:38:20 +00001073 if (_cmpname(self, v, name, namelen)) {
Fredrik Lundh06d12682001-01-24 07:59:11 +00001074 *code = v;
1075 return 1;
1076 }
1077 incr = incr << 1;
1078 if (incr > mask)
1079 incr = incr ^ code_poly;
1080 }
1081}
1082
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001083static const _PyUnicode_Name_CAPI hashAPI =
Fredrik Lundh06d12682001-01-24 07:59:11 +00001084{
1085 sizeof(_PyUnicode_Name_CAPI),
Andrew MacIntyre74a3bec2002-06-13 11:55:14 +00001086 _getucname,
Fredrik Lundhb95896b2001-02-18 22:06:17 +00001087 _getcode
Fredrik Lundh06d12682001-01-24 07:59:11 +00001088};
1089
1090/* -------------------------------------------------------------------- */
1091/* Python bindings */
1092
Hye-Shik Changcf18a5d2005-04-04 16:32:07 +00001093PyDoc_STRVAR(unicodedata_name__doc__,
1094"name(unichr[, default])\n\
1095Returns the name assigned to the Unicode character unichr as a\n\
1096string. If no name is defined, default is returned, or, if not\n\
1097given, ValueError is raised.");
1098
Fredrik Lundh06d12682001-01-24 07:59:11 +00001099static PyObject *
1100unicodedata_name(PyObject* self, PyObject* args)
1101{
1102 char name[NAME_MAXLEN];
Walter Dörwaldf342bfc2008-06-03 11:45:02 +00001103 Py_UCS4 c;
Fredrik Lundh06d12682001-01-24 07:59:11 +00001104
1105 PyUnicodeObject* v;
1106 PyObject* defobj = NULL;
1107 if (!PyArg_ParseTuple(args, "O!|O:name", &PyUnicode_Type, &v, &defobj))
1108 return NULL;
1109
Walter Dörwaldf342bfc2008-06-03 11:45:02 +00001110 c = getuchar(v);
1111 if (c == (Py_UCS4)-1)
1112 return NULL;
Fredrik Lundh06d12682001-01-24 07:59:11 +00001113
Walter Dörwaldf342bfc2008-06-03 11:45:02 +00001114 if (!_getucname(self, c, name, sizeof(name))) {
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001115 if (defobj == NULL) {
1116 PyErr_SetString(PyExc_ValueError, "no such name");
Fredrik Lundh06d12682001-01-24 07:59:11 +00001117 return NULL;
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001118 }
1119 else {
1120 Py_INCREF(defobj);
1121 return defobj;
1122 }
Fredrik Lundh06d12682001-01-24 07:59:11 +00001123 }
1124
Walter Dörwald4254e762007-06-05 16:04:09 +00001125 return PyUnicode_FromString(name);
Fredrik Lundh06d12682001-01-24 07:59:11 +00001126}
1127
Hye-Shik Changcf18a5d2005-04-04 16:32:07 +00001128PyDoc_STRVAR(unicodedata_lookup__doc__,
1129"lookup(name)\n\
1130\n\
1131Look up character by name. If a character with the\n\
1132given name is found, return the corresponding Unicode\n\
1133character. If not found, KeyError is raised.");
1134
Fredrik Lundh06d12682001-01-24 07:59:11 +00001135static PyObject *
1136unicodedata_lookup(PyObject* self, PyObject* args)
1137{
1138 Py_UCS4 code;
Guido van Rossum806c2462007-08-06 23:33:07 +00001139 Py_UNICODE str[2];
Fredrik Lundh06d12682001-01-24 07:59:11 +00001140
1141 char* name;
1142 int namelen;
1143 if (!PyArg_ParseTuple(args, "s#:lookup", &name, &namelen))
1144 return NULL;
1145
Martin v. Löwis480f1bb2006-03-09 23:38:20 +00001146 if (!_getcode(self, name, namelen, &code)) {
Guido van Rossum806c2462007-08-06 23:33:07 +00001147 PyErr_Format(PyExc_KeyError, "undefined character name '%s'",
1148 name);
Fredrik Lundh06d12682001-01-24 07:59:11 +00001149 return NULL;
1150 }
1151
Guido van Rossum806c2462007-08-06 23:33:07 +00001152#ifndef Py_UNICODE_WIDE
1153 if (code >= 0x10000) {
1154 str[0] = 0xd800 + ((code - 0x10000) >> 10);
1155 str[1] = 0xdc00 + ((code - 0x10000) & 0x3ff);
1156 return PyUnicode_FromUnicode(str, 2);
1157 }
1158#endif
Fredrik Lundh06d12682001-01-24 07:59:11 +00001159 str[0] = (Py_UNICODE) code;
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001160 return PyUnicode_FromUnicode(str, 1);
Fredrik Lundh06d12682001-01-24 07:59:11 +00001161}
1162
Guido van Rossum2a70a3a2000-03-10 23:10:21 +00001163/* XXX Add doc strings. */
1164
1165static PyMethodDef unicodedata_functions[] = {
Hye-Shik Changcf18a5d2005-04-04 16:32:07 +00001166 {"decimal", unicodedata_decimal, METH_VARARGS, unicodedata_decimal__doc__},
1167 {"digit", unicodedata_digit, METH_VARARGS, unicodedata_digit__doc__},
1168 {"numeric", unicodedata_numeric, METH_VARARGS, unicodedata_numeric__doc__},
1169 {"category", unicodedata_category, METH_VARARGS,
1170 unicodedata_category__doc__},
1171 {"bidirectional", unicodedata_bidirectional, METH_VARARGS,
1172 unicodedata_bidirectional__doc__},
1173 {"combining", unicodedata_combining, METH_VARARGS,
1174 unicodedata_combining__doc__},
1175 {"mirrored", unicodedata_mirrored, METH_VARARGS,
1176 unicodedata_mirrored__doc__},
1177 {"east_asian_width", unicodedata_east_asian_width, METH_VARARGS,
1178 unicodedata_east_asian_width__doc__},
1179 {"decomposition", unicodedata_decomposition, METH_VARARGS,
1180 unicodedata_decomposition__doc__},
1181 {"name", unicodedata_name, METH_VARARGS, unicodedata_name__doc__},
1182 {"lookup", unicodedata_lookup, METH_VARARGS, unicodedata_lookup__doc__},
1183 {"normalize", unicodedata_normalize, METH_VARARGS,
1184 unicodedata_normalize__doc__},
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001185 {NULL, NULL} /* sentinel */
Guido van Rossum2a70a3a2000-03-10 23:10:21 +00001186};
1187
Martin v. Löwis5bd7c022006-03-10 11:20:04 +00001188static PyTypeObject UCD_Type = {
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001189 /* The ob_type field must be initialized in the module init function
1190 * to be portable to Windows without using C++. */
1191 PyVarObject_HEAD_INIT(NULL, 0)
1192 "unicodedata.UCD", /*tp_name*/
1193 sizeof(PreviousDBVersion), /*tp_basicsize*/
1194 0, /*tp_itemsize*/
1195 /* methods */
1196 (destructor)PyObject_Del, /*tp_dealloc*/
1197 0, /*tp_print*/
1198 0, /*tp_getattr*/
1199 0, /*tp_setattr*/
1200 0, /*tp_reserved*/
1201 0, /*tp_repr*/
1202 0, /*tp_as_number*/
1203 0, /*tp_as_sequence*/
1204 0, /*tp_as_mapping*/
1205 0, /*tp_hash*/
Martin v. Löwis5bd7c022006-03-10 11:20:04 +00001206 0, /*tp_call*/
1207 0, /*tp_str*/
1208 PyObject_GenericGetAttr,/*tp_getattro*/
1209 0, /*tp_setattro*/
1210 0, /*tp_as_buffer*/
1211 Py_TPFLAGS_DEFAULT, /*tp_flags*/
1212 0, /*tp_doc*/
1213 0, /*tp_traverse*/
1214 0, /*tp_clear*/
1215 0, /*tp_richcompare*/
1216 0, /*tp_weaklistoffset*/
1217 0, /*tp_iter*/
1218 0, /*tp_iternext*/
1219 unicodedata_functions, /*tp_methods*/
1220 DB_members, /*tp_members*/
1221 0, /*tp_getset*/
1222 0, /*tp_base*/
1223 0, /*tp_dict*/
1224 0, /*tp_descr_get*/
1225 0, /*tp_descr_set*/
1226 0, /*tp_dictoffset*/
1227 0, /*tp_init*/
1228 0, /*tp_alloc*/
1229 0, /*tp_new*/
1230 0, /*tp_free*/
1231 0, /*tp_is_gc*/
1232};
Martin v. Löwis480f1bb2006-03-09 23:38:20 +00001233
Hye-Shik Changcf18a5d2005-04-04 16:32:07 +00001234PyDoc_STRVAR(unicodedata_docstring,
1235"This module provides access to the Unicode Character Database which\n\
1236defines character properties for all Unicode characters. The data in\n\
1237this database is based on the UnicodeData.txt file version\n\
Ezio Melotti4c5475d2010-03-22 23:16:42 +000012385.2.0 which is publically available from ftp://ftp.unicode.org/.\n\
Hye-Shik Changcf18a5d2005-04-04 16:32:07 +00001239\n\
1240The module uses the same names and symbols as defined by the\n\
Ezio Melottid96b2f22010-03-23 00:39:22 +00001241UnicodeData File Format 5.2.0 (see\n\
1242http://www.unicode.org/reports/tr44/tr44-4.html).");
Fredrik Lundh06d12682001-01-24 07:59:11 +00001243
Martin v. Löwis1a214512008-06-11 05:26:20 +00001244
1245static struct PyModuleDef unicodedatamodule = {
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001246 PyModuleDef_HEAD_INIT,
1247 "unicodedata",
1248 unicodedata_docstring,
1249 -1,
1250 unicodedata_functions,
1251 NULL,
1252 NULL,
1253 NULL,
1254 NULL
Martin v. Löwis1a214512008-06-11 05:26:20 +00001255};
1256
Mark Hammond62b1ab12002-07-23 06:31:15 +00001257PyMODINIT_FUNC
Martin v. Löwis1a214512008-06-11 05:26:20 +00001258PyInit_unicodedata(void)
Guido van Rossum2a70a3a2000-03-10 23:10:21 +00001259{
Fred Drakea2bd8d32002-04-03 21:39:26 +00001260 PyObject *m, *v;
Fredrik Lundh06d12682001-01-24 07:59:11 +00001261
Christian Heimes90aa7642007-12-19 02:45:37 +00001262 Py_TYPE(&UCD_Type) = &PyType_Type;
Martin v. Löwis5bd7c022006-03-10 11:20:04 +00001263
Martin v. Löwis1a214512008-06-11 05:26:20 +00001264 m = PyModule_Create(&unicodedatamodule);
Fredrik Lundh06d12682001-01-24 07:59:11 +00001265 if (!m)
Martin v. Löwis1a214512008-06-11 05:26:20 +00001266 return NULL;
Fredrik Lundh06d12682001-01-24 07:59:11 +00001267
Martin v. Löwisb5c980b2002-11-25 09:13:37 +00001268 PyModule_AddStringConstant(m, "unidata_version", UNIDATA_VERSION);
Martin v. Löwis0e2f9b22006-03-10 11:29:32 +00001269 Py_INCREF(&UCD_Type);
Martin v. Löwis5bd7c022006-03-10 11:20:04 +00001270 PyModule_AddObject(m, "UCD", (PyObject*)&UCD_Type);
Martin v. Löwisb5c980b2002-11-25 09:13:37 +00001271
Martin v. Löwis480f1bb2006-03-09 23:38:20 +00001272 /* Previous versions */
1273 v = new_previous_version("3.2.0", get_change_3_2_0, normalization_3_2_0);
1274 if (v != NULL)
Martin v. Löwis5bd7c022006-03-10 11:20:04 +00001275 PyModule_AddObject(m, "ucd_3_2_0", v);
Martin v. Löwis480f1bb2006-03-09 23:38:20 +00001276
Fredrik Lundh06d12682001-01-24 07:59:11 +00001277 /* Export C API */
Benjamin Petersonb173f782009-05-05 22:31:58 +00001278 v = PyCapsule_New((void *)&hashAPI, PyUnicodeData_CAPSULE_NAME, NULL);
Fred Drakea2bd8d32002-04-03 21:39:26 +00001279 if (v != NULL)
1280 PyModule_AddObject(m, "ucnhash_CAPI", v);
Martin v. Löwis1a214512008-06-11 05:26:20 +00001281 return m;
Guido van Rossum2a70a3a2000-03-10 23:10:21 +00001282}
Martin v. Löwis7d41e292002-11-23 12:22:32 +00001283
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001284/*
Martin v. Löwis7d41e292002-11-23 12:22:32 +00001285Local variables:
1286c-basic-offset: 4
Martin v. Löwis677bde22002-11-23 22:08:15 +00001287indent-tabs-mode: nil
Martin v. Löwis7d41e292002-11-23 12:22:32 +00001288End:
1289*/