| /* |
| * cjkcodecs.h: common header for cjkcodecs |
| * |
| * Written by Hye-Shik Chang <perky@FreeBSD.org> |
| */ |
| |
| #ifndef _CJKCODECS_H_ |
| #define _CJKCODECS_H_ |
| |
| #define PY_SSIZE_T_CLEAN |
| #include "Python.h" |
| #include "multibytecodec.h" |
| |
| |
| /* a unicode "undefined" codepoint */ |
| #define UNIINV 0xFFFE |
| |
| /* internal-use DBCS codepoints which aren't used by any charsets */ |
| #define NOCHAR 0xFFFF |
| #define MULTIC 0xFFFE |
| #define DBCINV 0xFFFD |
| |
| /* shorter macros to save source size of mapping tables */ |
| #define U UNIINV |
| #define N NOCHAR |
| #define M MULTIC |
| #define D DBCINV |
| |
| struct dbcs_index { |
| const ucs2_t *map; |
| unsigned char bottom, top; |
| }; |
| typedef struct dbcs_index decode_map; |
| |
| struct widedbcs_index { |
| const ucs4_t *map; |
| unsigned char bottom, top; |
| }; |
| typedef struct widedbcs_index widedecode_map; |
| |
| struct unim_index { |
| const DBCHAR *map; |
| unsigned char bottom, top; |
| }; |
| typedef struct unim_index encode_map; |
| |
| struct unim_index_bytebased { |
| const unsigned char *map; |
| unsigned char bottom, top; |
| }; |
| |
| struct dbcs_map { |
| const char *charset; |
| const struct unim_index *encmap; |
| const struct dbcs_index *decmap; |
| }; |
| |
| struct pair_encodemap { |
| ucs4_t uniseq; |
| DBCHAR code; |
| }; |
| |
| static const MultibyteCodec *codec_list; |
| static const struct dbcs_map *mapping_list; |
| |
| #define CODEC_INIT(encoding) \ |
| static int encoding##_codec_init(const void *config) |
| |
| #define ENCODER_INIT(encoding) \ |
| static int encoding##_encode_init( \ |
| MultibyteCodec_State *state, const void *config) |
| #define ENCODER(encoding) \ |
| static Py_ssize_t encoding##_encode( \ |
| MultibyteCodec_State *state, const void *config, \ |
| const Py_UNICODE **inbuf, Py_ssize_t inleft, \ |
| unsigned char **outbuf, Py_ssize_t outleft, int flags) |
| #define ENCODER_RESET(encoding) \ |
| static Py_ssize_t encoding##_encode_reset( \ |
| MultibyteCodec_State *state, const void *config, \ |
| unsigned char **outbuf, Py_ssize_t outleft) |
| |
| #define DECODER_INIT(encoding) \ |
| static int encoding##_decode_init( \ |
| MultibyteCodec_State *state, const void *config) |
| #define DECODER(encoding) \ |
| static Py_ssize_t encoding##_decode( \ |
| MultibyteCodec_State *state, const void *config, \ |
| const unsigned char **inbuf, Py_ssize_t inleft, \ |
| Py_UNICODE **outbuf, Py_ssize_t outleft) |
| #define DECODER_RESET(encoding) \ |
| static Py_ssize_t encoding##_decode_reset( \ |
| MultibyteCodec_State *state, const void *config) |
| |
| #if Py_UNICODE_SIZE == 4 |
| #define UCS4INVALID(code) \ |
| if ((code) > 0xFFFF) \ |
| return 1; |
| #else |
| #define UCS4INVALID(code) \ |
| if (0) ; |
| #endif |
| |
| #define NEXT_IN(i) \ |
| (*inbuf) += (i); \ |
| (inleft) -= (i); |
| #define NEXT_OUT(o) \ |
| (*outbuf) += (o); \ |
| (outleft) -= (o); |
| #define NEXT(i, o) \ |
| NEXT_IN(i) NEXT_OUT(o) |
| |
| #define REQUIRE_INBUF(n) \ |
| if (inleft < (n)) \ |
| return MBERR_TOOFEW; |
| #define REQUIRE_OUTBUF(n) \ |
| if (outleft < (n)) \ |
| return MBERR_TOOSMALL; |
| |
| #define IN1 ((*inbuf)[0]) |
| #define IN2 ((*inbuf)[1]) |
| #define IN3 ((*inbuf)[2]) |
| #define IN4 ((*inbuf)[3]) |
| |
| #define OUT1(c) ((*outbuf)[0]) = (c); |
| #define OUT2(c) ((*outbuf)[1]) = (c); |
| #define OUT3(c) ((*outbuf)[2]) = (c); |
| #define OUT4(c) ((*outbuf)[3]) = (c); |
| |
| #define WRITE1(c1) \ |
| REQUIRE_OUTBUF(1) \ |
| (*outbuf)[0] = (c1); |
| #define WRITE2(c1, c2) \ |
| REQUIRE_OUTBUF(2) \ |
| (*outbuf)[0] = (c1); \ |
| (*outbuf)[1] = (c2); |
| #define WRITE3(c1, c2, c3) \ |
| REQUIRE_OUTBUF(3) \ |
| (*outbuf)[0] = (c1); \ |
| (*outbuf)[1] = (c2); \ |
| (*outbuf)[2] = (c3); |
| #define WRITE4(c1, c2, c3, c4) \ |
| REQUIRE_OUTBUF(4) \ |
| (*outbuf)[0] = (c1); \ |
| (*outbuf)[1] = (c2); \ |
| (*outbuf)[2] = (c3); \ |
| (*outbuf)[3] = (c4); |
| |
| #if Py_UNICODE_SIZE == 2 |
| # define WRITEUCS4(c) \ |
| REQUIRE_OUTBUF(2) \ |
| (*outbuf)[0] = 0xd800 + (((c) - 0x10000) >> 10); \ |
| (*outbuf)[1] = 0xdc00 + (((c) - 0x10000) & 0x3ff); \ |
| NEXT_OUT(2) |
| #else |
| # define WRITEUCS4(c) \ |
| REQUIRE_OUTBUF(1) \ |
| **outbuf = (Py_UNICODE)(c); \ |
| NEXT_OUT(1) |
| #endif |
| |
| #define _TRYMAP_ENC(m, assi, val) \ |
| ((m)->map != NULL && (val) >= (m)->bottom && \ |
| (val)<= (m)->top && ((assi) = (m)->map[(val) - \ |
| (m)->bottom]) != NOCHAR) |
| #define TRYMAP_ENC_COND(charset, assi, uni) \ |
| _TRYMAP_ENC(&charset##_encmap[(uni) >> 8], assi, (uni) & 0xff) |
| #define TRYMAP_ENC(charset, assi, uni) \ |
| if TRYMAP_ENC_COND(charset, assi, uni) |
| |
| #define _TRYMAP_DEC(m, assi, val) \ |
| ((m)->map != NULL && (val) >= (m)->bottom && \ |
| (val)<= (m)->top && ((assi) = (m)->map[(val) - \ |
| (m)->bottom]) != UNIINV) |
| #define TRYMAP_DEC(charset, assi, c1, c2) \ |
| if _TRYMAP_DEC(&charset##_decmap[c1], assi, c2) |
| |
| #define _TRYMAP_ENC_MPLANE(m, assplane, asshi, asslo, val) \ |
| ((m)->map != NULL && (val) >= (m)->bottom && \ |
| (val)<= (m)->top && \ |
| ((assplane) = (m)->map[((val) - (m)->bottom)*3]) != 0 && \ |
| (((asshi) = (m)->map[((val) - (m)->bottom)*3 + 1]), 1) && \ |
| (((asslo) = (m)->map[((val) - (m)->bottom)*3 + 2]), 1)) |
| #define TRYMAP_ENC_MPLANE(charset, assplane, asshi, asslo, uni) \ |
| if _TRYMAP_ENC_MPLANE(&charset##_encmap[(uni) >> 8], \ |
| assplane, asshi, asslo, (uni) & 0xff) |
| #define TRYMAP_DEC_MPLANE(charset, assi, plane, c1, c2) \ |
| if _TRYMAP_DEC(&charset##_decmap[plane][c1], assi, c2) |
| |
| #if Py_UNICODE_SIZE == 2 |
| #define DECODE_SURROGATE(c) \ |
| if (c >> 10 == 0xd800 >> 10) { /* high surrogate */ \ |
| REQUIRE_INBUF(2) \ |
| if (IN2 >> 10 == 0xdc00 >> 10) { /* low surrogate */ \ |
| c = 0x10000 + ((ucs4_t)(c - 0xd800) << 10) + \ |
| ((ucs4_t)(IN2) - 0xdc00); \ |
| } \ |
| } |
| #define GET_INSIZE(c) ((c) > 0xffff ? 2 : 1) |
| #else |
| #define DECODE_SURROGATE(c) {;} |
| #define GET_INSIZE(c) 1 |
| #endif |
| |
| #define BEGIN_MAPPINGS_LIST static const struct dbcs_map _mapping_list[] = { |
| #define MAPPING_ENCONLY(enc) {#enc, (void*)enc##_encmap, NULL}, |
| #define MAPPING_DECONLY(enc) {#enc, NULL, (void*)enc##_decmap}, |
| #define MAPPING_ENCDEC(enc) {#enc, (void*)enc##_encmap, (void*)enc##_decmap}, |
| #define END_MAPPINGS_LIST \ |
| {"", NULL, NULL} }; \ |
| static const struct dbcs_map *mapping_list = \ |
| (const struct dbcs_map *)_mapping_list; |
| |
| #define BEGIN_CODECS_LIST static const MultibyteCodec _codec_list[] = { |
| #define _STATEFUL_METHODS(enc) \ |
| enc##_encode, \ |
| enc##_encode_init, \ |
| enc##_encode_reset, \ |
| enc##_decode, \ |
| enc##_decode_init, \ |
| enc##_decode_reset, |
| #define _STATELESS_METHODS(enc) \ |
| enc##_encode, NULL, NULL, \ |
| enc##_decode, NULL, NULL, |
| #define CODEC_STATEFUL(enc) { \ |
| #enc, NULL, NULL, \ |
| _STATEFUL_METHODS(enc) \ |
| }, |
| #define CODEC_STATELESS(enc) { \ |
| #enc, NULL, NULL, \ |
| _STATELESS_METHODS(enc) \ |
| }, |
| #define CODEC_STATELESS_WINIT(enc) { \ |
| #enc, NULL, \ |
| enc##_codec_init, \ |
| _STATELESS_METHODS(enc) \ |
| }, |
| #define END_CODECS_LIST \ |
| {"", NULL,} }; \ |
| static const MultibyteCodec *codec_list = \ |
| (const MultibyteCodec *)_codec_list; |
| |
| |
| |
| static PyObject * |
| getmultibytecodec(void) |
| { |
| static PyObject *cofunc = NULL; |
| |
| if (cofunc == NULL) { |
| PyObject *mod = PyImport_ImportModuleNoBlock("_multibytecodec"); |
| if (mod == NULL) |
| return NULL; |
| cofunc = PyObject_GetAttrString(mod, "__create_codec"); |
| Py_DECREF(mod); |
| } |
| return cofunc; |
| } |
| |
| static PyObject * |
| getcodec(PyObject *self, PyObject *encoding) |
| { |
| PyObject *codecobj, *r, *cofunc; |
| const MultibyteCodec *codec; |
| const char *enc; |
| |
| if (!PyUnicode_Check(encoding)) { |
| PyErr_SetString(PyExc_TypeError, |
| "encoding name must be a string."); |
| return NULL; |
| } |
| enc = _PyUnicode_AsString(encoding); |
| if (enc == NULL) |
| return NULL; |
| |
| cofunc = getmultibytecodec(); |
| if (cofunc == NULL) |
| return NULL; |
| |
| for (codec = codec_list; codec->encoding[0]; codec++) |
| if (strcmp(codec->encoding, enc) == 0) |
| break; |
| |
| if (codec->encoding[0] == '\0') { |
| PyErr_SetString(PyExc_LookupError, |
| "no such codec is supported."); |
| return NULL; |
| } |
| |
| codecobj = PyCapsule_New((void *)codec, PyMultibyteCodec_CAPSULE_NAME, NULL); |
| if (codecobj == NULL) |
| return NULL; |
| |
| r = PyObject_CallFunctionObjArgs(cofunc, codecobj, NULL); |
| Py_DECREF(codecobj); |
| |
| return r; |
| } |
| |
| static struct PyMethodDef __methods[] = { |
| {"getcodec", (PyCFunction)getcodec, METH_O, ""}, |
| {NULL, NULL}, |
| }; |
| |
| static int |
| register_maps(PyObject *module) |
| { |
| const struct dbcs_map *h; |
| |
| for (h = mapping_list; h->charset[0] != '\0'; h++) { |
| char mhname[256] = "__map_"; |
| int r; |
| strcpy(mhname + sizeof("__map_") - 1, h->charset); |
| r = PyModule_AddObject(module, mhname, |
| PyCapsule_New((void *)h, PyMultibyteCodec_CAPSULE_NAME, NULL)); |
| if (r == -1) |
| return -1; |
| } |
| return 0; |
| } |
| |
| #ifdef USING_BINARY_PAIR_SEARCH |
| static DBCHAR |
| find_pairencmap(ucs2_t body, ucs2_t modifier, |
| const struct pair_encodemap *haystack, int haystacksize) |
| { |
| int pos, min, max; |
| ucs4_t value = body << 16 | modifier; |
| |
| min = 0; |
| max = haystacksize; |
| |
| for (pos = haystacksize >> 1; min != max; pos = (min + max) >> 1) |
| if (value < haystack[pos].uniseq) { |
| if (max == pos) break; |
| else max = pos; |
| } |
| else if (value > haystack[pos].uniseq) { |
| if (min == pos) break; |
| else min = pos; |
| } |
| else |
| break; |
| |
| if (value == haystack[pos].uniseq) |
| return haystack[pos].code; |
| else |
| return DBCINV; |
| } |
| #endif |
| |
| #ifdef USING_IMPORTED_MAPS |
| #define IMPORT_MAP(locale, charset, encmap, decmap) \ |
| importmap("_codecs_" #locale, "__map_" #charset, \ |
| (const void**)encmap, (const void**)decmap) |
| |
| static int |
| importmap(const char *modname, const char *symbol, |
| const void **encmap, const void **decmap) |
| { |
| PyObject *o, *mod; |
| |
| mod = PyImport_ImportModule((char *)modname); |
| if (mod == NULL) |
| return -1; |
| |
| o = PyObject_GetAttrString(mod, (char*)symbol); |
| if (o == NULL) |
| goto errorexit; |
| else if (!PyCapsule_IsValid(o, PyMultibyteCodec_CAPSULE_NAME)) { |
| PyErr_SetString(PyExc_ValueError, |
| "map data must be a Capsule."); |
| goto errorexit; |
| } |
| else { |
| struct dbcs_map *map; |
| map = PyCapsule_GetPointer(o, PyMultibyteCodec_CAPSULE_NAME); |
| if (encmap != NULL) |
| *encmap = map->encmap; |
| if (decmap != NULL) |
| *decmap = map->decmap; |
| Py_DECREF(o); |
| } |
| |
| Py_DECREF(mod); |
| return 0; |
| |
| errorexit: |
| Py_DECREF(mod); |
| return -1; |
| } |
| #endif |
| |
| #define I_AM_A_MODULE_FOR(loc) \ |
| static struct PyModuleDef __module = { \ |
| PyModuleDef_HEAD_INIT, \ |
| "_codecs_"#loc, \ |
| NULL, \ |
| 0, \ |
| __methods, \ |
| NULL, \ |
| NULL, \ |
| NULL, \ |
| NULL \ |
| }; \ |
| PyObject* \ |
| PyInit__codecs_##loc(void) \ |
| { \ |
| PyObject *m = PyModule_Create(&__module); \ |
| if (m != NULL) \ |
| (void)register_maps(m); \ |
| return m; \ |
| } |
| |
| #endif |