| /* |
| * Copyright (C) 2005 The Android Open Source Project |
| * |
| * Licensed under the Apache License, Version 2.0 (the "License"); |
| * you may not use this file except in compliance with the License. |
| * You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| |
| #include <utils/Unicode.h> |
| |
| #include <stddef.h> |
| |
| #ifdef HAVE_WINSOCK |
| # undef nhtol |
| # undef htonl |
| # undef nhtos |
| # undef htons |
| |
| # ifdef HAVE_LITTLE_ENDIAN |
| # define ntohl(x) ( ((x) << 24) | (((x) >> 24) & 255) | (((x) << 8) & 0xff0000) | (((x) >> 8) & 0xff00) ) |
| # define htonl(x) ntohl(x) |
| # define ntohs(x) ( (((x) << 8) & 0xff00) | (((x) >> 8) & 255) ) |
| # define htons(x) ntohs(x) |
| # else |
| # define ntohl(x) (x) |
| # define htonl(x) (x) |
| # define ntohs(x) (x) |
| # define htons(x) (x) |
| # endif |
| #else |
| # include <netinet/in.h> |
| #endif |
| |
| extern "C" { |
| |
| static const char32_t kByteMask = 0x000000BF; |
| static const char32_t kByteMark = 0x00000080; |
| |
| // Surrogates aren't valid for UTF-32 characters, so define some |
| // constants that will let us screen them out. |
| static const char32_t kUnicodeSurrogateHighStart = 0x0000D800; |
| static const char32_t kUnicodeSurrogateHighEnd = 0x0000DBFF; |
| static const char32_t kUnicodeSurrogateLowStart = 0x0000DC00; |
| static const char32_t kUnicodeSurrogateLowEnd = 0x0000DFFF; |
| static const char32_t kUnicodeSurrogateStart = kUnicodeSurrogateHighStart; |
| static const char32_t kUnicodeSurrogateEnd = kUnicodeSurrogateLowEnd; |
| static const char32_t kUnicodeMaxCodepoint = 0x0010FFFF; |
| |
| // Mask used to set appropriate bits in first byte of UTF-8 sequence, |
| // indexed by number of bytes in the sequence. |
| // 0xxxxxxx |
| // -> (00-7f) 7bit. Bit mask for the first byte is 0x00000000 |
| // 110yyyyx 10xxxxxx |
| // -> (c0-df)(80-bf) 11bit. Bit mask is 0x000000C0 |
| // 1110yyyy 10yxxxxx 10xxxxxx |
| // -> (e0-ef)(80-bf)(80-bf) 16bit. Bit mask is 0x000000E0 |
| // 11110yyy 10yyxxxx 10xxxxxx 10xxxxxx |
| // -> (f0-f7)(80-bf)(80-bf)(80-bf) 21bit. Bit mask is 0x000000F0 |
| static const char32_t kFirstByteMark[] = { |
| 0x00000000, 0x00000000, 0x000000C0, 0x000000E0, 0x000000F0 |
| }; |
| |
| // -------------------------------------------------------------------------- |
| // UTF-32 |
| // -------------------------------------------------------------------------- |
| |
| /** |
| * Return number of UTF-8 bytes required for the character. If the character |
| * is invalid, return size of 0. |
| */ |
| static inline size_t utf32_codepoint_utf8_length(char32_t srcChar) |
| { |
| // Figure out how many bytes the result will require. |
| if (srcChar < 0x00000080) { |
| return 1; |
| } else if (srcChar < 0x00000800) { |
| return 2; |
| } else if (srcChar < 0x00010000) { |
| if ((srcChar < kUnicodeSurrogateStart) || (srcChar > kUnicodeSurrogateEnd)) { |
| return 3; |
| } else { |
| // Surrogates are invalid UTF-32 characters. |
| return 0; |
| } |
| } |
| // Max code point for Unicode is 0x0010FFFF. |
| else if (srcChar <= kUnicodeMaxCodepoint) { |
| return 4; |
| } else { |
| // Invalid UTF-32 character. |
| return 0; |
| } |
| } |
| |
| // Write out the source character to <dstP>. |
| |
| static inline void utf32_codepoint_to_utf8(uint8_t* dstP, char32_t srcChar, size_t bytes) |
| { |
| dstP += bytes; |
| switch (bytes) |
| { /* note: everything falls through. */ |
| case 4: *--dstP = (uint8_t)((srcChar | kByteMark) & kByteMask); srcChar >>= 6; |
| case 3: *--dstP = (uint8_t)((srcChar | kByteMark) & kByteMask); srcChar >>= 6; |
| case 2: *--dstP = (uint8_t)((srcChar | kByteMark) & kByteMask); srcChar >>= 6; |
| case 1: *--dstP = (uint8_t)(srcChar | kFirstByteMark[bytes]); |
| } |
| } |
| |
| size_t strlen32(const char32_t *s) |
| { |
| const char32_t *ss = s; |
| while ( *ss ) |
| ss++; |
| return ss-s; |
| } |
| |
| size_t strnlen32(const char32_t *s, size_t maxlen) |
| { |
| const char32_t *ss = s; |
| while ((maxlen > 0) && *ss) { |
| ss++; |
| maxlen--; |
| } |
| return ss-s; |
| } |
| |
| static inline int32_t utf32_at_internal(const char* cur, size_t *num_read) |
| { |
| const char first_char = *cur; |
| if ((first_char & 0x80) == 0) { // ASCII |
| *num_read = 1; |
| return *cur; |
| } |
| cur++; |
| char32_t mask, to_ignore_mask; |
| size_t num_to_read = 0; |
| char32_t utf32 = first_char; |
| for (num_to_read = 1, mask = 0x40, to_ignore_mask = 0xFFFFFF80; |
| (first_char & mask); |
| num_to_read++, to_ignore_mask |= mask, mask >>= 1) { |
| // 0x3F == 00111111 |
| utf32 = (utf32 << 6) + (*cur++ & 0x3F); |
| } |
| to_ignore_mask |= mask; |
| utf32 &= ~(to_ignore_mask << (6 * (num_to_read - 1))); |
| |
| *num_read = num_to_read; |
| return static_cast<int32_t>(utf32); |
| } |
| |
| int32_t utf32_from_utf8_at(const char *src, size_t src_len, size_t index, size_t *next_index) |
| { |
| if (index >= src_len) { |
| return -1; |
| } |
| size_t dummy_index; |
| if (next_index == NULL) { |
| next_index = &dummy_index; |
| } |
| size_t num_read; |
| int32_t ret = utf32_at_internal(src + index, &num_read); |
| if (ret >= 0) { |
| *next_index = index + num_read; |
| } |
| |
| return ret; |
| } |
| |
| ssize_t utf32_to_utf8_length(const char32_t *src, size_t src_len) |
| { |
| if (src == NULL || src_len == 0) { |
| return -1; |
| } |
| |
| size_t ret = 0; |
| const char32_t *end = src + src_len; |
| while (src < end) { |
| ret += utf32_codepoint_utf8_length(*src++); |
| } |
| return ret; |
| } |
| |
| void utf32_to_utf8(const char32_t* src, size_t src_len, char* dst) |
| { |
| if (src == NULL || src_len == 0 || dst == NULL) { |
| return; |
| } |
| |
| const char32_t *cur_utf32 = src; |
| const char32_t *end_utf32 = src + src_len; |
| char *cur = dst; |
| while (cur_utf32 < end_utf32) { |
| size_t len = utf32_codepoint_utf8_length(*cur_utf32); |
| utf32_codepoint_to_utf8((uint8_t *)cur, *cur_utf32++, len); |
| cur += len; |
| } |
| *cur = '\0'; |
| } |
| |
| // -------------------------------------------------------------------------- |
| // UTF-16 |
| // -------------------------------------------------------------------------- |
| |
| int strcmp16(const char16_t *s1, const char16_t *s2) |
| { |
| char16_t ch; |
| int d = 0; |
| |
| while ( 1 ) { |
| d = (int)(ch = *s1++) - (int)*s2++; |
| if ( d || !ch ) |
| break; |
| } |
| |
| return d; |
| } |
| |
| int strncmp16(const char16_t *s1, const char16_t *s2, size_t n) |
| { |
| char16_t ch; |
| int d = 0; |
| |
| while ( n-- ) { |
| d = (int)(ch = *s1++) - (int)*s2++; |
| if ( d || !ch ) |
| break; |
| } |
| |
| return d; |
| } |
| |
| char16_t *strcpy16(char16_t *dst, const char16_t *src) |
| { |
| char16_t *q = dst; |
| const char16_t *p = src; |
| char16_t ch; |
| |
| do { |
| *q++ = ch = *p++; |
| } while ( ch ); |
| |
| return dst; |
| } |
| |
| size_t strlen16(const char16_t *s) |
| { |
| const char16_t *ss = s; |
| while ( *ss ) |
| ss++; |
| return ss-s; |
| } |
| |
| |
| char16_t *strncpy16(char16_t *dst, const char16_t *src, size_t n) |
| { |
| char16_t *q = dst; |
| const char16_t *p = src; |
| char ch; |
| |
| while (n) { |
| n--; |
| *q++ = ch = *p++; |
| if ( !ch ) |
| break; |
| } |
| |
| *q = 0; |
| |
| return dst; |
| } |
| |
| size_t strnlen16(const char16_t *s, size_t maxlen) |
| { |
| const char16_t *ss = s; |
| |
| /* Important: the maxlen test must precede the reference through ss; |
| since the byte beyond the maximum may segfault */ |
| while ((maxlen > 0) && *ss) { |
| ss++; |
| maxlen--; |
| } |
| return ss-s; |
| } |
| |
| int strzcmp16(const char16_t *s1, size_t n1, const char16_t *s2, size_t n2) |
| { |
| const char16_t* e1 = s1+n1; |
| const char16_t* e2 = s2+n2; |
| |
| while (s1 < e1 && s2 < e2) { |
| const int d = (int)*s1++ - (int)*s2++; |
| if (d) { |
| return d; |
| } |
| } |
| |
| return n1 < n2 |
| ? (0 - (int)*s2) |
| : (n1 > n2 |
| ? ((int)*s1 - 0) |
| : 0); |
| } |
| |
| int strzcmp16_h_n(const char16_t *s1H, size_t n1, const char16_t *s2N, size_t n2) |
| { |
| const char16_t* e1 = s1H+n1; |
| const char16_t* e2 = s2N+n2; |
| |
| while (s1H < e1 && s2N < e2) { |
| const char16_t c2 = ntohs(*s2N); |
| const int d = (int)*s1H++ - (int)c2; |
| s2N++; |
| if (d) { |
| return d; |
| } |
| } |
| |
| return n1 < n2 |
| ? (0 - (int)ntohs(*s2N)) |
| : (n1 > n2 |
| ? ((int)*s1H - 0) |
| : 0); |
| } |
| |
| void utf16_to_utf8(const char16_t* src, size_t src_len, char* dst) |
| { |
| if (src == NULL || src_len == 0 || dst == NULL) { |
| return; |
| } |
| |
| const char16_t* cur_utf16 = src; |
| const char16_t* const end_utf16 = src + src_len; |
| char *cur = dst; |
| while (cur_utf16 < end_utf16) { |
| char32_t utf32; |
| // surrogate pairs |
| if ((*cur_utf16 & 0xFC00) == 0xD800) { |
| utf32 = (*cur_utf16++ - 0xD800) << 10; |
| utf32 |= *cur_utf16++ - 0xDC00; |
| utf32 += 0x10000; |
| } else { |
| utf32 = (char32_t) *cur_utf16++; |
| } |
| const size_t len = utf32_codepoint_utf8_length(utf32); |
| utf32_codepoint_to_utf8((uint8_t*)cur, utf32, len); |
| cur += len; |
| } |
| *cur = '\0'; |
| } |
| |
| // -------------------------------------------------------------------------- |
| // UTF-8 |
| // -------------------------------------------------------------------------- |
| |
| ssize_t utf8_length(const char *src) |
| { |
| const char *cur = src; |
| size_t ret = 0; |
| while (*cur != '\0') { |
| const char first_char = *cur++; |
| if ((first_char & 0x80) == 0) { // ASCII |
| ret += 1; |
| continue; |
| } |
| // (UTF-8's character must not be like 10xxxxxx, |
| // but 110xxxxx, 1110xxxx, ... or 1111110x) |
| if ((first_char & 0x40) == 0) { |
| return -1; |
| } |
| |
| int32_t mask, to_ignore_mask; |
| size_t num_to_read = 0; |
| char32_t utf32 = 0; |
| for (num_to_read = 1, mask = 0x40, to_ignore_mask = 0x80; |
| num_to_read < 5 && (first_char & mask); |
| num_to_read++, to_ignore_mask |= mask, mask >>= 1) { |
| if ((*cur & 0xC0) != 0x80) { // must be 10xxxxxx |
| return -1; |
| } |
| // 0x3F == 00111111 |
| utf32 = (utf32 << 6) + (*cur++ & 0x3F); |
| } |
| // "first_char" must be (110xxxxx - 11110xxx) |
| if (num_to_read == 5) { |
| return -1; |
| } |
| to_ignore_mask |= mask; |
| utf32 |= ((~to_ignore_mask) & first_char) << (6 * (num_to_read - 1)); |
| if (utf32 > kUnicodeMaxCodepoint) { |
| return -1; |
| } |
| |
| ret += num_to_read; |
| } |
| return ret; |
| } |
| |
| ssize_t utf16_to_utf8_length(const char16_t *src, size_t src_len) |
| { |
| if (src == NULL || src_len == 0) { |
| return -1; |
| } |
| |
| size_t ret = 0; |
| const char16_t* const end = src + src_len; |
| while (src < end) { |
| if ((*src & 0xFC00) == 0xD800 && (src + 1) < end |
| && (*++src & 0xFC00) == 0xDC00) { |
| // surrogate pairs are always 4 bytes. |
| ret += 4; |
| src++; |
| } else { |
| ret += utf32_codepoint_utf8_length((char32_t) *src++); |
| } |
| } |
| return ret; |
| } |
| |
| /** |
| * Returns 1-4 based on the number of leading bits. |
| * |
| * 1111 -> 4 |
| * 1110 -> 3 |
| * 110x -> 2 |
| * 10xx -> 1 |
| * 0xxx -> 1 |
| */ |
| static inline size_t utf8_codepoint_len(uint8_t ch) |
| { |
| return ((0xe5000000 >> ((ch >> 3) & 0x1e)) & 3) + 1; |
| } |
| |
| static inline void utf8_shift_and_mask(uint32_t* codePoint, const uint8_t byte) |
| { |
| *codePoint <<= 6; |
| *codePoint |= 0x3F & byte; |
| } |
| |
| size_t utf8_to_utf32_length(const char *src, size_t src_len) |
| { |
| if (src == NULL || src_len == 0) { |
| return 0; |
| } |
| size_t ret = 0; |
| const char* cur; |
| const char* end; |
| size_t num_to_skip; |
| for (cur = src, end = src + src_len, num_to_skip = 1; |
| cur < end; |
| cur += num_to_skip, ret++) { |
| const char first_char = *cur; |
| num_to_skip = 1; |
| if ((first_char & 0x80) == 0) { // ASCII |
| continue; |
| } |
| int32_t mask; |
| |
| for (mask = 0x40; (first_char & mask); num_to_skip++, mask >>= 1) { |
| } |
| } |
| return ret; |
| } |
| |
| void utf8_to_utf32(const char* src, size_t src_len, char32_t* dst) |
| { |
| if (src == NULL || src_len == 0 || dst == NULL) { |
| return; |
| } |
| |
| const char* cur = src; |
| const char* const end = src + src_len; |
| char32_t* cur_utf32 = dst; |
| while (cur < end) { |
| size_t num_read; |
| *cur_utf32++ = static_cast<char32_t>(utf32_at_internal(cur, &num_read)); |
| cur += num_read; |
| } |
| *cur_utf32 = 0; |
| } |
| |
| static inline uint32_t utf8_to_utf32_codepoint(const uint8_t *src, size_t length) |
| { |
| uint32_t unicode; |
| |
| switch (length) |
| { |
| case 1: |
| return src[0]; |
| case 2: |
| unicode = src[0] & 0x1f; |
| utf8_shift_and_mask(&unicode, src[1]); |
| return unicode; |
| case 3: |
| unicode = src[0] & 0x0f; |
| utf8_shift_and_mask(&unicode, src[1]); |
| utf8_shift_and_mask(&unicode, src[2]); |
| return unicode; |
| case 4: |
| unicode = src[0] & 0x07; |
| utf8_shift_and_mask(&unicode, src[1]); |
| utf8_shift_and_mask(&unicode, src[2]); |
| utf8_shift_and_mask(&unicode, src[3]); |
| return unicode; |
| default: |
| return 0xffff; |
| } |
| |
| //printf("Char at %p: len=%d, utf-16=%p\n", src, length, (void*)result); |
| } |
| |
| ssize_t utf8_to_utf16_length(const uint8_t* u8str, size_t u8len) |
| { |
| const uint8_t* const u8end = u8str + u8len; |
| const uint8_t* u8cur = u8str; |
| |
| /* Validate that the UTF-8 is the correct len */ |
| size_t u16measuredLen = 0; |
| while (u8cur < u8end) { |
| u16measuredLen++; |
| int u8charLen = utf8_codepoint_len(*u8cur); |
| uint32_t codepoint = utf8_to_utf32_codepoint(u8cur, u8charLen); |
| if (codepoint > 0xFFFF) u16measuredLen++; // this will be a surrogate pair in utf16 |
| u8cur += u8charLen; |
| } |
| |
| /** |
| * Make sure that we ended where we thought we would and the output UTF-16 |
| * will be exactly how long we were told it would be. |
| */ |
| if (u8cur != u8end) { |
| return -1; |
| } |
| |
| return u16measuredLen; |
| } |
| |
| char16_t* utf8_to_utf16_no_null_terminator(const uint8_t* u8str, size_t u8len, char16_t* u16str) |
| { |
| const uint8_t* const u8end = u8str + u8len; |
| const uint8_t* u8cur = u8str; |
| char16_t* u16cur = u16str; |
| |
| while (u8cur < u8end) { |
| size_t u8len = utf8_codepoint_len(*u8cur); |
| uint32_t codepoint = utf8_to_utf32_codepoint(u8cur, u8len); |
| |
| // Convert the UTF32 codepoint to one or more UTF16 codepoints |
| if (codepoint <= 0xFFFF) { |
| // Single UTF16 character |
| *u16cur++ = (char16_t) codepoint; |
| } else { |
| // Multiple UTF16 characters with surrogates |
| codepoint = codepoint - 0x10000; |
| *u16cur++ = (char16_t) ((codepoint >> 10) + 0xD800); |
| *u16cur++ = (char16_t) ((codepoint & 0x3FF) + 0xDC00); |
| } |
| |
| u8cur += u8len; |
| } |
| return u16cur; |
| } |
| |
| void utf8_to_utf16(const uint8_t* u8str, size_t u8len, char16_t* u16str) { |
| char16_t* end = utf8_to_utf16_no_null_terminator(u8str, u8len, u16str); |
| *end = 0; |
| } |
| |
| char16_t* utf8_to_utf16_n(const uint8_t* src, size_t srcLen, char16_t* dst, size_t dstLen) { |
| const uint8_t* const u8end = src + srcLen; |
| const uint8_t* u8cur = src; |
| const uint16_t* const u16end = dst + dstLen; |
| char16_t* u16cur = dst; |
| |
| while (u8cur < u8end && u16cur < u16end) { |
| size_t u8len = utf8_codepoint_len(*u8cur); |
| uint32_t codepoint = utf8_to_utf32_codepoint(u8cur, u8len); |
| |
| // Convert the UTF32 codepoint to one or more UTF16 codepoints |
| if (codepoint <= 0xFFFF) { |
| // Single UTF16 character |
| *u16cur++ = (char16_t) codepoint; |
| } else { |
| // Multiple UTF16 characters with surrogates |
| codepoint = codepoint - 0x10000; |
| *u16cur++ = (char16_t) ((codepoint >> 10) + 0xD800); |
| if (u16cur >= u16end) { |
| // Ooops... not enough room for this surrogate pair. |
| return u16cur-1; |
| } |
| *u16cur++ = (char16_t) ((codepoint & 0x3FF) + 0xDC00); |
| } |
| |
| u8cur += u8len; |
| } |
| return u16cur; |
| } |
| |
| } |