Kenny Root | ba0165b | 2010-11-09 14:37:23 -0800 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (C) 2005 The Android Open Source Project |
| 3 | * |
| 4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | * you may not use this file except in compliance with the License. |
| 6 | * You may obtain a copy of the License at |
| 7 | * |
| 8 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | * |
| 10 | * Unless required by applicable law or agreed to in writing, software |
| 11 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | * See the License for the specific language governing permissions and |
| 14 | * limitations under the License. |
| 15 | */ |
| 16 | |
Mark Salyzyn | cfd5b08 | 2016-10-17 14:28:00 -0700 | [diff] [blame] | 17 | #define LOG_TAG "unicode" |
| 18 | |
Mathias Agopian | 22dbf39 | 2017-02-28 15:06:51 -0800 | [diff] [blame] | 19 | #include <utils/Unicode.h> |
Sergio Giro | 9de6776 | 2016-07-20 20:01:33 +0100 | [diff] [blame] | 20 | #include <limits.h> |
Kenny Root | ba0165b | 2010-11-09 14:37:23 -0800 | [diff] [blame] | 21 | |
Mark Salyzyn | 30f991f | 2017-01-10 13:19:54 -0800 | [diff] [blame] | 22 | #include <log/log.h> |
Mark Salyzyn | ff2dcd9 | 2016-09-28 15:54:45 -0700 | [diff] [blame] | 23 | |
Elliott Hughes | adbf442 | 2015-07-29 17:45:24 -0700 | [diff] [blame] | 24 | #if defined(_WIN32) |
Kenny Root | ba0165b | 2010-11-09 14:37:23 -0800 | [diff] [blame] | 25 | # undef nhtol |
| 26 | # undef htonl |
| 27 | # undef nhtos |
| 28 | # undef htons |
| 29 | |
Elliott Hughes | 97ac0e1 | 2014-11-21 23:01:59 -0800 | [diff] [blame] | 30 | # define ntohl(x) ( ((x) << 24) | (((x) >> 24) & 255) | (((x) << 8) & 0xff0000) | (((x) >> 8) & 0xff00) ) |
| 31 | # define htonl(x) ntohl(x) |
| 32 | # define ntohs(x) ( (((x) << 8) & 0xff00) | (((x) >> 8) & 255) ) |
| 33 | # define htons(x) ntohs(x) |
Kenny Root | ba0165b | 2010-11-09 14:37:23 -0800 | [diff] [blame] | 34 | #else |
| 35 | # include <netinet/in.h> |
| 36 | #endif |
| 37 | |
| 38 | extern "C" { |
| 39 | |
| 40 | static const char32_t kByteMask = 0x000000BF; |
| 41 | static const char32_t kByteMark = 0x00000080; |
| 42 | |
| 43 | // Surrogates aren't valid for UTF-32 characters, so define some |
| 44 | // constants that will let us screen them out. |
| 45 | static const char32_t kUnicodeSurrogateHighStart = 0x0000D800; |
Andreas Gampe | a53c815 | 2014-11-24 09:42:07 -0800 | [diff] [blame] | 46 | // Unused, here for completeness: |
| 47 | // static const char32_t kUnicodeSurrogateHighEnd = 0x0000DBFF; |
| 48 | // static const char32_t kUnicodeSurrogateLowStart = 0x0000DC00; |
Kenny Root | ba0165b | 2010-11-09 14:37:23 -0800 | [diff] [blame] | 49 | static const char32_t kUnicodeSurrogateLowEnd = 0x0000DFFF; |
| 50 | static const char32_t kUnicodeSurrogateStart = kUnicodeSurrogateHighStart; |
| 51 | static const char32_t kUnicodeSurrogateEnd = kUnicodeSurrogateLowEnd; |
| 52 | static const char32_t kUnicodeMaxCodepoint = 0x0010FFFF; |
| 53 | |
| 54 | // Mask used to set appropriate bits in first byte of UTF-8 sequence, |
| 55 | // indexed by number of bytes in the sequence. |
| 56 | // 0xxxxxxx |
| 57 | // -> (00-7f) 7bit. Bit mask for the first byte is 0x00000000 |
| 58 | // 110yyyyx 10xxxxxx |
| 59 | // -> (c0-df)(80-bf) 11bit. Bit mask is 0x000000C0 |
| 60 | // 1110yyyy 10yxxxxx 10xxxxxx |
| 61 | // -> (e0-ef)(80-bf)(80-bf) 16bit. Bit mask is 0x000000E0 |
| 62 | // 11110yyy 10yyxxxx 10xxxxxx 10xxxxxx |
| 63 | // -> (f0-f7)(80-bf)(80-bf)(80-bf) 21bit. Bit mask is 0x000000F0 |
| 64 | static const char32_t kFirstByteMark[] = { |
| 65 | 0x00000000, 0x00000000, 0x000000C0, 0x000000E0, 0x000000F0 |
| 66 | }; |
| 67 | |
| 68 | // -------------------------------------------------------------------------- |
| 69 | // UTF-32 |
| 70 | // -------------------------------------------------------------------------- |
| 71 | |
| 72 | /** |
| 73 | * Return number of UTF-8 bytes required for the character. If the character |
| 74 | * is invalid, return size of 0. |
| 75 | */ |
| 76 | static inline size_t utf32_codepoint_utf8_length(char32_t srcChar) |
| 77 | { |
| 78 | // Figure out how many bytes the result will require. |
| 79 | if (srcChar < 0x00000080) { |
| 80 | return 1; |
| 81 | } else if (srcChar < 0x00000800) { |
| 82 | return 2; |
| 83 | } else if (srcChar < 0x00010000) { |
| 84 | if ((srcChar < kUnicodeSurrogateStart) || (srcChar > kUnicodeSurrogateEnd)) { |
| 85 | return 3; |
| 86 | } else { |
| 87 | // Surrogates are invalid UTF-32 characters. |
| 88 | return 0; |
| 89 | } |
| 90 | } |
| 91 | // Max code point for Unicode is 0x0010FFFF. |
| 92 | else if (srcChar <= kUnicodeMaxCodepoint) { |
| 93 | return 4; |
| 94 | } else { |
| 95 | // Invalid UTF-32 character. |
| 96 | return 0; |
| 97 | } |
| 98 | } |
| 99 | |
| 100 | // Write out the source character to <dstP>. |
| 101 | |
| 102 | static inline void utf32_codepoint_to_utf8(uint8_t* dstP, char32_t srcChar, size_t bytes) |
| 103 | { |
| 104 | dstP += bytes; |
| 105 | switch (bytes) |
| 106 | { /* note: everything falls through. */ |
| 107 | case 4: *--dstP = (uint8_t)((srcChar | kByteMark) & kByteMask); srcChar >>= 6; |
| 108 | case 3: *--dstP = (uint8_t)((srcChar | kByteMark) & kByteMask); srcChar >>= 6; |
| 109 | case 2: *--dstP = (uint8_t)((srcChar | kByteMark) & kByteMask); srcChar >>= 6; |
| 110 | case 1: *--dstP = (uint8_t)(srcChar | kFirstByteMark[bytes]); |
| 111 | } |
| 112 | } |
| 113 | |
| 114 | size_t strlen32(const char32_t *s) |
| 115 | { |
| 116 | const char32_t *ss = s; |
| 117 | while ( *ss ) |
| 118 | ss++; |
| 119 | return ss-s; |
| 120 | } |
| 121 | |
| 122 | size_t strnlen32(const char32_t *s, size_t maxlen) |
| 123 | { |
| 124 | const char32_t *ss = s; |
| 125 | while ((maxlen > 0) && *ss) { |
| 126 | ss++; |
| 127 | maxlen--; |
| 128 | } |
| 129 | return ss-s; |
| 130 | } |
| 131 | |
| 132 | static inline int32_t utf32_at_internal(const char* cur, size_t *num_read) |
| 133 | { |
| 134 | const char first_char = *cur; |
| 135 | if ((first_char & 0x80) == 0) { // ASCII |
| 136 | *num_read = 1; |
| 137 | return *cur; |
| 138 | } |
| 139 | cur++; |
| 140 | char32_t mask, to_ignore_mask; |
| 141 | size_t num_to_read = 0; |
| 142 | char32_t utf32 = first_char; |
| 143 | for (num_to_read = 1, mask = 0x40, to_ignore_mask = 0xFFFFFF80; |
| 144 | (first_char & mask); |
| 145 | num_to_read++, to_ignore_mask |= mask, mask >>= 1) { |
| 146 | // 0x3F == 00111111 |
| 147 | utf32 = (utf32 << 6) + (*cur++ & 0x3F); |
| 148 | } |
| 149 | to_ignore_mask |= mask; |
| 150 | utf32 &= ~(to_ignore_mask << (6 * (num_to_read - 1))); |
| 151 | |
| 152 | *num_read = num_to_read; |
| 153 | return static_cast<int32_t>(utf32); |
| 154 | } |
| 155 | |
| 156 | int32_t utf32_from_utf8_at(const char *src, size_t src_len, size_t index, size_t *next_index) |
| 157 | { |
| 158 | if (index >= src_len) { |
| 159 | return -1; |
| 160 | } |
| 161 | size_t dummy_index; |
| 162 | if (next_index == NULL) { |
| 163 | next_index = &dummy_index; |
| 164 | } |
| 165 | size_t num_read; |
| 166 | int32_t ret = utf32_at_internal(src + index, &num_read); |
| 167 | if (ret >= 0) { |
| 168 | *next_index = index + num_read; |
| 169 | } |
| 170 | |
| 171 | return ret; |
| 172 | } |
| 173 | |
| 174 | ssize_t utf32_to_utf8_length(const char32_t *src, size_t src_len) |
| 175 | { |
| 176 | if (src == NULL || src_len == 0) { |
| 177 | return -1; |
| 178 | } |
| 179 | |
| 180 | size_t ret = 0; |
| 181 | const char32_t *end = src + src_len; |
| 182 | while (src < end) { |
Adam Vartanian | 47efc67 | 2017-08-14 15:51:29 +0100 | [diff] [blame] | 183 | size_t char_len = utf32_codepoint_utf8_length(*src++); |
| 184 | if (SSIZE_MAX - char_len < ret) { |
| 185 | // If this happens, we would overflow the ssize_t type when |
| 186 | // returning from this function, so we cannot express how |
| 187 | // long this string is in an ssize_t. |
| 188 | android_errorWriteLog(0x534e4554, "37723026"); |
| 189 | return -1; |
| 190 | } |
| 191 | ret += char_len; |
Kenny Root | ba0165b | 2010-11-09 14:37:23 -0800 | [diff] [blame] | 192 | } |
| 193 | return ret; |
| 194 | } |
| 195 | |
Sergio Giro | 1cfa56d | 2016-06-28 18:02:29 +0100 | [diff] [blame] | 196 | void utf32_to_utf8(const char32_t* src, size_t src_len, char* dst, size_t dst_len) |
Kenny Root | ba0165b | 2010-11-09 14:37:23 -0800 | [diff] [blame] | 197 | { |
| 198 | if (src == NULL || src_len == 0 || dst == NULL) { |
| 199 | return; |
| 200 | } |
| 201 | |
| 202 | const char32_t *cur_utf32 = src; |
| 203 | const char32_t *end_utf32 = src + src_len; |
| 204 | char *cur = dst; |
| 205 | while (cur_utf32 < end_utf32) { |
| 206 | size_t len = utf32_codepoint_utf8_length(*cur_utf32); |
Sergio Giro | 1cfa56d | 2016-06-28 18:02:29 +0100 | [diff] [blame] | 207 | LOG_ALWAYS_FATAL_IF(dst_len < len, "%zu < %zu", dst_len, len); |
Kenny Root | ba0165b | 2010-11-09 14:37:23 -0800 | [diff] [blame] | 208 | utf32_codepoint_to_utf8((uint8_t *)cur, *cur_utf32++, len); |
| 209 | cur += len; |
Sergio Giro | 1cfa56d | 2016-06-28 18:02:29 +0100 | [diff] [blame] | 210 | dst_len -= len; |
Kenny Root | ba0165b | 2010-11-09 14:37:23 -0800 | [diff] [blame] | 211 | } |
Sergio Giro | 1cfa56d | 2016-06-28 18:02:29 +0100 | [diff] [blame] | 212 | LOG_ALWAYS_FATAL_IF(dst_len < 1, "dst_len < 1: %zu < 1", dst_len); |
Kenny Root | ba0165b | 2010-11-09 14:37:23 -0800 | [diff] [blame] | 213 | *cur = '\0'; |
| 214 | } |
| 215 | |
| 216 | // -------------------------------------------------------------------------- |
| 217 | // UTF-16 |
| 218 | // -------------------------------------------------------------------------- |
| 219 | |
| 220 | int strcmp16(const char16_t *s1, const char16_t *s2) |
| 221 | { |
| 222 | char16_t ch; |
| 223 | int d = 0; |
| 224 | |
| 225 | while ( 1 ) { |
| 226 | d = (int)(ch = *s1++) - (int)*s2++; |
| 227 | if ( d || !ch ) |
| 228 | break; |
| 229 | } |
| 230 | |
| 231 | return d; |
| 232 | } |
| 233 | |
| 234 | int strncmp16(const char16_t *s1, const char16_t *s2, size_t n) |
| 235 | { |
| 236 | char16_t ch; |
| 237 | int d = 0; |
| 238 | |
Michael Wright | 5bacef3 | 2016-05-09 14:43:31 +0100 | [diff] [blame] | 239 | if (n == 0) { |
| 240 | return 0; |
Kenny Root | ba0165b | 2010-11-09 14:37:23 -0800 | [diff] [blame] | 241 | } |
| 242 | |
Michael Wright | 5bacef3 | 2016-05-09 14:43:31 +0100 | [diff] [blame] | 243 | do { |
| 244 | d = (int)(ch = *s1++) - (int)*s2++; |
| 245 | if ( d || !ch ) { |
| 246 | break; |
| 247 | } |
| 248 | } while (--n); |
| 249 | |
Kenny Root | ba0165b | 2010-11-09 14:37:23 -0800 | [diff] [blame] | 250 | return d; |
| 251 | } |
| 252 | |
| 253 | char16_t *strcpy16(char16_t *dst, const char16_t *src) |
| 254 | { |
| 255 | char16_t *q = dst; |
| 256 | const char16_t *p = src; |
| 257 | char16_t ch; |
| 258 | |
| 259 | do { |
| 260 | *q++ = ch = *p++; |
| 261 | } while ( ch ); |
| 262 | |
| 263 | return dst; |
| 264 | } |
| 265 | |
| 266 | size_t strlen16(const char16_t *s) |
| 267 | { |
| 268 | const char16_t *ss = s; |
| 269 | while ( *ss ) |
| 270 | ss++; |
| 271 | return ss-s; |
| 272 | } |
| 273 | |
| 274 | |
| 275 | char16_t *strncpy16(char16_t *dst, const char16_t *src, size_t n) |
| 276 | { |
| 277 | char16_t *q = dst; |
| 278 | const char16_t *p = src; |
| 279 | char ch; |
| 280 | |
| 281 | while (n) { |
| 282 | n--; |
| 283 | *q++ = ch = *p++; |
| 284 | if ( !ch ) |
| 285 | break; |
| 286 | } |
| 287 | |
| 288 | *q = 0; |
| 289 | |
| 290 | return dst; |
| 291 | } |
| 292 | |
| 293 | size_t strnlen16(const char16_t *s, size_t maxlen) |
| 294 | { |
| 295 | const char16_t *ss = s; |
| 296 | |
| 297 | /* Important: the maxlen test must precede the reference through ss; |
| 298 | since the byte beyond the maximum may segfault */ |
| 299 | while ((maxlen > 0) && *ss) { |
| 300 | ss++; |
| 301 | maxlen--; |
| 302 | } |
| 303 | return ss-s; |
| 304 | } |
| 305 | |
Michael Wright | 5bacef3 | 2016-05-09 14:43:31 +0100 | [diff] [blame] | 306 | char16_t* strstr16(const char16_t* src, const char16_t* target) |
| 307 | { |
Branislav Rankov | bf3fff1 | 2017-10-12 15:08:42 +0200 | [diff] [blame] | 308 | const char16_t needle = *target; |
| 309 | if (needle == '\0') return (char16_t*)src; |
| 310 | |
| 311 | const size_t target_len = strlen16(++target); |
| 312 | do { |
Michael Wright | 5bacef3 | 2016-05-09 14:43:31 +0100 | [diff] [blame] | 313 | do { |
Branislav Rankov | bf3fff1 | 2017-10-12 15:08:42 +0200 | [diff] [blame] | 314 | if (*src == '\0') { |
| 315 | return nullptr; |
| 316 | } |
Michael Wright | 5bacef3 | 2016-05-09 14:43:31 +0100 | [diff] [blame] | 317 | } while (*src++ != needle); |
Branislav Rankov | bf3fff1 | 2017-10-12 15:08:42 +0200 | [diff] [blame] | 318 | } while (strncmp16(src, target, target_len) != 0); |
| 319 | src--; |
Michael Wright | 5bacef3 | 2016-05-09 14:43:31 +0100 | [diff] [blame] | 320 | |
| 321 | return (char16_t*)src; |
| 322 | } |
| 323 | |
Kenny Root | ba0165b | 2010-11-09 14:37:23 -0800 | [diff] [blame] | 324 | int strzcmp16(const char16_t *s1, size_t n1, const char16_t *s2, size_t n2) |
| 325 | { |
| 326 | const char16_t* e1 = s1+n1; |
| 327 | const char16_t* e2 = s2+n2; |
| 328 | |
| 329 | while (s1 < e1 && s2 < e2) { |
| 330 | const int d = (int)*s1++ - (int)*s2++; |
| 331 | if (d) { |
| 332 | return d; |
| 333 | } |
| 334 | } |
| 335 | |
| 336 | return n1 < n2 |
| 337 | ? (0 - (int)*s2) |
| 338 | : (n1 > n2 |
| 339 | ? ((int)*s1 - 0) |
| 340 | : 0); |
| 341 | } |
| 342 | |
| 343 | int strzcmp16_h_n(const char16_t *s1H, size_t n1, const char16_t *s2N, size_t n2) |
| 344 | { |
| 345 | const char16_t* e1 = s1H+n1; |
| 346 | const char16_t* e2 = s2N+n2; |
| 347 | |
| 348 | while (s1H < e1 && s2N < e2) { |
| 349 | const char16_t c2 = ntohs(*s2N); |
| 350 | const int d = (int)*s1H++ - (int)c2; |
| 351 | s2N++; |
| 352 | if (d) { |
| 353 | return d; |
| 354 | } |
| 355 | } |
| 356 | |
| 357 | return n1 < n2 |
| 358 | ? (0 - (int)ntohs(*s2N)) |
| 359 | : (n1 > n2 |
| 360 | ? ((int)*s1H - 0) |
| 361 | : 0); |
| 362 | } |
| 363 | |
Sergio Giro | 1cfa56d | 2016-06-28 18:02:29 +0100 | [diff] [blame] | 364 | void utf16_to_utf8(const char16_t* src, size_t src_len, char* dst, size_t dst_len) |
Kenny Root | ba0165b | 2010-11-09 14:37:23 -0800 | [diff] [blame] | 365 | { |
| 366 | if (src == NULL || src_len == 0 || dst == NULL) { |
| 367 | return; |
| 368 | } |
| 369 | |
| 370 | const char16_t* cur_utf16 = src; |
| 371 | const char16_t* const end_utf16 = src + src_len; |
| 372 | char *cur = dst; |
| 373 | while (cur_utf16 < end_utf16) { |
| 374 | char32_t utf32; |
| 375 | // surrogate pairs |
Cylen Yao | 72299bf | 2014-06-04 19:11:27 +0800 | [diff] [blame] | 376 | if((*cur_utf16 & 0xFC00) == 0xD800 && (cur_utf16 + 1) < end_utf16 |
| 377 | && (*(cur_utf16 + 1) & 0xFC00) == 0xDC00) { |
Kenny Root | ba0165b | 2010-11-09 14:37:23 -0800 | [diff] [blame] | 378 | utf32 = (*cur_utf16++ - 0xD800) << 10; |
| 379 | utf32 |= *cur_utf16++ - 0xDC00; |
| 380 | utf32 += 0x10000; |
| 381 | } else { |
| 382 | utf32 = (char32_t) *cur_utf16++; |
| 383 | } |
| 384 | const size_t len = utf32_codepoint_utf8_length(utf32); |
Sergio Giro | 1cfa56d | 2016-06-28 18:02:29 +0100 | [diff] [blame] | 385 | LOG_ALWAYS_FATAL_IF(dst_len < len, "%zu < %zu", dst_len, len); |
Kenny Root | ba0165b | 2010-11-09 14:37:23 -0800 | [diff] [blame] | 386 | utf32_codepoint_to_utf8((uint8_t*)cur, utf32, len); |
| 387 | cur += len; |
Sergio Giro | 1cfa56d | 2016-06-28 18:02:29 +0100 | [diff] [blame] | 388 | dst_len -= len; |
Kenny Root | ba0165b | 2010-11-09 14:37:23 -0800 | [diff] [blame] | 389 | } |
Sergio Giro | 1cfa56d | 2016-06-28 18:02:29 +0100 | [diff] [blame] | 390 | LOG_ALWAYS_FATAL_IF(dst_len < 1, "%zu < 1", dst_len); |
Kenny Root | ba0165b | 2010-11-09 14:37:23 -0800 | [diff] [blame] | 391 | *cur = '\0'; |
| 392 | } |
| 393 | |
| 394 | // -------------------------------------------------------------------------- |
| 395 | // UTF-8 |
| 396 | // -------------------------------------------------------------------------- |
| 397 | |
| 398 | ssize_t utf8_length(const char *src) |
| 399 | { |
| 400 | const char *cur = src; |
| 401 | size_t ret = 0; |
| 402 | while (*cur != '\0') { |
| 403 | const char first_char = *cur++; |
| 404 | if ((first_char & 0x80) == 0) { // ASCII |
| 405 | ret += 1; |
| 406 | continue; |
| 407 | } |
| 408 | // (UTF-8's character must not be like 10xxxxxx, |
| 409 | // but 110xxxxx, 1110xxxx, ... or 1111110x) |
| 410 | if ((first_char & 0x40) == 0) { |
| 411 | return -1; |
| 412 | } |
| 413 | |
| 414 | int32_t mask, to_ignore_mask; |
| 415 | size_t num_to_read = 0; |
| 416 | char32_t utf32 = 0; |
| 417 | for (num_to_read = 1, mask = 0x40, to_ignore_mask = 0x80; |
| 418 | num_to_read < 5 && (first_char & mask); |
| 419 | num_to_read++, to_ignore_mask |= mask, mask >>= 1) { |
| 420 | if ((*cur & 0xC0) != 0x80) { // must be 10xxxxxx |
| 421 | return -1; |
| 422 | } |
| 423 | // 0x3F == 00111111 |
| 424 | utf32 = (utf32 << 6) + (*cur++ & 0x3F); |
| 425 | } |
| 426 | // "first_char" must be (110xxxxx - 11110xxx) |
| 427 | if (num_to_read == 5) { |
| 428 | return -1; |
| 429 | } |
| 430 | to_ignore_mask |= mask; |
| 431 | utf32 |= ((~to_ignore_mask) & first_char) << (6 * (num_to_read - 1)); |
| 432 | if (utf32 > kUnicodeMaxCodepoint) { |
| 433 | return -1; |
| 434 | } |
| 435 | |
| 436 | ret += num_to_read; |
| 437 | } |
| 438 | return ret; |
| 439 | } |
| 440 | |
| 441 | ssize_t utf16_to_utf8_length(const char16_t *src, size_t src_len) |
| 442 | { |
| 443 | if (src == NULL || src_len == 0) { |
| 444 | return -1; |
| 445 | } |
| 446 | |
| 447 | size_t ret = 0; |
| 448 | const char16_t* const end = src + src_len; |
| 449 | while (src < end) { |
Adam Vartanian | 47efc67 | 2017-08-14 15:51:29 +0100 | [diff] [blame] | 450 | size_t char_len; |
Kenny Root | ba0165b | 2010-11-09 14:37:23 -0800 | [diff] [blame] | 451 | if ((*src & 0xFC00) == 0xD800 && (src + 1) < end |
Sergio Giro | 1cfa56d | 2016-06-28 18:02:29 +0100 | [diff] [blame] | 452 | && (*(src + 1) & 0xFC00) == 0xDC00) { |
Kenny Root | ba0165b | 2010-11-09 14:37:23 -0800 | [diff] [blame] | 453 | // surrogate pairs are always 4 bytes. |
Adam Vartanian | 47efc67 | 2017-08-14 15:51:29 +0100 | [diff] [blame] | 454 | char_len = 4; |
Sergio Giro | 1cfa56d | 2016-06-28 18:02:29 +0100 | [diff] [blame] | 455 | src += 2; |
Kenny Root | ba0165b | 2010-11-09 14:37:23 -0800 | [diff] [blame] | 456 | } else { |
Adam Vartanian | 47efc67 | 2017-08-14 15:51:29 +0100 | [diff] [blame] | 457 | char_len = utf32_codepoint_utf8_length((char32_t)*src++); |
Kenny Root | ba0165b | 2010-11-09 14:37:23 -0800 | [diff] [blame] | 458 | } |
Adam Vartanian | 47efc67 | 2017-08-14 15:51:29 +0100 | [diff] [blame] | 459 | if (SSIZE_MAX - char_len < ret) { |
| 460 | // If this happens, we would overflow the ssize_t type when |
| 461 | // returning from this function, so we cannot express how |
| 462 | // long this string is in an ssize_t. |
| 463 | android_errorWriteLog(0x534e4554, "37723026"); |
| 464 | return -1; |
| 465 | } |
| 466 | ret += char_len; |
Kenny Root | ba0165b | 2010-11-09 14:37:23 -0800 | [diff] [blame] | 467 | } |
| 468 | return ret; |
| 469 | } |
| 470 | |
| 471 | /** |
| 472 | * Returns 1-4 based on the number of leading bits. |
| 473 | * |
| 474 | * 1111 -> 4 |
| 475 | * 1110 -> 3 |
| 476 | * 110x -> 2 |
| 477 | * 10xx -> 1 |
| 478 | * 0xxx -> 1 |
| 479 | */ |
| 480 | static inline size_t utf8_codepoint_len(uint8_t ch) |
| 481 | { |
| 482 | return ((0xe5000000 >> ((ch >> 3) & 0x1e)) & 3) + 1; |
| 483 | } |
| 484 | |
| 485 | static inline void utf8_shift_and_mask(uint32_t* codePoint, const uint8_t byte) |
| 486 | { |
| 487 | *codePoint <<= 6; |
| 488 | *codePoint |= 0x3F & byte; |
| 489 | } |
| 490 | |
| 491 | size_t utf8_to_utf32_length(const char *src, size_t src_len) |
| 492 | { |
| 493 | if (src == NULL || src_len == 0) { |
| 494 | return 0; |
| 495 | } |
| 496 | size_t ret = 0; |
| 497 | const char* cur; |
| 498 | const char* end; |
| 499 | size_t num_to_skip; |
| 500 | for (cur = src, end = src + src_len, num_to_skip = 1; |
| 501 | cur < end; |
| 502 | cur += num_to_skip, ret++) { |
| 503 | const char first_char = *cur; |
| 504 | num_to_skip = 1; |
| 505 | if ((first_char & 0x80) == 0) { // ASCII |
| 506 | continue; |
| 507 | } |
| 508 | int32_t mask; |
| 509 | |
| 510 | for (mask = 0x40; (first_char & mask); num_to_skip++, mask >>= 1) { |
| 511 | } |
| 512 | } |
| 513 | return ret; |
| 514 | } |
| 515 | |
| 516 | void utf8_to_utf32(const char* src, size_t src_len, char32_t* dst) |
| 517 | { |
| 518 | if (src == NULL || src_len == 0 || dst == NULL) { |
| 519 | return; |
| 520 | } |
| 521 | |
| 522 | const char* cur = src; |
| 523 | const char* const end = src + src_len; |
| 524 | char32_t* cur_utf32 = dst; |
| 525 | while (cur < end) { |
| 526 | size_t num_read; |
| 527 | *cur_utf32++ = static_cast<char32_t>(utf32_at_internal(cur, &num_read)); |
| 528 | cur += num_read; |
| 529 | } |
| 530 | *cur_utf32 = 0; |
| 531 | } |
| 532 | |
| 533 | static inline uint32_t utf8_to_utf32_codepoint(const uint8_t *src, size_t length) |
| 534 | { |
| 535 | uint32_t unicode; |
| 536 | |
| 537 | switch (length) |
| 538 | { |
| 539 | case 1: |
| 540 | return src[0]; |
| 541 | case 2: |
| 542 | unicode = src[0] & 0x1f; |
| 543 | utf8_shift_and_mask(&unicode, src[1]); |
| 544 | return unicode; |
| 545 | case 3: |
| 546 | unicode = src[0] & 0x0f; |
| 547 | utf8_shift_and_mask(&unicode, src[1]); |
| 548 | utf8_shift_and_mask(&unicode, src[2]); |
| 549 | return unicode; |
| 550 | case 4: |
| 551 | unicode = src[0] & 0x07; |
| 552 | utf8_shift_and_mask(&unicode, src[1]); |
| 553 | utf8_shift_and_mask(&unicode, src[2]); |
| 554 | utf8_shift_and_mask(&unicode, src[3]); |
| 555 | return unicode; |
| 556 | default: |
| 557 | return 0xffff; |
| 558 | } |
| 559 | |
| 560 | //printf("Char at %p: len=%d, utf-16=%p\n", src, length, (void*)result); |
| 561 | } |
| 562 | |
Sergio Giro | 9de6776 | 2016-07-20 20:01:33 +0100 | [diff] [blame] | 563 | ssize_t utf8_to_utf16_length(const uint8_t* u8str, size_t u8len, bool overreadIsFatal) |
Kenny Root | ba0165b | 2010-11-09 14:37:23 -0800 | [diff] [blame] | 564 | { |
| 565 | const uint8_t* const u8end = u8str + u8len; |
| 566 | const uint8_t* u8cur = u8str; |
| 567 | |
| 568 | /* Validate that the UTF-8 is the correct len */ |
| 569 | size_t u16measuredLen = 0; |
| 570 | while (u8cur < u8end) { |
| 571 | u16measuredLen++; |
| 572 | int u8charLen = utf8_codepoint_len(*u8cur); |
Sergio Giro | 9de6776 | 2016-07-20 20:01:33 +0100 | [diff] [blame] | 573 | // Malformed utf8, some characters are beyond the end. |
| 574 | // Cases: |
| 575 | // If u8charLen == 1, this becomes u8cur >= u8end, which cannot happen as u8cur < u8end, |
| 576 | // then this condition fail and we continue, as expected. |
| 577 | // If u8charLen == 2, this becomes u8cur + 1 >= u8end, which fails only if |
| 578 | // u8cur == u8end - 1, that is, there was only one remaining character to read but we need |
| 579 | // 2 of them. This condition holds and we return -1, as expected. |
| 580 | if (u8cur + u8charLen - 1 >= u8end) { |
| 581 | if (overreadIsFatal) { |
| 582 | LOG_ALWAYS_FATAL("Attempt to overread computing length of utf8 string"); |
| 583 | } else { |
| 584 | return -1; |
| 585 | } |
| 586 | } |
Kenny Root | ba0165b | 2010-11-09 14:37:23 -0800 | [diff] [blame] | 587 | uint32_t codepoint = utf8_to_utf32_codepoint(u8cur, u8charLen); |
| 588 | if (codepoint > 0xFFFF) u16measuredLen++; // this will be a surrogate pair in utf16 |
| 589 | u8cur += u8charLen; |
| 590 | } |
| 591 | |
| 592 | /** |
| 593 | * Make sure that we ended where we thought we would and the output UTF-16 |
| 594 | * will be exactly how long we were told it would be. |
| 595 | */ |
| 596 | if (u8cur != u8end) { |
| 597 | return -1; |
| 598 | } |
| 599 | |
| 600 | return u16measuredLen; |
| 601 | } |
| 602 | |
Sergio Giro | 9de6776 | 2016-07-20 20:01:33 +0100 | [diff] [blame] | 603 | char16_t* utf8_to_utf16(const uint8_t* u8str, size_t u8len, char16_t* u16str, size_t u16len) { |
| 604 | // A value > SSIZE_MAX is probably a negative value returned as an error and casted. |
| 605 | LOG_ALWAYS_FATAL_IF(u16len == 0 || u16len > SSIZE_MAX, "u16len is %zu", u16len); |
| 606 | char16_t* end = utf8_to_utf16_no_null_terminator(u8str, u8len, u16str, u16len - 1); |
Jeff Brown | aa983c9 | 2011-10-07 13:28:18 -0700 | [diff] [blame] | 607 | *end = 0; |
Sergio Giro | 9de6776 | 2016-07-20 20:01:33 +0100 | [diff] [blame] | 608 | return end; |
Kenny Root | ba0165b | 2010-11-09 14:37:23 -0800 | [diff] [blame] | 609 | } |
| 610 | |
Sergio Giro | 9de6776 | 2016-07-20 20:01:33 +0100 | [diff] [blame] | 611 | char16_t* utf8_to_utf16_no_null_terminator( |
| 612 | const uint8_t* src, size_t srcLen, char16_t* dst, size_t dstLen) { |
| 613 | if (dstLen == 0) { |
| 614 | return dst; |
| 615 | } |
| 616 | // A value > SSIZE_MAX is probably a negative value returned as an error and casted. |
| 617 | LOG_ALWAYS_FATAL_IF(dstLen > SSIZE_MAX, "dstLen is %zu", dstLen); |
Dianne Hackborn | 0f10d0a | 2013-07-31 16:04:39 -0700 | [diff] [blame] | 618 | const uint8_t* const u8end = src + srcLen; |
| 619 | const uint8_t* u8cur = src; |
Mark Salyzyn | 5bed803 | 2014-04-30 11:10:46 -0700 | [diff] [blame] | 620 | const char16_t* const u16end = dst + dstLen; |
Dianne Hackborn | 0f10d0a | 2013-07-31 16:04:39 -0700 | [diff] [blame] | 621 | char16_t* u16cur = dst; |
| 622 | |
| 623 | while (u8cur < u8end && u16cur < u16end) { |
| 624 | size_t u8len = utf8_codepoint_len(*u8cur); |
| 625 | uint32_t codepoint = utf8_to_utf32_codepoint(u8cur, u8len); |
| 626 | |
| 627 | // Convert the UTF32 codepoint to one or more UTF16 codepoints |
| 628 | if (codepoint <= 0xFFFF) { |
| 629 | // Single UTF16 character |
| 630 | *u16cur++ = (char16_t) codepoint; |
| 631 | } else { |
| 632 | // Multiple UTF16 characters with surrogates |
| 633 | codepoint = codepoint - 0x10000; |
| 634 | *u16cur++ = (char16_t) ((codepoint >> 10) + 0xD800); |
| 635 | if (u16cur >= u16end) { |
| 636 | // Ooops... not enough room for this surrogate pair. |
| 637 | return u16cur-1; |
| 638 | } |
| 639 | *u16cur++ = (char16_t) ((codepoint & 0x3FF) + 0xDC00); |
| 640 | } |
| 641 | |
| 642 | u8cur += u8len; |
| 643 | } |
| 644 | return u16cur; |
| 645 | } |
| 646 | |
Kenny Root | ba0165b | 2010-11-09 14:37:23 -0800 | [diff] [blame] | 647 | } |