Denys Vlasenko | 42a8fd0 | 2009-07-11 21:36:13 +0200 | [diff] [blame] | 1 | /* vi: set sw=4 ts=4: */ |
| 2 | /* |
| 3 | * Unicode support routines. |
| 4 | * |
| 5 | * Copyright (C) 2008 Denys Vlasenko |
| 6 | * |
| 7 | * Licensed under GPL version 2, see file LICENSE in this tarball for details. |
| 8 | */ |
| 9 | #include "libbb.h" |
Denys Vlasenko | fda8f57 | 2009-07-11 22:26:48 +0200 | [diff] [blame^] | 10 | # include "unicode.h" |
Denys Vlasenko | 42a8fd0 | 2009-07-11 21:36:13 +0200 | [diff] [blame] | 11 | |
Denys Vlasenko | fda8f57 | 2009-07-11 22:26:48 +0200 | [diff] [blame^] | 12 | size_t FAST_FUNC bb_mbstrlen(const char *string) |
| 13 | { |
| 14 | size_t width = mbstowcs(NULL, string, INT_MAX); |
| 15 | if (width == (size_t)-1L) |
| 16 | return strlen(string); |
| 17 | return width; |
| 18 | } |
Denys Vlasenko | 42a8fd0 | 2009-07-11 21:36:13 +0200 | [diff] [blame] | 19 | |
| 20 | #if !ENABLE_LOCALE_SUPPORT |
Denys Vlasenko | 42a8fd0 | 2009-07-11 21:36:13 +0200 | [diff] [blame] | 21 | |
Denys Vlasenko | fda8f57 | 2009-07-11 22:26:48 +0200 | [diff] [blame^] | 22 | /* Crude "locale support" which knows only C and Unicode locales */ |
| 23 | |
| 24 | /* unicode_is_enabled: |
| 25 | * 0: not known yet, |
Denys Vlasenko | 42a8fd0 | 2009-07-11 21:36:13 +0200 | [diff] [blame] | 26 | * 1: not unicode (IOW: assuming one char == one byte) |
| 27 | * 2: unicode |
| 28 | */ |
| 29 | # if !ENABLE_FEATURE_CHECK_UNICODE_IN_ENV |
| 30 | # define unicode_is_enabled 2 |
| 31 | # else |
| 32 | static smallint unicode_is_enabled; |
| 33 | void FAST_FUNC check_unicode_in_env(void) |
| 34 | { |
| 35 | char *lang; |
| 36 | |
| 37 | if (unicode_is_enabled) |
| 38 | return; |
| 39 | unicode_is_enabled = 1; |
| 40 | |
| 41 | lang = getenv("LANG"); |
| 42 | if (!lang || !strstr(lang, ".utf8")) |
| 43 | return; |
| 44 | |
| 45 | unicode_is_enabled = 2; |
| 46 | } |
| 47 | # endif |
| 48 | |
| 49 | static size_t wcrtomb_internal(char *s, wchar_t wc) |
| 50 | { |
Denys Vlasenko | fda8f57 | 2009-07-11 22:26:48 +0200 | [diff] [blame^] | 51 | int n; |
Denys Vlasenko | 42a8fd0 | 2009-07-11 21:36:13 +0200 | [diff] [blame] | 52 | uint32_t v = wc; |
| 53 | |
| 54 | if (v <= 0x7f) { |
| 55 | *s = v; |
| 56 | return 1; |
| 57 | } |
| 58 | |
Denys Vlasenko | fda8f57 | 2009-07-11 22:26:48 +0200 | [diff] [blame^] | 59 | /* RFC 3629 says that Unicode ends at 10FFFF, |
| 60 | * but we cover entire 32 bits */ |
Denys Vlasenko | 42a8fd0 | 2009-07-11 21:36:13 +0200 | [diff] [blame] | 61 | |
Denys Vlasenko | fda8f57 | 2009-07-11 22:26:48 +0200 | [diff] [blame^] | 62 | n = 2; |
Denys Vlasenko | 42a8fd0 | 2009-07-11 21:36:13 +0200 | [diff] [blame] | 63 | /* 4000000-FFFFFFFF -> 111111tt 10tttttt 10zzzzzz 10zzyyyy 10yyyyxx 10xxxxxx */ |
Denys Vlasenko | fda8f57 | 2009-07-11 22:26:48 +0200 | [diff] [blame^] | 64 | if (v >= 0x4000000) { |
| 65 | s[5] = (wc & 0x3f) | 0x80; |
| 66 | wc = (uint32_t)wc >> 6; /* ensuring that high bits are 0 */ |
| 67 | n++; |
| 68 | } |
| 69 | /* 200000-3FFFFFF -> 111110tt 10zzzzzz 10zzyyyy 10yyyyxx 10xxxxxx */ |
| 70 | if (v >= 0x200000) { |
| 71 | s[4] = (wc & 0x3f) | 0x80; |
| 72 | wc >>= 6; |
| 73 | n++; |
| 74 | } |
| 75 | /* 10000-1FFFFF -> 11110zzz 10zzyyyy 10yyyyxx 10xxxxxx */ |
| 76 | if (v >= 0x10000) { |
| 77 | s[3] = (wc & 0x3f) | 0x80; |
| 78 | wc >>= 6; |
| 79 | n++; |
| 80 | } |
| 81 | /* 800-FFFF -> 1110yyyy 10yyyyxx 10xxxxxx */ |
| 82 | if (v >= 0x800) { |
| 83 | s[2] = (wc & 0x3f) | 0x80; |
| 84 | wc >>= 6; |
| 85 | n++; |
| 86 | } |
| 87 | s[1] = (wc & 0x3f) | 0x80; |
| 88 | wc >>= 6; |
| 89 | s[0] = wc | (uint8_t)(0x3f00 >> n); |
| 90 | return n; |
Denys Vlasenko | 42a8fd0 | 2009-07-11 21:36:13 +0200 | [diff] [blame] | 91 | } |
| 92 | |
| 93 | size_t FAST_FUNC wcrtomb(char *s, wchar_t wc, mbstate_t *ps UNUSED_PARAM) |
| 94 | { |
| 95 | if (unicode_is_enabled != 2) { |
| 96 | *s = wc; |
| 97 | return 1; |
| 98 | } |
| 99 | |
| 100 | return wcrtomb_internal(s, wc); |
| 101 | } |
| 102 | |
| 103 | size_t FAST_FUNC wcstombs(char *dest, const wchar_t *src, size_t n) |
| 104 | { |
| 105 | size_t org_n = n; |
| 106 | |
| 107 | if (unicode_is_enabled != 2) { |
| 108 | while (n) { |
| 109 | wchar_t c = *src++; |
| 110 | *dest++ = c; |
| 111 | if (c == 0) |
| 112 | break; |
| 113 | n--; |
| 114 | } |
| 115 | return org_n - n; |
| 116 | } |
| 117 | |
| 118 | while (n >= MB_CUR_MAX) { |
| 119 | wchar_t wc = *src++; |
| 120 | size_t len = wcrtomb_internal(dest, wc); |
| 121 | |
| 122 | if (wc == L'\0') |
| 123 | return org_n - n; |
| 124 | dest += len; |
| 125 | n -= len; |
| 126 | } |
| 127 | while (n) { |
| 128 | char tbuf[MB_CUR_MAX]; |
| 129 | wchar_t wc = *src++; |
| 130 | size_t len = wcrtomb_internal(tbuf, wc); |
| 131 | |
| 132 | if (len > n) |
| 133 | len = n; |
| 134 | memcpy(dest, tbuf, len); |
| 135 | if (wc == L'\0') |
| 136 | return org_n - n; |
| 137 | dest += len; |
| 138 | n -= len; |
| 139 | } |
| 140 | return org_n - n; |
| 141 | } |
| 142 | |
| 143 | size_t FAST_FUNC mbstowcs(wchar_t *dest, const char *src, size_t n) |
| 144 | { |
| 145 | size_t org_n = n; |
| 146 | |
| 147 | if (unicode_is_enabled != 2) { |
| 148 | while (n) { |
| 149 | unsigned char c = *src++; |
Denys Vlasenko | fda8f57 | 2009-07-11 22:26:48 +0200 | [diff] [blame^] | 150 | |
| 151 | if (dest) |
| 152 | *dest++ = c; |
Denys Vlasenko | 42a8fd0 | 2009-07-11 21:36:13 +0200 | [diff] [blame] | 153 | if (c == 0) |
| 154 | break; |
| 155 | n--; |
| 156 | } |
| 157 | return org_n - n; |
| 158 | } |
| 159 | |
| 160 | while (n) { |
| 161 | int bytes; |
| 162 | unsigned c = (unsigned char) *src++; |
| 163 | |
| 164 | if (c <= 0x7f) { |
Denys Vlasenko | fda8f57 | 2009-07-11 22:26:48 +0200 | [diff] [blame^] | 165 | if (dest) |
| 166 | *dest++ = c; |
Denys Vlasenko | 42a8fd0 | 2009-07-11 21:36:13 +0200 | [diff] [blame] | 167 | if (c == '\0') |
| 168 | break; |
| 169 | n--; |
| 170 | continue; |
| 171 | } |
| 172 | |
| 173 | /* 80-7FF -> 110yyyxx 10xxxxxx */ |
| 174 | /* 800-FFFF -> 1110yyyy 10yyyyxx 10xxxxxx */ |
| 175 | /* 10000-1FFFFF -> 11110zzz 10zzyyyy 10yyyyxx 10xxxxxx */ |
| 176 | /* 200000-3FFFFFF -> 111110tt 10zzzzzz 10zzyyyy 10yyyyxx 10xxxxxx */ |
| 177 | /* 4000000-FFFFFFFF -> 111111tt 10tttttt 10zzzzzz 10zzyyyy 10yyyyxx 10xxxxxx */ |
| 178 | bytes = 0; |
| 179 | do { |
| 180 | c <<= 1; |
| 181 | bytes++; |
| 182 | } while ((c & 0x80) && bytes < 6); |
| 183 | if (bytes == 1) |
| 184 | return (size_t) -1L; |
| 185 | c = (uint8_t)(c) >> bytes; |
| 186 | |
| 187 | while (--bytes) { |
| 188 | unsigned ch = (unsigned char) *src++; |
| 189 | if ((ch & 0xc0) != 0x80) { |
| 190 | return (size_t) -1L; |
| 191 | } |
| 192 | c = (c << 6) + (ch & 0x3f); |
| 193 | } |
| 194 | |
| 195 | /* TODO */ |
| 196 | /* Need to check that c isn't produced by overlong encoding */ |
| 197 | /* Example: 11000000 10000000 converts to NUL */ |
| 198 | /* 11110000 10000000 10000100 10000000 converts to 0x100 */ |
| 199 | /* correct encoding: 11000100 10000000 */ |
| 200 | if (c <= 0x7f) { /* crude check */ |
| 201 | return (size_t) -1L; |
| 202 | //or maybe: c = 0xfffd; /* replacement character */ |
| 203 | } |
| 204 | |
Denys Vlasenko | fda8f57 | 2009-07-11 22:26:48 +0200 | [diff] [blame^] | 205 | if (dest) |
| 206 | *dest++ = c; |
Denys Vlasenko | 42a8fd0 | 2009-07-11 21:36:13 +0200 | [diff] [blame] | 207 | n--; |
| 208 | } |
| 209 | |
| 210 | return org_n - n; |
| 211 | } |
| 212 | |
| 213 | int FAST_FUNC iswspace(wint_t wc) |
| 214 | { |
| 215 | return (unsigned)wc <= 0x7f && isspace(wc); |
| 216 | } |
| 217 | |
| 218 | int FAST_FUNC iswalnum(wint_t wc) |
| 219 | { |
| 220 | return (unsigned)wc <= 0x7f && isalnum(wc); |
| 221 | } |
| 222 | |
| 223 | int FAST_FUNC iswpunct(wint_t wc) |
| 224 | { |
| 225 | return (unsigned)wc <= 0x7f && ispunct(wc); |
| 226 | } |
| 227 | |
| 228 | #endif |