Denys Vlasenko | 42a8fd0 | 2009-07-11 21:36:13 +0200 | [diff] [blame] | 1 | /* vi: set sw=4 ts=4: */ |
| 2 | /* |
| 3 | * Unicode support routines. |
| 4 | * |
Denys Vlasenko | f6106e6 | 2009-07-16 02:27:04 +0200 | [diff] [blame^] | 5 | * Copyright (C) 2009 Denys Vlasenko |
Denys Vlasenko | 42a8fd0 | 2009-07-11 21:36:13 +0200 | [diff] [blame] | 6 | * |
| 7 | * Licensed under GPL version 2, see file LICENSE in this tarball for details. |
| 8 | */ |
| 9 | #include "libbb.h" |
Denys Vlasenko | fda8f57 | 2009-07-11 22:26:48 +0200 | [diff] [blame] | 10 | # include "unicode.h" |
Denys Vlasenko | 42a8fd0 | 2009-07-11 21:36:13 +0200 | [diff] [blame] | 11 | |
Denys Vlasenko | fda8f57 | 2009-07-11 22:26:48 +0200 | [diff] [blame] | 12 | size_t FAST_FUNC bb_mbstrlen(const char *string) |
| 13 | { |
| 14 | size_t width = mbstowcs(NULL, string, INT_MAX); |
| 15 | if (width == (size_t)-1L) |
| 16 | return strlen(string); |
| 17 | return width; |
| 18 | } |
Denys Vlasenko | 42a8fd0 | 2009-07-11 21:36:13 +0200 | [diff] [blame] | 19 | |
| 20 | #if !ENABLE_LOCALE_SUPPORT |
Denys Vlasenko | 42a8fd0 | 2009-07-11 21:36:13 +0200 | [diff] [blame] | 21 | |
Denys Vlasenko | fda8f57 | 2009-07-11 22:26:48 +0200 | [diff] [blame] | 22 | /* Crude "locale support" which knows only C and Unicode locales */ |
| 23 | |
| 24 | /* unicode_is_enabled: |
| 25 | * 0: not known yet, |
Denys Vlasenko | 42a8fd0 | 2009-07-11 21:36:13 +0200 | [diff] [blame] | 26 | * 1: not unicode (IOW: assuming one char == one byte) |
| 27 | * 2: unicode |
| 28 | */ |
| 29 | # if !ENABLE_FEATURE_CHECK_UNICODE_IN_ENV |
| 30 | # define unicode_is_enabled 2 |
| 31 | # else |
| 32 | static smallint unicode_is_enabled; |
| 33 | void FAST_FUNC check_unicode_in_env(void) |
| 34 | { |
| 35 | char *lang; |
| 36 | |
| 37 | if (unicode_is_enabled) |
| 38 | return; |
| 39 | unicode_is_enabled = 1; |
| 40 | |
| 41 | lang = getenv("LANG"); |
| 42 | if (!lang || !strstr(lang, ".utf8")) |
| 43 | return; |
| 44 | |
| 45 | unicode_is_enabled = 2; |
| 46 | } |
| 47 | # endif |
| 48 | |
| 49 | static size_t wcrtomb_internal(char *s, wchar_t wc) |
| 50 | { |
Denys Vlasenko | fda8f57 | 2009-07-11 22:26:48 +0200 | [diff] [blame] | 51 | int n; |
Denys Vlasenko | 42a8fd0 | 2009-07-11 21:36:13 +0200 | [diff] [blame] | 52 | uint32_t v = wc; |
| 53 | |
| 54 | if (v <= 0x7f) { |
| 55 | *s = v; |
| 56 | return 1; |
| 57 | } |
| 58 | |
Denys Vlasenko | fda8f57 | 2009-07-11 22:26:48 +0200 | [diff] [blame] | 59 | /* RFC 3629 says that Unicode ends at 10FFFF, |
| 60 | * but we cover entire 32 bits */ |
Denys Vlasenko | 42a8fd0 | 2009-07-11 21:36:13 +0200 | [diff] [blame] | 61 | |
Denys Vlasenko | fda8f57 | 2009-07-11 22:26:48 +0200 | [diff] [blame] | 62 | n = 2; |
Denys Vlasenko | 42a8fd0 | 2009-07-11 21:36:13 +0200 | [diff] [blame] | 63 | /* 4000000-FFFFFFFF -> 111111tt 10tttttt 10zzzzzz 10zzyyyy 10yyyyxx 10xxxxxx */ |
Denys Vlasenko | fda8f57 | 2009-07-11 22:26:48 +0200 | [diff] [blame] | 64 | if (v >= 0x4000000) { |
| 65 | s[5] = (wc & 0x3f) | 0x80; |
| 66 | wc = (uint32_t)wc >> 6; /* ensuring that high bits are 0 */ |
| 67 | n++; |
| 68 | } |
| 69 | /* 200000-3FFFFFF -> 111110tt 10zzzzzz 10zzyyyy 10yyyyxx 10xxxxxx */ |
| 70 | if (v >= 0x200000) { |
| 71 | s[4] = (wc & 0x3f) | 0x80; |
| 72 | wc >>= 6; |
| 73 | n++; |
| 74 | } |
| 75 | /* 10000-1FFFFF -> 11110zzz 10zzyyyy 10yyyyxx 10xxxxxx */ |
| 76 | if (v >= 0x10000) { |
| 77 | s[3] = (wc & 0x3f) | 0x80; |
| 78 | wc >>= 6; |
| 79 | n++; |
| 80 | } |
| 81 | /* 800-FFFF -> 1110yyyy 10yyyyxx 10xxxxxx */ |
| 82 | if (v >= 0x800) { |
| 83 | s[2] = (wc & 0x3f) | 0x80; |
| 84 | wc >>= 6; |
| 85 | n++; |
| 86 | } |
Denys Vlasenko | f6106e6 | 2009-07-16 02:27:04 +0200 | [diff] [blame^] | 87 | /* 80-7FF -> 110yyyxx 10xxxxxx */ |
Denys Vlasenko | fda8f57 | 2009-07-11 22:26:48 +0200 | [diff] [blame] | 88 | s[1] = (wc & 0x3f) | 0x80; |
| 89 | wc >>= 6; |
| 90 | s[0] = wc | (uint8_t)(0x3f00 >> n); |
| 91 | return n; |
Denys Vlasenko | 42a8fd0 | 2009-07-11 21:36:13 +0200 | [diff] [blame] | 92 | } |
| 93 | |
| 94 | size_t FAST_FUNC wcrtomb(char *s, wchar_t wc, mbstate_t *ps UNUSED_PARAM) |
| 95 | { |
| 96 | if (unicode_is_enabled != 2) { |
| 97 | *s = wc; |
| 98 | return 1; |
| 99 | } |
| 100 | |
| 101 | return wcrtomb_internal(s, wc); |
| 102 | } |
| 103 | |
| 104 | size_t FAST_FUNC wcstombs(char *dest, const wchar_t *src, size_t n) |
| 105 | { |
| 106 | size_t org_n = n; |
| 107 | |
| 108 | if (unicode_is_enabled != 2) { |
| 109 | while (n) { |
| 110 | wchar_t c = *src++; |
| 111 | *dest++ = c; |
| 112 | if (c == 0) |
| 113 | break; |
| 114 | n--; |
| 115 | } |
| 116 | return org_n - n; |
| 117 | } |
| 118 | |
| 119 | while (n >= MB_CUR_MAX) { |
| 120 | wchar_t wc = *src++; |
| 121 | size_t len = wcrtomb_internal(dest, wc); |
| 122 | |
| 123 | if (wc == L'\0') |
| 124 | return org_n - n; |
| 125 | dest += len; |
| 126 | n -= len; |
| 127 | } |
| 128 | while (n) { |
| 129 | char tbuf[MB_CUR_MAX]; |
| 130 | wchar_t wc = *src++; |
| 131 | size_t len = wcrtomb_internal(tbuf, wc); |
| 132 | |
| 133 | if (len > n) |
| 134 | len = n; |
| 135 | memcpy(dest, tbuf, len); |
| 136 | if (wc == L'\0') |
| 137 | return org_n - n; |
| 138 | dest += len; |
| 139 | n -= len; |
| 140 | } |
| 141 | return org_n - n; |
| 142 | } |
| 143 | |
| 144 | size_t FAST_FUNC mbstowcs(wchar_t *dest, const char *src, size_t n) |
| 145 | { |
| 146 | size_t org_n = n; |
| 147 | |
| 148 | if (unicode_is_enabled != 2) { |
| 149 | while (n) { |
| 150 | unsigned char c = *src++; |
Denys Vlasenko | fda8f57 | 2009-07-11 22:26:48 +0200 | [diff] [blame] | 151 | |
| 152 | if (dest) |
| 153 | *dest++ = c; |
Denys Vlasenko | 42a8fd0 | 2009-07-11 21:36:13 +0200 | [diff] [blame] | 154 | if (c == 0) |
| 155 | break; |
| 156 | n--; |
| 157 | } |
| 158 | return org_n - n; |
| 159 | } |
| 160 | |
| 161 | while (n) { |
| 162 | int bytes; |
| 163 | unsigned c = (unsigned char) *src++; |
| 164 | |
| 165 | if (c <= 0x7f) { |
Denys Vlasenko | fda8f57 | 2009-07-11 22:26:48 +0200 | [diff] [blame] | 166 | if (dest) |
| 167 | *dest++ = c; |
Denys Vlasenko | 42a8fd0 | 2009-07-11 21:36:13 +0200 | [diff] [blame] | 168 | if (c == '\0') |
| 169 | break; |
| 170 | n--; |
| 171 | continue; |
| 172 | } |
| 173 | |
| 174 | /* 80-7FF -> 110yyyxx 10xxxxxx */ |
| 175 | /* 800-FFFF -> 1110yyyy 10yyyyxx 10xxxxxx */ |
| 176 | /* 10000-1FFFFF -> 11110zzz 10zzyyyy 10yyyyxx 10xxxxxx */ |
| 177 | /* 200000-3FFFFFF -> 111110tt 10zzzzzz 10zzyyyy 10yyyyxx 10xxxxxx */ |
| 178 | /* 4000000-FFFFFFFF -> 111111tt 10tttttt 10zzzzzz 10zzyyyy 10yyyyxx 10xxxxxx */ |
| 179 | bytes = 0; |
| 180 | do { |
| 181 | c <<= 1; |
| 182 | bytes++; |
| 183 | } while ((c & 0x80) && bytes < 6); |
| 184 | if (bytes == 1) |
| 185 | return (size_t) -1L; |
| 186 | c = (uint8_t)(c) >> bytes; |
| 187 | |
| 188 | while (--bytes) { |
| 189 | unsigned ch = (unsigned char) *src++; |
| 190 | if ((ch & 0xc0) != 0x80) { |
| 191 | return (size_t) -1L; |
| 192 | } |
| 193 | c = (c << 6) + (ch & 0x3f); |
| 194 | } |
| 195 | |
| 196 | /* TODO */ |
| 197 | /* Need to check that c isn't produced by overlong encoding */ |
| 198 | /* Example: 11000000 10000000 converts to NUL */ |
| 199 | /* 11110000 10000000 10000100 10000000 converts to 0x100 */ |
| 200 | /* correct encoding: 11000100 10000000 */ |
| 201 | if (c <= 0x7f) { /* crude check */ |
| 202 | return (size_t) -1L; |
| 203 | //or maybe: c = 0xfffd; /* replacement character */ |
| 204 | } |
| 205 | |
Denys Vlasenko | fda8f57 | 2009-07-11 22:26:48 +0200 | [diff] [blame] | 206 | if (dest) |
| 207 | *dest++ = c; |
Denys Vlasenko | 42a8fd0 | 2009-07-11 21:36:13 +0200 | [diff] [blame] | 208 | n--; |
| 209 | } |
| 210 | |
| 211 | return org_n - n; |
| 212 | } |
| 213 | |
| 214 | int FAST_FUNC iswspace(wint_t wc) |
| 215 | { |
| 216 | return (unsigned)wc <= 0x7f && isspace(wc); |
| 217 | } |
| 218 | |
| 219 | int FAST_FUNC iswalnum(wint_t wc) |
| 220 | { |
| 221 | return (unsigned)wc <= 0x7f && isalnum(wc); |
| 222 | } |
| 223 | |
| 224 | int FAST_FUNC iswpunct(wint_t wc) |
| 225 | { |
| 226 | return (unsigned)wc <= 0x7f && ispunct(wc); |
| 227 | } |
| 228 | |
| 229 | #endif |