blob: 5e9fdf7fc88526b140167f4d825a7950c23d4f05 [file] [log] [blame]
Elliott Hughes2faa5f12012-01-30 14:42:07 -08001/*
2 * Copyright (C) 2011 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
Elliott Hughes814e4032011-08-23 12:07:56 -070016
17#include "utf.h"
18
Elliott Hughes07ed66b2012-12-12 18:34:25 -080019#include "base/logging.h"
Ian Rogers2dd0e2c2013-01-24 12:42:14 -080020#include "mirror/array.h"
Ian Rogers4f6ad8a2013-03-18 15:27:28 -070021#include "mirror/object-inl.h"
Ian Rogersa6724902013-09-23 09:23:37 -070022#include "utf-inl.h"
Elliott Hughesb465ab02011-08-24 11:21:21 -070023
Elliott Hughes814e4032011-08-23 12:07:56 -070024namespace art {
25
Bruce Hoult1646d7a2015-10-28 15:06:12 +030026// This is used only from debugger and test code.
Elliott Hughes814e4032011-08-23 12:07:56 -070027size_t CountModifiedUtf8Chars(const char* utf8) {
Bruce Hoult1646d7a2015-10-28 15:06:12 +030028 return CountModifiedUtf8Chars(utf8, strlen(utf8));
29}
30
31/*
32 * This does not validate UTF8 rules (nor did older code). But it gets the right answer
33 * for valid UTF-8 and that's fine because it's used only to size a buffer for later
34 * conversion.
35 *
36 * Modified UTF-8 consists of a series of bytes up to 21 bit Unicode code points as follows:
37 * U+0001 - U+007F 0xxxxxxx
38 * U+0080 - U+07FF 110xxxxx 10xxxxxx
39 * U+0800 - U+FFFF 1110xxxx 10xxxxxx 10xxxxxx
40 * U+10000 - U+1FFFFF 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
41 *
42 * U+0000 is encoded using the 2nd form to avoid nulls inside strings (this differs from
43 * standard UTF-8).
44 * The four byte encoding converts to two utf16 characters.
45 */
46size_t CountModifiedUtf8Chars(const char* utf8, size_t byte_count) {
47 DCHECK_LE(byte_count, strlen(utf8));
Elliott Hughes814e4032011-08-23 12:07:56 -070048 size_t len = 0;
Bruce Hoult1646d7a2015-10-28 15:06:12 +030049 const char* end = utf8 + byte_count;
50 for (; utf8 < end; ++utf8) {
51 int ic = *utf8;
Elliott Hughes814e4032011-08-23 12:07:56 -070052 len++;
Bruce Hoult1646d7a2015-10-28 15:06:12 +030053 if (LIKELY((ic & 0x80) == 0)) {
54 // One-byte encoding.
Elliott Hughes814e4032011-08-23 12:07:56 -070055 continue;
56 }
Bruce Hoult1646d7a2015-10-28 15:06:12 +030057 // Two- or three-byte encoding.
Elliott Hughes814e4032011-08-23 12:07:56 -070058 utf8++;
59 if ((ic & 0x20) == 0) {
Bruce Hoult1646d7a2015-10-28 15:06:12 +030060 // Two-byte encoding.
Elliott Hughes814e4032011-08-23 12:07:56 -070061 continue;
62 }
Elliott Hughes814e4032011-08-23 12:07:56 -070063 utf8++;
Narayan Kamatha5afcfc2015-01-29 20:06:46 +000064 if ((ic & 0x10) == 0) {
Bruce Hoult1646d7a2015-10-28 15:06:12 +030065 // Three-byte encoding.
Narayan Kamatha5afcfc2015-01-29 20:06:46 +000066 continue;
67 }
68
Bruce Hoult1646d7a2015-10-28 15:06:12 +030069 // Four-byte encoding: needs to be converted into a surrogate
Narayan Kamatha5afcfc2015-01-29 20:06:46 +000070 // pair.
71 utf8++;
72 len++;
Elliott Hughes814e4032011-08-23 12:07:56 -070073 }
74 return len;
75}
76
Bruce Hoult1646d7a2015-10-28 15:06:12 +030077// This is used only from debugger and test code.
Elliott Hughes814e4032011-08-23 12:07:56 -070078void ConvertModifiedUtf8ToUtf16(uint16_t* utf16_data_out, const char* utf8_data_in) {
79 while (*utf8_data_in != '\0') {
Narayan Kamatha5afcfc2015-01-29 20:06:46 +000080 const uint32_t ch = GetUtf16FromUtf8(&utf8_data_in);
81 const uint16_t leading = GetLeadingUtf16Char(ch);
82 const uint16_t trailing = GetTrailingUtf16Char(ch);
83
84 *utf16_data_out++ = leading;
85 if (trailing != 0) {
86 *utf16_data_out++ = trailing;
87 }
Elliott Hughes814e4032011-08-23 12:07:56 -070088 }
89}
90
Bruce Hoult1646d7a2015-10-28 15:06:12 +030091void ConvertModifiedUtf8ToUtf16(uint16_t* utf16_data_out, size_t out_chars,
92 const char* utf8_data_in, size_t in_bytes) {
93 const char *in_start = utf8_data_in;
94 const char *in_end = utf8_data_in + in_bytes;
95 uint16_t *out_p = utf16_data_out;
96
97 if (LIKELY(out_chars == in_bytes)) {
98 // Common case where all characters are ASCII.
99 for (const char *p = in_start; p < in_end;) {
100 // Safe even if char is signed because ASCII characters always have
101 // the high bit cleared.
102 *out_p++ = dchecked_integral_cast<uint16_t>(*p++);
103 }
104 return;
105 }
106
107 // String contains non-ASCII characters.
108 for (const char *p = in_start; p < in_end;) {
109 const uint32_t ch = GetUtf16FromUtf8(&p);
110 const uint16_t leading = GetLeadingUtf16Char(ch);
111 const uint16_t trailing = GetTrailingUtf16Char(ch);
112
113 *out_p++ = leading;
114 if (trailing != 0) {
115 *out_p++ = trailing;
116 }
117 }
118}
119
120void ConvertUtf16ToModifiedUtf8(char* utf8_out, size_t byte_count,
121 const uint16_t* utf16_in, size_t char_count) {
122 if (LIKELY(byte_count == char_count)) {
123 // Common case where all characters are ASCII.
124 const uint16_t *utf16_end = utf16_in + char_count;
125 for (const uint16_t *p = utf16_in; p < utf16_end;) {
126 *utf8_out++ = dchecked_integral_cast<char>(*p++);
127 }
128 return;
129 }
130
131 // String contains non-ASCII characters.
Elliott Hughesb465ab02011-08-24 11:21:21 -0700132 while (char_count--) {
Narayan Kamathe16dad12015-02-13 11:49:22 +0000133 const uint16_t ch = *utf16_in++;
Elliott Hughesb465ab02011-08-24 11:21:21 -0700134 if (ch > 0 && ch <= 0x7f) {
135 *utf8_out++ = ch;
136 } else {
Bruce Hoult1646d7a2015-10-28 15:06:12 +0300137 // Char_count == 0 here implies we've encountered an unpaired
Narayan Kamathe16dad12015-02-13 11:49:22 +0000138 // surrogate and we have no choice but to encode it as 3-byte UTF
139 // sequence. Note that unpaired surrogates can occur as a part of
140 // "normal" operation.
141 if ((ch >= 0xd800 && ch <= 0xdbff) && (char_count > 0)) {
142 const uint16_t ch2 = *utf16_in;
143
144 // Check if the other half of the pair is within the expected
145 // range. If it isn't, we will have to emit both "halves" as
146 // separate 3 byte sequences.
147 if (ch2 >= 0xdc00 && ch2 <= 0xdfff) {
148 utf16_in++;
149 char_count--;
150 const uint32_t code_point = (ch << 10) + ch2 - 0x035fdc00;
151 *utf8_out++ = (code_point >> 18) | 0xf0;
152 *utf8_out++ = ((code_point >> 12) & 0x3f) | 0x80;
153 *utf8_out++ = ((code_point >> 6) & 0x3f) | 0x80;
154 *utf8_out++ = (code_point & 0x3f) | 0x80;
155 continue;
156 }
157 }
158
Elliott Hughesb465ab02011-08-24 11:21:21 -0700159 if (ch > 0x07ff) {
Narayan Kamathe16dad12015-02-13 11:49:22 +0000160 // Three byte encoding.
Elliott Hughesb465ab02011-08-24 11:21:21 -0700161 *utf8_out++ = (ch >> 12) | 0xe0;
162 *utf8_out++ = ((ch >> 6) & 0x3f) | 0x80;
163 *utf8_out++ = (ch & 0x3f) | 0x80;
164 } else /*(ch > 0x7f || ch == 0)*/ {
Narayan Kamathe16dad12015-02-13 11:49:22 +0000165 // Two byte encoding.
Elliott Hughesb465ab02011-08-24 11:21:21 -0700166 *utf8_out++ = (ch >> 6) | 0xc0;
167 *utf8_out++ = (ch & 0x3f) | 0x80;
168 }
169 }
170 }
171}
172
Elliott Hughes814e4032011-08-23 12:07:56 -0700173int32_t ComputeUtf16Hash(const uint16_t* chars, size_t char_count) {
Ian Rogers8f41dc32014-10-30 15:16:16 -0700174 uint32_t hash = 0;
Elliott Hughes814e4032011-08-23 12:07:56 -0700175 while (char_count--) {
176 hash = hash * 31 + *chars++;
177 }
Ian Rogers8f41dc32014-10-30 15:16:16 -0700178 return static_cast<int32_t>(hash);
Elliott Hughes814e4032011-08-23 12:07:56 -0700179}
180
Vladimir Markocac5a7e2016-02-22 10:39:50 +0000181int32_t ComputeUtf16HashFromModifiedUtf8(const char* utf8, size_t utf16_length) {
182 uint32_t hash = 0;
183 while (utf16_length != 0u) {
184 const uint32_t pair = GetUtf16FromUtf8(&utf8);
185 const uint16_t first = GetLeadingUtf16Char(pair);
186 hash = hash * 31 + first;
187 --utf16_length;
188 const uint16_t second = GetTrailingUtf16Char(pair);
189 if (second != 0) {
190 hash = hash * 31 + second;
191 DCHECK_NE(utf16_length, 0u);
192 --utf16_length;
193 }
194 }
195 return static_cast<int32_t>(hash);
196}
197
Mathieu Chartier208a5cb2015-12-02 15:44:07 -0800198uint32_t ComputeModifiedUtf8Hash(const char* chars) {
199 uint32_t hash = 0;
Ian Rogers68b56852014-08-29 20:19:11 -0700200 while (*chars != '\0') {
Mathieu Chartiere7c9a8c2014-11-06 16:35:45 -0800201 hash = hash * 31 + *chars++;
Ian Rogers68b56852014-08-29 20:19:11 -0700202 }
Ian Rogers8f41dc32014-10-30 15:16:16 -0700203 return static_cast<int32_t>(hash);
Ian Rogers68b56852014-08-29 20:19:11 -0700204}
205
Vladimir Markoa48aef42014-12-03 17:53:53 +0000206int CompareModifiedUtf8ToUtf16AsCodePointValues(const char* utf8, const uint16_t* utf16,
207 size_t utf16_length) {
Ian Rogers637c65b2013-05-31 11:46:00 -0700208 for (;;) {
Vladimir Markoa48aef42014-12-03 17:53:53 +0000209 if (*utf8 == '\0') {
210 return (utf16_length == 0) ? 0 : -1;
211 } else if (utf16_length == 0) {
Ian Rogers637c65b2013-05-31 11:46:00 -0700212 return 1;
213 }
214
Narayan Kamatha5afcfc2015-01-29 20:06:46 +0000215 const uint32_t pair = GetUtf16FromUtf8(&utf8);
Ian Rogers637c65b2013-05-31 11:46:00 -0700216
Narayan Kamatha5afcfc2015-01-29 20:06:46 +0000217 // First compare the leading utf16 char.
218 const uint16_t lhs = GetLeadingUtf16Char(pair);
219 const uint16_t rhs = *utf16++;
220 --utf16_length;
221 if (lhs != rhs) {
222 return lhs > rhs ? 1 : -1;
223 }
224
225 // Then compare the trailing utf16 char. First check if there
226 // are any characters left to consume.
227 const uint16_t lhs2 = GetTrailingUtf16Char(pair);
228 if (lhs2 != 0) {
229 if (utf16_length == 0) {
230 return 1;
231 }
232
233 const uint16_t rhs2 = *utf16++;
234 --utf16_length;
235 if (lhs2 != rhs2) {
236 return lhs2 > rhs2 ? 1 : -1;
237 }
Ian Rogers637c65b2013-05-31 11:46:00 -0700238 }
239 }
240}
241
Elliott Hughes814e4032011-08-23 12:07:56 -0700242size_t CountUtf8Bytes(const uint16_t* chars, size_t char_count) {
243 size_t result = 0;
Bruce Hoult1646d7a2015-10-28 15:06:12 +0300244 const uint16_t *end = chars + char_count;
245 while (chars < end) {
Narayan Kamathe16dad12015-02-13 11:49:22 +0000246 const uint16_t ch = *chars++;
Bruce Hoult1646d7a2015-10-28 15:06:12 +0300247 if (LIKELY(ch != 0 && ch < 0x80)) {
248 result++;
249 continue;
250 }
251 if (ch < 0x800) {
252 result += 2;
253 continue;
254 }
255 if (ch >= 0xd800 && ch < 0xdc00) {
256 if (chars < end) {
Narayan Kamathe16dad12015-02-13 11:49:22 +0000257 const uint16_t ch2 = *chars;
258 // If we find a properly paired surrogate, we emit it as a 4 byte
259 // UTF sequence. If we find an unpaired leading or trailing surrogate,
260 // we emit it as a 3 byte sequence like would have done earlier.
Bruce Hoult1646d7a2015-10-28 15:06:12 +0300261 if (ch2 >= 0xdc00 && ch2 < 0xe000) {
Narayan Kamathe16dad12015-02-13 11:49:22 +0000262 chars++;
Narayan Kamathe16dad12015-02-13 11:49:22 +0000263 result += 4;
Bruce Hoult1646d7a2015-10-28 15:06:12 +0300264 continue;
Narayan Kamathe16dad12015-02-13 11:49:22 +0000265 }
Elliott Hughes814e4032011-08-23 12:07:56 -0700266 }
Elliott Hughes814e4032011-08-23 12:07:56 -0700267 }
Bruce Hoult1646d7a2015-10-28 15:06:12 +0300268 result += 3;
Elliott Hughes814e4032011-08-23 12:07:56 -0700269 }
270 return result;
271}
272
273} // namespace art