Split UTF functions from String8/16
Split out all the UTF-8/16/32 handling code from String8/16 to its own
file to allow better reuse of code.
Change-Id: If9ce63920edc75472c38da4adce0d13cda9ad2f7
diff --git a/include/utils/String16.h b/include/utils/String16.h
index 07a0c11..584f53f 100644
--- a/include/utils/String16.h
+++ b/include/utils/String16.h
@@ -19,39 +19,12 @@
#include <utils/Errors.h>
#include <utils/SharedBuffer.h>
-
-#include <stdint.h>
-#include <sys/types.h>
+#include <utils/Unicode.h>
// ---------------------------------------------------------------------------
extern "C" {
-typedef uint16_t char16_t;
-
-// Standard string functions on char16 strings.
-int strcmp16(const char16_t *, const char16_t *);
-int strncmp16(const char16_t *s1, const char16_t *s2, size_t n);
-size_t strlen16(const char16_t *);
-size_t strnlen16(const char16_t *, size_t);
-char16_t *strcpy16(char16_t *, const char16_t *);
-char16_t *strncpy16(char16_t *, const char16_t *, size_t);
-
-// Version of comparison that supports embedded nulls.
-// This is different than strncmp() because we don't stop
-// at a nul character and consider the strings to be different
-// if the lengths are different (thus we need to supply the
-// lengths of both strings). This can also be used when
-// your string is not nul-terminated as it will have the
-// equivalent result as strcmp16 (unlike strncmp16).
-int strzcmp16(const char16_t *s1, size_t n1, const char16_t *s2, size_t n2);
-
-// Version of strzcmp16 for comparing strings in different endianness.
-int strzcmp16_h_n(const char16_t *s1H, size_t n1, const char16_t *s2N, size_t n2);
-
-// Convert UTF-8 to UTF-16 including surrogate pairs
-void utf8_to_utf16(const uint8_t *src, size_t srcLen, char16_t* dst, const size_t dstLen);
-
}
// ---------------------------------------------------------------------------
diff --git a/include/utils/String8.h b/include/utils/String8.h
index cef8eca..b36f128 100644
--- a/include/utils/String8.h
+++ b/include/utils/String8.h
@@ -18,122 +18,16 @@
#define ANDROID_STRING8_H
#include <utils/Errors.h>
+#include <utils/SharedBuffer.h>
+#include <utils/Unicode.h>
-// Need this for the char16_t type; String8.h should not
-// be depedent on the String16 class.
-#include <utils/String16.h>
-
-#include <stdint.h>
-#include <string.h>
-#include <sys/types.h>
-
-// ---------------------------------------------------------------------------
-
-extern "C" {
-
-typedef uint32_t char32_t;
-
-size_t strlen32(const char32_t *);
-size_t strnlen32(const char32_t *, size_t);
-
-/*
- * Returns the length of "src" when "src" is valid UTF-8 string.
- * Returns 0 if src is NULL, 0-length string or non UTF-8 string.
- * This function should be used to determine whether "src" is valid UTF-8
- * characters with valid unicode codepoints. "src" must be null-terminated.
- *
- * If you are going to use other GetUtf... functions defined in this header
- * with string which may not be valid UTF-8 with valid codepoint (form 0 to
- * 0x10FFFF), you should use this function before calling others, since the
- * other functions do not check whether the string is valid UTF-8 or not.
- *
- * If you do not care whether "src" is valid UTF-8 or not, you should use
- * strlen() as usual, which should be much faster.
- */
-size_t utf8_length(const char *src);
-
-/*
- * Returns the UTF-32 length of "src".
- */
-size_t utf32_length(const char *src, size_t src_len);
-
-/*
- * Returns the UTF-8 length of "src".
- */
-size_t utf8_length_from_utf16(const char16_t *src, size_t src_len);
-
-/*
- * Returns the UTF-8 length of "src".
- */
-size_t utf8_length_from_utf32(const char32_t *src, size_t src_len);
-
-/*
- * Returns the unicode value at "index".
- * Returns -1 when the index is invalid (equals to or more than "src_len").
- * If returned value is positive, it is able to be converted to char32_t, which
- * is unsigned. Then, if "next_index" is not NULL, the next index to be used is
- * stored in "next_index". "next_index" can be NULL.
- */
-int32_t utf32_at(const char *src, size_t src_len,
- size_t index, size_t *next_index);
-
-/*
- * Stores a UTF-32 string converted from "src" in "dst", if "dst_length" is not
- * large enough to store the string, the part of the "src" string is stored
- * into "dst".
- * Returns the size actually used for storing the string.
- * "dst" is not null-terminated when dst_len is fully used (like strncpy).
- */
-size_t utf8_to_utf32(const char* src, size_t src_len,
- char32_t* dst, size_t dst_len);
-
-/*
- * Stores a UTF-8 string converted from "src" in "dst", if "dst_length" is not
- * large enough to store the string, the part of the "src" string is stored
- * into "dst" as much as possible. See the examples for more detail.
- * Returns the size actually used for storing the string.
- * dst" is not null-terminated when dst_len is fully used (like strncpy).
- *
- * Example 1
- * "src" == \u3042\u3044 (\xE3\x81\x82\xE3\x81\x84)
- * "src_len" == 2
- * "dst_len" >= 7
- * ->
- * Returned value == 6
- * "dst" becomes \xE3\x81\x82\xE3\x81\x84\0
- * (note that "dst" is null-terminated)
- *
- * Example 2
- * "src" == \u3042\u3044 (\xE3\x81\x82\xE3\x81\x84)
- * "src_len" == 2
- * "dst_len" == 5
- * ->
- * Returned value == 3
- * "dst" becomes \xE3\x81\x82\0
- * (note that "dst" is null-terminated, but \u3044 is not stored in "dst"
- * since "dst" does not have enough size to store the character)
- *
- * Example 3
- * "src" == \u3042\u3044 (\xE3\x81\x82\xE3\x81\x84)
- * "src_len" == 2
- * "dst_len" == 6
- * ->
- * Returned value == 6
- * "dst" becomes \xE3\x81\x82\xE3\x81\x84
- * (note that "dst" is NOT null-terminated, like strncpy)
- */
-size_t utf32_to_utf8(const char32_t* src, size_t src_len,
- char* dst, size_t dst_len);
-
-size_t utf16_to_utf8(const char16_t* src, size_t src_len,
- char* dst, size_t dst_len);
-
-}
+#include <string.h> // for strcmp
// ---------------------------------------------------------------------------
namespace android {
+class String16;
class TextOutput;
//! This is a string holding UTF-8 characters. Does not allow the value more
@@ -182,7 +76,7 @@
size_t getUtf32Length() const;
int32_t getUtf32At(size_t index,
size_t *next_index) const;
- size_t getUtf32(char32_t* dst, size_t dst_len) const;
+ void getUtf32(char32_t* dst) const;
inline String8& operator=(const String8& other);
inline String8& operator=(const char* other);
diff --git a/include/utils/Unicode.h b/include/utils/Unicode.h
new file mode 100644
index 0000000..6afb291
--- /dev/null
+++ b/include/utils/Unicode.h
@@ -0,0 +1,161 @@
+/*
+ * Copyright (C) 2005 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef ANDROID_UNICODE_H
+#define ANDROID_UNICODE_H
+
+#include <sys/types.h>
+#include <stdint.h>
+
+extern "C" {
+
+typedef uint32_t char32_t;
+typedef uint16_t char16_t;
+
+// Standard string functions on char16_t strings.
+int strcmp16(const char16_t *, const char16_t *);
+int strncmp16(const char16_t *s1, const char16_t *s2, size_t n);
+size_t strlen16(const char16_t *);
+size_t strnlen16(const char16_t *, size_t);
+char16_t *strcpy16(char16_t *, const char16_t *);
+char16_t *strncpy16(char16_t *, const char16_t *, size_t);
+
+// Version of comparison that supports embedded nulls.
+// This is different than strncmp() because we don't stop
+// at a nul character and consider the strings to be different
+// if the lengths are different (thus we need to supply the
+// lengths of both strings). This can also be used when
+// your string is not nul-terminated as it will have the
+// equivalent result as strcmp16 (unlike strncmp16).
+int strzcmp16(const char16_t *s1, size_t n1, const char16_t *s2, size_t n2);
+
+// Version of strzcmp16 for comparing strings in different endianness.
+int strzcmp16_h_n(const char16_t *s1H, size_t n1, const char16_t *s2N, size_t n2);
+
+// Standard string functions on char32_t strings.
+size_t strlen32(const char32_t *);
+size_t strnlen32(const char32_t *, size_t);
+
+/**
+ * Measure the length of a UTF-32 string in UTF-8. If the string is invalid
+ * such as containing a surrogate character, -1 will be returned.
+ */
+ssize_t utf32_to_utf8_length(const char32_t *src, size_t src_len);
+
+/**
+ * Stores a UTF-8 string converted from "src" in "dst", if "dst_length" is not
+ * large enough to store the string, the part of the "src" string is stored
+ * into "dst" as much as possible. See the examples for more detail.
+ * Returns the size actually used for storing the string.
+ * dst" is not null-terminated when dst_len is fully used (like strncpy).
+ *
+ * Example 1
+ * "src" == \u3042\u3044 (\xE3\x81\x82\xE3\x81\x84)
+ * "src_len" == 2
+ * "dst_len" >= 7
+ * ->
+ * Returned value == 6
+ * "dst" becomes \xE3\x81\x82\xE3\x81\x84\0
+ * (note that "dst" is null-terminated)
+ *
+ * Example 2
+ * "src" == \u3042\u3044 (\xE3\x81\x82\xE3\x81\x84)
+ * "src_len" == 2
+ * "dst_len" == 5
+ * ->
+ * Returned value == 3
+ * "dst" becomes \xE3\x81\x82\0
+ * (note that "dst" is null-terminated, but \u3044 is not stored in "dst"
+ * since "dst" does not have enough size to store the character)
+ *
+ * Example 3
+ * "src" == \u3042\u3044 (\xE3\x81\x82\xE3\x81\x84)
+ * "src_len" == 2
+ * "dst_len" == 6
+ * ->
+ * Returned value == 6
+ * "dst" becomes \xE3\x81\x82\xE3\x81\x84
+ * (note that "dst" is NOT null-terminated, like strncpy)
+ */
+void utf32_to_utf8(const char32_t* src, size_t src_len, char* dst);
+
+/**
+ * Returns the unicode value at "index".
+ * Returns -1 when the index is invalid (equals to or more than "src_len").
+ * If returned value is positive, it is able to be converted to char32_t, which
+ * is unsigned. Then, if "next_index" is not NULL, the next index to be used is
+ * stored in "next_index". "next_index" can be NULL.
+ */
+int32_t utf32_from_utf8_at(const char *src, size_t src_len, size_t index, size_t *next_index);
+
+
+/**
+ * Returns the UTF-8 length of UTF-16 string "src".
+ */
+ssize_t utf16_to_utf8_length(const char16_t *src, size_t src_len);
+
+/**
+ * Converts a UTF-16 string to UTF-8. The destination buffer must be large
+ * enough to fit the UTF-16 as measured by utf16_to_utf8_length with an added
+ * NULL terminator.
+ */
+void utf16_to_utf8(const char16_t* src, size_t src_len, char* dst);
+
+/**
+ * Returns the length of "src" when "src" is valid UTF-8 string.
+ * Returns 0 if src is NULL or 0-length string. Returns -1 when the source
+ * is an invalid string.
+ *
+ * This function should be used to determine whether "src" is valid UTF-8
+ * characters with valid unicode codepoints. "src" must be null-terminated.
+ *
+ * If you are going to use other utf8_to_... functions defined in this header
+ * with string which may not be valid UTF-8 with valid codepoint (form 0 to
+ * 0x10FFFF), you should use this function before calling others, since the
+ * other functions do not check whether the string is valid UTF-8 or not.
+ *
+ * If you do not care whether "src" is valid UTF-8 or not, you should use
+ * strlen() as usual, which should be much faster.
+ */
+ssize_t utf8_length(const char *src);
+
+/**
+ * Measure the length of a UTF-32 string.
+ */
+size_t utf8_to_utf32_length(const char *src, size_t src_len);
+
+/**
+ * Stores a UTF-32 string converted from "src" in "dst". "dst" must be large
+ * enough to store the entire converted string as measured by
+ * utf8_to_utf32_length plus space for a NULL terminator.
+ */
+void utf8_to_utf32(const char* src, size_t src_len, char32_t* dst);
+
+/**
+ * Returns the UTF-16 length of UTF-8 string "src".
+ */
+ssize_t utf8_to_utf16_length(const uint8_t* src, size_t srcLen);
+
+/**
+ * Convert UTF-8 to UTF-16 including surrogate pairs. The destination buffer
+ * must be large enough to hold the result as measured by utf8_to_utf16_length
+ * plus an added NULL terminator.
+ */
+void utf8_to_utf16(const uint8_t* src, size_t srcLen, char16_t* dst);
+
+}
+
+#endif
diff --git a/libs/utils/Android.mk b/libs/utils/Android.mk
index eb75ed8..05a9674 100644
--- a/libs/utils/Android.mk
+++ b/libs/utils/Android.mk
@@ -41,6 +41,7 @@
TextOutput.cpp \
Threads.cpp \
Timers.cpp \
+ Unicode.cpp \
VectorImpl.cpp \
ZipFileCRO.cpp \
ZipFileRO.cpp \
diff --git a/libs/utils/ResourceTypes.cpp b/libs/utils/ResourceTypes.cpp
index f287298..bbf5093 100644
--- a/libs/utils/ResourceTypes.cpp
+++ b/libs/utils/ResourceTypes.cpp
@@ -444,15 +444,51 @@
}
}
-#define DECODE_LENGTH(str, chrsz, len) \
- len = *(str); \
- if (*(str)&(1<<(chrsz*8-1))) { \
- (str)++; \
- len = (((len)&((1<<(chrsz*8-1))-1))<<(chrsz*8)) + *(str); \
- } \
- (str)++;
+/**
+ * Strings in UTF-16 format have length indicated by a length encoded in the
+ * stored data. It is either 1 or 2 characters of length data. This allows a
+ * maximum length of 0x7FFFFFF (2147483647 bytes), but if you're storing that
+ * much data in a string, you're abusing them.
+ *
+ * If the high bit is set, then there are two characters or 4 bytes of length
+ * data encoded. In that case, drop the high bit of the first character and
+ * add it together with the next character.
+ */
+static inline size_t
+decodeLength(const char16_t** str)
+{
+ size_t len = **str;
+ if ((len & 0x8000) != 0) {
+ (*str)++;
+ len = ((len & 0x7FFF) << 16) | **str;
+ }
+ (*str)++;
+ return len;
+}
-const uint16_t* ResStringPool::stringAt(size_t idx, size_t* outLen) const
+/**
+ * Strings in UTF-8 format have length indicated by a length encoded in the
+ * stored data. It is either 1 or 2 characters of length data. This allows a
+ * maximum length of 0x7FFF (32767 bytes), but you should consider storing
+ * text in another way if you're using that much data in a single string.
+ *
+ * If the high bit is set, then there are two characters or 2 bytes of length
+ * data encoded. In that case, drop the high bit of the first character and
+ * add it together with the next character.
+ */
+static inline size_t
+decodeLength(const uint8_t** str)
+{
+ size_t len = **str;
+ if ((len & 0x80) != 0) {
+ (*str)++;
+ len = ((len & 0x7F) << 8) | **str;
+ }
+ (*str)++;
+ return len;
+}
+
+const uint16_t* ResStringPool::stringAt(size_t idx, size_t* u16len) const
{
if (mError == NO_ERROR && idx < mHeader->stringCount) {
const bool isUTF8 = (mHeader->flags&ResStringPool_header::UTF8_FLAG) != 0;
@@ -461,37 +497,51 @@
if (!isUTF8) {
const char16_t* strings = (char16_t*)mStrings;
const char16_t* str = strings+off;
- DECODE_LENGTH(str, sizeof(char16_t), *outLen)
- if ((uint32_t)(str+*outLen-strings) < mStringPoolSize) {
+
+ *u16len = decodeLength(&str);
+ if ((uint32_t)(str+*u16len-strings) < mStringPoolSize) {
return str;
} else {
LOGW("Bad string block: string #%d extends to %d, past end at %d\n",
- (int)idx, (int)(str+*outLen-strings), (int)mStringPoolSize);
+ (int)idx, (int)(str+*u16len-strings), (int)mStringPoolSize);
}
} else {
const uint8_t* strings = (uint8_t*)mStrings;
- const uint8_t* str = strings+off;
- DECODE_LENGTH(str, sizeof(uint8_t), *outLen)
- size_t encLen;
- DECODE_LENGTH(str, sizeof(uint8_t), encLen)
- if ((uint32_t)(str+encLen-strings) < mStringPoolSize) {
+ const uint8_t* u8str = strings+off;
+
+ *u16len = decodeLength(&u8str);
+ size_t u8len = decodeLength(&u8str);
+
+ // encLen must be less than 0x7FFF due to encoding.
+ if ((uint32_t)(u8str+u8len-strings) < mStringPoolSize) {
AutoMutex lock(mDecodeLock);
+
if (mCache[idx] != NULL) {
return mCache[idx];
}
- char16_t *u16str = (char16_t *)calloc(*outLen+1, sizeof(char16_t));
+
+ ssize_t actualLen = utf8_to_utf16_length(u8str, u8len);
+ if (actualLen < 0 || (size_t)actualLen != *u16len) {
+ LOGW("Bad string block: string #%lld decoded length is not correct "
+ "%lld vs %llu\n",
+ (long long)idx, (long long)actualLen, (long long)*u16len);
+ return NULL;
+ }
+
+ char16_t *u16str = (char16_t *)calloc(*u16len+1, sizeof(char16_t));
if (!u16str) {
LOGW("No memory when trying to allocate decode cache for string #%d\n",
(int)idx);
return NULL;
}
- const unsigned char *u8src = reinterpret_cast<const unsigned char *>(str);
- utf8_to_utf16(u8src, encLen, u16str, *outLen);
+
+ utf8_to_utf16(u8str, u8len, u16str);
mCache[idx] = u16str;
return u16str;
} else {
- LOGW("Bad string block: string #%d extends to %d, past end at %d\n",
- (int)idx, (int)(str+encLen-strings), (int)mStringPoolSize);
+ LOGW("Bad string block: string #%lld extends to %lld, past end at %lld\n",
+ (long long)idx, (long long)(u8str+u8len-strings),
+ (long long)mStringPoolSize);
}
}
} else {
@@ -512,9 +562,8 @@
if (isUTF8) {
const uint8_t* strings = (uint8_t*)mStrings;
const uint8_t* str = strings+off;
- DECODE_LENGTH(str, sizeof(uint8_t), *outLen)
- size_t encLen;
- DECODE_LENGTH(str, sizeof(uint8_t), encLen)
+ *outLen = decodeLength(&str);
+ size_t encLen = decodeLength(&str);
if ((uint32_t)(str+encLen-strings) < mStringPoolSize) {
return (const char*)str;
} else {
diff --git a/libs/utils/String16.cpp b/libs/utils/String16.cpp
index eab7b2b..4ce1664 100644
--- a/libs/utils/String16.cpp
+++ b/libs/utils/String16.cpp
@@ -18,228 +18,17 @@
#include <utils/Debug.h>
#include <utils/Log.h>
+#include <utils/Unicode.h>
#include <utils/String8.h>
#include <utils/TextOutput.h>
#include <utils/threads.h>
#include <private/utils/Static.h>
-#ifdef HAVE_WINSOCK
-# undef nhtol
-# undef htonl
-# undef nhtos
-# undef htons
-
-# ifdef HAVE_LITTLE_ENDIAN
-# define ntohl(x) ( ((x) << 24) | (((x) >> 24) & 255) | (((x) << 8) & 0xff0000) | (((x) >> 8) & 0xff00) )
-# define htonl(x) ntohl(x)
-# define ntohs(x) ( (((x) << 8) & 0xff00) | (((x) >> 8) & 255) )
-# define htons(x) ntohs(x)
-# else
-# define ntohl(x) (x)
-# define htonl(x) (x)
-# define ntohs(x) (x)
-# define htons(x) (x)
-# endif
-#else
-# include <netinet/in.h>
-#endif
-
#include <memory.h>
#include <stdio.h>
#include <ctype.h>
-// ---------------------------------------------------------------------------
-
-int strcmp16(const char16_t *s1, const char16_t *s2)
-{
- char16_t ch;
- int d = 0;
-
- while ( 1 ) {
- d = (int)(ch = *s1++) - (int)*s2++;
- if ( d || !ch )
- break;
- }
-
- return d;
-}
-
-int strncmp16(const char16_t *s1, const char16_t *s2, size_t n)
-{
- char16_t ch;
- int d = 0;
-
- while ( n-- ) {
- d = (int)(ch = *s1++) - (int)*s2++;
- if ( d || !ch )
- break;
- }
-
- return d;
-}
-
-char16_t *strcpy16(char16_t *dst, const char16_t *src)
-{
- char16_t *q = dst;
- const char16_t *p = src;
- char16_t ch;
-
- do {
- *q++ = ch = *p++;
- } while ( ch );
-
- return dst;
-}
-
-size_t strlen16(const char16_t *s)
-{
- const char16_t *ss = s;
- while ( *ss )
- ss++;
- return ss-s;
-}
-
-
-char16_t *strncpy16(char16_t *dst, const char16_t *src, size_t n)
-{
- char16_t *q = dst;
- const char16_t *p = src;
- char ch;
-
- while (n) {
- n--;
- *q++ = ch = *p++;
- if ( !ch )
- break;
- }
-
- *q = 0;
-
- return dst;
-}
-
-size_t strnlen16(const char16_t *s, size_t maxlen)
-{
- const char16_t *ss = s;
-
- /* Important: the maxlen test must precede the reference through ss;
- since the byte beyond the maximum may segfault */
- while ((maxlen > 0) && *ss) {
- ss++;
- maxlen--;
- }
- return ss-s;
-}
-
-int strzcmp16(const char16_t *s1, size_t n1, const char16_t *s2, size_t n2)
-{
- const char16_t* e1 = s1+n1;
- const char16_t* e2 = s2+n2;
-
- while (s1 < e1 && s2 < e2) {
- const int d = (int)*s1++ - (int)*s2++;
- if (d) {
- return d;
- }
- }
-
- return n1 < n2
- ? (0 - (int)*s2)
- : (n1 > n2
- ? ((int)*s1 - 0)
- : 0);
-}
-
-int strzcmp16_h_n(const char16_t *s1H, size_t n1, const char16_t *s2N, size_t n2)
-{
- const char16_t* e1 = s1H+n1;
- const char16_t* e2 = s2N+n2;
-
- while (s1H < e1 && s2N < e2) {
- const char16_t c2 = ntohs(*s2N);
- const int d = (int)*s1H++ - (int)c2;
- s2N++;
- if (d) {
- return d;
- }
- }
-
- return n1 < n2
- ? (0 - (int)ntohs(*s2N))
- : (n1 > n2
- ? ((int)*s1H - 0)
- : 0);
-}
-
-static inline size_t
-utf8_char_len(uint8_t ch)
-{
- return ((0xe5000000 >> ((ch >> 3) & 0x1e)) & 3) + 1;
-}
-
-#define UTF8_SHIFT_AND_MASK(unicode, byte) (unicode)<<=6; (unicode) |= (0x3f & (byte));
-
-static inline uint32_t
-utf8_to_utf32(const uint8_t *src, size_t length)
-{
- uint32_t unicode;
-
- switch (length)
- {
- case 1:
- return src[0];
- case 2:
- unicode = src[0] & 0x1f;
- UTF8_SHIFT_AND_MASK(unicode, src[1])
- return unicode;
- case 3:
- unicode = src[0] & 0x0f;
- UTF8_SHIFT_AND_MASK(unicode, src[1])
- UTF8_SHIFT_AND_MASK(unicode, src[2])
- return unicode;
- case 4:
- unicode = src[0] & 0x07;
- UTF8_SHIFT_AND_MASK(unicode, src[1])
- UTF8_SHIFT_AND_MASK(unicode, src[2])
- UTF8_SHIFT_AND_MASK(unicode, src[3])
- return unicode;
- default:
- return 0xffff;
- }
-
- //printf("Char at %p: len=%d, utf-16=%p\n", src, length, (void*)result);
-}
-
-void
-utf8_to_utf16(const uint8_t *src, size_t srcLen,
- char16_t* dst, const size_t dstLen)
-{
- const uint8_t* const end = src + srcLen;
- const char16_t* const dstEnd = dst + dstLen;
- while (src < end && dst < dstEnd) {
- size_t len = utf8_char_len(*src);
- uint32_t codepoint = utf8_to_utf32((const uint8_t*)src, len);
-
- // Convert the UTF32 codepoint to one or more UTF16 codepoints
- if (codepoint <= 0xFFFF) {
- // Single UTF16 character
- *dst++ = (char16_t) codepoint;
- } else {
- // Multiple UTF16 characters with surrogates
- codepoint = codepoint - 0x10000;
- *dst++ = (char16_t) ((codepoint >> 10) + 0xD800);
- *dst++ = (char16_t) ((codepoint & 0x3FF) + 0xDC00);
- }
-
- src += len;
- }
- if (dst < dstEnd) {
- *dst = 0;
- }
-}
-
-// ---------------------------------------------------------------------------
namespace android {
@@ -270,37 +59,33 @@
// ---------------------------------------------------------------------------
-static char16_t* allocFromUTF8(const char* in, size_t len)
+static char16_t* allocFromUTF8(const char* u8str, size_t u8len)
{
- if (len == 0) return getEmptyString();
-
- size_t chars = 0;
- const char* end = in+len;
- const char* p = in;
-
- while (p < end) {
- chars++;
- int utf8len = utf8_char_len(*p);
- uint32_t codepoint = utf8_to_utf32((const uint8_t*)p, utf8len);
- if (codepoint > 0xFFFF) chars++; // this will be a surrogate pair in utf16
- p += utf8len;
+ if (u8len == 0) return getEmptyString();
+
+ const uint8_t* u8cur = (const uint8_t*) u8str;
+
+ const ssize_t u16len = utf8_to_utf16_length(u8cur, u8len);
+ if (u16len < 0) {
+ return getEmptyString();
}
-
- size_t bufSize = (chars+1)*sizeof(char16_t);
- SharedBuffer* buf = SharedBuffer::alloc(bufSize);
+
+ const uint8_t* const u8end = u8cur + u8len;
+
+ SharedBuffer* buf = SharedBuffer::alloc(sizeof(char16_t)*(u16len+1));
if (buf) {
- p = in;
- char16_t* str = (char16_t*)buf->data();
-
- utf8_to_utf16((const uint8_t*)p, len, str, bufSize);
+ u8cur = (const uint8_t*) u8str;
+ char16_t* u16str = (char16_t*)buf->data();
+
+ utf8_to_utf16(u8cur, u8len, u16str);
//printf("Created UTF-16 string from UTF-8 \"%s\":", in);
//printHexData(1, str, buf->size(), 16, 1);
//printf("\n");
- return str;
+ return u16str;
}
-
+
return getEmptyString();
}
diff --git a/libs/utils/String8.cpp b/libs/utils/String8.cpp
index 6358fc4..c8dc083 100644
--- a/libs/utils/String8.cpp
+++ b/libs/utils/String8.cpp
@@ -17,6 +17,8 @@
#include <utils/String8.h>
#include <utils/Log.h>
+#include <utils/Unicode.h>
+#include <utils/SharedBuffer.h>
#include <utils/String16.h>
#include <utils/TextOutput.h>
#include <utils/threads.h>
@@ -34,94 +36,10 @@
namespace android {
-static const char32_t kByteMask = 0x000000BF;
-static const char32_t kByteMark = 0x00000080;
-
-// Surrogates aren't valid for UTF-32 characters, so define some
-// constants that will let us screen them out.
-static const char32_t kUnicodeSurrogateHighStart = 0x0000D800;
-static const char32_t kUnicodeSurrogateHighEnd = 0x0000DBFF;
-static const char32_t kUnicodeSurrogateLowStart = 0x0000DC00;
-static const char32_t kUnicodeSurrogateLowEnd = 0x0000DFFF;
-static const char32_t kUnicodeSurrogateStart = kUnicodeSurrogateHighStart;
-static const char32_t kUnicodeSurrogateEnd = kUnicodeSurrogateLowEnd;
-static const char32_t kUnicodeMaxCodepoint = 0x0010FFFF;
-
-// Mask used to set appropriate bits in first byte of UTF-8 sequence,
-// indexed by number of bytes in the sequence.
-// 0xxxxxxx
-// -> (00-7f) 7bit. Bit mask for the first byte is 0x00000000
-// 110yyyyx 10xxxxxx
-// -> (c0-df)(80-bf) 11bit. Bit mask is 0x000000C0
-// 1110yyyy 10yxxxxx 10xxxxxx
-// -> (e0-ef)(80-bf)(80-bf) 16bit. Bit mask is 0x000000E0
-// 11110yyy 10yyxxxx 10xxxxxx 10xxxxxx
-// -> (f0-f7)(80-bf)(80-bf)(80-bf) 21bit. Bit mask is 0x000000F0
-static const char32_t kFirstByteMark[] = {
- 0x00000000, 0x00000000, 0x000000C0, 0x000000E0, 0x000000F0
-};
-
// Separator used by resource paths. This is not platform dependent contrary
// to OS_PATH_SEPARATOR.
#define RES_PATH_SEPARATOR '/'
-// Return number of utf8 bytes required for the character.
-static size_t utf32_to_utf8_bytes(char32_t srcChar)
-{
- size_t bytesToWrite;
-
- // Figure out how many bytes the result will require.
- if (srcChar < 0x00000080)
- {
- bytesToWrite = 1;
- }
- else if (srcChar < 0x00000800)
- {
- bytesToWrite = 2;
- }
- else if (srcChar < 0x00010000)
- {
- if ((srcChar < kUnicodeSurrogateStart)
- || (srcChar > kUnicodeSurrogateEnd))
- {
- bytesToWrite = 3;
- }
- else
- {
- // Surrogates are invalid UTF-32 characters.
- return 0;
- }
- }
- // Max code point for Unicode is 0x0010FFFF.
- else if (srcChar <= kUnicodeMaxCodepoint)
- {
- bytesToWrite = 4;
- }
- else
- {
- // Invalid UTF-32 character.
- return 0;
- }
-
- return bytesToWrite;
-}
-
-// Write out the source character to <dstP>.
-
-static void utf32_to_utf8(uint8_t* dstP, char32_t srcChar, size_t bytes)
-{
- dstP += bytes;
- switch (bytes)
- { /* note: everything falls through. */
- case 4: *--dstP = (uint8_t)((srcChar | kByteMark) & kByteMask); srcChar >>= 6;
- case 3: *--dstP = (uint8_t)((srcChar | kByteMark) & kByteMask); srcChar >>= 6;
- case 2: *--dstP = (uint8_t)((srcChar | kByteMark) & kByteMask); srcChar >>= 6;
- case 1: *--dstP = (uint8_t)(srcChar | kFirstByteMark[bytes]);
- }
-}
-
-// ---------------------------------------------------------------------------
-
static SharedBuffer* gEmptyStringBuf = NULL;
static char* gEmptyString = NULL;
@@ -175,62 +93,47 @@
return getEmptyString();
}
-template<typename T, typename L>
-static char* allocFromUTF16OrUTF32(const T* in, L len)
-{
- if (len == 0) return getEmptyString();
-
- size_t bytes = 0;
- const T* end = in+len;
- const T* p = in;
-
- while (p < end) {
- bytes += utf32_to_utf8_bytes(*p);
- p++;
- }
-
- SharedBuffer* buf = SharedBuffer::alloc(bytes+1);
- LOG_ASSERT(buf, "Unable to allocate shared buffer");
- if (buf) {
- p = in;
- char* str = (char*)buf->data();
- char* d = str;
- while (p < end) {
- const T c = *p++;
- size_t len = utf32_to_utf8_bytes(c);
- utf32_to_utf8((uint8_t*)d, c, len);
- d += len;
- }
- *d = 0;
-
- return str;
- }
-
- return getEmptyString();
-}
-
static char* allocFromUTF16(const char16_t* in, size_t len)
{
if (len == 0) return getEmptyString();
- const size_t bytes = utf8_length_from_utf16(in, len);
+ const ssize_t bytes = utf16_to_utf8_length(in, len);
+ if (bytes < 0) {
+ return getEmptyString();
+ }
SharedBuffer* buf = SharedBuffer::alloc(bytes+1);
LOG_ASSERT(buf, "Unable to allocate shared buffer");
- if (buf) {
- char* str = (char*)buf->data();
-
- utf16_to_utf8(in, len, str, bytes+1);
-
- return str;
+ if (!buf) {
+ return getEmptyString();
}
- return getEmptyString();
+ char* str = (char*)buf->data();
+ utf16_to_utf8(in, len, str);
+ return str;
}
static char* allocFromUTF32(const char32_t* in, size_t len)
{
- return allocFromUTF16OrUTF32<char32_t, size_t>(in, len);
+ if (len == 0) {
+ return getEmptyString();
+ }
+
+ const ssize_t bytes = utf32_to_utf8_length(in, len);
+ if (bytes < 0) {
+ return getEmptyString();
+ }
+
+ SharedBuffer* buf = SharedBuffer::alloc(bytes+1);
+ LOG_ASSERT(buf, "Unable to allocate shared buffer");
+ if (!buf) {
+ return getEmptyString();
+ }
+
+ char* str = (char*) buf->data();
+ utf32_to_utf8(in, len, str);
+
+ return str;
}
// ---------------------------------------------------------------------------
@@ -510,17 +413,17 @@
size_t String8::getUtf32Length() const
{
- return utf32_length(mString, length());
+ return utf8_to_utf32_length(mString, length());
}
int32_t String8::getUtf32At(size_t index, size_t *next_index) const
{
- return utf32_at(mString, length(), index, next_index);
+ return utf32_from_utf8_at(mString, length(), index, next_index);
}
-size_t String8::getUtf32(char32_t* dst, size_t dst_len) const
+void String8::getUtf32(char32_t* dst) const
{
- return utf8_to_utf32(mString, length(), dst, dst_len);
+ utf8_to_utf32(mString, length(), dst);
}
TextOutput& operator<<(TextOutput& to, const String8& val)
@@ -705,241 +608,3 @@
}
}; // namespace android
-
-// ---------------------------------------------------------------------------
-
-size_t strlen32(const char32_t *s)
-{
- const char32_t *ss = s;
- while ( *ss )
- ss++;
- return ss-s;
-}
-
-size_t strnlen32(const char32_t *s, size_t maxlen)
-{
- const char32_t *ss = s;
- while ((maxlen > 0) && *ss) {
- ss++;
- maxlen--;
- }
- return ss-s;
-}
-
-size_t utf8_length(const char *src)
-{
- const char *cur = src;
- size_t ret = 0;
- while (*cur != '\0') {
- const char first_char = *cur++;
- if ((first_char & 0x80) == 0) { // ASCII
- ret += 1;
- continue;
- }
- // (UTF-8's character must not be like 10xxxxxx,
- // but 110xxxxx, 1110xxxx, ... or 1111110x)
- if ((first_char & 0x40) == 0) {
- return 0;
- }
-
- int32_t mask, to_ignore_mask;
- size_t num_to_read = 0;
- char32_t utf32 = 0;
- for (num_to_read = 1, mask = 0x40, to_ignore_mask = 0x80;
- num_to_read < 5 && (first_char & mask);
- num_to_read++, to_ignore_mask |= mask, mask >>= 1) {
- if ((*cur & 0xC0) != 0x80) { // must be 10xxxxxx
- return 0;
- }
- // 0x3F == 00111111
- utf32 = (utf32 << 6) + (*cur++ & 0x3F);
- }
- // "first_char" must be (110xxxxx - 11110xxx)
- if (num_to_read == 5) {
- return 0;
- }
- to_ignore_mask |= mask;
- utf32 |= ((~to_ignore_mask) & first_char) << (6 * (num_to_read - 1));
- if (utf32 > android::kUnicodeMaxCodepoint) {
- return 0;
- }
-
- ret += num_to_read;
- }
- return ret;
-}
-
-size_t utf32_length(const char *src, size_t src_len)
-{
- if (src == NULL || src_len == 0) {
- return 0;
- }
- size_t ret = 0;
- const char* cur;
- const char* end;
- size_t num_to_skip;
- for (cur = src, end = src + src_len, num_to_skip = 1;
- cur < end;
- cur += num_to_skip, ret++) {
- const char first_char = *cur;
- num_to_skip = 1;
- if ((first_char & 0x80) == 0) { // ASCII
- continue;
- }
- int32_t mask;
-
- for (mask = 0x40; (first_char & mask); num_to_skip++, mask >>= 1) {
- }
- }
- return ret;
-}
-
-size_t utf8_length_from_utf32(const char32_t *src, size_t src_len)
-{
- if (src == NULL || src_len == 0) {
- return 0;
- }
- size_t ret = 0;
- const char32_t *end = src + src_len;
- while (src < end) {
- ret += android::utf32_to_utf8_bytes(*src++);
- }
- return ret;
-}
-
-size_t utf8_length_from_utf16(const char16_t *src, size_t src_len)
-{
- if (src == NULL || src_len == 0) {
- return 0;
- }
- size_t ret = 0;
- const char16_t* const end = src + src_len;
- while (src < end) {
- if ((*src & 0xFC00) == 0xD800 && (src + 1) < end
- && (*++src & 0xFC00) == 0xDC00) {
- // surrogate pairs are always 4 bytes.
- ret += 4;
- src++;
- } else {
- ret += android::utf32_to_utf8_bytes((char32_t) *src++);
- }
- }
- return ret;
-}
-
-static int32_t utf32_at_internal(const char* cur, size_t *num_read)
-{
- const char first_char = *cur;
- if ((first_char & 0x80) == 0) { // ASCII
- *num_read = 1;
- return *cur;
- }
- cur++;
- char32_t mask, to_ignore_mask;
- size_t num_to_read = 0;
- char32_t utf32 = first_char;
- for (num_to_read = 1, mask = 0x40, to_ignore_mask = 0xFFFFFF80;
- (first_char & mask);
- num_to_read++, to_ignore_mask |= mask, mask >>= 1) {
- // 0x3F == 00111111
- utf32 = (utf32 << 6) + (*cur++ & 0x3F);
- }
- to_ignore_mask |= mask;
- utf32 &= ~(to_ignore_mask << (6 * (num_to_read - 1)));
-
- *num_read = num_to_read;
- return static_cast<int32_t>(utf32);
-}
-
-int32_t utf32_at(const char *src, size_t src_len,
- size_t index, size_t *next_index)
-{
- if (index >= src_len) {
- return -1;
- }
- size_t dummy_index;
- if (next_index == NULL) {
- next_index = &dummy_index;
- }
- size_t num_read;
- int32_t ret = utf32_at_internal(src + index, &num_read);
- if (ret >= 0) {
- *next_index = index + num_read;
- }
-
- return ret;
-}
-
-size_t utf8_to_utf32(const char* src, size_t src_len,
- char32_t* dst, size_t dst_len)
-{
- if (src == NULL || src_len == 0 || dst == NULL || dst_len == 0) {
- return 0;
- }
-
- const char* cur = src;
- const char* end = src + src_len;
- char32_t* cur_utf32 = dst;
- const char32_t* end_utf32 = dst + dst_len;
- while (cur_utf32 < end_utf32 && cur < end) {
- size_t num_read;
- *cur_utf32++ =
- static_cast<char32_t>(utf32_at_internal(cur, &num_read));
- cur += num_read;
- }
- if (cur_utf32 < end_utf32) {
- *cur_utf32 = 0;
- }
- return static_cast<size_t>(cur_utf32 - dst);
-}
-
-size_t utf32_to_utf8(const char32_t* src, size_t src_len,
- char* dst, size_t dst_len)
-{
- if (src == NULL || src_len == 0 || dst == NULL || dst_len == 0) {
- return 0;
- }
- const char32_t *cur_utf32 = src;
- const char32_t *end_utf32 = src + src_len;
- char *cur = dst;
- const char *end = dst + dst_len;
- while (cur_utf32 < end_utf32 && cur < end) {
- size_t len = android::utf32_to_utf8_bytes(*cur_utf32);
- android::utf32_to_utf8((uint8_t *)cur, *cur_utf32++, len);
- cur += len;
- }
- if (cur < end) {
- *cur = '\0';
- }
- return cur - dst;
-}
-
-size_t utf16_to_utf8(const char16_t* src, size_t src_len,
- char* dst, size_t dst_len)
-{
- if (src == NULL || src_len == 0 || dst == NULL || dst_len == 0) {
- return 0;
- }
- const char16_t* cur_utf16 = src;
- const char16_t* const end_utf16 = src + src_len;
- char *cur = dst;
- const char* const end = dst + dst_len;
- while (cur_utf16 < end_utf16 && cur < end) {
- char32_t utf32;
- // surrogate pairs
- if ((*cur_utf16 & 0xFC00) == 0xD800 && (cur_utf16 + 1) < end_utf16) {
- utf32 = (*cur_utf16++ - 0xD800) << 10;
- utf32 |= *cur_utf16++ - 0xDC00;
- utf32 += 0x10000;
- } else {
- utf32 = (char32_t) *cur_utf16++;
- }
- size_t len = android::utf32_to_utf8_bytes(utf32);
- android::utf32_to_utf8((uint8_t*)cur, utf32, len);
- cur += len;
- }
- if (cur < end) {
- *cur = '\0';
- }
- return cur - dst;
-}
diff --git a/libs/utils/Unicode.cpp b/libs/utils/Unicode.cpp
new file mode 100644
index 0000000..78c61b4
--- /dev/null
+++ b/libs/utils/Unicode.cpp
@@ -0,0 +1,575 @@
+/*
+ * Copyright (C) 2005 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <utils/Unicode.h>
+
+#include <stddef.h>
+
+#ifdef HAVE_WINSOCK
+# undef nhtol
+# undef htonl
+# undef nhtos
+# undef htons
+
+# ifdef HAVE_LITTLE_ENDIAN
+# define ntohl(x) ( ((x) << 24) | (((x) >> 24) & 255) | (((x) << 8) & 0xff0000) | (((x) >> 8) & 0xff00) )
+# define htonl(x) ntohl(x)
+# define ntohs(x) ( (((x) << 8) & 0xff00) | (((x) >> 8) & 255) )
+# define htons(x) ntohs(x)
+# else
+# define ntohl(x) (x)
+# define htonl(x) (x)
+# define ntohs(x) (x)
+# define htons(x) (x)
+# endif
+#else
+# include <netinet/in.h>
+#endif
+
+extern "C" {
+
+static const char32_t kByteMask = 0x000000BF;
+static const char32_t kByteMark = 0x00000080;
+
+// Surrogates aren't valid for UTF-32 characters, so define some
+// constants that will let us screen them out.
+static const char32_t kUnicodeSurrogateHighStart = 0x0000D800;
+static const char32_t kUnicodeSurrogateHighEnd = 0x0000DBFF;
+static const char32_t kUnicodeSurrogateLowStart = 0x0000DC00;
+static const char32_t kUnicodeSurrogateLowEnd = 0x0000DFFF;
+static const char32_t kUnicodeSurrogateStart = kUnicodeSurrogateHighStart;
+static const char32_t kUnicodeSurrogateEnd = kUnicodeSurrogateLowEnd;
+static const char32_t kUnicodeMaxCodepoint = 0x0010FFFF;
+
+// Mask used to set appropriate bits in first byte of UTF-8 sequence,
+// indexed by number of bytes in the sequence.
+// 0xxxxxxx
+// -> (00-7f) 7bit. Bit mask for the first byte is 0x00000000
+// 110yyyyx 10xxxxxx
+// -> (c0-df)(80-bf) 11bit. Bit mask is 0x000000C0
+// 1110yyyy 10yxxxxx 10xxxxxx
+// -> (e0-ef)(80-bf)(80-bf) 16bit. Bit mask is 0x000000E0
+// 11110yyy 10yyxxxx 10xxxxxx 10xxxxxx
+// -> (f0-f7)(80-bf)(80-bf)(80-bf) 21bit. Bit mask is 0x000000F0
+static const char32_t kFirstByteMark[] = {
+ 0x00000000, 0x00000000, 0x000000C0, 0x000000E0, 0x000000F0
+};
+
+// --------------------------------------------------------------------------
+// UTF-32
+// --------------------------------------------------------------------------
+
+/**
+ * Return number of UTF-8 bytes required for the character. If the character
+ * is invalid, return size of 0.
+ */
+static inline size_t utf32_codepoint_utf8_length(char32_t srcChar)
+{
+ // Figure out how many bytes the result will require.
+ if (srcChar < 0x00000080) {
+ return 1;
+ } else if (srcChar < 0x00000800) {
+ return 2;
+ } else if (srcChar < 0x00010000) {
+ if ((srcChar < kUnicodeSurrogateStart) || (srcChar > kUnicodeSurrogateEnd)) {
+ return 3;
+ } else {
+ // Surrogates are invalid UTF-32 characters.
+ return 0;
+ }
+ }
+ // Max code point for Unicode is 0x0010FFFF.
+ else if (srcChar <= kUnicodeMaxCodepoint) {
+ return 4;
+ } else {
+ // Invalid UTF-32 character.
+ return 0;
+ }
+}
+
+// Write out the source character to <dstP>.
+
+static inline void utf32_codepoint_to_utf8(uint8_t* dstP, char32_t srcChar, size_t bytes)
+{
+ dstP += bytes;
+ switch (bytes)
+ { /* note: everything falls through. */
+ case 4: *--dstP = (uint8_t)((srcChar | kByteMark) & kByteMask); srcChar >>= 6;
+ case 3: *--dstP = (uint8_t)((srcChar | kByteMark) & kByteMask); srcChar >>= 6;
+ case 2: *--dstP = (uint8_t)((srcChar | kByteMark) & kByteMask); srcChar >>= 6;
+ case 1: *--dstP = (uint8_t)(srcChar | kFirstByteMark[bytes]);
+ }
+}
+
+size_t strlen32(const char32_t *s)
+{
+ const char32_t *ss = s;
+ while ( *ss )
+ ss++;
+ return ss-s;
+}
+
+size_t strnlen32(const char32_t *s, size_t maxlen)
+{
+ const char32_t *ss = s;
+ while ((maxlen > 0) && *ss) {
+ ss++;
+ maxlen--;
+ }
+ return ss-s;
+}
+
+static inline int32_t utf32_at_internal(const char* cur, size_t *num_read)
+{
+ const char first_char = *cur;
+ if ((first_char & 0x80) == 0) { // ASCII
+ *num_read = 1;
+ return *cur;
+ }
+ cur++;
+ char32_t mask, to_ignore_mask;
+ size_t num_to_read = 0;
+ char32_t utf32 = first_char;
+ for (num_to_read = 1, mask = 0x40, to_ignore_mask = 0xFFFFFF80;
+ (first_char & mask);
+ num_to_read++, to_ignore_mask |= mask, mask >>= 1) {
+ // 0x3F == 00111111
+ utf32 = (utf32 << 6) + (*cur++ & 0x3F);
+ }
+ to_ignore_mask |= mask;
+ utf32 &= ~(to_ignore_mask << (6 * (num_to_read - 1)));
+
+ *num_read = num_to_read;
+ return static_cast<int32_t>(utf32);
+}
+
+int32_t utf32_from_utf8_at(const char *src, size_t src_len, size_t index, size_t *next_index)
+{
+ if (index >= src_len) {
+ return -1;
+ }
+ size_t dummy_index;
+ if (next_index == NULL) {
+ next_index = &dummy_index;
+ }
+ size_t num_read;
+ int32_t ret = utf32_at_internal(src + index, &num_read);
+ if (ret >= 0) {
+ *next_index = index + num_read;
+ }
+
+ return ret;
+}
+
+ssize_t utf32_to_utf8_length(const char32_t *src, size_t src_len)
+{
+ if (src == NULL || src_len == 0) {
+ return -1;
+ }
+
+ size_t ret = 0;
+ const char32_t *end = src + src_len;
+ while (src < end) {
+ ret += utf32_codepoint_utf8_length(*src++);
+ }
+ return ret;
+}
+
+void utf32_to_utf8(const char32_t* src, size_t src_len, char* dst)
+{
+ if (src == NULL || src_len == 0 || dst == NULL) {
+ return;
+ }
+
+ const char32_t *cur_utf32 = src;
+ const char32_t *end_utf32 = src + src_len;
+ char *cur = dst;
+ while (cur_utf32 < end_utf32) {
+ size_t len = utf32_codepoint_utf8_length(*cur_utf32);
+ utf32_codepoint_to_utf8((uint8_t *)cur, *cur_utf32++, len);
+ cur += len;
+ }
+ *cur = '\0';
+}
+
+// --------------------------------------------------------------------------
+// UTF-16
+// --------------------------------------------------------------------------
+
+int strcmp16(const char16_t *s1, const char16_t *s2)
+{
+ char16_t ch;
+ int d = 0;
+
+ while ( 1 ) {
+ d = (int)(ch = *s1++) - (int)*s2++;
+ if ( d || !ch )
+ break;
+ }
+
+ return d;
+}
+
+int strncmp16(const char16_t *s1, const char16_t *s2, size_t n)
+{
+ char16_t ch;
+ int d = 0;
+
+ while ( n-- ) {
+ d = (int)(ch = *s1++) - (int)*s2++;
+ if ( d || !ch )
+ break;
+ }
+
+ return d;
+}
+
+char16_t *strcpy16(char16_t *dst, const char16_t *src)
+{
+ char16_t *q = dst;
+ const char16_t *p = src;
+ char16_t ch;
+
+ do {
+ *q++ = ch = *p++;
+ } while ( ch );
+
+ return dst;
+}
+
+size_t strlen16(const char16_t *s)
+{
+ const char16_t *ss = s;
+ while ( *ss )
+ ss++;
+ return ss-s;
+}
+
+
+char16_t *strncpy16(char16_t *dst, const char16_t *src, size_t n)
+{
+ char16_t *q = dst;
+ const char16_t *p = src;
+ char ch;
+
+ while (n) {
+ n--;
+ *q++ = ch = *p++;
+ if ( !ch )
+ break;
+ }
+
+ *q = 0;
+
+ return dst;
+}
+
+size_t strnlen16(const char16_t *s, size_t maxlen)
+{
+ const char16_t *ss = s;
+
+ /* Important: the maxlen test must precede the reference through ss;
+ since the byte beyond the maximum may segfault */
+ while ((maxlen > 0) && *ss) {
+ ss++;
+ maxlen--;
+ }
+ return ss-s;
+}
+
+int strzcmp16(const char16_t *s1, size_t n1, const char16_t *s2, size_t n2)
+{
+ const char16_t* e1 = s1+n1;
+ const char16_t* e2 = s2+n2;
+
+ while (s1 < e1 && s2 < e2) {
+ const int d = (int)*s1++ - (int)*s2++;
+ if (d) {
+ return d;
+ }
+ }
+
+ return n1 < n2
+ ? (0 - (int)*s2)
+ : (n1 > n2
+ ? ((int)*s1 - 0)
+ : 0);
+}
+
+int strzcmp16_h_n(const char16_t *s1H, size_t n1, const char16_t *s2N, size_t n2)
+{
+ const char16_t* e1 = s1H+n1;
+ const char16_t* e2 = s2N+n2;
+
+ while (s1H < e1 && s2N < e2) {
+ const char16_t c2 = ntohs(*s2N);
+ const int d = (int)*s1H++ - (int)c2;
+ s2N++;
+ if (d) {
+ return d;
+ }
+ }
+
+ return n1 < n2
+ ? (0 - (int)ntohs(*s2N))
+ : (n1 > n2
+ ? ((int)*s1H - 0)
+ : 0);
+}
+
+void utf16_to_utf8(const char16_t* src, size_t src_len, char* dst)
+{
+ if (src == NULL || src_len == 0 || dst == NULL) {
+ return;
+ }
+
+ const char16_t* cur_utf16 = src;
+ const char16_t* const end_utf16 = src + src_len;
+ char *cur = dst;
+ while (cur_utf16 < end_utf16) {
+ char32_t utf32;
+ // surrogate pairs
+ if ((*cur_utf16 & 0xFC00) == 0xD800) {
+ utf32 = (*cur_utf16++ - 0xD800) << 10;
+ utf32 |= *cur_utf16++ - 0xDC00;
+ utf32 += 0x10000;
+ } else {
+ utf32 = (char32_t) *cur_utf16++;
+ }
+ const size_t len = utf32_codepoint_utf8_length(utf32);
+ utf32_codepoint_to_utf8((uint8_t*)cur, utf32, len);
+ cur += len;
+ }
+ *cur = '\0';
+}
+
+// --------------------------------------------------------------------------
+// UTF-8
+// --------------------------------------------------------------------------
+
+ssize_t utf8_length(const char *src)
+{
+ const char *cur = src;
+ size_t ret = 0;
+ while (*cur != '\0') {
+ const char first_char = *cur++;
+ if ((first_char & 0x80) == 0) { // ASCII
+ ret += 1;
+ continue;
+ }
+ // (UTF-8's character must not be like 10xxxxxx,
+ // but 110xxxxx, 1110xxxx, ... or 1111110x)
+ if ((first_char & 0x40) == 0) {
+ return -1;
+ }
+
+ int32_t mask, to_ignore_mask;
+ size_t num_to_read = 0;
+ char32_t utf32 = 0;
+ for (num_to_read = 1, mask = 0x40, to_ignore_mask = 0x80;
+ num_to_read < 5 && (first_char & mask);
+ num_to_read++, to_ignore_mask |= mask, mask >>= 1) {
+ if ((*cur & 0xC0) != 0x80) { // must be 10xxxxxx
+ return -1;
+ }
+ // 0x3F == 00111111
+ utf32 = (utf32 << 6) + (*cur++ & 0x3F);
+ }
+ // "first_char" must be (110xxxxx - 11110xxx)
+ if (num_to_read == 5) {
+ return -1;
+ }
+ to_ignore_mask |= mask;
+ utf32 |= ((~to_ignore_mask) & first_char) << (6 * (num_to_read - 1));
+ if (utf32 > kUnicodeMaxCodepoint) {
+ return -1;
+ }
+
+ ret += num_to_read;
+ }
+ return ret;
+}
+
+ssize_t utf16_to_utf8_length(const char16_t *src, size_t src_len)
+{
+ if (src == NULL || src_len == 0) {
+ return -1;
+ }
+
+ size_t ret = 0;
+ const char16_t* const end = src + src_len;
+ while (src < end) {
+ if ((*src & 0xFC00) == 0xD800 && (src + 1) < end
+ && (*++src & 0xFC00) == 0xDC00) {
+ // surrogate pairs are always 4 bytes.
+ ret += 4;
+ src++;
+ } else {
+ ret += utf32_codepoint_utf8_length((char32_t) *src++);
+ }
+ }
+ return ret;
+}
+
+/**
+ * Returns 1-4 based on the number of leading bits.
+ *
+ * 1111 -> 4
+ * 1110 -> 3
+ * 110x -> 2
+ * 10xx -> 1
+ * 0xxx -> 1
+ */
+static inline size_t utf8_codepoint_len(uint8_t ch)
+{
+ return ((0xe5000000 >> ((ch >> 3) & 0x1e)) & 3) + 1;
+}
+
+static inline void utf8_shift_and_mask(uint32_t* codePoint, const uint8_t byte)
+{
+ *codePoint <<= 6;
+ *codePoint |= 0x3F & byte;
+}
+
+size_t utf8_to_utf32_length(const char *src, size_t src_len)
+{
+ if (src == NULL || src_len == 0) {
+ return 0;
+ }
+ size_t ret = 0;
+ const char* cur;
+ const char* end;
+ size_t num_to_skip;
+ for (cur = src, end = src + src_len, num_to_skip = 1;
+ cur < end;
+ cur += num_to_skip, ret++) {
+ const char first_char = *cur;
+ num_to_skip = 1;
+ if ((first_char & 0x80) == 0) { // ASCII
+ continue;
+ }
+ int32_t mask;
+
+ for (mask = 0x40; (first_char & mask); num_to_skip++, mask >>= 1) {
+ }
+ }
+ return ret;
+}
+
+void utf8_to_utf32(const char* src, size_t src_len, char32_t* dst)
+{
+ if (src == NULL || src_len == 0 || dst == NULL) {
+ return;
+ }
+
+ const char* cur = src;
+ const char* const end = src + src_len;
+ char32_t* cur_utf32 = dst;
+ while (cur < end) {
+ size_t num_read;
+ *cur_utf32++ = static_cast<char32_t>(utf32_at_internal(cur, &num_read));
+ cur += num_read;
+ }
+ *cur_utf32 = 0;
+}
+
+static inline uint32_t utf8_to_utf32_codepoint(const uint8_t *src, size_t length)
+{
+ uint32_t unicode;
+
+ switch (length)
+ {
+ case 1:
+ return src[0];
+ case 2:
+ unicode = src[0] & 0x1f;
+ utf8_shift_and_mask(&unicode, src[1]);
+ return unicode;
+ case 3:
+ unicode = src[0] & 0x0f;
+ utf8_shift_and_mask(&unicode, src[1]);
+ utf8_shift_and_mask(&unicode, src[2]);
+ return unicode;
+ case 4:
+ unicode = src[0] & 0x07;
+ utf8_shift_and_mask(&unicode, src[1]);
+ utf8_shift_and_mask(&unicode, src[2]);
+ utf8_shift_and_mask(&unicode, src[3]);
+ return unicode;
+ default:
+ return 0xffff;
+ }
+
+ //printf("Char at %p: len=%d, utf-16=%p\n", src, length, (void*)result);
+}
+
+ssize_t utf8_to_utf16_length(const uint8_t* u8str, size_t u8len)
+{
+ const uint8_t* const u8end = u8str + u8len;
+ const uint8_t* u8cur = u8str;
+
+ /* Validate that the UTF-8 is the correct len */
+ size_t u16measuredLen = 0;
+ while (u8cur < u8end) {
+ u16measuredLen++;
+ int u8charLen = utf8_codepoint_len(*u8cur);
+ uint32_t codepoint = utf8_to_utf32_codepoint(u8cur, u8charLen);
+ if (codepoint > 0xFFFF) u16measuredLen++; // this will be a surrogate pair in utf16
+ u8cur += u8charLen;
+ }
+
+ /**
+ * Make sure that we ended where we thought we would and the output UTF-16
+ * will be exactly how long we were told it would be.
+ */
+ if (u8cur != u8end) {
+ return -1;
+ }
+
+ return u16measuredLen;
+}
+
+/**
+ * Convert a UTF-8 string to UTF-16. The destination UTF-16 buffer must have
+ * space for NULL at the end.
+ */
+void utf8_to_utf16(const uint8_t* u8str, size_t u8len, char16_t* u16str)
+{
+ const uint8_t* const u8end = u8str + u8len;
+ const uint8_t* u8cur = u8str;
+ char16_t* u16cur = u16str;
+
+ while (u8cur < u8end) {
+ size_t u8len = utf8_codepoint_len(*u8cur);
+ uint32_t codepoint = utf8_to_utf32_codepoint(u8cur, u8len);
+
+ // Convert the UTF32 codepoint to one or more UTF16 codepoints
+ if (codepoint <= 0xFFFF) {
+ // Single UTF16 character
+ *u16cur++ = (char16_t) codepoint;
+ } else {
+ // Multiple UTF16 characters with surrogates
+ codepoint = codepoint - 0x10000;
+ *u16cur++ = (char16_t) ((codepoint >> 10) + 0xD800);
+ *u16cur++ = (char16_t) ((codepoint & 0x3FF) + 0xDC00);
+ }
+
+ u8cur += u8len;
+ }
+ *u16cur = 0;
+}
+
+}
diff --git a/libs/utils/tests/Android.mk b/libs/utils/tests/Android.mk
index 00077ee..72d4876 100644
--- a/libs/utils/tests/Android.mk
+++ b/libs/utils/tests/Android.mk
@@ -8,7 +8,8 @@
test_src_files := \
ObbFile_test.cpp \
Looper_test.cpp \
- String8_test.cpp
+ String8_test.cpp \
+ Unicode_test.cpp
shared_libraries := \
libz \
diff --git a/libs/utils/tests/Unicode_test.cpp b/libs/utils/tests/Unicode_test.cpp
new file mode 100644
index 0000000..18c130c
--- /dev/null
+++ b/libs/utils/tests/Unicode_test.cpp
@@ -0,0 +1,115 @@
+/*
+ * Copyright (C) 2010 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define LOG_TAG "Unicode_test"
+#include <utils/Log.h>
+#include <utils/Unicode.h>
+
+#include <gtest/gtest.h>
+
+namespace android {
+
+class UnicodeTest : public testing::Test {
+protected:
+ virtual void SetUp() {
+ }
+
+ virtual void TearDown() {
+ }
+};
+
+TEST_F(UnicodeTest, UTF8toUTF16ZeroLength) {
+ ssize_t measured;
+
+ const uint8_t str[] = { };
+
+ measured = utf8_to_utf16_length(str, 0);
+ EXPECT_EQ(0, measured)
+ << "Zero length input should return zero length output.";
+}
+
+TEST_F(UnicodeTest, UTF8toUTF16ASCIILength) {
+ ssize_t measured;
+
+ // U+0030 or ASCII '0'
+ const uint8_t str[] = { 0x30 };
+
+ measured = utf8_to_utf16_length(str, sizeof(str));
+ EXPECT_EQ(1, measured)
+ << "ASCII glyphs should have a length of 1 char16_t";
+}
+
+TEST_F(UnicodeTest, UTF8toUTF16Plane1Length) {
+ ssize_t measured;
+
+ // U+2323 SMILE
+ const uint8_t str[] = { 0xE2, 0x8C, 0xA3 };
+
+ measured = utf8_to_utf16_length(str, sizeof(str));
+ EXPECT_EQ(1, measured)
+ << "Plane 1 glyphs should have a length of 1 char16_t";
+}
+
+TEST_F(UnicodeTest, UTF8toUTF16SurrogateLength) {
+ ssize_t measured;
+
+ // U+10000
+ const uint8_t str[] = { 0xF0, 0x90, 0x80, 0x80 };
+
+ measured = utf8_to_utf16_length(str, sizeof(str));
+ EXPECT_EQ(2, measured)
+ << "Surrogate pairs should have a length of 2 char16_t";
+}
+
+TEST_F(UnicodeTest, UTF8toUTF16TruncatedUTF8) {
+ ssize_t measured;
+
+ // Truncated U+2323 SMILE
+ // U+2323 SMILE
+ const uint8_t str[] = { 0xE2, 0x8C };
+
+ measured = utf8_to_utf16_length(str, sizeof(str));
+ EXPECT_EQ(-1, measured)
+ << "Truncated UTF-8 should return -1 to indicate invalid";
+}
+
+TEST_F(UnicodeTest, UTF8toUTF16Normal) {
+ const uint8_t str[] = {
+ 0x30, // U+0030, 1 UTF-16 character
+ 0xC4, 0x80, // U+0100, 1 UTF-16 character
+ 0xE2, 0x8C, 0xA3, // U+2323, 1 UTF-16 character
+ 0xF0, 0x90, 0x80, 0x80, // U+10000, 2 UTF-16 character
+ };
+
+ char16_t output[1 + 1 + 1 + 2 + 1]; // Room for NULL
+
+ utf8_to_utf16(str, sizeof(str), output);
+
+ EXPECT_EQ(0x0030, output[0])
+ << "should be U+0030";
+ EXPECT_EQ(0x0100, output[1])
+ << "should be U+0100";
+ EXPECT_EQ(0x2323, output[2])
+ << "should be U+2323";
+ EXPECT_EQ(0xD800, output[3])
+ << "should be first half of surrogate U+10000";
+ EXPECT_EQ(0xDC00, output[4])
+ << "should be second half of surrogate U+10000";
+ EXPECT_EQ(NULL, output[5])
+ << "should be NULL terminated";
+}
+
+}