Split *AndAdjustOffset() functions into their own header, to restore utf_string_conversions.h to a simple, readable state.
BUG=4010
TEST=none
Review URL: http://codereview.chromium.org/387012
git-svn-id: svn://svn.chromium.org/chrome/trunk/src@31928 0039d316-1c4b-4281-b951-d872f2087c98
CrOS-Libchrome-Original-Commit: b9f9383602610ef1f94e9bee2d91d7c044e16051
diff --git a/base/base.gyp b/base/base.gyp
index bcb88f6..b954e3b 100644
--- a/base/base.gyp
+++ b/base/base.gyp
@@ -332,6 +332,10 @@
'tracked_objects.cc',
'tracked_objects.h',
'tuple.h',
+ 'utf_offset_string_conversions.cc',
+ 'utf_offset_string_conversions.h',
+ 'utf_string_conversion_utils.cc',
+ 'utf_string_conversion_utils.h',
'utf_string_conversions.cc',
'utf_string_conversions.h',
'unix_domain_socket_posix.cc',
@@ -655,6 +659,7 @@
'timer_unittest.cc',
'tracked_objects_unittest.cc',
'tuple_unittest.cc',
+ 'utf_offset_string_conversions_unittest.cc',
'utf_string_conversions_unittest.cc',
'values_unittest.cc',
'version_unittest.cc',
diff --git a/base/i18n/icu_string_conversions.h b/base/i18n/icu_string_conversions.h
index 7b0c77e..79d7e02 100644
--- a/base/i18n/icu_string_conversions.h
+++ b/base/i18n/icu_string_conversions.h
@@ -8,7 +8,6 @@
#include <string>
#include "base/string16.h"
-#include "base/string_piece.h"
namespace base {
diff --git a/base/utf_offset_string_conversions.cc b/base/utf_offset_string_conversions.cc
new file mode 100644
index 0000000..69b572e
--- /dev/null
+++ b/base/utf_offset_string_conversions.cc
@@ -0,0 +1,122 @@
+// Copyright (c) 2009 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "base/utf_offset_string_conversions.h"
+
+#include "base/string_piece.h"
+#include "base/utf_string_conversion_utils.h"
+
+using base::PrepareForUTF16Or32Output;
+using base::ReadUnicodeCharacter;
+using base::WriteUnicodeCharacter;
+
+// Generalized Unicode converter -----------------------------------------------
+
+// Converts the given source Unicode character type to the given destination
+// Unicode character type as a STL string. The given input buffer and size
+// determine the source, and the given output STL string will be replaced by
+// the result.
+template<typename SRC_CHAR>
+bool ConvertUnicode(const SRC_CHAR* src,
+ size_t src_len,
+ std::wstring* output,
+ size_t* offset_for_adjustment) {
+ size_t output_offset =
+ (offset_for_adjustment && *offset_for_adjustment < src_len) ?
+ *offset_for_adjustment : std::wstring::npos;
+
+ // ICU requires 32-bit numbers.
+ bool success = true;
+ int32 src_len32 = static_cast<int32>(src_len);
+ for (int32 i = 0; i < src_len32; i++) {
+ uint32 code_point;
+ size_t original_i = i;
+ size_t chars_written = 0;
+ if (ReadUnicodeCharacter(src, src_len32, &i, &code_point)) {
+ chars_written = WriteUnicodeCharacter(code_point, output);
+ } else {
+ // TODO(jungshik): consider adding 'Replacement character' (U+FFFD)
+ // in place of an invalid codepoint.
+ success = false;
+ }
+ if ((output_offset != std::wstring::npos) &&
+ (*offset_for_adjustment > original_i)) {
+ // NOTE: ReadUnicodeCharacter() adjusts |i| to point _at_ the last
+ // character read, not after it (so that incrementing it in the loop
+ // increment will place it at the right location), so we need to account
+ // for that in determining the amount that was read.
+ if (*offset_for_adjustment <= static_cast<size_t>(i))
+ output_offset = std::wstring::npos;
+ else
+ output_offset += chars_written - (i - original_i + 1);
+ }
+ }
+
+ if (offset_for_adjustment)
+ *offset_for_adjustment = output_offset;
+ return success;
+}
+
+// UTF-8 <-> Wide --------------------------------------------------------------
+
+bool UTF8ToWideAndAdjustOffset(const char* src,
+ size_t src_len,
+ std::wstring* output,
+ size_t* offset_for_adjustment) {
+ PrepareForUTF16Or32Output(src, src_len, output);
+ return ConvertUnicode(src, src_len, output, offset_for_adjustment);
+}
+
+std::wstring UTF8ToWideAndAdjustOffset(const base::StringPiece& utf8,
+ size_t* offset_for_adjustment) {
+ std::wstring ret;
+ UTF8ToWideAndAdjustOffset(utf8.data(), utf8.length(), &ret,
+ offset_for_adjustment);
+ return ret;
+}
+
+// UTF-16 <-> Wide -------------------------------------------------------------
+
+#if defined(WCHAR_T_IS_UTF16)
+
+// When wide == UTF-16, then conversions are a NOP.
+bool UTF16ToWideAndAdjustOffset(const char16* src,
+ size_t src_len,
+ std::wstring* output,
+ size_t* offset_for_adjustment) {
+ output->assign(src, src_len);
+ if (offset_for_adjustment && (*offset_for_adjustment >= src_len))
+ *offset_for_adjustment = std::wstring::npos;
+ return true;
+}
+
+std::wstring UTF16ToWideAndAdjustOffset(const string16& utf16,
+ size_t* offset_for_adjustment) {
+ if (offset_for_adjustment && (*offset_for_adjustment >= utf16.length()))
+ *offset_for_adjustment = std::wstring::npos;
+ return utf16;
+}
+
+#elif defined(WCHAR_T_IS_UTF32)
+
+bool UTF16ToWideAndAdjustOffset(const char16* src,
+ size_t src_len,
+ std::wstring* output,
+ size_t* offset_for_adjustment) {
+ output->clear();
+ // Assume that normally we won't have any non-BMP characters so the counts
+ // will be the same.
+ output->reserve(src_len);
+ return ConvertUnicode(src, src_len, output, offset_for_adjustment);
+}
+
+std::wstring UTF16ToWideAndAdjustOffset(const string16& utf16,
+ size_t* offset_for_adjustment) {
+ std::wstring ret;
+ UTF16ToWideAndAdjustOffset(utf16.data(), utf16.length(), &ret,
+ offset_for_adjustment);
+ return ret;
+}
+
+#endif // defined(WCHAR_T_IS_UTF32)
diff --git a/base/utf_offset_string_conversions.h b/base/utf_offset_string_conversions.h
new file mode 100644
index 0000000..0a1a682
--- /dev/null
+++ b/base/utf_offset_string_conversions.h
@@ -0,0 +1,35 @@
+// Copyright (c) 2009 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef BASE_UTF_OFFSET_STRING_CONVERSIONS_H_
+#define BASE_UTF_OFFSET_STRING_CONVERSIONS_H_
+
+#include <string>
+
+#include "base/string16.h"
+
+namespace base {
+class StringPiece;
+}
+
+// Like the conversions in utf_string_conversions.h, but also take offsets into
+// the source strings, which will be adjusted to point at the same logical place
+// in the result strings. If this isn't possible because the offsets point past
+// the end of the source strings or into the middle of multibyte sequences, they
+// will be set to std::wstring::npos. |offset_for_adjustment| may be NULL.
+bool UTF8ToWideAndAdjustOffset(const char* src,
+ size_t src_len,
+ std::wstring* output,
+ size_t* offset_for_adjustment);
+std::wstring UTF8ToWideAndAdjustOffset(const base::StringPiece& utf8,
+ size_t* offset_for_adjustment);
+
+bool UTF16ToWideAndAdjustOffset(const char16* src,
+ size_t src_len,
+ std::wstring* output,
+ size_t* offset_for_adjustment);
+std::wstring UTF16ToWideAndAdjustOffset(const string16& utf16,
+ size_t* offset_for_adjustment);
+
+#endif // BASE_UTF_OFFSET_STRING_CONVERSIONS_H_
diff --git a/base/utf_offset_string_conversions_unittest.cc b/base/utf_offset_string_conversions_unittest.cc
new file mode 100644
index 0000000..00d87d3
--- /dev/null
+++ b/base/utf_offset_string_conversions_unittest.cc
@@ -0,0 +1,80 @@
+// Copyright (c) 2009 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "base/logging.h"
+#include "base/string_piece.h"
+#include "base/utf_offset_string_conversions.h"
+#include "testing/gtest/include/gtest/gtest.h"
+
+namespace base {
+
+namespace {
+
+// Given a null-terminated string of wchar_t with each wchar_t representing
+// a UTF-16 code unit, returns a string16 made up of wchar_t's in the input.
+// Each wchar_t should be <= 0xFFFF and a non-BMP character (> U+FFFF)
+// should be represented as a surrogate pair (two UTF-16 units)
+// *even* where wchar_t is 32-bit (Linux and Mac).
+//
+// This is to help write tests for functions with string16 params until
+// the C++ 0x UTF-16 literal is well-supported by compilers.
+string16 BuildString16(const wchar_t* s) {
+#if defined(WCHAR_T_IS_UTF16)
+ return string16(s);
+#elif defined(WCHAR_T_IS_UTF32)
+ string16 u16;
+ while (*s != 0) {
+ DCHECK(static_cast<unsigned int>(*s) <= 0xFFFFu);
+ u16.push_back(*s++);
+ }
+ return u16;
+#endif
+}
+
+} // namespace
+
+TEST(UTFOffsetStringConversionsTest, AdjustOffset) {
+ struct UTF8ToWideCase {
+ const char* utf8;
+ size_t input_offset;
+ size_t output_offset;
+ } utf8_to_wide_cases[] = {
+ {"", 0, std::wstring::npos},
+ {"\xe4\xbd\xa0\xe5\xa5\xbd", 1, std::wstring::npos},
+ {"\xe4\xbd\xa0\xe5\xa5\xbd", 3, 1},
+ {"\xed\xb0\x80z", 3, 0},
+ {"A\xF0\x90\x8C\x80z", 1, 1},
+ {"A\xF0\x90\x8C\x80z", 2, std::wstring::npos},
+#if defined(WCHAR_T_IS_UTF16)
+ {"A\xF0\x90\x8C\x80z", 5, 3},
+#elif defined(WCHAR_T_IS_UTF32)
+ {"A\xF0\x90\x8C\x80z", 5, 2},
+#endif
+ };
+ for (size_t i = 0; i < ARRAYSIZE_UNSAFE(utf8_to_wide_cases); ++i) {
+ size_t offset = utf8_to_wide_cases[i].input_offset;
+ UTF8ToWideAndAdjustOffset(utf8_to_wide_cases[i].utf8, &offset);
+ EXPECT_EQ(utf8_to_wide_cases[i].output_offset, offset);
+ }
+
+#if defined(WCHAR_T_IS_UTF32)
+ struct UTF16ToWideCase {
+ const wchar_t* wide;
+ size_t input_offset;
+ size_t output_offset;
+ } utf16_to_wide_cases[] = {
+ {L"\xD840\xDC00\x4E00", 0, 0},
+ {L"\xD840\xDC00\x4E00", 1, std::wstring::npos},
+ {L"\xD840\xDC00\x4E00", 2, 1},
+ };
+ for (size_t i = 0; i < ARRAYSIZE_UNSAFE(utf16_to_wide_cases); ++i) {
+ size_t offset = utf16_to_wide_cases[i].input_offset;
+ UTF16ToWideAndAdjustOffset(BuildString16(utf16_to_wide_cases[i].wide),
+ &offset);
+ EXPECT_EQ(utf16_to_wide_cases[i].output_offset, offset);
+ }
+#endif
+}
+
+} // namaspace base
diff --git a/base/utf_string_conversion_utils.cc b/base/utf_string_conversion_utils.cc
new file mode 100644
index 0000000..ebd8fd9
--- /dev/null
+++ b/base/utf_string_conversion_utils.cc
@@ -0,0 +1,148 @@
+// Copyright (c) 2009 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "base/utf_string_conversion_utils.h"
+
+#include "base/third_party/icu/icu_utf.h"
+
+namespace base {
+
+// ReadUnicodeCharacter --------------------------------------------------------
+
+bool ReadUnicodeCharacter(const char* src,
+ int32 src_len,
+ int32* char_index,
+ uint32* code_point_out) {
+ // U8_NEXT expects to be able to use -1 to signal an error, so we must
+ // use a signed type for code_point. But this function returns false
+ // on error anyway, so code_point_out is unsigned.
+ int32 code_point;
+ CBU8_NEXT(src, *char_index, src_len, code_point);
+ *code_point_out = static_cast<uint32>(code_point);
+
+ // The ICU macro above moves to the next char, we want to point to the last
+ // char consumed.
+ (*char_index)--;
+
+ // Validate the decoded value.
+ return IsValidCodepoint(code_point);
+}
+
+bool ReadUnicodeCharacter(const char16* src,
+ int32 src_len,
+ int32* char_index,
+ uint32* code_point) {
+ if (CBU16_IS_SURROGATE(src[*char_index])) {
+ if (!CBU16_IS_SURROGATE_LEAD(src[*char_index]) ||
+ *char_index + 1 >= src_len ||
+ !CBU16_IS_TRAIL(src[*char_index + 1])) {
+ // Invalid surrogate pair.
+ return false;
+ }
+
+ // Valid surrogate pair.
+ *code_point = CBU16_GET_SUPPLEMENTARY(src[*char_index],
+ src[*char_index + 1]);
+ (*char_index)++;
+ } else {
+ // Not a surrogate, just one 16-bit word.
+ *code_point = src[*char_index];
+ }
+
+ return IsValidCodepoint(*code_point);
+}
+
+#if defined(WCHAR_T_IS_UTF32)
+bool ReadUnicodeCharacter(const wchar_t* src,
+ int32 src_len,
+ int32* char_index,
+ uint32* code_point) {
+ // Conversion is easy since the source is 32-bit.
+ *code_point = src[*char_index];
+
+ // Validate the value.
+ return IsValidCodepoint(*code_point);
+}
+#endif // defined(WCHAR_T_IS_UTF32)
+
+// WriteUnicodeCharacter -------------------------------------------------------
+
+size_t WriteUnicodeCharacter(uint32 code_point, std::string* output) {
+ if (code_point <= 0x7f) {
+ // Fast path the common case of one byte.
+ output->push_back(code_point);
+ return 1;
+ }
+
+
+ // CBU8_APPEND_UNSAFE can append up to 4 bytes.
+ size_t char_offset = output->length();
+ size_t original_char_offset = char_offset;
+ output->resize(char_offset + CBU8_MAX_LENGTH);
+
+ CBU8_APPEND_UNSAFE(&(*output)[0], char_offset, code_point);
+
+ // CBU8_APPEND_UNSAFE will advance our pointer past the inserted character, so
+ // it will represent the new length of the string.
+ output->resize(char_offset);
+ return char_offset - original_char_offset;
+}
+
+size_t WriteUnicodeCharacter(uint32 code_point, string16* output) {
+ if (CBU16_LENGTH(code_point) == 1) {
+ // Thie code point is in the Basic Multilingual Plane (BMP).
+ output->push_back(static_cast<char16>(code_point));
+ return 1;
+ }
+ // Non-BMP characters use a double-character encoding.
+ size_t char_offset = output->length();
+ output->resize(char_offset + CBU16_MAX_LENGTH);
+ CBU16_APPEND_UNSAFE(&(*output)[0], char_offset, code_point);
+ return CBU16_MAX_LENGTH;
+}
+
+// Generalized Unicode converter -----------------------------------------------
+
+template<typename CHAR>
+void PrepareForUTF8Output(const CHAR* src,
+ size_t src_len,
+ std::string* output) {
+ output->clear();
+ if (src_len == 0)
+ return;
+ if (src[0] < 0x80) {
+ // Assume that the entire input will be ASCII.
+ output->reserve(src_len);
+ } else {
+ // Assume that the entire input is non-ASCII and will have 3 bytes per char.
+ output->reserve(src_len * 3);
+ }
+}
+
+// Instantiate versions we know callers will need.
+template void PrepareForUTF8Output(const wchar_t*, size_t, std::string*);
+template void PrepareForUTF8Output(const char16*, size_t, std::string*);
+
+template<typename STRING>
+void PrepareForUTF16Or32Output(const char* src,
+ size_t src_len,
+ STRING* output) {
+ output->clear();
+ if (src_len == 0)
+ return;
+ if (static_cast<unsigned char>(src[0]) < 0x80) {
+ // Assume the input is all ASCII, which means 1:1 correspondence.
+ output->reserve(src_len);
+ } else {
+ // Otherwise assume that the UTF-8 sequences will have 2 bytes for each
+ // character.
+ output->reserve(src_len / 2);
+ }
+}
+
+// Instantiate versions we know callers will need.
+template void PrepareForUTF16Or32Output(const char*, size_t, std::wstring*);
+template void PrepareForUTF16Or32Output(const char*, size_t, string16*);
+
+} // namespace base
diff --git a/base/utf_string_conversion_utils.h b/base/utf_string_conversion_utils.h
new file mode 100644
index 0000000..a8a76c5
--- /dev/null
+++ b/base/utf_string_conversion_utils.h
@@ -0,0 +1,86 @@
+// Copyright (c) 2009 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef BASE_UTF_STRING_CONVERSION_UTILS_H_
+#define BASE_UTF_STRING_CONVERSION_UTILS_H_
+
+// This should only be used by the various UTF string conversion files.
+
+#include "base/string16.h"
+
+namespace base {
+
+inline bool IsValidCodepoint(uint32 code_point) {
+ // Excludes the surrogate code points ([0xD800, 0xDFFF]) and
+ // codepoints larger than 0x10FFFF (the highest codepoint allowed).
+ // Non-characters and unassigned codepoints are allowed.
+ return code_point < 0xD800u ||
+ (code_point >= 0xE000u && code_point <= 0x10FFFFu);
+}
+
+// ReadUnicodeCharacter --------------------------------------------------------
+
+// Reads a UTF-8 stream, placing the next code point into the given output
+// |*code_point|. |src| represents the entire string to read, and |*char_index|
+// is the character offset within the string to start reading at. |*char_index|
+// will be updated to index the last character read, such that incrementing it
+// (as in a for loop) will take the reader to the next character.
+//
+// Returns true on success. On false, |*code_point| will be invalid.
+bool ReadUnicodeCharacter(const char* src,
+ int32 src_len,
+ int32* char_index,
+ uint32* code_point_out);
+
+// Reads a UTF-16 character. The usage is the same as the 8-bit version above.
+bool ReadUnicodeCharacter(const char16* src,
+ int32 src_len,
+ int32* char_index,
+ uint32* code_point);
+
+#if defined(WCHAR_T_IS_UTF32)
+// Reads UTF-32 character. The usage is the same as the 8-bit version above.
+bool ReadUnicodeCharacter(const wchar_t* src,
+ int32 src_len,
+ int32* char_index,
+ uint32* code_point);
+#endif // defined(WCHAR_T_IS_UTF32)
+
+// WriteUnicodeCharacter -------------------------------------------------------
+
+// Appends a UTF-8 character to the given 8-bit string. Returns the number of
+// bytes written.
+size_t WriteUnicodeCharacter(uint32 code_point, std::string* output);
+
+// Appends the given code point as a UTF-16 character to the given 16-bit
+// string. Returns the number of 16-bit values written.
+size_t WriteUnicodeCharacter(uint32 code_point, string16* output);
+
+#if defined(WCHAR_T_IS_UTF32)
+// Appends the given UTF-32 character to the given 32-bit string. Returns the
+// number of 32-bit values written.
+inline size_t WriteUnicodeCharacter(uint32 code_point, std::wstring* output) {
+ // This is the easy case, just append the character.
+ output->push_back(code_point);
+ return 1;
+}
+#endif // defined(WCHAR_T_IS_UTF32)
+
+// Generalized Unicode converter -----------------------------------------------
+
+// Guesses the length of the output in UTF-8 in bytes, clears that output
+// string, and reserves that amount of space. We assume that the input
+// character types are unsigned, which will be true for UTF-16 and -32 on our
+// systems.
+template<typename CHAR>
+void PrepareForUTF8Output(const CHAR* src, size_t src_len, std::string* output);
+
+// Prepares an output buffer (containing either UTF-16 or -32 data) given some
+// UTF-8 input that will be converted to it. See PrepareForUTF8Output().
+template<typename STRING>
+void PrepareForUTF16Or32Output(const char* src, size_t src_len, STRING* output);
+
+} // namespace base
+
+#endif // BASE_UTF_STRING_CONVERSION_UTILS_H_
diff --git a/base/utf_string_conversions.cc b/base/utf_string_conversions.cc
index ee52f47..7376933 100644
--- a/base/utf_string_conversions.cc
+++ b/base/utf_string_conversions.cc
@@ -4,133 +4,16 @@
#include "base/utf_string_conversions.h"
-#include <vector>
+#include "base/string_piece.h"
+#include "base/utf_string_conversion_utils.h"
-#include "base/basictypes.h"
-#include "base/logging.h"
-#include "base/string_util.h"
-#include "base/third_party/icu/icu_utf.h"
+using base::PrepareForUTF8Output;
+using base::PrepareForUTF16Or32Output;
+using base::ReadUnicodeCharacter;
+using base::WriteUnicodeCharacter;
namespace {
-inline bool IsValidCodepoint(uint32 code_point) {
- // Excludes the surrogate code points ([0xD800, 0xDFFF]) and
- // codepoints larger than 0x10FFFF (the highest codepoint allowed).
- // Non-characters and unassigned codepoints are allowed.
- return code_point < 0xD800u ||
- (code_point >= 0xE000u && code_point <= 0x10FFFFu);
-}
-
-// ReadUnicodeCharacter --------------------------------------------------------
-
-// Reads a UTF-8 stream, placing the next code point into the given output
-// |*code_point|. |src| represents the entire string to read, and |*char_index|
-// is the character offset within the string to start reading at. |*char_index|
-// will be updated to index the last character read, such that incrementing it
-// (as in a for loop) will take the reader to the next character.
-//
-// Returns true on success. On false, |*code_point| will be invalid.
-bool ReadUnicodeCharacter(const char* src, int32 src_len,
- int32* char_index, uint32* code_point_out) {
- // U8_NEXT expects to be able to use -1 to signal an error, so we must
- // use a signed type for code_point. But this function returns false
- // on error anyway, so code_point_out is unsigned.
- int32 code_point;
- CBU8_NEXT(src, *char_index, src_len, code_point);
- *code_point_out = static_cast<uint32>(code_point);
-
- // The ICU macro above moves to the next char, we want to point to the last
- // char consumed.
- (*char_index)--;
-
- // Validate the decoded value.
- return IsValidCodepoint(code_point);
-}
-
-// Reads a UTF-16 character. The usage is the same as the 8-bit version above.
-bool ReadUnicodeCharacter(const char16* src, int32 src_len,
- int32* char_index, uint32* code_point) {
- if (CBU16_IS_SURROGATE(src[*char_index])) {
- if (!CBU16_IS_SURROGATE_LEAD(src[*char_index]) ||
- *char_index + 1 >= src_len ||
- !CBU16_IS_TRAIL(src[*char_index + 1])) {
- // Invalid surrogate pair.
- return false;
- }
-
- // Valid surrogate pair.
- *code_point = CBU16_GET_SUPPLEMENTARY(src[*char_index],
- src[*char_index + 1]);
- (*char_index)++;
- } else {
- // Not a surrogate, just one 16-bit word.
- *code_point = src[*char_index];
- }
-
- return IsValidCodepoint(*code_point);
-}
-
-#if defined(WCHAR_T_IS_UTF32)
-// Reads UTF-32 character. The usage is the same as the 8-bit version above.
-bool ReadUnicodeCharacter(const wchar_t* src, int32 src_len,
- int32* char_index, uint32* code_point) {
- // Conversion is easy since the source is 32-bit.
- *code_point = src[*char_index];
-
- // Validate the value.
- return IsValidCodepoint(*code_point);
-}
-#endif // defined(WCHAR_T_IS_UTF32)
-
-// WriteUnicodeCharacter -------------------------------------------------------
-
-// Appends a UTF-8 character to the given 8-bit string. Returns the number of
-// bytes written.
-size_t WriteUnicodeCharacter(uint32 code_point, std::string* output) {
- if (code_point <= 0x7f) {
- // Fast path the common case of one byte.
- output->push_back(code_point);
- return 1;
- }
-
- // CBU8_APPEND_UNSAFE can append up to 4 bytes.
- size_t char_offset = output->length();
- size_t original_char_offset = char_offset;
- output->resize(char_offset + CBU8_MAX_LENGTH);
-
- CBU8_APPEND_UNSAFE(&(*output)[0], char_offset, code_point);
-
- // CBU8_APPEND_UNSAFE will advance our pointer past the inserted character, so
- // it will represent the new length of the string.
- output->resize(char_offset);
- return char_offset - original_char_offset;
-}
-
-// Appends the given code point as a UTF-16 character to the given 16-bit
-// string. Returns the number of 16-bit values written.
-size_t WriteUnicodeCharacter(uint32 code_point, string16* output) {
- if (CBU16_LENGTH(code_point) == 1) {
- // Thie code point is in the Basic Multilingual Plane (BMP).
- output->push_back(static_cast<char16>(code_point));
- return 1;
- }
- // Non-BMP characters use a double-character encoding.
- size_t char_offset = output->length();
- output->resize(char_offset + CBU16_MAX_LENGTH);
- CBU16_APPEND_UNSAFE(&(*output)[0], char_offset, code_point);
- return CBU16_MAX_LENGTH;
-}
-
-#if defined(WCHAR_T_IS_UTF32)
-// Appends the given UTF-32 character to the given 32-bit string. Returns the
-// number of 32-bit values written.
-inline size_t WriteUnicodeCharacter(uint32 code_point, std::wstring* output) {
- // This is the easy case, just append the character.
- output->push_back(code_point);
- return 1;
-}
-#endif // defined(WCHAR_T_IS_UTF32)
-
// Generalized Unicode converter -----------------------------------------------
// Converts the given source Unicode character type to the given destination
@@ -140,90 +23,31 @@
template<typename SRC_CHAR, typename DEST_STRING>
bool ConvertUnicode(const SRC_CHAR* src,
size_t src_len,
- DEST_STRING* output,
- size_t* offset_for_adjustment) {
- size_t output_offset =
- (offset_for_adjustment && *offset_for_adjustment < src_len) ?
- *offset_for_adjustment : DEST_STRING::npos;
-
+ DEST_STRING* output) {
// ICU requires 32-bit numbers.
bool success = true;
int32 src_len32 = static_cast<int32>(src_len);
for (int32 i = 0; i < src_len32; i++) {
uint32 code_point;
- size_t original_i = i;
- size_t chars_written = 0;
if (ReadUnicodeCharacter(src, src_len32, &i, &code_point)) {
- chars_written = WriteUnicodeCharacter(code_point, output);
+ WriteUnicodeCharacter(code_point, output);
} else {
// TODO(jungshik): consider adding 'Replacement character' (U+FFFD)
// in place of an invalid codepoint.
success = false;
}
- if ((output_offset != DEST_STRING::npos) &&
- (*offset_for_adjustment > original_i)) {
- // NOTE: ReadUnicodeCharacter() adjusts |i| to point _at_ the last
- // character read, not after it (so that incrementing it in the loop
- // increment will place it at the right location), so we need to account
- // for that in determining the amount that was read.
- if (*offset_for_adjustment <= static_cast<size_t>(i))
- output_offset = DEST_STRING::npos;
- else
- output_offset += chars_written - (i - original_i + 1);
- }
}
- if (offset_for_adjustment)
- *offset_for_adjustment = output_offset;
return success;
}
-// Guesses the length of the output in UTF-8 in bytes, clears that output
-// string, and reserves that amount of space. We assume that the input
-// character types are unsigned, which will be true for UTF-16 and -32 on our
-// systems.
-template<typename CHAR>
-void PrepareForUTF8Output(const CHAR* src,
- size_t src_len,
- std::string* output) {
- output->clear();
- if (src_len == 0)
- return;
- if (src[0] < 0x80) {
- // Assume that the entire input will be ASCII.
- output->reserve(src_len);
- } else {
- // Assume that the entire input is non-ASCII and will have 3 bytes per char.
- output->reserve(src_len * 3);
- }
-}
-
-// Prepares an output buffer (containing either UTF-16 or -32 data) given some
-// UTF-8 input that will be converted to it. See PrepareForUTF8Output().
-template<typename STRING>
-void PrepareForUTF16Or32Output(const char* src,
- size_t src_len,
- STRING* output) {
- output->clear();
- if (src_len == 0)
- return;
- if (static_cast<unsigned char>(src[0]) < 0x80) {
- // Assume the input is all ASCII, which means 1:1 correspondence.
- output->reserve(src_len);
- } else {
- // Otherwise assume that the UTF-8 sequences will have 2 bytes for each
- // character.
- output->reserve(src_len / 2);
- }
-}
-
} // namespace
// UTF-8 <-> Wide --------------------------------------------------------------
bool WideToUTF8(const wchar_t* src, size_t src_len, std::string* output) {
PrepareForUTF8Output(src, src_len, output);
- return ConvertUnicode<wchar_t, std::string>(src, src_len, output, NULL);
+ return ConvertUnicode(src, src_len, output);
}
std::string WideToUTF8(const std::wstring& wide) {
@@ -234,20 +58,14 @@
return ret;
}
-bool UTF8ToWideAndAdjustOffset(const char* src,
- size_t src_len,
- std::wstring* output,
- size_t* offset_for_adjustment) {
+bool UTF8ToWide(const char* src, size_t src_len, std::wstring* output) {
PrepareForUTF16Or32Output(src, src_len, output);
- return ConvertUnicode<char, std::wstring>(src, src_len, output,
- offset_for_adjustment);
+ return ConvertUnicode(src, src_len, output);
}
-std::wstring UTF8ToWideAndAdjustOffset(const base::StringPiece& utf8,
- size_t* offset_for_adjustment) {
+std::wstring UTF8ToWide(const base::StringPiece& utf8) {
std::wstring ret;
- UTF8ToWideAndAdjustOffset(utf8.data(), utf8.length(), &ret,
- offset_for_adjustment);
+ UTF8ToWide(utf8.data(), utf8.length(), &ret);
return ret;
}
@@ -265,20 +83,12 @@
return wide;
}
-bool UTF16ToWideAndAdjustOffset(const char16* src,
- size_t src_len,
- std::wstring* output,
- size_t* offset_for_adjustment) {
+bool UTF16ToWide(const char16* src, size_t src_len, std::wstring* output) {
output->assign(src, src_len);
- if (offset_for_adjustment && (*offset_for_adjustment >= src_len))
- *offset_for_adjustment = std::wstring::npos;
return true;
}
-std::wstring UTF16ToWideAndAdjustOffset(const string16& utf16,
- size_t* offset_for_adjustment) {
- if (offset_for_adjustment && (*offset_for_adjustment >= utf16.length()))
- *offset_for_adjustment = std::wstring::npos;
+std::wstring UTF16ToWide(const string16& utf16) {
return utf16;
}
@@ -289,7 +99,7 @@
// Assume that normally we won't have any non-BMP characters so the counts
// will be the same.
output->reserve(src_len);
- return ConvertUnicode<wchar_t, string16>(src, src_len, output, NULL);
+ return ConvertUnicode(src, src_len, output);
}
string16 WideToUTF16(const std::wstring& wide) {
@@ -298,23 +108,17 @@
return ret;
}
-bool UTF16ToWideAndAdjustOffset(const char16* src,
- size_t src_len,
- std::wstring* output,
- size_t* offset_for_adjustment) {
+bool UTF16ToWide(const char16* src, size_t src_len, std::wstring* output) {
output->clear();
// Assume that normally we won't have any non-BMP characters so the counts
// will be the same.
output->reserve(src_len);
- return ConvertUnicode<char16, std::wstring>(src, src_len, output,
- offset_for_adjustment);
+ return ConvertUnicode(src, src_len, output);
}
-std::wstring UTF16ToWideAndAdjustOffset(const string16& utf16,
- size_t* offset_for_adjustment) {
+std::wstring UTF16ToWide(const string16& utf16) {
std::wstring ret;
- UTF16ToWideAndAdjustOffset(utf16.data(), utf16.length(), &ret,
- offset_for_adjustment);
+ UTF16ToWide(utf16.data(), utf16.length(), &ret);
return ret;
}
@@ -326,7 +130,7 @@
bool UTF8ToUTF16(const char* src, size_t src_len, string16* output) {
PrepareForUTF16Or32Output(src, src_len, output);
- return ConvertUnicode<char, string16>(src, src_len, output, NULL);
+ return ConvertUnicode(src, src_len, output);
}
string16 UTF8ToUTF16(const std::string& utf8) {
@@ -339,7 +143,7 @@
bool UTF16ToUTF8(const char16* src, size_t src_len, std::string* output) {
PrepareForUTF8Output(src, src_len, output);
- return ConvertUnicode<char16, std::string>(src, src_len, output, NULL);
+ return ConvertUnicode(src, src_len, output);
}
std::string UTF16ToUTF8(const string16& utf16) {
diff --git a/base/utf_string_conversions.h b/base/utf_string_conversions.h
index 7069f83..71e2cb2 100644
--- a/base/utf_string_conversions.h
+++ b/base/utf_string_conversions.h
@@ -8,26 +8,10 @@
#include <string>
#include "base/string16.h"
-#include "base/string_piece.h"
-// Like the conversions below, but also takes an offset into the source string,
-// which will be adjusted to point at the same logical place in the result
-// string. If this isn't possible because it points past the end of the source
-// string or into the middle of a multibyte sequence, it will be set to
-// std::wstring::npos. |offset_for_adjustment| may be NULL.
-bool UTF8ToWideAndAdjustOffset(const char* src,
- size_t src_len,
- std::wstring* output,
- size_t* offset_for_adjustment);
-std::wstring UTF8ToWideAndAdjustOffset(const base::StringPiece& utf8,
- size_t* offset_for_adjustment);
-
-bool UTF16ToWideAndAdjustOffset(const char16* src,
- size_t src_len,
- std::wstring* output,
- size_t* offset_for_adjustment);
-std::wstring UTF16ToWideAndAdjustOffset(const string16& utf16,
- size_t* offset_for_adjustment);
+namespace base {
+class StringPiece;
+}
// These convert between UTF-8, -16, and -32 strings. They are potentially slow,
// so avoid unnecessary conversions. The low-level versions return a boolean
@@ -44,22 +28,13 @@
// some situations.
bool WideToUTF8(const wchar_t* src, size_t src_len, std::string* output);
std::string WideToUTF8(const std::wstring& wide);
-inline bool UTF8ToWide(const char* src, size_t src_len, std::wstring* output) {
- return UTF8ToWideAndAdjustOffset(src, src_len, output, NULL);
-}
-inline std::wstring UTF8ToWide(const base::StringPiece& utf8) {
- return UTF8ToWideAndAdjustOffset(utf8, NULL);
-}
+bool UTF8ToWide(const char* src, size_t src_len, std::wstring* output);
+std::wstring UTF8ToWide(const base::StringPiece& utf8);
bool WideToUTF16(const wchar_t* src, size_t src_len, string16* output);
string16 WideToUTF16(const std::wstring& wide);
-inline bool UTF16ToWide(const char16* src, size_t src_len,
- std::wstring* output) {
- return UTF16ToWideAndAdjustOffset(src, src_len, output, NULL);
-}
-inline std::wstring UTF16ToWide(const string16& utf16) {
- return UTF16ToWideAndAdjustOffset(utf16, NULL);
-}
+bool UTF16ToWide(const char16* src, size_t src_len, std::wstring* output);
+std::wstring UTF16ToWide(const string16& utf16);
bool UTF8ToUTF16(const char* src, size_t src_len, string16* output);
string16 UTF8ToUTF16(const std::string& utf8);
diff --git a/base/utf_string_conversions_unittest.cc b/base/utf_string_conversions_unittest.cc
index ca79ec5..1918997 100644
--- a/base/utf_string_conversions_unittest.cc
+++ b/base/utf_string_conversions_unittest.cc
@@ -226,47 +226,4 @@
EXPECT_EQ(expected, converted);
}
-TEST(UTFStringConversionsTest, AdjustOffset) {
- struct UTF8ToWideCase {
- const char* utf8;
- size_t input_offset;
- size_t output_offset;
- } utf8_to_wide_cases[] = {
- {"", 0, std::wstring::npos},
- {"\xe4\xbd\xa0\xe5\xa5\xbd", 1, std::wstring::npos},
- {"\xe4\xbd\xa0\xe5\xa5\xbd", 3, 1},
- {"\xed\xb0\x80z", 3, 0},
- {"A\xF0\x90\x8C\x80z", 1, 1},
- {"A\xF0\x90\x8C\x80z", 2, std::wstring::npos},
-#if defined(WCHAR_T_IS_UTF16)
- {"A\xF0\x90\x8C\x80z", 5, 3},
-#elif defined(WCHAR_T_IS_UTF32)
- {"A\xF0\x90\x8C\x80z", 5, 2},
-#endif
- };
- for (size_t i = 0; i < ARRAYSIZE_UNSAFE(utf8_to_wide_cases); ++i) {
- size_t offset = utf8_to_wide_cases[i].input_offset;
- UTF8ToWideAndAdjustOffset(utf8_to_wide_cases[i].utf8, &offset);
- EXPECT_EQ(utf8_to_wide_cases[i].output_offset, offset);
- }
-
-#if defined(WCHAR_T_IS_UTF32)
- struct UTF16ToWideCase {
- const wchar_t* wide;
- size_t input_offset;
- size_t output_offset;
- } utf16_to_wide_cases[] = {
- {L"\xD840\xDC00\x4E00", 0, 0},
- {L"\xD840\xDC00\x4E00", 1, std::wstring::npos},
- {L"\xD840\xDC00\x4E00", 2, 1},
- };
- for (size_t i = 0; i < ARRAYSIZE_UNSAFE(utf16_to_wide_cases); ++i) {
- size_t offset = utf16_to_wide_cases[i].input_offset;
- UTF16ToWideAndAdjustOffset(BuildString16(utf16_to_wide_cases[i].wide),
- &offset);
- EXPECT_EQ(utf16_to_wide_cases[i].output_offset, offset);
- }
-#endif
-}
-
} // namaspace base