Add UTF-8 check for JSON deserializer.
Add tests for IsStringUTF8
Make IsStringUTF8 accept std::string/std::wstring rather than char*/wchar_t*
Review URL: http://codereview.chromium.org/4268
git-svn-id: svn://svn.chromium.org/chrome/trunk/src@2610 0039d316-1c4b-4281-b951-d872f2087c98
CrOS-Libchrome-Original-Commit: c9ec45429c64884c35f83b74131c0e3ae5b2bbe9
diff --git a/base/json_reader.cc b/base/json_reader.cc
index 72ce364..8d0cab4 100644
--- a/base/json_reader.cc
+++ b/base/json_reader.cc
@@ -83,8 +83,11 @@
Value** root,
bool check_root,
bool allow_trailing_comma) {
- // Assume input is UTF8. The conversion from UTF8 to wstring removes null
- // bytes for us (a good thing).
+ // The input must be in UTF-8.
+ if (!IsStringUTF8(json.c_str()))
+ return false;
+ // The conversion from UTF8 to wstring removes null bytes for us
+ // (a good thing).
std::wstring json_wide(UTF8ToWide(json));
const wchar_t* json_cstr = json_wide.c_str();
diff --git a/base/json_reader_unittest.cc b/base/json_reader_unittest.cc
index 6c60a07..c2d6a42 100644
--- a/base/json_reader_unittest.cc
+++ b/base/json_reader_unittest.cc
@@ -481,6 +481,13 @@
ASSERT_EQ(L"\x7f51\x9875", str_val);
delete root;
+ // Test invalid utf8 encoded input
+ root = NULL;
+ ASSERT_FALSE(JSONReader::JsonToValue("\"345\xb0\xa1\xb0\xa2\"", &root,
+ false, false));
+ ASSERT_FALSE(JSONReader::JsonToValue("\"123\xc0\x81\"", &root,
+ false, false));
+
// Test invalid root objects.
root = NULL;
ASSERT_FALSE(JSONReader::Read("null", &root, false));
diff --git a/base/string_util.cc b/base/string_util.cc
index 2f39104..223c485 100644
--- a/base/string_util.cc
+++ b/base/string_util.cc
@@ -517,7 +517,7 @@
// originally been UTF-8, but has been converted to wide characters because
// that's what we (and Windows) use internally.
template<typename CHAR>
-static bool IsStringUTF8T(const CHAR* str) {
+static bool IsStringUTF8T(const CHAR* str, int length) {
bool overlong = false;
bool surrogate = false;
bool nonchar = false;
@@ -532,7 +532,7 @@
// are left in the sequence
int positions_left = 0;
- for (int i = 0; str[i] != 0; i++) {
+ for (int i = 0; i < length; i++) {
// This whole function assume an unsigned value so force its conversion to
// an unsigned value.
typename ToUnsigned<CHAR>::Unsigned c = str[i];
@@ -556,6 +556,7 @@
slower = 0xA0;
} else if (c == 0xEF) {
// EF BF [BE-BF] : non-character
+ // TODO(jungshik): EF B7 [90-AF] should be checked as well.
nonchar = true;
}
} else if (c <= 0xF4) {
@@ -599,12 +600,12 @@
return true;
}
-bool IsStringUTF8(const char* str) {
- return IsStringUTF8T(str);
+bool IsStringUTF8(const std::string& str) {
+ return IsStringUTF8T(str.data(), str.length());
}
-bool IsStringWideUTF8(const wchar_t* str) {
- return IsStringUTF8T(str);
+bool IsStringWideUTF8(const std::wstring& str) {
+ return IsStringUTF8T(str.data(), str.length());
}
template<typename Iter>
diff --git a/base/string_util.h b/base/string_util.h
index 8c28f3b..a9f08c4 100644
--- a/base/string_util.h
+++ b/base/string_util.h
@@ -227,8 +227,8 @@
// first case) or characters that use only 8-bits and whose 8-bit
// representation looks like a UTF-8 string (the second case).
bool IsString8Bit(const std::wstring& str);
-bool IsStringUTF8(const char* str);
-bool IsStringWideUTF8(const wchar_t* str);
+bool IsStringUTF8(const std::string& str);
+bool IsStringWideUTF8(const std::wstring& str);
bool IsStringASCII(const std::wstring& str);
bool IsStringASCII(const std::string& str);
diff --git a/base/string_util_unittest.cc b/base/string_util_unittest.cc
index 89cec22..e438ebb 100644
--- a/base/string_util_unittest.cc
+++ b/base/string_util_unittest.cc
@@ -113,6 +113,71 @@
}
}
+
+TEST(StringUtilTest, IsStringUTF8) {
+ EXPECT_TRUE(IsStringUTF8("abc"));
+ EXPECT_TRUE(IsStringUTF8("\xc2\x81"));
+ EXPECT_TRUE(IsStringUTF8("\xe1\x80\xbf"));
+ EXPECT_TRUE(IsStringUTF8("\xf1\x80\xa0\xbf"));
+ EXPECT_TRUE(IsStringUTF8("a\xc2\x81\xe1\x80\xbf\xf1\x80\xa0\xbf"));
+ EXPECT_TRUE(IsStringUTF8("\xef\xbb\xbf" "abc")); // UTF-8 BOM
+
+
+ // surrogate code points
+ EXPECT_FALSE(IsStringUTF8("\xed\xa0\x80\xed\xbf\xbf"));
+ EXPECT_FALSE(IsStringUTF8("\xed\xa0\x8f"));
+ EXPECT_FALSE(IsStringUTF8("\xed\xbf\xbf"));
+
+ // overlong sequences
+ EXPECT_FALSE(IsStringUTF8("\xc0\x80")); // U+0000
+ EXPECT_FALSE(IsStringUTF8("\xc1\x80\xc1\x81")); // "AB"
+ EXPECT_FALSE(IsStringUTF8("\xe0\x80\x80")); // U+0000
+ EXPECT_FALSE(IsStringUTF8("\xe0\x82\x80")); // U+0080
+ EXPECT_FALSE(IsStringUTF8("\xe0\x9f\xbf")); // U+07ff
+ EXPECT_FALSE(IsStringUTF8("\xf0\x80\x80\x8D")); // U+000D
+ EXPECT_FALSE(IsStringUTF8("\xf0\x80\x82\x91")); // U+0091
+ EXPECT_FALSE(IsStringUTF8("\xf0\x80\xa0\x80")); // U+0800
+ EXPECT_FALSE(IsStringUTF8("\xf0\x8f\xbb\xbf")); // U+FEFF (BOM)
+ EXPECT_FALSE(IsStringUTF8("\xf8\x80\x80\x80\xbf")); // U+003F
+ EXPECT_FALSE(IsStringUTF8("\xfc\x80\x80\x80\xa0\xa5")); // U+00A5
+
+ // Beyond U+10FFFF (the upper limit of Unicode codespace)
+ EXPECT_FALSE(IsStringUTF8("\xf4\x90\x80\x80")); // U+110000
+ EXPECT_FALSE(IsStringUTF8("\xf8\xa0\xbf\x80\xbf")); // 5 bytes
+ EXPECT_FALSE(IsStringUTF8("\xfc\x9c\xbf\x80\xbf\x80")); // 6 bytes
+
+ // BOMs in UTF-16(BE|LE) and UTF-32(BE|LE)
+ EXPECT_FALSE(IsStringUTF8("\xfe\xff"));
+ EXPECT_FALSE(IsStringUTF8("\xff\xfe"));
+ EXPECT_FALSE(IsStringUTF8(std::string("\x00\x00\xfe\xff", 4)));
+ EXPECT_FALSE(IsStringUTF8("\xff\xfe\x00\x00"));
+
+ // Non-characters : U+xxFFF[EF] where xx is 0x00 through 0x10 and <FDD0,FDEF>
+ EXPECT_FALSE(IsStringUTF8("\xef\xbf\xbe")); // U+FFFE)
+ EXPECT_FALSE(IsStringUTF8("\xf0\x8f\xbf\xbe")); // U+1FFFE
+ EXPECT_FALSE(IsStringUTF8("\xf3\xbf\xbf\xbf")); // U+10FFFF
+
+ // This should also be false, but currently we pass them through.
+ // Disable them for now.
+#if 0
+ EXPECT_FALSE(IsStringUTF8("\xef\xb7\x90")); // U+FDD0
+ EXPECT_FALSE(IsStringUTF8("\xef\xb7\xaf")); // U+FDEF
+#endif
+
+ // Strings in legacy encodings. We can certainly make up strings
+ // in a legacy encoding that are valid in UTF-8, but in real data,
+ // most of them are invalid as UTF-8.
+ EXPECT_FALSE(IsStringUTF8("caf\xe9")); // cafe with U+00E9 in ISO-8859-1
+ EXPECT_FALSE(IsStringUTF8("\xb0\xa1\xb0\xa2")); // U+AC00, U+AC001 in EUC-KR
+ EXPECT_FALSE(IsStringUTF8("\xa7\x41\xa6\x6e")); // U+4F60 U+597D in Big5
+ // "abc" with U+201[CD] in windows-125[0-8]
+ EXPECT_FALSE(IsStringUTF8("\x93" "abc\x94"));
+ // U+0639 U+064E U+0644 U+064E in ISO-8859-6
+ EXPECT_FALSE(IsStringUTF8("\xd9\xee\xe4\xee"));
+ // U+03B3 U+03B5 U+03B9 U+03AC in ISO-8859-7
+ EXPECT_FALSE(IsStringUTF8("\xe3\xe5\xe9\xdC"));
+}
+
static const wchar_t* const kConvertRoundtripCases[] = {
L"Google Video",
// "网页 图片 资讯更多 »"