clang-format: Don't remove 'unknown' tokens.

In certain macros or incorrect string literals, the token stream can
contain 'unknown' tokens, e.g. a single backslash or a set of empty
ticks. clang-format simply treated them as whitespace and removed them
prior to this patch.

This fixes llvm.org/PR17215

git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@192490 91177308-0d34-0410-b5e6-96231b3b80d8
diff --git a/unittests/Format/FormatTest.cpp b/unittests/Format/FormatTest.cpp
index c77616c..59596bc 100644
--- a/unittests/Format/FormatTest.cpp
+++ b/unittests/Format/FormatTest.cpp
@@ -1890,6 +1890,16 @@
   EXPECT_EQ("#define A B", format("#  \\\n define  \\\n    A  \\\n       B"));
 }
 
+TEST_F(FormatTest, DoesntRemoveUnknownTokens) {
+  verifyFormat("#define A \\x20");
+  verifyFormat("#define A \\ x20");
+  EXPECT_EQ("#define A \\ x20", format("#define A \\   x20"));
+  verifyFormat("#define A ''");
+  verifyFormat("#define A ''qqq");
+  verifyFormat("#define A `qqq");
+  verifyFormat("f(\"aaaa, bbbb, \"\\\"ccccc\\\"\");");
+}
+
 TEST_F(FormatTest, IndentsPPDirectiveInReducedSpace) {
   verifyFormat("#define A(BB)", getLLVMStyleWithColumns(13));
   verifyFormat("#define A( \\\n    BB)", getLLVMStyleWithColumns(12));