[Lexer] Improve Lexer::getSourceText() when the given range deals with function macro arguments.

This is a modified version of a patch by Manuel Klimek.

git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@182055 91177308-0d34-0410-b5e6-96231b3b80d8
diff --git a/unittests/Lex/LexerTest.cpp b/unittests/Lex/LexerTest.cpp
index c9b1840..a8e25cb 100644
--- a/unittests/Lex/LexerTest.cpp
+++ b/unittests/Lex/LexerTest.cpp
@@ -28,30 +28,6 @@
 
 namespace {
 
-// The test fixture.
-class LexerTest : public ::testing::Test {
-protected:
-  LexerTest()
-    : FileMgr(FileMgrOpts),
-      DiagID(new DiagnosticIDs()),
-      Diags(DiagID, new DiagnosticOptions, new IgnoringDiagConsumer()),
-      SourceMgr(Diags, FileMgr),
-      TargetOpts(new TargetOptions) 
-  {
-    TargetOpts->Triple = "x86_64-apple-darwin11.1.0";
-    Target = TargetInfo::CreateTargetInfo(Diags, &*TargetOpts);
-  }
-
-  FileSystemOptions FileMgrOpts;
-  FileManager FileMgr;
-  IntrusiveRefCntPtr<DiagnosticIDs> DiagID;
-  DiagnosticsEngine Diags;
-  SourceManager SourceMgr;
-  LangOptions LangOpts;
-  IntrusiveRefCntPtr<TargetOptions> TargetOpts;
-  IntrusiveRefCntPtr<TargetInfo> Target;
-};
-
 class VoidModuleLoader : public ModuleLoader {
   virtual ModuleLoadResult loadModule(SourceLocation ImportLoc, 
                                       ModuleIdPath Path,
@@ -66,51 +42,245 @@
                                  bool Complain) { }
 };
 
-TEST_F(LexerTest, LexAPI) {
-  const char *source =
-    "#define M(x) [x]\n"
-    "#define N(x) x\n"
-    "#define INN(x) x\n"
-    "#define NOF1 INN(val)\n"
-    "#define NOF2 val\n"
-    "M(foo) N([bar])\n"
-    "N(INN(val)) N(NOF1) N(NOF2) N(val)";
-
-  MemoryBuffer *buf = MemoryBuffer::getMemBuffer(source);
-  (void)SourceMgr.createMainFileIDForMemBuffer(buf);
-
-  VoidModuleLoader ModLoader;
-  HeaderSearch HeaderInfo(new HeaderSearchOptions, FileMgr, Diags, LangOpts, 
-                          Target.getPtr());
-  Preprocessor PP(new PreprocessorOptions(), Diags, LangOpts, Target.getPtr(),
-                  SourceMgr, HeaderInfo, ModLoader,
-                  /*IILookup =*/ 0,
-                  /*OwnsHeaderSearch =*/false,
-                  /*DelayInitialization =*/ false);
-  PP.EnterMainSourceFile();
-
-  std::vector<Token> toks;
-  while (1) {
-    Token tok;
-    PP.Lex(tok);
-    if (tok.is(tok::eof))
-      break;
-    toks.push_back(tok);
+// The test fixture.
+class LexerTest : public ::testing::Test {
+protected:
+  LexerTest()
+    : FileMgr(FileMgrOpts),
+      DiagID(new DiagnosticIDs()),
+      Diags(DiagID, new DiagnosticOptions, new IgnoringDiagConsumer()),
+      SourceMgr(Diags, FileMgr),
+      TargetOpts(new TargetOptions) 
+  {
+    TargetOpts->Triple = "x86_64-apple-darwin11.1.0";
+    Target = TargetInfo::CreateTargetInfo(Diags, &*TargetOpts);
   }
 
-  // Make sure we got the tokens that we expected.
-  ASSERT_EQ(10U, toks.size());
-  ASSERT_EQ(tok::l_square, toks[0].getKind());
-  ASSERT_EQ(tok::identifier, toks[1].getKind());
-  ASSERT_EQ(tok::r_square, toks[2].getKind());
-  ASSERT_EQ(tok::l_square, toks[3].getKind());
-  ASSERT_EQ(tok::identifier, toks[4].getKind());
-  ASSERT_EQ(tok::r_square, toks[5].getKind());
-  ASSERT_EQ(tok::identifier, toks[6].getKind());
-  ASSERT_EQ(tok::identifier, toks[7].getKind());
-  ASSERT_EQ(tok::identifier, toks[8].getKind());
-  ASSERT_EQ(tok::identifier, toks[9].getKind());
-  
+  std::vector<Token> CheckLex(StringRef Source,
+                              ArrayRef<tok::TokenKind> ExpectedTokens) {
+    MemoryBuffer *buf = MemoryBuffer::getMemBuffer(Source);
+    (void) SourceMgr.createMainFileIDForMemBuffer(buf);
+
+    VoidModuleLoader ModLoader;
+    HeaderSearch HeaderInfo(new HeaderSearchOptions, FileMgr, Diags, LangOpts,
+                            Target.getPtr());
+    Preprocessor PP(new PreprocessorOptions(), Diags, LangOpts, Target.getPtr(),
+                    SourceMgr, HeaderInfo, ModLoader, /*IILookup =*/ 0,
+                    /*OwnsHeaderSearch =*/ false,
+                    /*DelayInitialization =*/ false);
+    PP.EnterMainSourceFile();
+
+    std::vector<Token> toks;
+    while (1) {
+      Token tok;
+      PP.Lex(tok);
+      if (tok.is(tok::eof))
+        break;
+      toks.push_back(tok);
+    }
+
+    EXPECT_EQ(ExpectedTokens.size(), toks.size());
+    for (unsigned i = 0, e = ExpectedTokens.size(); i != e; ++i) {
+      EXPECT_EQ(ExpectedTokens[i], toks[i].getKind());
+    }
+
+    return toks;
+  }
+
+  std::string getSourceText(Token Begin, Token End) {
+    bool Invalid;
+    StringRef Str =
+        Lexer::getSourceText(CharSourceRange::getTokenRange(SourceRange(
+                                    Begin.getLocation(), End.getLocation())),
+                             SourceMgr, LangOpts, &Invalid);
+    if (Invalid)
+      return "<INVALID>";
+    return Str;
+  }
+
+  FileSystemOptions FileMgrOpts;
+  FileManager FileMgr;
+  IntrusiveRefCntPtr<DiagnosticIDs> DiagID;
+  DiagnosticsEngine Diags;
+  SourceManager SourceMgr;
+  LangOptions LangOpts;
+  IntrusiveRefCntPtr<TargetOptions> TargetOpts;
+  IntrusiveRefCntPtr<TargetInfo> Target;
+};
+
+TEST_F(LexerTest, GetSourceTextExpandsToMaximumInMacroArgument) {
+  std::vector<tok::TokenKind> ExpectedTokens;
+  ExpectedTokens.push_back(tok::identifier);
+  ExpectedTokens.push_back(tok::l_paren);
+  ExpectedTokens.push_back(tok::identifier);
+  ExpectedTokens.push_back(tok::r_paren);
+
+  std::vector<Token> toks = CheckLex("#define M(x) x\n"
+                                     "M(f(M(i)))",
+                                     ExpectedTokens);
+
+  EXPECT_EQ("M(i)", getSourceText(toks[2], toks[2]));
+}
+
+TEST_F(LexerTest, GetSourceTextExpandsToMaximumInMacroArgumentForEndOfMacro) {
+  std::vector<tok::TokenKind> ExpectedTokens;
+  ExpectedTokens.push_back(tok::identifier);
+  ExpectedTokens.push_back(tok::identifier);
+
+  std::vector<Token> toks = CheckLex("#define M(x) x\n"
+                                     "M(M(i) c)",
+                                     ExpectedTokens);
+
+  EXPECT_EQ("M(i)", getSourceText(toks[0], toks[0]));
+}
+
+TEST_F(LexerTest, GetSourceTextExpandsInMacroArgumentForBeginOfMacro) {
+  std::vector<tok::TokenKind> ExpectedTokens;
+  ExpectedTokens.push_back(tok::identifier);
+  ExpectedTokens.push_back(tok::identifier);
+  ExpectedTokens.push_back(tok::identifier);
+
+  std::vector<Token> toks = CheckLex("#define M(x) x\n"
+                                     "M(c c M(i))",
+                                     ExpectedTokens);
+
+  EXPECT_EQ("c M(i)", getSourceText(toks[1], toks[2]));
+}
+
+TEST_F(LexerTest, GetSourceTextExpandsInMacroArgumentForEndOfMacro) {
+  std::vector<tok::TokenKind> ExpectedTokens;
+  ExpectedTokens.push_back(tok::identifier);
+  ExpectedTokens.push_back(tok::identifier);
+  ExpectedTokens.push_back(tok::identifier);
+
+  std::vector<Token> toks = CheckLex("#define M(x) x\n"
+                                     "M(M(i) c c)",
+                                     ExpectedTokens);
+
+  EXPECT_EQ("M(i) c", getSourceText(toks[0], toks[1]));
+}
+
+TEST_F(LexerTest, GetSourceTextInSeparateFnMacros) {
+  std::vector<tok::TokenKind> ExpectedTokens;
+  ExpectedTokens.push_back(tok::identifier);
+  ExpectedTokens.push_back(tok::identifier);
+  ExpectedTokens.push_back(tok::identifier);
+  ExpectedTokens.push_back(tok::identifier);
+
+  std::vector<Token> toks = CheckLex("#define M(x) x\n"
+                                     "M(c M(i)) M(M(i) c)",
+                                     ExpectedTokens);
+
+  EXPECT_EQ("<INVALID>", getSourceText(toks[1], toks[2]));
+}
+
+TEST_F(LexerTest, GetSourceTextWorksAcrossTokenPastes) {
+  std::vector<tok::TokenKind> ExpectedTokens;
+  ExpectedTokens.push_back(tok::identifier);
+  ExpectedTokens.push_back(tok::l_paren);
+  ExpectedTokens.push_back(tok::identifier);
+  ExpectedTokens.push_back(tok::r_paren);
+
+  std::vector<Token> toks = CheckLex("#define M(x) x\n"
+                                     "#define C(x) M(x##c)\n"
+                                     "M(f(C(i)))",
+                                     ExpectedTokens);
+
+  EXPECT_EQ("C(i)", getSourceText(toks[2], toks[2]));
+}
+
+TEST_F(LexerTest, GetSourceTextExpandsAcrossMultipleMacroCalls) {
+  std::vector<tok::TokenKind> ExpectedTokens;
+  ExpectedTokens.push_back(tok::identifier);
+  ExpectedTokens.push_back(tok::l_paren);
+  ExpectedTokens.push_back(tok::identifier);
+  ExpectedTokens.push_back(tok::r_paren);
+
+  std::vector<Token> toks = CheckLex("#define M(x) x\n"
+                                     "f(M(M(i)))",
+                                     ExpectedTokens);
+  EXPECT_EQ("M(M(i))", getSourceText(toks[2], toks[2]));
+}
+
+TEST_F(LexerTest, GetSourceTextInMiddleOfMacroArgument) {
+  std::vector<tok::TokenKind> ExpectedTokens;
+  ExpectedTokens.push_back(tok::identifier);
+  ExpectedTokens.push_back(tok::l_paren);
+  ExpectedTokens.push_back(tok::identifier);
+  ExpectedTokens.push_back(tok::r_paren);
+
+  std::vector<Token> toks = CheckLex("#define M(x) x\n"
+                                     "M(f(i))",
+                                     ExpectedTokens);
+  EXPECT_EQ("i", getSourceText(toks[2], toks[2]));
+}
+
+TEST_F(LexerTest, GetSourceTextExpandsAroundDifferentMacroCalls) {
+  std::vector<tok::TokenKind> ExpectedTokens;
+  ExpectedTokens.push_back(tok::identifier);
+  ExpectedTokens.push_back(tok::l_paren);
+  ExpectedTokens.push_back(tok::identifier);
+  ExpectedTokens.push_back(tok::r_paren);
+
+  std::vector<Token> toks = CheckLex("#define M(x) x\n"
+                                     "#define C(x) x\n"
+                                     "f(C(M(i)))",
+                                     ExpectedTokens);
+  EXPECT_EQ("C(M(i))", getSourceText(toks[2], toks[2]));
+}
+
+TEST_F(LexerTest, GetSourceTextOnlyExpandsIfFirstTokenInMacro) {
+  std::vector<tok::TokenKind> ExpectedTokens;
+  ExpectedTokens.push_back(tok::identifier);
+  ExpectedTokens.push_back(tok::l_paren);
+  ExpectedTokens.push_back(tok::identifier);
+  ExpectedTokens.push_back(tok::identifier);
+  ExpectedTokens.push_back(tok::r_paren);
+
+  std::vector<Token> toks = CheckLex("#define M(x) x\n"
+                                     "#define C(x) c x\n"
+                                     "f(C(M(i)))",
+                                     ExpectedTokens);
+  EXPECT_EQ("M(i)", getSourceText(toks[3], toks[3]));
+}
+
+TEST_F(LexerTest, GetSourceTextExpandsRecursively) {
+  std::vector<tok::TokenKind> ExpectedTokens;
+  ExpectedTokens.push_back(tok::identifier);
+  ExpectedTokens.push_back(tok::identifier);
+  ExpectedTokens.push_back(tok::l_paren);
+  ExpectedTokens.push_back(tok::identifier);
+  ExpectedTokens.push_back(tok::r_paren);
+
+  std::vector<Token> toks = CheckLex("#define M(x) x\n"
+                                     "#define C(x) c M(x)\n"
+                                     "C(f(M(i)))",
+                                     ExpectedTokens);
+  EXPECT_EQ("M(i)", getSourceText(toks[3], toks[3]));
+}
+
+TEST_F(LexerTest, LexAPI) {
+  std::vector<tok::TokenKind> ExpectedTokens;
+  ExpectedTokens.push_back(tok::l_square);
+  ExpectedTokens.push_back(tok::identifier);
+  ExpectedTokens.push_back(tok::r_square);
+  ExpectedTokens.push_back(tok::l_square);
+  ExpectedTokens.push_back(tok::identifier);
+  ExpectedTokens.push_back(tok::r_square);
+  ExpectedTokens.push_back(tok::identifier);
+  ExpectedTokens.push_back(tok::identifier);
+  ExpectedTokens.push_back(tok::identifier);
+  ExpectedTokens.push_back(tok::identifier);
+
+  std::vector<Token> toks = CheckLex("#define M(x) [x]\n"
+                                     "#define N(x) x\n"
+                                     "#define INN(x) x\n"
+                                     "#define NOF1 INN(val)\n"
+                                     "#define NOF2 val\n"
+                                     "M(foo) N([bar])\n"
+                                     "N(INN(val)) N(NOF1) N(NOF2) N(val)",
+                                     ExpectedTokens);
+
   SourceLocation lsqrLoc = toks[0].getLocation();
   SourceLocation idLoc = toks[1].getLocation();
   SourceLocation rsqrLoc = toks[2].getLocation();