[clangd] Use identifiers in file as completion candidates when build is not ready.

Summary:
o Lex the code to get the identifiers and put them into a "symbol" index.
o Adds a new completion mode without compilation/sema into code completion workflow.
o Make IncludeInserter work even when no compile command is present, by avoiding
inserting non-verbatim headers.

Reviewers: sammccall

Reviewed By: sammccall

Subscribers: ilya-biryukov, MaskRay, jkorous, arphaman, kadircet, jdoerfert, cfe-commits

Tags: #clang

Differential Revision: https://reviews.llvm.org/D60126

llvm-svn: 358159
diff --git a/clang-tools-extra/clangd/SourceCode.cpp b/clang-tools-extra/clangd/SourceCode.cpp
index 656ab1d..d90b04a 100644
--- a/clang-tools-extra/clangd/SourceCode.cpp
+++ b/clang-tools-extra/clangd/SourceCode.cpp
@@ -391,5 +391,29 @@
   return formatReplacements(Code, std::move(*CleanReplaces), Style);
 }
 
+llvm::StringMap<unsigned> collectIdentifiers(llvm::StringRef Content,
+                                             const format::FormatStyle &Style) {
+  SourceManagerForFile FileSM("dummy.cpp", Content);
+  auto &SM = FileSM.get();
+  auto FID = SM.getMainFileID();
+  Lexer Lex(FID, SM.getBuffer(FID), SM, format::getFormattingLangOpts(Style));
+  Token Tok;
+
+  llvm::StringMap<unsigned> Identifiers;
+  while (!Lex.LexFromRawLexer(Tok)) {
+    switch (Tok.getKind()) {
+    case tok::identifier:
+      ++Identifiers[Tok.getIdentifierInfo()->getName()];
+      break;
+    case tok::raw_identifier:
+      ++Identifiers[Tok.getRawIdentifier()];
+      break;
+    default:
+      continue;
+    }
+  }
+  return Identifiers;
+}
+
 } // namespace clangd
 } // namespace clang