Comment parsing: fix a bug where a line with whitespace between two paragraphs
would cause us to concatenate these paragraphs into a single one.
The no-op whitespace churn in test/Index test happened because these tests
don't use the correct approach for testing and are more strict than required
for they are testing.
git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@189126 91177308-0d34-0410-b5e6-96231b3b80d8
diff --git a/lib/AST/CommentParser.cpp b/lib/AST/CommentParser.cpp
index d89c79b..03e0101 100644
--- a/lib/AST/CommentParser.cpp
+++ b/lib/AST/CommentParser.cpp
@@ -16,6 +16,15 @@
#include "llvm/Support/ErrorHandling.h"
namespace clang {
+
+static inline bool isWhitespace(llvm::StringRef S) {
+ for (StringRef::const_iterator I = S.begin(), E = S.end(); I != E; ++I) {
+ if (!isWhitespace(*I))
+ return false;
+ }
+ return true;
+}
+
namespace comments {
/// Re-lexes a sequence of tok::text tokens.
@@ -594,6 +603,18 @@
consumeToken();
break; // Two newlines -- end of paragraph.
}
+ // Also allow [tok::newline, tok::text, tok::newline] if the middle
+ // tok::text is just whitespace.
+ if (Tok.is(tok::text) && isWhitespace(Tok.getText())) {
+ Token WhitespaceTok = Tok;
+ consumeToken();
+ if (Tok.is(tok::newline) || Tok.is(tok::eof)) {
+ consumeToken();
+ break;
+ }
+ // We have [tok::newline, tok::text, non-newline]. Put back tok::text.
+ putBack(WhitespaceTok);
+ }
if (Content.size() > 0)
Content.back()->addTrailingNewline();
continue;