Sam McCall | b536a2a | 2017-12-19 12:23:48 +0000 | [diff] [blame] | 1 | //===--- SourceCode.h - Manipulating source code as strings -----*- C++ -*-===// |
| 2 | // |
Chandler Carruth | 2946cd7 | 2019-01-19 08:50:56 +0000 | [diff] [blame] | 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
Sam McCall | b536a2a | 2017-12-19 12:23:48 +0000 | [diff] [blame] | 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | #include "SourceCode.h" |
| 9 | |
Sam McCall | a69698f | 2019-03-27 17:47:49 +0000 | [diff] [blame] | 10 | #include "Context.h" |
Sam McCall | 9fb22b2 | 2019-05-06 10:25:10 +0000 | [diff] [blame] | 11 | #include "FuzzyMatch.h" |
Marc-Andre Laperle | 1be6970 | 2018-07-05 19:35:01 +0000 | [diff] [blame] | 12 | #include "Logger.h" |
Sam McCall | a69698f | 2019-03-27 17:47:49 +0000 | [diff] [blame] | 13 | #include "Protocol.h" |
Marc-Andre Laperle | 1be6970 | 2018-07-05 19:35:01 +0000 | [diff] [blame] | 14 | #include "clang/AST/ASTContext.h" |
Marc-Andre Laperle | 63a1098 | 2018-02-21 02:39:08 +0000 | [diff] [blame] | 15 | #include "clang/Basic/SourceManager.h" |
Sam McCall | c316b22 | 2019-04-26 07:45:49 +0000 | [diff] [blame] | 16 | #include "clang/Basic/TokenKinds.h" |
| 17 | #include "clang/Format/Format.h" |
Marc-Andre Laperle | 1be6970 | 2018-07-05 19:35:01 +0000 | [diff] [blame] | 18 | #include "clang/Lex/Lexer.h" |
Haojian Wu | 9d34f45 | 2019-07-01 09:26:48 +0000 | [diff] [blame^] | 19 | #include "clang/Lex/Preprocessor.h" |
Ilya Biryukov | 4399878 | 2019-01-31 21:30:05 +0000 | [diff] [blame] | 20 | #include "llvm/ADT/None.h" |
Sam McCall | c316b22 | 2019-04-26 07:45:49 +0000 | [diff] [blame] | 21 | #include "llvm/ADT/StringExtras.h" |
Ilya Biryukov | 4399878 | 2019-01-31 21:30:05 +0000 | [diff] [blame] | 22 | #include "llvm/ADT/StringRef.h" |
Sam McCall | 9fb22b2 | 2019-05-06 10:25:10 +0000 | [diff] [blame] | 23 | #include "llvm/Support/Compiler.h" |
Simon Marchi | 766338a | 2018-03-21 14:36:46 +0000 | [diff] [blame] | 24 | #include "llvm/Support/Errc.h" |
| 25 | #include "llvm/Support/Error.h" |
Sam McCall | 8b25d22 | 2019-03-28 14:37:51 +0000 | [diff] [blame] | 26 | #include "llvm/Support/ErrorHandling.h" |
Marc-Andre Laperle | 1be6970 | 2018-07-05 19:35:01 +0000 | [diff] [blame] | 27 | #include "llvm/Support/Path.h" |
Sam McCall | c316b22 | 2019-04-26 07:45:49 +0000 | [diff] [blame] | 28 | #include <algorithm> |
Marc-Andre Laperle | 63a1098 | 2018-02-21 02:39:08 +0000 | [diff] [blame] | 29 | |
Sam McCall | b536a2a | 2017-12-19 12:23:48 +0000 | [diff] [blame] | 30 | namespace clang { |
| 31 | namespace clangd { |
Sam McCall | b536a2a | 2017-12-19 12:23:48 +0000 | [diff] [blame] | 32 | |
Sam McCall | a4962cc | 2018-04-27 11:59:28 +0000 | [diff] [blame] | 33 | // Here be dragons. LSP positions use columns measured in *UTF-16 code units*! |
| 34 | // Clangd uses UTF-8 and byte-offsets internally, so conversion is nontrivial. |
| 35 | |
| 36 | // Iterates over unicode codepoints in the (UTF-8) string. For each, |
| 37 | // invokes CB(UTF-8 length, UTF-16 length), and breaks if it returns true. |
| 38 | // Returns true if CB returned true, false if we hit the end of string. |
| 39 | template <typename Callback> |
Ilya Biryukov | f2001aa | 2019-01-07 15:45:19 +0000 | [diff] [blame] | 40 | static bool iterateCodepoints(llvm::StringRef U8, const Callback &CB) { |
Sam McCall | 8b25d22 | 2019-03-28 14:37:51 +0000 | [diff] [blame] | 41 | // A codepoint takes two UTF-16 code unit if it's astral (outside BMP). |
| 42 | // Astral codepoints are encoded as 4 bytes in UTF-8, starting with 11110xxx. |
Sam McCall | a4962cc | 2018-04-27 11:59:28 +0000 | [diff] [blame] | 43 | for (size_t I = 0; I < U8.size();) { |
| 44 | unsigned char C = static_cast<unsigned char>(U8[I]); |
| 45 | if (LLVM_LIKELY(!(C & 0x80))) { // ASCII character. |
| 46 | if (CB(1, 1)) |
| 47 | return true; |
| 48 | ++I; |
| 49 | continue; |
| 50 | } |
| 51 | // This convenient property of UTF-8 holds for all non-ASCII characters. |
Ilya Biryukov | f2001aa | 2019-01-07 15:45:19 +0000 | [diff] [blame] | 52 | size_t UTF8Length = llvm::countLeadingOnes(C); |
Sam McCall | a4962cc | 2018-04-27 11:59:28 +0000 | [diff] [blame] | 53 | // 0xxx is ASCII, handled above. 10xxx is a trailing byte, invalid here. |
| 54 | // 11111xxx is not valid UTF-8 at all. Assert because it's probably our bug. |
| 55 | assert((UTF8Length >= 2 && UTF8Length <= 4) && |
| 56 | "Invalid UTF-8, or transcoding bug?"); |
| 57 | I += UTF8Length; // Skip over all trailing bytes. |
| 58 | // A codepoint takes two UTF-16 code unit if it's astral (outside BMP). |
| 59 | // Astral codepoints are encoded as 4 bytes in UTF-8 (11110xxx ...) |
| 60 | if (CB(UTF8Length, UTF8Length == 4 ? 2 : 1)) |
| 61 | return true; |
| 62 | } |
| 63 | return false; |
| 64 | } |
| 65 | |
Sam McCall | 8b25d22 | 2019-03-28 14:37:51 +0000 | [diff] [blame] | 66 | // Returns the byte offset into the string that is an offset of \p Units in |
| 67 | // the specified encoding. |
| 68 | // Conceptually, this converts to the encoding, truncates to CodeUnits, |
| 69 | // converts back to UTF-8, and returns the length in bytes. |
| 70 | static size_t measureUnits(llvm::StringRef U8, int Units, OffsetEncoding Enc, |
| 71 | bool &Valid) { |
| 72 | Valid = Units >= 0; |
| 73 | if (Units <= 0) |
| 74 | return 0; |
Sam McCall | a4962cc | 2018-04-27 11:59:28 +0000 | [diff] [blame] | 75 | size_t Result = 0; |
Sam McCall | 8b25d22 | 2019-03-28 14:37:51 +0000 | [diff] [blame] | 76 | switch (Enc) { |
| 77 | case OffsetEncoding::UTF8: |
| 78 | Result = Units; |
| 79 | break; |
| 80 | case OffsetEncoding::UTF16: |
| 81 | Valid = iterateCodepoints(U8, [&](int U8Len, int U16Len) { |
| 82 | Result += U8Len; |
| 83 | Units -= U16Len; |
| 84 | return Units <= 0; |
| 85 | }); |
| 86 | if (Units < 0) // Offset in the middle of a surrogate pair. |
| 87 | Valid = false; |
| 88 | break; |
| 89 | case OffsetEncoding::UTF32: |
| 90 | Valid = iterateCodepoints(U8, [&](int U8Len, int U16Len) { |
| 91 | Result += U8Len; |
| 92 | Units--; |
| 93 | return Units <= 0; |
| 94 | }); |
| 95 | break; |
| 96 | case OffsetEncoding::UnsupportedEncoding: |
| 97 | llvm_unreachable("unsupported encoding"); |
| 98 | } |
Sam McCall | a4962cc | 2018-04-27 11:59:28 +0000 | [diff] [blame] | 99 | // Don't return an out-of-range index if we overran. |
Sam McCall | 8b25d22 | 2019-03-28 14:37:51 +0000 | [diff] [blame] | 100 | if (Result > U8.size()) { |
| 101 | Valid = false; |
| 102 | return U8.size(); |
| 103 | } |
| 104 | return Result; |
Sam McCall | a4962cc | 2018-04-27 11:59:28 +0000 | [diff] [blame] | 105 | } |
| 106 | |
Sam McCall | a69698f | 2019-03-27 17:47:49 +0000 | [diff] [blame] | 107 | Key<OffsetEncoding> kCurrentOffsetEncoding; |
Sam McCall | 8b25d22 | 2019-03-28 14:37:51 +0000 | [diff] [blame] | 108 | static OffsetEncoding lspEncoding() { |
Sam McCall | a69698f | 2019-03-27 17:47:49 +0000 | [diff] [blame] | 109 | auto *Enc = Context::current().get(kCurrentOffsetEncoding); |
Sam McCall | 8b25d22 | 2019-03-28 14:37:51 +0000 | [diff] [blame] | 110 | return Enc ? *Enc : OffsetEncoding::UTF16; |
Sam McCall | a69698f | 2019-03-27 17:47:49 +0000 | [diff] [blame] | 111 | } |
| 112 | |
Sam McCall | a4962cc | 2018-04-27 11:59:28 +0000 | [diff] [blame] | 113 | // Like most strings in clangd, the input is UTF-8 encoded. |
Ilya Biryukov | f2001aa | 2019-01-07 15:45:19 +0000 | [diff] [blame] | 114 | size_t lspLength(llvm::StringRef Code) { |
Sam McCall | a4962cc | 2018-04-27 11:59:28 +0000 | [diff] [blame] | 115 | size_t Count = 0; |
Sam McCall | 8b25d22 | 2019-03-28 14:37:51 +0000 | [diff] [blame] | 116 | switch (lspEncoding()) { |
| 117 | case OffsetEncoding::UTF8: |
| 118 | Count = Code.size(); |
| 119 | break; |
| 120 | case OffsetEncoding::UTF16: |
| 121 | iterateCodepoints(Code, [&](int U8Len, int U16Len) { |
| 122 | Count += U16Len; |
| 123 | return false; |
| 124 | }); |
| 125 | break; |
| 126 | case OffsetEncoding::UTF32: |
| 127 | iterateCodepoints(Code, [&](int U8Len, int U16Len) { |
| 128 | ++Count; |
| 129 | return false; |
| 130 | }); |
| 131 | break; |
| 132 | case OffsetEncoding::UnsupportedEncoding: |
| 133 | llvm_unreachable("unsupported encoding"); |
| 134 | } |
Sam McCall | a4962cc | 2018-04-27 11:59:28 +0000 | [diff] [blame] | 135 | return Count; |
| 136 | } |
| 137 | |
Ilya Biryukov | f2001aa | 2019-01-07 15:45:19 +0000 | [diff] [blame] | 138 | llvm::Expected<size_t> positionToOffset(llvm::StringRef Code, Position P, |
| 139 | bool AllowColumnsBeyondLineLength) { |
Sam McCall | b536a2a | 2017-12-19 12:23:48 +0000 | [diff] [blame] | 140 | if (P.line < 0) |
Ilya Biryukov | f2001aa | 2019-01-07 15:45:19 +0000 | [diff] [blame] | 141 | return llvm::make_error<llvm::StringError>( |
| 142 | llvm::formatv("Line value can't be negative ({0})", P.line), |
| 143 | llvm::errc::invalid_argument); |
Simon Marchi | 766338a | 2018-03-21 14:36:46 +0000 | [diff] [blame] | 144 | if (P.character < 0) |
Ilya Biryukov | f2001aa | 2019-01-07 15:45:19 +0000 | [diff] [blame] | 145 | return llvm::make_error<llvm::StringError>( |
| 146 | llvm::formatv("Character value can't be negative ({0})", P.character), |
| 147 | llvm::errc::invalid_argument); |
Sam McCall | b536a2a | 2017-12-19 12:23:48 +0000 | [diff] [blame] | 148 | size_t StartOfLine = 0; |
| 149 | for (int I = 0; I != P.line; ++I) { |
| 150 | size_t NextNL = Code.find('\n', StartOfLine); |
Ilya Biryukov | f2001aa | 2019-01-07 15:45:19 +0000 | [diff] [blame] | 151 | if (NextNL == llvm::StringRef::npos) |
| 152 | return llvm::make_error<llvm::StringError>( |
| 153 | llvm::formatv("Line value is out of range ({0})", P.line), |
| 154 | llvm::errc::invalid_argument); |
Sam McCall | b536a2a | 2017-12-19 12:23:48 +0000 | [diff] [blame] | 155 | StartOfLine = NextNL + 1; |
| 156 | } |
Sam McCall | a69698f | 2019-03-27 17:47:49 +0000 | [diff] [blame] | 157 | StringRef Line = |
| 158 | Code.substr(StartOfLine).take_until([](char C) { return C == '\n'; }); |
Simon Marchi | 766338a | 2018-03-21 14:36:46 +0000 | [diff] [blame] | 159 | |
Sam McCall | 8b25d22 | 2019-03-28 14:37:51 +0000 | [diff] [blame] | 160 | // P.character may be in UTF-16, transcode if necessary. |
Sam McCall | a4962cc | 2018-04-27 11:59:28 +0000 | [diff] [blame] | 161 | bool Valid; |
Sam McCall | 8b25d22 | 2019-03-28 14:37:51 +0000 | [diff] [blame] | 162 | size_t ByteInLine = measureUnits(Line, P.character, lspEncoding(), Valid); |
Sam McCall | a4962cc | 2018-04-27 11:59:28 +0000 | [diff] [blame] | 163 | if (!Valid && !AllowColumnsBeyondLineLength) |
Ilya Biryukov | f2001aa | 2019-01-07 15:45:19 +0000 | [diff] [blame] | 164 | return llvm::make_error<llvm::StringError>( |
Sam McCall | 8b25d22 | 2019-03-28 14:37:51 +0000 | [diff] [blame] | 165 | llvm::formatv("{0} offset {1} is invalid for line {2}", lspEncoding(), |
| 166 | P.character, P.line), |
Ilya Biryukov | f2001aa | 2019-01-07 15:45:19 +0000 | [diff] [blame] | 167 | llvm::errc::invalid_argument); |
Sam McCall | 8b25d22 | 2019-03-28 14:37:51 +0000 | [diff] [blame] | 168 | return StartOfLine + ByteInLine; |
Sam McCall | b536a2a | 2017-12-19 12:23:48 +0000 | [diff] [blame] | 169 | } |
| 170 | |
Ilya Biryukov | f2001aa | 2019-01-07 15:45:19 +0000 | [diff] [blame] | 171 | Position offsetToPosition(llvm::StringRef Code, size_t Offset) { |
Sam McCall | b536a2a | 2017-12-19 12:23:48 +0000 | [diff] [blame] | 172 | Offset = std::min(Code.size(), Offset); |
Ilya Biryukov | f2001aa | 2019-01-07 15:45:19 +0000 | [diff] [blame] | 173 | llvm::StringRef Before = Code.substr(0, Offset); |
Sam McCall | b536a2a | 2017-12-19 12:23:48 +0000 | [diff] [blame] | 174 | int Lines = Before.count('\n'); |
| 175 | size_t PrevNL = Before.rfind('\n'); |
Ilya Biryukov | f2001aa | 2019-01-07 15:45:19 +0000 | [diff] [blame] | 176 | size_t StartOfLine = (PrevNL == llvm::StringRef::npos) ? 0 : (PrevNL + 1); |
Ilya Biryukov | 7beea3a | 2018-02-14 10:52:04 +0000 | [diff] [blame] | 177 | Position Pos; |
| 178 | Pos.line = Lines; |
Sam McCall | 7189112 | 2018-10-23 11:51:53 +0000 | [diff] [blame] | 179 | Pos.character = lspLength(Before.substr(StartOfLine)); |
Ilya Biryukov | 7beea3a | 2018-02-14 10:52:04 +0000 | [diff] [blame] | 180 | return Pos; |
Sam McCall | b536a2a | 2017-12-19 12:23:48 +0000 | [diff] [blame] | 181 | } |
| 182 | |
Marc-Andre Laperle | 63a1098 | 2018-02-21 02:39:08 +0000 | [diff] [blame] | 183 | Position sourceLocToPosition(const SourceManager &SM, SourceLocation Loc) { |
Sam McCall | a4962cc | 2018-04-27 11:59:28 +0000 | [diff] [blame] | 184 | // We use the SourceManager's line tables, but its column number is in bytes. |
| 185 | FileID FID; |
| 186 | unsigned Offset; |
| 187 | std::tie(FID, Offset) = SM.getDecomposedSpellingLoc(Loc); |
Marc-Andre Laperle | 63a1098 | 2018-02-21 02:39:08 +0000 | [diff] [blame] | 188 | Position P; |
Sam McCall | a4962cc | 2018-04-27 11:59:28 +0000 | [diff] [blame] | 189 | P.line = static_cast<int>(SM.getLineNumber(FID, Offset)) - 1; |
| 190 | bool Invalid = false; |
Ilya Biryukov | f2001aa | 2019-01-07 15:45:19 +0000 | [diff] [blame] | 191 | llvm::StringRef Code = SM.getBufferData(FID, &Invalid); |
Sam McCall | a4962cc | 2018-04-27 11:59:28 +0000 | [diff] [blame] | 192 | if (!Invalid) { |
| 193 | auto ColumnInBytes = SM.getColumnNumber(FID, Offset) - 1; |
| 194 | auto LineSoFar = Code.substr(Offset - ColumnInBytes, ColumnInBytes); |
Sam McCall | 7189112 | 2018-10-23 11:51:53 +0000 | [diff] [blame] | 195 | P.character = lspLength(LineSoFar); |
Sam McCall | a4962cc | 2018-04-27 11:59:28 +0000 | [diff] [blame] | 196 | } |
Marc-Andre Laperle | 63a1098 | 2018-02-21 02:39:08 +0000 | [diff] [blame] | 197 | return P; |
| 198 | } |
| 199 | |
Haojian Wu | 92c3257 | 2019-06-25 08:01:46 +0000 | [diff] [blame] | 200 | llvm::Optional<Range> getTokenRange(const SourceManager &SM, |
| 201 | const LangOptions &LangOpts, |
| 202 | SourceLocation TokLoc) { |
| 203 | if (!TokLoc.isValid()) |
| 204 | return llvm::None; |
| 205 | SourceLocation End = Lexer::getLocForEndOfToken(TokLoc, 0, SM, LangOpts); |
| 206 | if (!End.isValid()) |
| 207 | return llvm::None; |
| 208 | return halfOpenToRange(SM, CharSourceRange::getCharRange(TokLoc, End)); |
| 209 | } |
| 210 | |
Ilya Biryukov | 4399878 | 2019-01-31 21:30:05 +0000 | [diff] [blame] | 211 | bool isValidFileRange(const SourceManager &Mgr, SourceRange R) { |
| 212 | if (!R.getBegin().isValid() || !R.getEnd().isValid()) |
| 213 | return false; |
| 214 | |
| 215 | FileID BeginFID; |
| 216 | size_t BeginOffset = 0; |
| 217 | std::tie(BeginFID, BeginOffset) = Mgr.getDecomposedLoc(R.getBegin()); |
| 218 | |
| 219 | FileID EndFID; |
| 220 | size_t EndOffset = 0; |
| 221 | std::tie(EndFID, EndOffset) = Mgr.getDecomposedLoc(R.getEnd()); |
| 222 | |
| 223 | return BeginFID.isValid() && BeginFID == EndFID && BeginOffset <= EndOffset; |
| 224 | } |
| 225 | |
| 226 | bool halfOpenRangeContains(const SourceManager &Mgr, SourceRange R, |
| 227 | SourceLocation L) { |
| 228 | assert(isValidFileRange(Mgr, R)); |
| 229 | |
| 230 | FileID BeginFID; |
| 231 | size_t BeginOffset = 0; |
| 232 | std::tie(BeginFID, BeginOffset) = Mgr.getDecomposedLoc(R.getBegin()); |
| 233 | size_t EndOffset = Mgr.getFileOffset(R.getEnd()); |
| 234 | |
| 235 | FileID LFid; |
| 236 | size_t LOffset; |
| 237 | std::tie(LFid, LOffset) = Mgr.getDecomposedLoc(L); |
| 238 | return BeginFID == LFid && BeginOffset <= LOffset && LOffset < EndOffset; |
| 239 | } |
| 240 | |
| 241 | bool halfOpenRangeTouches(const SourceManager &Mgr, SourceRange R, |
| 242 | SourceLocation L) { |
| 243 | return L == R.getEnd() || halfOpenRangeContains(Mgr, R, L); |
| 244 | } |
| 245 | |
| 246 | llvm::Optional<SourceRange> toHalfOpenFileRange(const SourceManager &Mgr, |
| 247 | const LangOptions &LangOpts, |
| 248 | SourceRange R) { |
| 249 | auto Begin = Mgr.getFileLoc(R.getBegin()); |
| 250 | if (Begin.isInvalid()) |
| 251 | return llvm::None; |
| 252 | auto End = Mgr.getFileLoc(R.getEnd()); |
| 253 | if (End.isInvalid()) |
| 254 | return llvm::None; |
| 255 | End = Lexer::getLocForEndOfToken(End, 0, Mgr, LangOpts); |
| 256 | |
| 257 | SourceRange Result(Begin, End); |
| 258 | if (!isValidFileRange(Mgr, Result)) |
| 259 | return llvm::None; |
| 260 | return Result; |
| 261 | } |
| 262 | |
| 263 | llvm::StringRef toSourceCode(const SourceManager &SM, SourceRange R) { |
| 264 | assert(isValidFileRange(SM, R)); |
| 265 | bool Invalid = false; |
| 266 | auto *Buf = SM.getBuffer(SM.getFileID(R.getBegin()), &Invalid); |
| 267 | assert(!Invalid); |
| 268 | |
| 269 | size_t BeginOffset = SM.getFileOffset(R.getBegin()); |
| 270 | size_t EndOffset = SM.getFileOffset(R.getEnd()); |
| 271 | return Buf->getBuffer().substr(BeginOffset, EndOffset - BeginOffset); |
| 272 | } |
| 273 | |
Ilya Biryukov | cce67a3 | 2019-01-29 14:17:36 +0000 | [diff] [blame] | 274 | llvm::Expected<SourceLocation> sourceLocationInMainFile(const SourceManager &SM, |
| 275 | Position P) { |
| 276 | llvm::StringRef Code = SM.getBuffer(SM.getMainFileID())->getBuffer(); |
| 277 | auto Offset = |
| 278 | positionToOffset(Code, P, /*AllowColumnBeyondLineLength=*/false); |
| 279 | if (!Offset) |
| 280 | return Offset.takeError(); |
| 281 | return SM.getLocForStartOfFile(SM.getMainFileID()).getLocWithOffset(*Offset); |
| 282 | } |
| 283 | |
Ilya Biryukov | 71028b8 | 2018-03-12 15:28:22 +0000 | [diff] [blame] | 284 | Range halfOpenToRange(const SourceManager &SM, CharSourceRange R) { |
| 285 | // Clang is 1-based, LSP uses 0-based indexes. |
| 286 | Position Begin = sourceLocToPosition(SM, R.getBegin()); |
| 287 | Position End = sourceLocToPosition(SM, R.getEnd()); |
| 288 | |
| 289 | return {Begin, End}; |
| 290 | } |
| 291 | |
Ilya Biryukov | f2001aa | 2019-01-07 15:45:19 +0000 | [diff] [blame] | 292 | std::pair<size_t, size_t> offsetToClangLineColumn(llvm::StringRef Code, |
Sam McCall | a4962cc | 2018-04-27 11:59:28 +0000 | [diff] [blame] | 293 | size_t Offset) { |
| 294 | Offset = std::min(Code.size(), Offset); |
Ilya Biryukov | f2001aa | 2019-01-07 15:45:19 +0000 | [diff] [blame] | 295 | llvm::StringRef Before = Code.substr(0, Offset); |
Sam McCall | a4962cc | 2018-04-27 11:59:28 +0000 | [diff] [blame] | 296 | int Lines = Before.count('\n'); |
| 297 | size_t PrevNL = Before.rfind('\n'); |
Ilya Biryukov | f2001aa | 2019-01-07 15:45:19 +0000 | [diff] [blame] | 298 | size_t StartOfLine = (PrevNL == llvm::StringRef::npos) ? 0 : (PrevNL + 1); |
Sam McCall | a4962cc | 2018-04-27 11:59:28 +0000 | [diff] [blame] | 299 | return {Lines + 1, Offset - StartOfLine + 1}; |
| 300 | } |
| 301 | |
Ilya Biryukov | 4399878 | 2019-01-31 21:30:05 +0000 | [diff] [blame] | 302 | std::pair<StringRef, StringRef> splitQualifiedName(StringRef QName) { |
Marc-Andre Laperle | b387b6e | 2018-04-23 20:00:52 +0000 | [diff] [blame] | 303 | size_t Pos = QName.rfind("::"); |
Ilya Biryukov | f2001aa | 2019-01-07 15:45:19 +0000 | [diff] [blame] | 304 | if (Pos == llvm::StringRef::npos) |
| 305 | return {llvm::StringRef(), QName}; |
Marc-Andre Laperle | b387b6e | 2018-04-23 20:00:52 +0000 | [diff] [blame] | 306 | return {QName.substr(0, Pos + 2), QName.substr(Pos + 2)}; |
| 307 | } |
| 308 | |
Ilya Biryukov | f2001aa | 2019-01-07 15:45:19 +0000 | [diff] [blame] | 309 | TextEdit replacementToEdit(llvm::StringRef Code, |
| 310 | const tooling::Replacement &R) { |
Eric Liu | 9133ecd | 2018-05-11 12:12:08 +0000 | [diff] [blame] | 311 | Range ReplacementRange = { |
| 312 | offsetToPosition(Code, R.getOffset()), |
| 313 | offsetToPosition(Code, R.getOffset() + R.getLength())}; |
| 314 | return {ReplacementRange, R.getReplacementText()}; |
| 315 | } |
| 316 | |
Ilya Biryukov | f2001aa | 2019-01-07 15:45:19 +0000 | [diff] [blame] | 317 | std::vector<TextEdit> replacementsToEdits(llvm::StringRef Code, |
Eric Liu | 9133ecd | 2018-05-11 12:12:08 +0000 | [diff] [blame] | 318 | const tooling::Replacements &Repls) { |
| 319 | std::vector<TextEdit> Edits; |
| 320 | for (const auto &R : Repls) |
| 321 | Edits.push_back(replacementToEdit(Code, R)); |
| 322 | return Edits; |
| 323 | } |
| 324 | |
Ilya Biryukov | f2001aa | 2019-01-07 15:45:19 +0000 | [diff] [blame] | 325 | llvm::Optional<std::string> getCanonicalPath(const FileEntry *F, |
| 326 | const SourceManager &SourceMgr) { |
Kadir Cetinkaya | dd67793 | 2018-12-19 10:46:21 +0000 | [diff] [blame] | 327 | if (!F) |
| 328 | return None; |
Simon Marchi | 25f1f73 | 2018-08-10 22:27:53 +0000 | [diff] [blame] | 329 | |
Ilya Biryukov | f2001aa | 2019-01-07 15:45:19 +0000 | [diff] [blame] | 330 | llvm::SmallString<128> FilePath = F->getName(); |
| 331 | if (!llvm::sys::path::is_absolute(FilePath)) { |
Kadir Cetinkaya | dd67793 | 2018-12-19 10:46:21 +0000 | [diff] [blame] | 332 | if (auto EC = |
Duncan P. N. Exon Smith | db8a742 | 2019-03-26 22:32:06 +0000 | [diff] [blame] | 333 | SourceMgr.getFileManager().getVirtualFileSystem().makeAbsolute( |
Kadir Cetinkaya | dd67793 | 2018-12-19 10:46:21 +0000 | [diff] [blame] | 334 | FilePath)) { |
| 335 | elog("Could not turn relative path '{0}' to absolute: {1}", FilePath, |
| 336 | EC.message()); |
Sam McCall | c008af6 | 2018-10-20 15:30:37 +0000 | [diff] [blame] | 337 | return None; |
Marc-Andre Laperle | 1be6970 | 2018-07-05 19:35:01 +0000 | [diff] [blame] | 338 | } |
| 339 | } |
Simon Marchi | 25f1f73 | 2018-08-10 22:27:53 +0000 | [diff] [blame] | 340 | |
Kadir Cetinkaya | dd67793 | 2018-12-19 10:46:21 +0000 | [diff] [blame] | 341 | // Handle the symbolic link path case where the current working directory |
| 342 | // (getCurrentWorkingDirectory) is a symlink./ We always want to the real |
| 343 | // file path (instead of the symlink path) for the C++ symbols. |
| 344 | // |
| 345 | // Consider the following example: |
| 346 | // |
| 347 | // src dir: /project/src/foo.h |
| 348 | // current working directory (symlink): /tmp/build -> /project/src/ |
| 349 | // |
| 350 | // The file path of Symbol is "/project/src/foo.h" instead of |
| 351 | // "/tmp/build/foo.h" |
| 352 | if (const DirectoryEntry *Dir = SourceMgr.getFileManager().getDirectory( |
Ilya Biryukov | f2001aa | 2019-01-07 15:45:19 +0000 | [diff] [blame] | 353 | llvm::sys::path::parent_path(FilePath))) { |
| 354 | llvm::SmallString<128> RealPath; |
| 355 | llvm::StringRef DirName = SourceMgr.getFileManager().getCanonicalName(Dir); |
| 356 | llvm::sys::path::append(RealPath, DirName, |
| 357 | llvm::sys::path::filename(FilePath)); |
Kadir Cetinkaya | dd67793 | 2018-12-19 10:46:21 +0000 | [diff] [blame] | 358 | return RealPath.str().str(); |
Simon Marchi | 25f1f73 | 2018-08-10 22:27:53 +0000 | [diff] [blame] | 359 | } |
| 360 | |
Kadir Cetinkaya | dd67793 | 2018-12-19 10:46:21 +0000 | [diff] [blame] | 361 | return FilePath.str().str(); |
Marc-Andre Laperle | 1be6970 | 2018-07-05 19:35:01 +0000 | [diff] [blame] | 362 | } |
| 363 | |
Kadir Cetinkaya | 2f84d91 | 2018-08-08 08:59:29 +0000 | [diff] [blame] | 364 | TextEdit toTextEdit(const FixItHint &FixIt, const SourceManager &M, |
| 365 | const LangOptions &L) { |
| 366 | TextEdit Result; |
| 367 | Result.range = |
| 368 | halfOpenToRange(M, Lexer::makeFileCharRange(FixIt.RemoveRange, M, L)); |
| 369 | Result.newText = FixIt.CodeToInsert; |
| 370 | return Result; |
| 371 | } |
| 372 | |
Haojian Wu | aa3ed5a | 2019-01-25 15:14:03 +0000 | [diff] [blame] | 373 | bool isRangeConsecutive(const Range &Left, const Range &Right) { |
Kadir Cetinkaya | a9c9d00 | 2018-08-13 08:23:01 +0000 | [diff] [blame] | 374 | return Left.end.line == Right.start.line && |
| 375 | Left.end.character == Right.start.character; |
| 376 | } |
| 377 | |
Ilya Biryukov | f2001aa | 2019-01-07 15:45:19 +0000 | [diff] [blame] | 378 | FileDigest digest(llvm::StringRef Content) { |
Kadir Cetinkaya | d08eab4 | 2018-11-27 16:08:53 +0000 | [diff] [blame] | 379 | return llvm::SHA1::hash({(const uint8_t *)Content.data(), Content.size()}); |
| 380 | } |
| 381 | |
Ilya Biryukov | f2001aa | 2019-01-07 15:45:19 +0000 | [diff] [blame] | 382 | llvm::Optional<FileDigest> digestFile(const SourceManager &SM, FileID FID) { |
Kadir Cetinkaya | d08eab4 | 2018-11-27 16:08:53 +0000 | [diff] [blame] | 383 | bool Invalid = false; |
Ilya Biryukov | f2001aa | 2019-01-07 15:45:19 +0000 | [diff] [blame] | 384 | llvm::StringRef Content = SM.getBufferData(FID, &Invalid); |
Kadir Cetinkaya | d08eab4 | 2018-11-27 16:08:53 +0000 | [diff] [blame] | 385 | if (Invalid) |
| 386 | return None; |
| 387 | return digest(Content); |
| 388 | } |
| 389 | |
Eric Liu | dd66277 | 2019-01-28 14:01:55 +0000 | [diff] [blame] | 390 | format::FormatStyle getFormatStyleForFile(llvm::StringRef File, |
| 391 | llvm::StringRef Content, |
| 392 | llvm::vfs::FileSystem *FS) { |
| 393 | auto Style = format::getStyle(format::DefaultFormatStyle, File, |
| 394 | format::DefaultFallbackStyle, Content, FS); |
| 395 | if (!Style) { |
| 396 | log("getStyle() failed for file {0}: {1}. Fallback is LLVM style.", File, |
| 397 | Style.takeError()); |
| 398 | Style = format::getLLVMStyle(); |
| 399 | } |
| 400 | return *Style; |
| 401 | } |
| 402 | |
Haojian Wu | 12e194c | 2019-02-06 15:24:50 +0000 | [diff] [blame] | 403 | llvm::Expected<tooling::Replacements> |
| 404 | cleanupAndFormat(StringRef Code, const tooling::Replacements &Replaces, |
| 405 | const format::FormatStyle &Style) { |
| 406 | auto CleanReplaces = cleanupAroundReplacements(Code, Replaces, Style); |
| 407 | if (!CleanReplaces) |
| 408 | return CleanReplaces; |
| 409 | return formatReplacements(Code, std::move(*CleanReplaces), Style); |
| 410 | } |
| 411 | |
Sam McCall | c316b22 | 2019-04-26 07:45:49 +0000 | [diff] [blame] | 412 | template <typename Action> |
| 413 | static void lex(llvm::StringRef Code, const format::FormatStyle &Style, |
| 414 | Action A) { |
| 415 | // FIXME: InMemoryFileAdapter crashes unless the buffer is null terminated! |
| 416 | std::string NullTerminatedCode = Code.str(); |
| 417 | SourceManagerForFile FileSM("dummy.cpp", NullTerminatedCode); |
Eric Liu | 00d99bd | 2019-04-11 09:36:36 +0000 | [diff] [blame] | 418 | auto &SM = FileSM.get(); |
| 419 | auto FID = SM.getMainFileID(); |
| 420 | Lexer Lex(FID, SM.getBuffer(FID), SM, format::getFormattingLangOpts(Style)); |
| 421 | Token Tok; |
| 422 | |
Sam McCall | c316b22 | 2019-04-26 07:45:49 +0000 | [diff] [blame] | 423 | while (!Lex.LexFromRawLexer(Tok)) |
| 424 | A(Tok); |
| 425 | } |
| 426 | |
| 427 | llvm::StringMap<unsigned> collectIdentifiers(llvm::StringRef Content, |
| 428 | const format::FormatStyle &Style) { |
Eric Liu | 00d99bd | 2019-04-11 09:36:36 +0000 | [diff] [blame] | 429 | llvm::StringMap<unsigned> Identifiers; |
Sam McCall | c316b22 | 2019-04-26 07:45:49 +0000 | [diff] [blame] | 430 | lex(Content, Style, [&](const clang::Token &Tok) { |
Eric Liu | 00d99bd | 2019-04-11 09:36:36 +0000 | [diff] [blame] | 431 | switch (Tok.getKind()) { |
| 432 | case tok::identifier: |
| 433 | ++Identifiers[Tok.getIdentifierInfo()->getName()]; |
| 434 | break; |
| 435 | case tok::raw_identifier: |
| 436 | ++Identifiers[Tok.getRawIdentifier()]; |
| 437 | break; |
| 438 | default: |
Sam McCall | c316b22 | 2019-04-26 07:45:49 +0000 | [diff] [blame] | 439 | break; |
Eric Liu | 00d99bd | 2019-04-11 09:36:36 +0000 | [diff] [blame] | 440 | } |
Sam McCall | c316b22 | 2019-04-26 07:45:49 +0000 | [diff] [blame] | 441 | }); |
Eric Liu | 00d99bd | 2019-04-11 09:36:36 +0000 | [diff] [blame] | 442 | return Identifiers; |
| 443 | } |
| 444 | |
Sam McCall | c316b22 | 2019-04-26 07:45:49 +0000 | [diff] [blame] | 445 | namespace { |
| 446 | enum NamespaceEvent { |
| 447 | BeginNamespace, // namespace <ns> {. Payload is resolved <ns>. |
| 448 | EndNamespace, // } // namespace <ns>. Payload is resolved *outer* namespace. |
| 449 | UsingDirective // using namespace <ns>. Payload is unresolved <ns>. |
| 450 | }; |
| 451 | // Scans C++ source code for constructs that change the visible namespaces. |
| 452 | void parseNamespaceEvents( |
| 453 | llvm::StringRef Code, const format::FormatStyle &Style, |
| 454 | llvm::function_ref<void(NamespaceEvent, llvm::StringRef)> Callback) { |
| 455 | |
| 456 | // Stack of enclosing namespaces, e.g. {"clang", "clangd"} |
| 457 | std::vector<std::string> Enclosing; // Contains e.g. "clang", "clangd" |
| 458 | // Stack counts open braces. true if the brace opened a namespace. |
| 459 | std::vector<bool> BraceStack; |
| 460 | |
| 461 | enum { |
| 462 | Default, |
| 463 | Namespace, // just saw 'namespace' |
| 464 | NamespaceName, // just saw 'namespace' NSName |
| 465 | Using, // just saw 'using' |
| 466 | UsingNamespace, // just saw 'using namespace' |
| 467 | UsingNamespaceName, // just saw 'using namespace' NSName |
| 468 | } State = Default; |
| 469 | std::string NSName; |
| 470 | |
| 471 | lex(Code, Style, [&](const clang::Token &Tok) { |
| 472 | switch(Tok.getKind()) { |
| 473 | case tok::raw_identifier: |
| 474 | // In raw mode, this could be a keyword or a name. |
| 475 | switch (State) { |
| 476 | case UsingNamespace: |
| 477 | case UsingNamespaceName: |
| 478 | NSName.append(Tok.getRawIdentifier()); |
| 479 | State = UsingNamespaceName; |
| 480 | break; |
| 481 | case Namespace: |
| 482 | case NamespaceName: |
| 483 | NSName.append(Tok.getRawIdentifier()); |
| 484 | State = NamespaceName; |
| 485 | break; |
| 486 | case Using: |
| 487 | State = |
| 488 | (Tok.getRawIdentifier() == "namespace") ? UsingNamespace : Default; |
| 489 | break; |
| 490 | case Default: |
| 491 | NSName.clear(); |
| 492 | if (Tok.getRawIdentifier() == "namespace") |
| 493 | State = Namespace; |
| 494 | else if (Tok.getRawIdentifier() == "using") |
| 495 | State = Using; |
| 496 | break; |
| 497 | } |
| 498 | break; |
| 499 | case tok::coloncolon: |
| 500 | // This can come at the beginning or in the middle of a namespace name. |
| 501 | switch (State) { |
| 502 | case UsingNamespace: |
| 503 | case UsingNamespaceName: |
| 504 | NSName.append("::"); |
| 505 | State = UsingNamespaceName; |
| 506 | break; |
| 507 | case NamespaceName: |
| 508 | NSName.append("::"); |
| 509 | State = NamespaceName; |
| 510 | break; |
| 511 | case Namespace: // Not legal here. |
| 512 | case Using: |
| 513 | case Default: |
| 514 | State = Default; |
| 515 | break; |
| 516 | } |
| 517 | break; |
| 518 | case tok::l_brace: |
| 519 | // Record which { started a namespace, so we know when } ends one. |
| 520 | if (State == NamespaceName) { |
| 521 | // Parsed: namespace <name> { |
| 522 | BraceStack.push_back(true); |
| 523 | Enclosing.push_back(NSName); |
| 524 | Callback(BeginNamespace, llvm::join(Enclosing, "::")); |
| 525 | } else { |
| 526 | // This case includes anonymous namespaces (State = Namespace). |
| 527 | // For our purposes, they're not namespaces and we ignore them. |
| 528 | BraceStack.push_back(false); |
| 529 | } |
| 530 | State = Default; |
| 531 | break; |
| 532 | case tok::r_brace: |
| 533 | // If braces are unmatched, we're going to be confused, but don't crash. |
| 534 | if (!BraceStack.empty()) { |
| 535 | if (BraceStack.back()) { |
| 536 | // Parsed: } // namespace |
| 537 | Enclosing.pop_back(); |
| 538 | Callback(EndNamespace, llvm::join(Enclosing, "::")); |
| 539 | } |
| 540 | BraceStack.pop_back(); |
| 541 | } |
| 542 | break; |
| 543 | case tok::semi: |
| 544 | if (State == UsingNamespaceName) |
| 545 | // Parsed: using namespace <name> ; |
| 546 | Callback(UsingDirective, llvm::StringRef(NSName)); |
| 547 | State = Default; |
| 548 | break; |
| 549 | default: |
| 550 | State = Default; |
| 551 | break; |
| 552 | } |
| 553 | }); |
| 554 | } |
| 555 | |
| 556 | // Returns the prefix namespaces of NS: {"" ... NS}. |
| 557 | llvm::SmallVector<llvm::StringRef, 8> ancestorNamespaces(llvm::StringRef NS) { |
| 558 | llvm::SmallVector<llvm::StringRef, 8> Results; |
| 559 | Results.push_back(NS.take_front(0)); |
| 560 | NS.split(Results, "::", /*MaxSplit=*/-1, /*KeepEmpty=*/false); |
| 561 | for (llvm::StringRef &R : Results) |
| 562 | R = NS.take_front(R.end() - NS.begin()); |
| 563 | return Results; |
| 564 | } |
| 565 | |
| 566 | } // namespace |
| 567 | |
| 568 | std::vector<std::string> visibleNamespaces(llvm::StringRef Code, |
| 569 | const format::FormatStyle &Style) { |
| 570 | std::string Current; |
| 571 | // Map from namespace to (resolved) namespaces introduced via using directive. |
| 572 | llvm::StringMap<llvm::StringSet<>> UsingDirectives; |
| 573 | |
| 574 | parseNamespaceEvents(Code, Style, |
| 575 | [&](NamespaceEvent Event, llvm::StringRef NS) { |
| 576 | switch (Event) { |
| 577 | case BeginNamespace: |
| 578 | case EndNamespace: |
| 579 | Current = NS; |
| 580 | break; |
| 581 | case UsingDirective: |
| 582 | if (NS.consume_front("::")) |
| 583 | UsingDirectives[Current].insert(NS); |
| 584 | else { |
| 585 | for (llvm::StringRef Enclosing : |
| 586 | ancestorNamespaces(Current)) { |
| 587 | if (Enclosing.empty()) |
| 588 | UsingDirectives[Current].insert(NS); |
| 589 | else |
| 590 | UsingDirectives[Current].insert( |
| 591 | (Enclosing + "::" + NS).str()); |
| 592 | } |
| 593 | } |
| 594 | break; |
| 595 | } |
| 596 | }); |
| 597 | |
| 598 | std::vector<std::string> Found; |
| 599 | for (llvm::StringRef Enclosing : ancestorNamespaces(Current)) { |
| 600 | Found.push_back(Enclosing); |
| 601 | auto It = UsingDirectives.find(Enclosing); |
| 602 | if (It != UsingDirectives.end()) |
| 603 | for (const auto& Used : It->second) |
| 604 | Found.push_back(Used.getKey()); |
| 605 | } |
| 606 | |
| 607 | |
| 608 | llvm::sort(Found, [&](const std::string &LHS, const std::string &RHS) { |
| 609 | if (Current == RHS) |
| 610 | return false; |
| 611 | if (Current == LHS) |
| 612 | return true; |
| 613 | return LHS < RHS; |
| 614 | }); |
| 615 | Found.erase(std::unique(Found.begin(), Found.end()), Found.end()); |
| 616 | return Found; |
| 617 | } |
| 618 | |
Sam McCall | 9fb22b2 | 2019-05-06 10:25:10 +0000 | [diff] [blame] | 619 | llvm::StringSet<> collectWords(llvm::StringRef Content) { |
| 620 | // We assume short words are not significant. |
| 621 | // We may want to consider other stopwords, e.g. language keywords. |
| 622 | // (A very naive implementation showed no benefit, but lexing might do better) |
| 623 | static constexpr int MinWordLength = 4; |
| 624 | |
| 625 | std::vector<CharRole> Roles(Content.size()); |
| 626 | calculateRoles(Content, Roles); |
| 627 | |
| 628 | llvm::StringSet<> Result; |
| 629 | llvm::SmallString<256> Word; |
| 630 | auto Flush = [&] { |
| 631 | if (Word.size() >= MinWordLength) { |
| 632 | for (char &C : Word) |
| 633 | C = llvm::toLower(C); |
| 634 | Result.insert(Word); |
| 635 | } |
| 636 | Word.clear(); |
| 637 | }; |
| 638 | for (unsigned I = 0; I < Content.size(); ++I) { |
| 639 | switch (Roles[I]) { |
| 640 | case Head: |
| 641 | Flush(); |
| 642 | LLVM_FALLTHROUGH; |
| 643 | case Tail: |
| 644 | Word.push_back(Content[I]); |
| 645 | break; |
| 646 | case Unknown: |
| 647 | case Separator: |
| 648 | Flush(); |
| 649 | break; |
| 650 | } |
| 651 | } |
| 652 | Flush(); |
| 653 | |
| 654 | return Result; |
| 655 | } |
| 656 | |
Haojian Wu | 9d34f45 | 2019-07-01 09:26:48 +0000 | [diff] [blame^] | 657 | llvm::Optional<DefinedMacro> locateMacroAt(SourceLocation Loc, |
| 658 | Preprocessor &PP) { |
| 659 | const auto &SM = PP.getSourceManager(); |
| 660 | const auto &LangOpts = PP.getLangOpts(); |
| 661 | Token Result; |
| 662 | if (Lexer::getRawToken(SM.getSpellingLoc(Loc), Result, SM, LangOpts, false)) |
| 663 | return None; |
| 664 | if (Result.is(tok::raw_identifier)) |
| 665 | PP.LookUpIdentifierInfo(Result); |
| 666 | IdentifierInfo *IdentifierInfo = Result.getIdentifierInfo(); |
| 667 | if (!IdentifierInfo || !IdentifierInfo->hadMacroDefinition()) |
| 668 | return None; |
| 669 | |
| 670 | std::pair<FileID, unsigned int> DecLoc = SM.getDecomposedExpansionLoc(Loc); |
| 671 | // Get the definition just before the searched location so that a macro |
| 672 | // referenced in a '#undef MACRO' can still be found. |
| 673 | SourceLocation BeforeSearchedLocation = |
| 674 | SM.getMacroArgExpandedLocation(SM.getLocForStartOfFile(DecLoc.first) |
| 675 | .getLocWithOffset(DecLoc.second - 1)); |
| 676 | MacroDefinition MacroDef = |
| 677 | PP.getMacroDefinitionAtLoc(IdentifierInfo, BeforeSearchedLocation); |
| 678 | if (auto *MI = MacroDef.getMacroInfo()) |
| 679 | return DefinedMacro{IdentifierInfo->getName(), MI}; |
| 680 | return None; |
| 681 | } |
| 682 | |
Sam McCall | b536a2a | 2017-12-19 12:23:48 +0000 | [diff] [blame] | 683 | } // namespace clangd |
| 684 | } // namespace clang |