blob: 9fcaed455194092257068bd7cbbc3cb0b106ca30 [file] [log] [blame]
Sam McCallb536a2a2017-12-19 12:23:48 +00001//===--- SourceCode.h - Manipulating source code as strings -----*- C++ -*-===//
2//
Chandler Carruth2946cd72019-01-19 08:50:56 +00003// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
Sam McCallb536a2a2017-12-19 12:23:48 +00006//
7//===----------------------------------------------------------------------===//
8#include "SourceCode.h"
9
Sam McCalla69698f2019-03-27 17:47:49 +000010#include "Context.h"
Sam McCall9fb22b22019-05-06 10:25:10 +000011#include "FuzzyMatch.h"
Marc-Andre Laperle1be69702018-07-05 19:35:01 +000012#include "Logger.h"
Sam McCalla69698f2019-03-27 17:47:49 +000013#include "Protocol.h"
Marc-Andre Laperle1be69702018-07-05 19:35:01 +000014#include "clang/AST/ASTContext.h"
Marc-Andre Laperle63a10982018-02-21 02:39:08 +000015#include "clang/Basic/SourceManager.h"
Sam McCallc316b222019-04-26 07:45:49 +000016#include "clang/Basic/TokenKinds.h"
17#include "clang/Format/Format.h"
Marc-Andre Laperle1be69702018-07-05 19:35:01 +000018#include "clang/Lex/Lexer.h"
Haojian Wu9d34f452019-07-01 09:26:48 +000019#include "clang/Lex/Preprocessor.h"
Ilya Biryukov43998782019-01-31 21:30:05 +000020#include "llvm/ADT/None.h"
Sam McCallc316b222019-04-26 07:45:49 +000021#include "llvm/ADT/StringExtras.h"
Ilya Biryukov43998782019-01-31 21:30:05 +000022#include "llvm/ADT/StringRef.h"
Sam McCall9fb22b22019-05-06 10:25:10 +000023#include "llvm/Support/Compiler.h"
Simon Marchi766338a2018-03-21 14:36:46 +000024#include "llvm/Support/Errc.h"
25#include "llvm/Support/Error.h"
Sam McCall8b25d222019-03-28 14:37:51 +000026#include "llvm/Support/ErrorHandling.h"
Marc-Andre Laperle1be69702018-07-05 19:35:01 +000027#include "llvm/Support/Path.h"
Sam McCallc316b222019-04-26 07:45:49 +000028#include <algorithm>
Marc-Andre Laperle63a10982018-02-21 02:39:08 +000029
Sam McCallb536a2a2017-12-19 12:23:48 +000030namespace clang {
31namespace clangd {
Sam McCallb536a2a2017-12-19 12:23:48 +000032
Sam McCalla4962cc2018-04-27 11:59:28 +000033// Here be dragons. LSP positions use columns measured in *UTF-16 code units*!
34// Clangd uses UTF-8 and byte-offsets internally, so conversion is nontrivial.
35
36// Iterates over unicode codepoints in the (UTF-8) string. For each,
37// invokes CB(UTF-8 length, UTF-16 length), and breaks if it returns true.
38// Returns true if CB returned true, false if we hit the end of string.
39template <typename Callback>
Ilya Biryukovf2001aa2019-01-07 15:45:19 +000040static bool iterateCodepoints(llvm::StringRef U8, const Callback &CB) {
Sam McCall8b25d222019-03-28 14:37:51 +000041 // A codepoint takes two UTF-16 code unit if it's astral (outside BMP).
42 // Astral codepoints are encoded as 4 bytes in UTF-8, starting with 11110xxx.
Sam McCalla4962cc2018-04-27 11:59:28 +000043 for (size_t I = 0; I < U8.size();) {
44 unsigned char C = static_cast<unsigned char>(U8[I]);
45 if (LLVM_LIKELY(!(C & 0x80))) { // ASCII character.
46 if (CB(1, 1))
47 return true;
48 ++I;
49 continue;
50 }
51 // This convenient property of UTF-8 holds for all non-ASCII characters.
Ilya Biryukovf2001aa2019-01-07 15:45:19 +000052 size_t UTF8Length = llvm::countLeadingOnes(C);
Sam McCalla4962cc2018-04-27 11:59:28 +000053 // 0xxx is ASCII, handled above. 10xxx is a trailing byte, invalid here.
54 // 11111xxx is not valid UTF-8 at all. Assert because it's probably our bug.
55 assert((UTF8Length >= 2 && UTF8Length <= 4) &&
56 "Invalid UTF-8, or transcoding bug?");
57 I += UTF8Length; // Skip over all trailing bytes.
58 // A codepoint takes two UTF-16 code unit if it's astral (outside BMP).
59 // Astral codepoints are encoded as 4 bytes in UTF-8 (11110xxx ...)
60 if (CB(UTF8Length, UTF8Length == 4 ? 2 : 1))
61 return true;
62 }
63 return false;
64}
65
Sam McCall8b25d222019-03-28 14:37:51 +000066// Returns the byte offset into the string that is an offset of \p Units in
67// the specified encoding.
68// Conceptually, this converts to the encoding, truncates to CodeUnits,
69// converts back to UTF-8, and returns the length in bytes.
70static size_t measureUnits(llvm::StringRef U8, int Units, OffsetEncoding Enc,
71 bool &Valid) {
72 Valid = Units >= 0;
73 if (Units <= 0)
74 return 0;
Sam McCalla4962cc2018-04-27 11:59:28 +000075 size_t Result = 0;
Sam McCall8b25d222019-03-28 14:37:51 +000076 switch (Enc) {
77 case OffsetEncoding::UTF8:
78 Result = Units;
79 break;
80 case OffsetEncoding::UTF16:
81 Valid = iterateCodepoints(U8, [&](int U8Len, int U16Len) {
82 Result += U8Len;
83 Units -= U16Len;
84 return Units <= 0;
85 });
86 if (Units < 0) // Offset in the middle of a surrogate pair.
87 Valid = false;
88 break;
89 case OffsetEncoding::UTF32:
90 Valid = iterateCodepoints(U8, [&](int U8Len, int U16Len) {
91 Result += U8Len;
92 Units--;
93 return Units <= 0;
94 });
95 break;
96 case OffsetEncoding::UnsupportedEncoding:
97 llvm_unreachable("unsupported encoding");
98 }
Sam McCalla4962cc2018-04-27 11:59:28 +000099 // Don't return an out-of-range index if we overran.
Sam McCall8b25d222019-03-28 14:37:51 +0000100 if (Result > U8.size()) {
101 Valid = false;
102 return U8.size();
103 }
104 return Result;
Sam McCalla4962cc2018-04-27 11:59:28 +0000105}
106
Sam McCalla69698f2019-03-27 17:47:49 +0000107Key<OffsetEncoding> kCurrentOffsetEncoding;
Sam McCall8b25d222019-03-28 14:37:51 +0000108static OffsetEncoding lspEncoding() {
Sam McCalla69698f2019-03-27 17:47:49 +0000109 auto *Enc = Context::current().get(kCurrentOffsetEncoding);
Sam McCall8b25d222019-03-28 14:37:51 +0000110 return Enc ? *Enc : OffsetEncoding::UTF16;
Sam McCalla69698f2019-03-27 17:47:49 +0000111}
112
Sam McCalla4962cc2018-04-27 11:59:28 +0000113// Like most strings in clangd, the input is UTF-8 encoded.
Ilya Biryukovf2001aa2019-01-07 15:45:19 +0000114size_t lspLength(llvm::StringRef Code) {
Sam McCalla4962cc2018-04-27 11:59:28 +0000115 size_t Count = 0;
Sam McCall8b25d222019-03-28 14:37:51 +0000116 switch (lspEncoding()) {
117 case OffsetEncoding::UTF8:
118 Count = Code.size();
119 break;
120 case OffsetEncoding::UTF16:
121 iterateCodepoints(Code, [&](int U8Len, int U16Len) {
122 Count += U16Len;
123 return false;
124 });
125 break;
126 case OffsetEncoding::UTF32:
127 iterateCodepoints(Code, [&](int U8Len, int U16Len) {
128 ++Count;
129 return false;
130 });
131 break;
132 case OffsetEncoding::UnsupportedEncoding:
133 llvm_unreachable("unsupported encoding");
134 }
Sam McCalla4962cc2018-04-27 11:59:28 +0000135 return Count;
136}
137
Ilya Biryukovf2001aa2019-01-07 15:45:19 +0000138llvm::Expected<size_t> positionToOffset(llvm::StringRef Code, Position P,
139 bool AllowColumnsBeyondLineLength) {
Sam McCallb536a2a2017-12-19 12:23:48 +0000140 if (P.line < 0)
Ilya Biryukovf2001aa2019-01-07 15:45:19 +0000141 return llvm::make_error<llvm::StringError>(
142 llvm::formatv("Line value can't be negative ({0})", P.line),
143 llvm::errc::invalid_argument);
Simon Marchi766338a2018-03-21 14:36:46 +0000144 if (P.character < 0)
Ilya Biryukovf2001aa2019-01-07 15:45:19 +0000145 return llvm::make_error<llvm::StringError>(
146 llvm::formatv("Character value can't be negative ({0})", P.character),
147 llvm::errc::invalid_argument);
Sam McCallb536a2a2017-12-19 12:23:48 +0000148 size_t StartOfLine = 0;
149 for (int I = 0; I != P.line; ++I) {
150 size_t NextNL = Code.find('\n', StartOfLine);
Ilya Biryukovf2001aa2019-01-07 15:45:19 +0000151 if (NextNL == llvm::StringRef::npos)
152 return llvm::make_error<llvm::StringError>(
153 llvm::formatv("Line value is out of range ({0})", P.line),
154 llvm::errc::invalid_argument);
Sam McCallb536a2a2017-12-19 12:23:48 +0000155 StartOfLine = NextNL + 1;
156 }
Sam McCalla69698f2019-03-27 17:47:49 +0000157 StringRef Line =
158 Code.substr(StartOfLine).take_until([](char C) { return C == '\n'; });
Simon Marchi766338a2018-03-21 14:36:46 +0000159
Sam McCall8b25d222019-03-28 14:37:51 +0000160 // P.character may be in UTF-16, transcode if necessary.
Sam McCalla4962cc2018-04-27 11:59:28 +0000161 bool Valid;
Sam McCall8b25d222019-03-28 14:37:51 +0000162 size_t ByteInLine = measureUnits(Line, P.character, lspEncoding(), Valid);
Sam McCalla4962cc2018-04-27 11:59:28 +0000163 if (!Valid && !AllowColumnsBeyondLineLength)
Ilya Biryukovf2001aa2019-01-07 15:45:19 +0000164 return llvm::make_error<llvm::StringError>(
Sam McCall8b25d222019-03-28 14:37:51 +0000165 llvm::formatv("{0} offset {1} is invalid for line {2}", lspEncoding(),
166 P.character, P.line),
Ilya Biryukovf2001aa2019-01-07 15:45:19 +0000167 llvm::errc::invalid_argument);
Sam McCall8b25d222019-03-28 14:37:51 +0000168 return StartOfLine + ByteInLine;
Sam McCallb536a2a2017-12-19 12:23:48 +0000169}
170
Ilya Biryukovf2001aa2019-01-07 15:45:19 +0000171Position offsetToPosition(llvm::StringRef Code, size_t Offset) {
Sam McCallb536a2a2017-12-19 12:23:48 +0000172 Offset = std::min(Code.size(), Offset);
Ilya Biryukovf2001aa2019-01-07 15:45:19 +0000173 llvm::StringRef Before = Code.substr(0, Offset);
Sam McCallb536a2a2017-12-19 12:23:48 +0000174 int Lines = Before.count('\n');
175 size_t PrevNL = Before.rfind('\n');
Ilya Biryukovf2001aa2019-01-07 15:45:19 +0000176 size_t StartOfLine = (PrevNL == llvm::StringRef::npos) ? 0 : (PrevNL + 1);
Ilya Biryukov7beea3a2018-02-14 10:52:04 +0000177 Position Pos;
178 Pos.line = Lines;
Sam McCall71891122018-10-23 11:51:53 +0000179 Pos.character = lspLength(Before.substr(StartOfLine));
Ilya Biryukov7beea3a2018-02-14 10:52:04 +0000180 return Pos;
Sam McCallb536a2a2017-12-19 12:23:48 +0000181}
182
Marc-Andre Laperle63a10982018-02-21 02:39:08 +0000183Position sourceLocToPosition(const SourceManager &SM, SourceLocation Loc) {
Sam McCalla4962cc2018-04-27 11:59:28 +0000184 // We use the SourceManager's line tables, but its column number is in bytes.
185 FileID FID;
186 unsigned Offset;
187 std::tie(FID, Offset) = SM.getDecomposedSpellingLoc(Loc);
Marc-Andre Laperle63a10982018-02-21 02:39:08 +0000188 Position P;
Sam McCalla4962cc2018-04-27 11:59:28 +0000189 P.line = static_cast<int>(SM.getLineNumber(FID, Offset)) - 1;
190 bool Invalid = false;
Ilya Biryukovf2001aa2019-01-07 15:45:19 +0000191 llvm::StringRef Code = SM.getBufferData(FID, &Invalid);
Sam McCalla4962cc2018-04-27 11:59:28 +0000192 if (!Invalid) {
193 auto ColumnInBytes = SM.getColumnNumber(FID, Offset) - 1;
194 auto LineSoFar = Code.substr(Offset - ColumnInBytes, ColumnInBytes);
Sam McCall71891122018-10-23 11:51:53 +0000195 P.character = lspLength(LineSoFar);
Sam McCalla4962cc2018-04-27 11:59:28 +0000196 }
Marc-Andre Laperle63a10982018-02-21 02:39:08 +0000197 return P;
198}
199
Haojian Wu92c32572019-06-25 08:01:46 +0000200llvm::Optional<Range> getTokenRange(const SourceManager &SM,
201 const LangOptions &LangOpts,
202 SourceLocation TokLoc) {
203 if (!TokLoc.isValid())
204 return llvm::None;
205 SourceLocation End = Lexer::getLocForEndOfToken(TokLoc, 0, SM, LangOpts);
206 if (!End.isValid())
207 return llvm::None;
208 return halfOpenToRange(SM, CharSourceRange::getCharRange(TokLoc, End));
209}
210
Ilya Biryukov43998782019-01-31 21:30:05 +0000211bool isValidFileRange(const SourceManager &Mgr, SourceRange R) {
212 if (!R.getBegin().isValid() || !R.getEnd().isValid())
213 return false;
214
215 FileID BeginFID;
216 size_t BeginOffset = 0;
217 std::tie(BeginFID, BeginOffset) = Mgr.getDecomposedLoc(R.getBegin());
218
219 FileID EndFID;
220 size_t EndOffset = 0;
221 std::tie(EndFID, EndOffset) = Mgr.getDecomposedLoc(R.getEnd());
222
223 return BeginFID.isValid() && BeginFID == EndFID && BeginOffset <= EndOffset;
224}
225
226bool halfOpenRangeContains(const SourceManager &Mgr, SourceRange R,
227 SourceLocation L) {
228 assert(isValidFileRange(Mgr, R));
229
230 FileID BeginFID;
231 size_t BeginOffset = 0;
232 std::tie(BeginFID, BeginOffset) = Mgr.getDecomposedLoc(R.getBegin());
233 size_t EndOffset = Mgr.getFileOffset(R.getEnd());
234
235 FileID LFid;
236 size_t LOffset;
237 std::tie(LFid, LOffset) = Mgr.getDecomposedLoc(L);
238 return BeginFID == LFid && BeginOffset <= LOffset && LOffset < EndOffset;
239}
240
241bool halfOpenRangeTouches(const SourceManager &Mgr, SourceRange R,
242 SourceLocation L) {
243 return L == R.getEnd() || halfOpenRangeContains(Mgr, R, L);
244}
245
246llvm::Optional<SourceRange> toHalfOpenFileRange(const SourceManager &Mgr,
247 const LangOptions &LangOpts,
248 SourceRange R) {
249 auto Begin = Mgr.getFileLoc(R.getBegin());
250 if (Begin.isInvalid())
251 return llvm::None;
252 auto End = Mgr.getFileLoc(R.getEnd());
253 if (End.isInvalid())
254 return llvm::None;
255 End = Lexer::getLocForEndOfToken(End, 0, Mgr, LangOpts);
256
257 SourceRange Result(Begin, End);
258 if (!isValidFileRange(Mgr, Result))
259 return llvm::None;
260 return Result;
261}
262
263llvm::StringRef toSourceCode(const SourceManager &SM, SourceRange R) {
264 assert(isValidFileRange(SM, R));
265 bool Invalid = false;
266 auto *Buf = SM.getBuffer(SM.getFileID(R.getBegin()), &Invalid);
267 assert(!Invalid);
268
269 size_t BeginOffset = SM.getFileOffset(R.getBegin());
270 size_t EndOffset = SM.getFileOffset(R.getEnd());
271 return Buf->getBuffer().substr(BeginOffset, EndOffset - BeginOffset);
272}
273
Ilya Biryukovcce67a32019-01-29 14:17:36 +0000274llvm::Expected<SourceLocation> sourceLocationInMainFile(const SourceManager &SM,
275 Position P) {
276 llvm::StringRef Code = SM.getBuffer(SM.getMainFileID())->getBuffer();
277 auto Offset =
278 positionToOffset(Code, P, /*AllowColumnBeyondLineLength=*/false);
279 if (!Offset)
280 return Offset.takeError();
281 return SM.getLocForStartOfFile(SM.getMainFileID()).getLocWithOffset(*Offset);
282}
283
Ilya Biryukov71028b82018-03-12 15:28:22 +0000284Range halfOpenToRange(const SourceManager &SM, CharSourceRange R) {
285 // Clang is 1-based, LSP uses 0-based indexes.
286 Position Begin = sourceLocToPosition(SM, R.getBegin());
287 Position End = sourceLocToPosition(SM, R.getEnd());
288
289 return {Begin, End};
290}
291
Ilya Biryukovf2001aa2019-01-07 15:45:19 +0000292std::pair<size_t, size_t> offsetToClangLineColumn(llvm::StringRef Code,
Sam McCalla4962cc2018-04-27 11:59:28 +0000293 size_t Offset) {
294 Offset = std::min(Code.size(), Offset);
Ilya Biryukovf2001aa2019-01-07 15:45:19 +0000295 llvm::StringRef Before = Code.substr(0, Offset);
Sam McCalla4962cc2018-04-27 11:59:28 +0000296 int Lines = Before.count('\n');
297 size_t PrevNL = Before.rfind('\n');
Ilya Biryukovf2001aa2019-01-07 15:45:19 +0000298 size_t StartOfLine = (PrevNL == llvm::StringRef::npos) ? 0 : (PrevNL + 1);
Sam McCalla4962cc2018-04-27 11:59:28 +0000299 return {Lines + 1, Offset - StartOfLine + 1};
300}
301
Ilya Biryukov43998782019-01-31 21:30:05 +0000302std::pair<StringRef, StringRef> splitQualifiedName(StringRef QName) {
Marc-Andre Laperleb387b6e2018-04-23 20:00:52 +0000303 size_t Pos = QName.rfind("::");
Ilya Biryukovf2001aa2019-01-07 15:45:19 +0000304 if (Pos == llvm::StringRef::npos)
305 return {llvm::StringRef(), QName};
Marc-Andre Laperleb387b6e2018-04-23 20:00:52 +0000306 return {QName.substr(0, Pos + 2), QName.substr(Pos + 2)};
307}
308
Ilya Biryukovf2001aa2019-01-07 15:45:19 +0000309TextEdit replacementToEdit(llvm::StringRef Code,
310 const tooling::Replacement &R) {
Eric Liu9133ecd2018-05-11 12:12:08 +0000311 Range ReplacementRange = {
312 offsetToPosition(Code, R.getOffset()),
313 offsetToPosition(Code, R.getOffset() + R.getLength())};
314 return {ReplacementRange, R.getReplacementText()};
315}
316
Ilya Biryukovf2001aa2019-01-07 15:45:19 +0000317std::vector<TextEdit> replacementsToEdits(llvm::StringRef Code,
Eric Liu9133ecd2018-05-11 12:12:08 +0000318 const tooling::Replacements &Repls) {
319 std::vector<TextEdit> Edits;
320 for (const auto &R : Repls)
321 Edits.push_back(replacementToEdit(Code, R));
322 return Edits;
323}
324
Ilya Biryukovf2001aa2019-01-07 15:45:19 +0000325llvm::Optional<std::string> getCanonicalPath(const FileEntry *F,
326 const SourceManager &SourceMgr) {
Kadir Cetinkayadd677932018-12-19 10:46:21 +0000327 if (!F)
328 return None;
Simon Marchi25f1f732018-08-10 22:27:53 +0000329
Ilya Biryukovf2001aa2019-01-07 15:45:19 +0000330 llvm::SmallString<128> FilePath = F->getName();
331 if (!llvm::sys::path::is_absolute(FilePath)) {
Kadir Cetinkayadd677932018-12-19 10:46:21 +0000332 if (auto EC =
Duncan P. N. Exon Smithdb8a7422019-03-26 22:32:06 +0000333 SourceMgr.getFileManager().getVirtualFileSystem().makeAbsolute(
Kadir Cetinkayadd677932018-12-19 10:46:21 +0000334 FilePath)) {
335 elog("Could not turn relative path '{0}' to absolute: {1}", FilePath,
336 EC.message());
Sam McCallc008af62018-10-20 15:30:37 +0000337 return None;
Marc-Andre Laperle1be69702018-07-05 19:35:01 +0000338 }
339 }
Simon Marchi25f1f732018-08-10 22:27:53 +0000340
Kadir Cetinkayadd677932018-12-19 10:46:21 +0000341 // Handle the symbolic link path case where the current working directory
342 // (getCurrentWorkingDirectory) is a symlink./ We always want to the real
343 // file path (instead of the symlink path) for the C++ symbols.
344 //
345 // Consider the following example:
346 //
347 // src dir: /project/src/foo.h
348 // current working directory (symlink): /tmp/build -> /project/src/
349 //
350 // The file path of Symbol is "/project/src/foo.h" instead of
351 // "/tmp/build/foo.h"
352 if (const DirectoryEntry *Dir = SourceMgr.getFileManager().getDirectory(
Ilya Biryukovf2001aa2019-01-07 15:45:19 +0000353 llvm::sys::path::parent_path(FilePath))) {
354 llvm::SmallString<128> RealPath;
355 llvm::StringRef DirName = SourceMgr.getFileManager().getCanonicalName(Dir);
356 llvm::sys::path::append(RealPath, DirName,
357 llvm::sys::path::filename(FilePath));
Kadir Cetinkayadd677932018-12-19 10:46:21 +0000358 return RealPath.str().str();
Simon Marchi25f1f732018-08-10 22:27:53 +0000359 }
360
Kadir Cetinkayadd677932018-12-19 10:46:21 +0000361 return FilePath.str().str();
Marc-Andre Laperle1be69702018-07-05 19:35:01 +0000362}
363
Kadir Cetinkaya2f84d912018-08-08 08:59:29 +0000364TextEdit toTextEdit(const FixItHint &FixIt, const SourceManager &M,
365 const LangOptions &L) {
366 TextEdit Result;
367 Result.range =
368 halfOpenToRange(M, Lexer::makeFileCharRange(FixIt.RemoveRange, M, L));
369 Result.newText = FixIt.CodeToInsert;
370 return Result;
371}
372
Haojian Wuaa3ed5a2019-01-25 15:14:03 +0000373bool isRangeConsecutive(const Range &Left, const Range &Right) {
Kadir Cetinkayaa9c9d002018-08-13 08:23:01 +0000374 return Left.end.line == Right.start.line &&
375 Left.end.character == Right.start.character;
376}
377
Ilya Biryukovf2001aa2019-01-07 15:45:19 +0000378FileDigest digest(llvm::StringRef Content) {
Kadir Cetinkayad08eab42018-11-27 16:08:53 +0000379 return llvm::SHA1::hash({(const uint8_t *)Content.data(), Content.size()});
380}
381
Ilya Biryukovf2001aa2019-01-07 15:45:19 +0000382llvm::Optional<FileDigest> digestFile(const SourceManager &SM, FileID FID) {
Kadir Cetinkayad08eab42018-11-27 16:08:53 +0000383 bool Invalid = false;
Ilya Biryukovf2001aa2019-01-07 15:45:19 +0000384 llvm::StringRef Content = SM.getBufferData(FID, &Invalid);
Kadir Cetinkayad08eab42018-11-27 16:08:53 +0000385 if (Invalid)
386 return None;
387 return digest(Content);
388}
389
Eric Liudd662772019-01-28 14:01:55 +0000390format::FormatStyle getFormatStyleForFile(llvm::StringRef File,
391 llvm::StringRef Content,
392 llvm::vfs::FileSystem *FS) {
393 auto Style = format::getStyle(format::DefaultFormatStyle, File,
394 format::DefaultFallbackStyle, Content, FS);
395 if (!Style) {
396 log("getStyle() failed for file {0}: {1}. Fallback is LLVM style.", File,
397 Style.takeError());
398 Style = format::getLLVMStyle();
399 }
400 return *Style;
401}
402
Haojian Wu12e194c2019-02-06 15:24:50 +0000403llvm::Expected<tooling::Replacements>
404cleanupAndFormat(StringRef Code, const tooling::Replacements &Replaces,
405 const format::FormatStyle &Style) {
406 auto CleanReplaces = cleanupAroundReplacements(Code, Replaces, Style);
407 if (!CleanReplaces)
408 return CleanReplaces;
409 return formatReplacements(Code, std::move(*CleanReplaces), Style);
410}
411
Sam McCallc316b222019-04-26 07:45:49 +0000412template <typename Action>
413static void lex(llvm::StringRef Code, const format::FormatStyle &Style,
414 Action A) {
415 // FIXME: InMemoryFileAdapter crashes unless the buffer is null terminated!
416 std::string NullTerminatedCode = Code.str();
417 SourceManagerForFile FileSM("dummy.cpp", NullTerminatedCode);
Eric Liu00d99bd2019-04-11 09:36:36 +0000418 auto &SM = FileSM.get();
419 auto FID = SM.getMainFileID();
420 Lexer Lex(FID, SM.getBuffer(FID), SM, format::getFormattingLangOpts(Style));
421 Token Tok;
422
Sam McCallc316b222019-04-26 07:45:49 +0000423 while (!Lex.LexFromRawLexer(Tok))
424 A(Tok);
425}
426
427llvm::StringMap<unsigned> collectIdentifiers(llvm::StringRef Content,
428 const format::FormatStyle &Style) {
Eric Liu00d99bd2019-04-11 09:36:36 +0000429 llvm::StringMap<unsigned> Identifiers;
Sam McCallc316b222019-04-26 07:45:49 +0000430 lex(Content, Style, [&](const clang::Token &Tok) {
Eric Liu00d99bd2019-04-11 09:36:36 +0000431 switch (Tok.getKind()) {
432 case tok::identifier:
433 ++Identifiers[Tok.getIdentifierInfo()->getName()];
434 break;
435 case tok::raw_identifier:
436 ++Identifiers[Tok.getRawIdentifier()];
437 break;
438 default:
Sam McCallc316b222019-04-26 07:45:49 +0000439 break;
Eric Liu00d99bd2019-04-11 09:36:36 +0000440 }
Sam McCallc316b222019-04-26 07:45:49 +0000441 });
Eric Liu00d99bd2019-04-11 09:36:36 +0000442 return Identifiers;
443}
444
Sam McCallc316b222019-04-26 07:45:49 +0000445namespace {
446enum NamespaceEvent {
447 BeginNamespace, // namespace <ns> {. Payload is resolved <ns>.
448 EndNamespace, // } // namespace <ns>. Payload is resolved *outer* namespace.
449 UsingDirective // using namespace <ns>. Payload is unresolved <ns>.
450};
451// Scans C++ source code for constructs that change the visible namespaces.
452void parseNamespaceEvents(
453 llvm::StringRef Code, const format::FormatStyle &Style,
454 llvm::function_ref<void(NamespaceEvent, llvm::StringRef)> Callback) {
455
456 // Stack of enclosing namespaces, e.g. {"clang", "clangd"}
457 std::vector<std::string> Enclosing; // Contains e.g. "clang", "clangd"
458 // Stack counts open braces. true if the brace opened a namespace.
459 std::vector<bool> BraceStack;
460
461 enum {
462 Default,
463 Namespace, // just saw 'namespace'
464 NamespaceName, // just saw 'namespace' NSName
465 Using, // just saw 'using'
466 UsingNamespace, // just saw 'using namespace'
467 UsingNamespaceName, // just saw 'using namespace' NSName
468 } State = Default;
469 std::string NSName;
470
471 lex(Code, Style, [&](const clang::Token &Tok) {
472 switch(Tok.getKind()) {
473 case tok::raw_identifier:
474 // In raw mode, this could be a keyword or a name.
475 switch (State) {
476 case UsingNamespace:
477 case UsingNamespaceName:
478 NSName.append(Tok.getRawIdentifier());
479 State = UsingNamespaceName;
480 break;
481 case Namespace:
482 case NamespaceName:
483 NSName.append(Tok.getRawIdentifier());
484 State = NamespaceName;
485 break;
486 case Using:
487 State =
488 (Tok.getRawIdentifier() == "namespace") ? UsingNamespace : Default;
489 break;
490 case Default:
491 NSName.clear();
492 if (Tok.getRawIdentifier() == "namespace")
493 State = Namespace;
494 else if (Tok.getRawIdentifier() == "using")
495 State = Using;
496 break;
497 }
498 break;
499 case tok::coloncolon:
500 // This can come at the beginning or in the middle of a namespace name.
501 switch (State) {
502 case UsingNamespace:
503 case UsingNamespaceName:
504 NSName.append("::");
505 State = UsingNamespaceName;
506 break;
507 case NamespaceName:
508 NSName.append("::");
509 State = NamespaceName;
510 break;
511 case Namespace: // Not legal here.
512 case Using:
513 case Default:
514 State = Default;
515 break;
516 }
517 break;
518 case tok::l_brace:
519 // Record which { started a namespace, so we know when } ends one.
520 if (State == NamespaceName) {
521 // Parsed: namespace <name> {
522 BraceStack.push_back(true);
523 Enclosing.push_back(NSName);
524 Callback(BeginNamespace, llvm::join(Enclosing, "::"));
525 } else {
526 // This case includes anonymous namespaces (State = Namespace).
527 // For our purposes, they're not namespaces and we ignore them.
528 BraceStack.push_back(false);
529 }
530 State = Default;
531 break;
532 case tok::r_brace:
533 // If braces are unmatched, we're going to be confused, but don't crash.
534 if (!BraceStack.empty()) {
535 if (BraceStack.back()) {
536 // Parsed: } // namespace
537 Enclosing.pop_back();
538 Callback(EndNamespace, llvm::join(Enclosing, "::"));
539 }
540 BraceStack.pop_back();
541 }
542 break;
543 case tok::semi:
544 if (State == UsingNamespaceName)
545 // Parsed: using namespace <name> ;
546 Callback(UsingDirective, llvm::StringRef(NSName));
547 State = Default;
548 break;
549 default:
550 State = Default;
551 break;
552 }
553 });
554}
555
556// Returns the prefix namespaces of NS: {"" ... NS}.
557llvm::SmallVector<llvm::StringRef, 8> ancestorNamespaces(llvm::StringRef NS) {
558 llvm::SmallVector<llvm::StringRef, 8> Results;
559 Results.push_back(NS.take_front(0));
560 NS.split(Results, "::", /*MaxSplit=*/-1, /*KeepEmpty=*/false);
561 for (llvm::StringRef &R : Results)
562 R = NS.take_front(R.end() - NS.begin());
563 return Results;
564}
565
566} // namespace
567
568std::vector<std::string> visibleNamespaces(llvm::StringRef Code,
569 const format::FormatStyle &Style) {
570 std::string Current;
571 // Map from namespace to (resolved) namespaces introduced via using directive.
572 llvm::StringMap<llvm::StringSet<>> UsingDirectives;
573
574 parseNamespaceEvents(Code, Style,
575 [&](NamespaceEvent Event, llvm::StringRef NS) {
576 switch (Event) {
577 case BeginNamespace:
578 case EndNamespace:
579 Current = NS;
580 break;
581 case UsingDirective:
582 if (NS.consume_front("::"))
583 UsingDirectives[Current].insert(NS);
584 else {
585 for (llvm::StringRef Enclosing :
586 ancestorNamespaces(Current)) {
587 if (Enclosing.empty())
588 UsingDirectives[Current].insert(NS);
589 else
590 UsingDirectives[Current].insert(
591 (Enclosing + "::" + NS).str());
592 }
593 }
594 break;
595 }
596 });
597
598 std::vector<std::string> Found;
599 for (llvm::StringRef Enclosing : ancestorNamespaces(Current)) {
600 Found.push_back(Enclosing);
601 auto It = UsingDirectives.find(Enclosing);
602 if (It != UsingDirectives.end())
603 for (const auto& Used : It->second)
604 Found.push_back(Used.getKey());
605 }
606
607
608 llvm::sort(Found, [&](const std::string &LHS, const std::string &RHS) {
609 if (Current == RHS)
610 return false;
611 if (Current == LHS)
612 return true;
613 return LHS < RHS;
614 });
615 Found.erase(std::unique(Found.begin(), Found.end()), Found.end());
616 return Found;
617}
618
Sam McCall9fb22b22019-05-06 10:25:10 +0000619llvm::StringSet<> collectWords(llvm::StringRef Content) {
620 // We assume short words are not significant.
621 // We may want to consider other stopwords, e.g. language keywords.
622 // (A very naive implementation showed no benefit, but lexing might do better)
623 static constexpr int MinWordLength = 4;
624
625 std::vector<CharRole> Roles(Content.size());
626 calculateRoles(Content, Roles);
627
628 llvm::StringSet<> Result;
629 llvm::SmallString<256> Word;
630 auto Flush = [&] {
631 if (Word.size() >= MinWordLength) {
632 for (char &C : Word)
633 C = llvm::toLower(C);
634 Result.insert(Word);
635 }
636 Word.clear();
637 };
638 for (unsigned I = 0; I < Content.size(); ++I) {
639 switch (Roles[I]) {
640 case Head:
641 Flush();
642 LLVM_FALLTHROUGH;
643 case Tail:
644 Word.push_back(Content[I]);
645 break;
646 case Unknown:
647 case Separator:
648 Flush();
649 break;
650 }
651 }
652 Flush();
653
654 return Result;
655}
656
Haojian Wu9d34f452019-07-01 09:26:48 +0000657llvm::Optional<DefinedMacro> locateMacroAt(SourceLocation Loc,
658 Preprocessor &PP) {
659 const auto &SM = PP.getSourceManager();
660 const auto &LangOpts = PP.getLangOpts();
661 Token Result;
662 if (Lexer::getRawToken(SM.getSpellingLoc(Loc), Result, SM, LangOpts, false))
663 return None;
664 if (Result.is(tok::raw_identifier))
665 PP.LookUpIdentifierInfo(Result);
666 IdentifierInfo *IdentifierInfo = Result.getIdentifierInfo();
667 if (!IdentifierInfo || !IdentifierInfo->hadMacroDefinition())
668 return None;
669
670 std::pair<FileID, unsigned int> DecLoc = SM.getDecomposedExpansionLoc(Loc);
671 // Get the definition just before the searched location so that a macro
672 // referenced in a '#undef MACRO' can still be found.
673 SourceLocation BeforeSearchedLocation =
674 SM.getMacroArgExpandedLocation(SM.getLocForStartOfFile(DecLoc.first)
675 .getLocWithOffset(DecLoc.second - 1));
676 MacroDefinition MacroDef =
677 PP.getMacroDefinitionAtLoc(IdentifierInfo, BeforeSearchedLocation);
678 if (auto *MI = MacroDef.getMacroInfo())
679 return DefinedMacro{IdentifierInfo->getName(), MI};
680 return None;
681}
682
Sam McCallb536a2a2017-12-19 12:23:48 +0000683} // namespace clangd
684} // namespace clang