introduce a new CharSourceRange class, and enhance the diagnostics routines
to use them instead of SourceRange.  CharSourceRange is just a SourceRange
plus a bool that indicates whether the range has the end character resolved
or whether the end location is the start of the end token.  While most of
the compiler wants to think of ranges that have ends that are the start of
the end token, the printf diagnostic stuff wants to highlight ranges within
tokens.

This is transparent to the diagnostic stuff.  To start taking advantage of
the new capabilities, you can do something like this:
  Diag(..) << CharSourceRange::getCharRange(Begin,End)





git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@106338 91177308-0d34-0410-b5e6-96231b3b80d8
diff --git a/include/clang/Basic/Diagnostic.h b/include/clang/Basic/Diagnostic.h
index a4221e0..1fe0d81 100644
--- a/include/clang/Basic/Diagnostic.h
+++ b/include/clang/Basic/Diagnostic.h
@@ -24,7 +24,6 @@
 
 namespace llvm {
   template <typename T> class SmallVectorImpl;
-  class raw_ostream;
 }
 
 namespace clang {
@@ -36,8 +35,6 @@
   class LangOptions;
   class PartialDiagnostic;
   class Preprocessor;
-  class SourceManager;
-  class SourceRange;
 
   // Import the diagnostic enums themselves.
   namespace diag {
@@ -98,8 +95,8 @@
 /// compilation.
 class FixItHint {
 public:
-  /// \brief Tokens that should be removed to correct the error.
-  SourceRange RemoveRange;
+  /// \brief Code that should be removed to correct the error.
+  CharSourceRange RemoveRange;
 
   /// \brief The location at which we should insert code to correct
   /// the error.
@@ -129,15 +126,18 @@
 
   /// \brief Create a code modification hint that removes the given
   /// source range.
-  static FixItHint CreateRemoval(SourceRange RemoveRange) {
+  static FixItHint CreateRemoval(CharSourceRange RemoveRange) {
     FixItHint Hint;
     Hint.RemoveRange = RemoveRange;
     return Hint;
   }
-
+  static FixItHint CreateRemoval(SourceRange RemoveRange) {
+    return CreateRemoval(CharSourceRange::getTokenRange(RemoveRange));
+  }
+  
   /// \brief Create a code modification hint that replaces the given
   /// source range with the given code string.
-  static FixItHint CreateReplacement(SourceRange RemoveRange,
+  static FixItHint CreateReplacement(CharSourceRange RemoveRange,
                                      llvm::StringRef Code) {
     FixItHint Hint;
     Hint.RemoveRange = RemoveRange;
@@ -145,6 +145,11 @@
     Hint.CodeToInsert = Code;
     return Hint;
   }
+  
+  static FixItHint CreateReplacement(SourceRange RemoveRange,
+                                     llvm::StringRef Code) {
+    return CreateReplacement(CharSourceRange::getTokenRange(RemoveRange), Code);
+  }
 };
 
 /// Diagnostic - This concrete class is used by the front-end to report
@@ -597,7 +602,7 @@
 
   /// DiagRanges - The list of ranges added to this diagnostic.  It currently
   /// only support 10 ranges, could easily be extended if needed.
-  SourceRange DiagRanges[10];
+  CharSourceRange DiagRanges[10];
 
   enum { MaxFixItHints = 3 };
 
@@ -696,7 +701,7 @@
     }
   }
 
-  void AddSourceRange(const SourceRange &R) const {
+  void AddSourceRange(const CharSourceRange &R) const {
     assert(NumRanges <
            sizeof(DiagObj->DiagRanges)/sizeof(DiagObj->DiagRanges[0]) &&
            "Too many arguments to diagnostic!");
@@ -767,11 +772,17 @@
   
 inline const DiagnosticBuilder &operator<<(const DiagnosticBuilder &DB,
                                            const SourceRange &R) {
-  DB.AddSourceRange(R);
+  DB.AddSourceRange(CharSourceRange::getTokenRange(R));
   return DB;
 }
 
 inline const DiagnosticBuilder &operator<<(const DiagnosticBuilder &DB,
+                                           const CharSourceRange &R) {
+  DB.AddSourceRange(R);
+  return DB;
+}
+  
+inline const DiagnosticBuilder &operator<<(const DiagnosticBuilder &DB,
                                            const FixItHint &Hint) {
   DB.AddFixItHint(Hint);
   return DB;
@@ -864,7 +875,7 @@
     return DiagObj->NumDiagRanges;
   }
 
-  SourceRange getRange(unsigned Idx) const {
+  const CharSourceRange &getRange(unsigned Idx) const {
     assert(Idx < DiagObj->NumDiagRanges && "Invalid diagnostic range index!");
     return DiagObj->DiagRanges[Idx];
   }
@@ -901,7 +912,7 @@
   Diagnostic::Level Level;
   FullSourceLoc Loc;
   std::string Message;
-  std::vector<SourceRange> Ranges;
+  std::vector<CharSourceRange> Ranges;
   std::vector<FixItHint> FixIts;
 
 public:
@@ -917,7 +928,7 @@
   const FullSourceLoc &getLocation() const { return Loc; }
   llvm::StringRef getMessage() const { return Message; }
   
-  typedef std::vector<SourceRange>::const_iterator range_iterator;
+  typedef std::vector<CharSourceRange>::const_iterator range_iterator;
   range_iterator range_begin() const { return Ranges.begin(); }
   range_iterator range_end() const { return Ranges.end(); }
   unsigned range_size() const { return Ranges.size(); }
diff --git a/include/clang/Basic/PartialDiagnostic.h b/include/clang/Basic/PartialDiagnostic.h
index 89fae87..cd0da97 100644
--- a/include/clang/Basic/PartialDiagnostic.h
+++ b/include/clang/Basic/PartialDiagnostic.h
@@ -59,7 +59,7 @@
   
     /// DiagRanges - The list of ranges added to this diagnostic.  It currently
     /// only support 10 ranges, could easily be extended if needed.
-    SourceRange DiagRanges[10];
+    CharSourceRange DiagRanges[10];
     
     enum { MaxFixItHints = 3 };
     
@@ -142,7 +142,7 @@
     DiagStorage = 0;
   }
   
-  void AddSourceRange(const SourceRange &R) const {
+  void AddSourceRange(const CharSourceRange &R) const {
     if (!DiagStorage)
       DiagStorage = getStorage();
 
@@ -264,10 +264,16 @@
 
   friend inline const PartialDiagnostic &operator<<(const PartialDiagnostic &PD,
                                                     const SourceRange &R) {
-    PD.AddSourceRange(R);
+    PD.AddSourceRange(CharSourceRange::getTokenRange(R));
     return PD;
   }
 
+  friend inline const PartialDiagnostic &operator<<(const PartialDiagnostic &PD,
+                                                    const CharSourceRange &R) {
+    PD.AddSourceRange(R);
+    return PD;
+  }
+  
   friend const PartialDiagnostic &operator<<(const PartialDiagnostic &PD,
                                              const FixItHint &Hint) {
     PD.AddFixItHint(Hint);
diff --git a/include/clang/Basic/SourceLocation.h b/include/clang/Basic/SourceLocation.h
index 0bbeffe..35f27fb 100644
--- a/include/clang/Basic/SourceLocation.h
+++ b/include/clang/Basic/SourceLocation.h
@@ -172,6 +172,56 @@
     return B != X.B || E != X.E;
   }
 };
+  
+/// CharSourceRange - This class represents a character granular source range.
+/// The underlying SourceRange can either specify the starting/ending character
+/// of the range, or it can specify the start or the range and the start of the
+/// last token of the range (a "token range").  In the token range case, the
+/// size of the last token must be measured to determine the actual end of the
+/// range.
+class CharSourceRange { 
+  SourceRange Range;
+  bool IsTokenRange;
+public:
+  CharSourceRange() : IsTokenRange(false) {}
+  CharSourceRange(SourceRange R, bool ITR) : Range(R),IsTokenRange(ITR){}
+
+  static CharSourceRange getTokenRange(SourceRange R) {
+    CharSourceRange Result;
+    Result.Range = R;
+    Result.IsTokenRange = true;
+    return Result;
+  }
+
+  static CharSourceRange getCharRange(SourceRange R) {
+    CharSourceRange Result;
+    Result.Range = R;
+    Result.IsTokenRange = false;
+    return Result;
+  }
+    
+  static CharSourceRange getTokenRange(SourceLocation B, SourceLocation E) {
+    return getTokenRange(SourceRange(B, E));
+  }
+  static CharSourceRange getCharRange(SourceLocation B, SourceLocation E) {
+    return getCharRange(SourceRange(B, E));
+  }
+  
+  /// isTokenRange - Return true if the end of this range specifies the start of
+  /// the last token.  Return false if the end of this range specifies the last
+  /// character in the range.
+  bool isTokenRange() const { return IsTokenRange; }
+  
+  SourceLocation getBegin() const { return Range.getBegin(); }
+  SourceLocation getEnd() const { return Range.getEnd(); }
+  const SourceRange &getAsRange() const { return Range; }
+ 
+  void setBegin(SourceLocation b) { Range.setBegin(b); }
+  void setEnd(SourceLocation e) { Range.setEnd(e); }
+  
+  bool isValid() const { return Range.isValid(); }
+  bool isInvalid() const { return !isValid(); }
+};
 
 /// FullSourceLoc - A SourceLocation and its associated SourceManager.  Useful
 /// for argument passing to functions that expect both objects.
diff --git a/include/clang/Frontend/TextDiagnosticPrinter.h b/include/clang/Frontend/TextDiagnosticPrinter.h
index ec4392f..f530294 100644
--- a/include/clang/Frontend/TextDiagnosticPrinter.h
+++ b/include/clang/Frontend/TextDiagnosticPrinter.h
@@ -18,14 +18,9 @@
 #include "clang/Basic/Diagnostic.h"
 #include "clang/Basic/SourceLocation.h"
 
-namespace llvm {
-  class raw_ostream;
-}
-
 namespace clang {
 class DiagnosticOptions;
 class LangOptions;
-class SourceManager;
 
 class TextDiagnosticPrinter : public DiagnosticClient {
   llvm::raw_ostream &OS;
@@ -60,14 +55,14 @@
 
   void PrintIncludeStack(SourceLocation Loc, const SourceManager &SM);
 
-  void HighlightRange(const SourceRange &R,
+  void HighlightRange(const CharSourceRange &R,
                       const SourceManager &SrcMgr,
                       unsigned LineNo, FileID FID,
                       std::string &CaretLine,
                       const std::string &SourceLine);
 
   void EmitCaretDiagnostic(SourceLocation Loc,
-                           SourceRange *Ranges, unsigned NumRanges,
+                           CharSourceRange *Ranges, unsigned NumRanges,
                            const SourceManager &SM,
                            const FixItHint *Hints,
                            unsigned NumHints,
diff --git a/include/clang/Rewrite/Rewriter.h b/include/clang/Rewrite/Rewriter.h
index adda866..0612a15 100644
--- a/include/clang/Rewrite/Rewriter.h
+++ b/include/clang/Rewrite/Rewriter.h
@@ -151,6 +151,7 @@
   /// getRangeSize - Return the size in bytes of the specified range if they
   /// are in the same file.  If not, this returns -1.
   int getRangeSize(SourceRange Range) const;
+  int getRangeSize(const CharSourceRange &Range) const;
 
   /// getRewrittenText - Return the rewritten form of the text in the specified
   /// range.  If the start or end of the range was unrewritable or if they are
diff --git a/lib/Basic/Diagnostic.cpp b/lib/Basic/Diagnostic.cpp
index a2480b1..641d87b 100644
--- a/lib/Basic/Diagnostic.cpp
+++ b/lib/Basic/Diagnostic.cpp
@@ -1043,8 +1043,7 @@
 
 StoredDiagnostic::StoredDiagnostic(Diagnostic::Level Level, 
                                    const DiagnosticInfo &Info)
-  : Level(Level), Loc(Info.getLocation()) 
-{
+  : Level(Level), Loc(Info.getLocation()) {
   llvm::SmallString<64> Message;
   Info.FormatDiagnostic(Message);
   this->Message.assign(Message.begin(), Message.end());
@@ -1131,6 +1130,7 @@
       
       WriteSourceLocation(OS, SM, R->getBegin());
       WriteSourceLocation(OS, SM, R->getEnd());
+      WriteUnsigned(OS, R->isTokenRange());
     }
   }
 
@@ -1159,6 +1159,7 @@
   for (fixit_iterator F = fixit_begin(), FEnd = fixit_end(); F != FEnd; ++F) {
     WriteSourceLocation(OS, SM, F->RemoveRange.getBegin());
     WriteSourceLocation(OS, SM, F->RemoveRange.getEnd());
+    WriteUnsigned(OS, F->RemoveRange.isTokenRange());
     WriteSourceLocation(OS, SM, F->InsertionLoc);
     WriteString(OS, F->CodeToInsert);
   }
@@ -1272,11 +1273,14 @@
     return Diag;
   for (unsigned I = 0; I != NumSourceRanges; ++I) {
     SourceLocation Begin, End;
+    unsigned IsTokenRange;
     if (ReadSourceLocation(FM, SM, Memory, MemoryEnd, Begin) ||
-        ReadSourceLocation(FM, SM, Memory, MemoryEnd, End))
+        ReadSourceLocation(FM, SM, Memory, MemoryEnd, End) ||
+        ReadUnsigned(Memory, MemoryEnd, IsTokenRange))
       return Diag;
 
-    Diag.Ranges.push_back(SourceRange(Begin, End));
+    Diag.Ranges.push_back(CharSourceRange(SourceRange(Begin, End),
+                                          IsTokenRange));
   }
 
   // Read the fix-it hints.
@@ -1285,9 +1289,10 @@
     return Diag;
   for (unsigned I = 0; I != NumFixIts; ++I) {
     SourceLocation RemoveBegin, RemoveEnd, InsertionLoc;
-    unsigned InsertLen = 0;
+    unsigned InsertLen = 0, RemoveIsTokenRange;
     if (ReadSourceLocation(FM, SM, Memory, MemoryEnd, RemoveBegin) ||
         ReadSourceLocation(FM, SM, Memory, MemoryEnd, RemoveEnd) ||
+        ReadUnsigned(Memory, MemoryEnd, RemoveIsTokenRange) ||
         ReadSourceLocation(FM, SM, Memory, MemoryEnd, InsertionLoc) ||
         ReadUnsigned(Memory, MemoryEnd, InsertLen) ||
         Memory + InsertLen > MemoryEnd) {
@@ -1296,7 +1301,8 @@
     }
 
     FixItHint Hint;
-    Hint.RemoveRange = SourceRange(RemoveBegin, RemoveEnd);
+    Hint.RemoveRange = CharSourceRange(SourceRange(RemoveBegin, RemoveEnd),
+                                       RemoveIsTokenRange);
     Hint.InsertionLoc = InsertionLoc;
     Hint.CodeToInsert.assign(Memory, Memory + InsertLen);
     Memory += InsertLen;
diff --git a/lib/Checker/PathDiagnostic.cpp b/lib/Checker/PathDiagnostic.cpp
index 963923c..6e6c749 100644
--- a/lib/Checker/PathDiagnostic.cpp
+++ b/lib/Checker/PathDiagnostic.cpp
@@ -107,7 +107,7 @@
     new PathDiagnosticEventPiece(Info.getLocation(), StrC.str());
 
   for (unsigned i = 0, e = Info.getNumRanges(); i != e; ++i)
-    P->addRange(Info.getRange(i));
+    P->addRange(Info.getRange(i).getAsRange());
   for (unsigned i = 0, e = Info.getNumFixItHints(); i != e; ++i)
     P->addFixItHint(Info.getFixItHint(i));
   D->push_front(P);
diff --git a/lib/Frontend/TextDiagnosticPrinter.cpp b/lib/Frontend/TextDiagnosticPrinter.cpp
index 6ccf4f1..3f1eb82 100644
--- a/lib/Frontend/TextDiagnosticPrinter.cpp
+++ b/lib/Frontend/TextDiagnosticPrinter.cpp
@@ -70,7 +70,7 @@
 
 /// HighlightRange - Given a SourceRange and a line number, highlight (with ~'s)
 /// any characters in LineNo that intersect the SourceRange.
-void TextDiagnosticPrinter::HighlightRange(const SourceRange &R,
+void TextDiagnosticPrinter::HighlightRange(const CharSourceRange &R,
                                            const SourceManager &SM,
                                            unsigned LineNo, FileID FID,
                                            std::string &CaretLine,
@@ -112,8 +112,10 @@
     if (EndColNo) {
       --EndColNo;  // Zero base the col #.
 
-      // Add in the length of the token, so that we cover multi-char tokens.
-      EndColNo += Lexer::MeasureTokenLength(End, SM, *LangOpts);
+      // Add in the length of the token, so that we cover multi-char tokens if
+      // this is a token range.
+      if (R.isTokenRange())
+        EndColNo += Lexer::MeasureTokenLength(End, SM, *LangOpts);
     } else {
       EndColNo = CaretLine.size();
     }
@@ -281,7 +283,7 @@
 }
 
 void TextDiagnosticPrinter::EmitCaretDiagnostic(SourceLocation Loc,
-                                                SourceRange *Ranges,
+                                                CharSourceRange *Ranges,
                                                 unsigned NumRanges,
                                                 const SourceManager &SM,
                                                 const FixItHint *Hints,
@@ -312,10 +314,12 @@
 
     // Map the ranges.
     for (unsigned i = 0; i != NumRanges; ++i) {
-      SourceLocation S = Ranges[i].getBegin(), E = Ranges[i].getEnd();
-      if (S.isMacroID()) S = SM.getImmediateSpellingLoc(S);
-      if (E.isMacroID()) E = SM.getImmediateSpellingLoc(E);
-      Ranges[i] = SourceRange(S, E);
+      CharSourceRange &R = Ranges[i];
+      SourceLocation S = R.getBegin(), E = R.getEnd();
+      if (S.isMacroID())
+        R.setBegin(SM.getImmediateSpellingLoc(S));
+      if (E.isMacroID())
+        R.setEnd(SM.getImmediateSpellingLoc(E));
     }
 
     if (!Suppressed) {
@@ -777,7 +781,9 @@
             continue;
 
           // Add in the length of the token, so that we cover multi-char tokens.
-          unsigned TokSize = Lexer::MeasureTokenLength(E, SM, *LangOpts);
+          unsigned TokSize = 0;
+          if (Info.getRange(i).isTokenRange())
+            TokSize = Lexer::MeasureTokenLength(E, SM, *LangOpts);
 
           OS << '{' << SM.getLineNumber(BInfo.first, BInfo.second) << ':'
              << SM.getColumnNumber(BInfo.first, BInfo.second) << '-'
@@ -904,15 +910,15 @@
     LastCaretDiagnosticWasNote = (Level == Diagnostic::Note);
 
     // Get the ranges into a local array we can hack on.
-    SourceRange Ranges[20];
+    CharSourceRange Ranges[20];
     unsigned NumRanges = Info.getNumRanges();
     assert(NumRanges < 20 && "Out of space");
     for (unsigned i = 0; i != NumRanges; ++i)
       Ranges[i] = Info.getRange(i);
 
     unsigned NumHints = Info.getNumFixItHints();
-    for (unsigned idx = 0; idx < NumHints; ++idx) {
-      const FixItHint &Hint = Info.getFixItHint(idx);
+    for (unsigned i = 0; i != NumHints; ++i) {
+      const FixItHint &Hint = Info.getFixItHint(i);
       if (Hint.RemoveRange.isValid()) {
         assert(NumRanges < 20 && "Out of space");
         Ranges[NumRanges++] = Hint.RemoveRange;
diff --git a/lib/Rewrite/Rewriter.cpp b/lib/Rewrite/Rewriter.cpp
index 376678a..8b0bf91 100644
--- a/lib/Rewrite/Rewriter.cpp
+++ b/lib/Rewrite/Rewriter.cpp
@@ -72,7 +72,7 @@
 
 /// getRangeSize - Return the size in bytes of the specified range if they
 /// are in the same file.  If not, this returns -1.
-int Rewriter::getRangeSize(SourceRange Range) const {
+int Rewriter::getRangeSize(const CharSourceRange &Range) const {
   if (!isRewritable(Range.getBegin()) ||
       !isRewritable(Range.getEnd())) return -1;
 
@@ -97,12 +97,18 @@
 
 
   // Adjust the end offset to the end of the last token, instead of being the
-  // start of the last token.
-  EndOff += Lexer::MeasureTokenLength(Range.getEnd(), *SourceMgr, *LangOpts);
+  // start of the last token if this is a token range.
+  if (Range.isTokenRange())
+    EndOff += Lexer::MeasureTokenLength(Range.getEnd(), *SourceMgr, *LangOpts);
 
   return EndOff-StartOff;
 }
 
+int Rewriter::getRangeSize(SourceRange Range) const {
+  return getRangeSize(CharSourceRange::getTokenRange(Range));
+}
+
+
 /// getRewrittenText - Return the rewritten form of the text in the specified
 /// range.  If the start or end of the range was unrewritable or if they are
 /// in different buffers, this returns an empty string.
diff --git a/tools/libclang/CIndex.cpp b/tools/libclang/CIndex.cpp
index 6410d37..e19cd42 100644
--- a/tools/libclang/CIndex.cpp
+++ b/tools/libclang/CIndex.cpp
@@ -177,12 +177,12 @@
 /// does the appropriate translation.
 CXSourceRange cxloc::translateSourceRange(const SourceManager &SM,
                                           const LangOptions &LangOpts,
-                                          SourceRange R) {
+                                          const CharSourceRange &R) {
   // We want the last character in this location, so we will adjust the
   // location accordingly.
   // FIXME: How do do this with a macro instantiation location?
   SourceLocation EndLoc = R.getEnd();
-  if (!EndLoc.isInvalid() && EndLoc.isFileID()) {
+  if (R.isTokenRange() && !EndLoc.isInvalid() && EndLoc.isFileID()) {
     unsigned Length = Lexer::MeasureTokenLength(EndLoc, SM, LangOpts);
     EndLoc = EndLoc.getFileLocWithOffset(Length);
   }
diff --git a/tools/libclang/CXSourceLocation.h b/tools/libclang/CXSourceLocation.h
index 66566c1..235f78c 100644
--- a/tools/libclang/CXSourceLocation.h
+++ b/tools/libclang/CXSourceLocation.h
@@ -50,14 +50,14 @@
 /// does the appropriate translation.
 CXSourceRange translateSourceRange(const SourceManager &SM, 
                                    const LangOptions &LangOpts,
-                                   SourceRange R);
+                                   const CharSourceRange &R);
   
 /// \brief Translate a Clang source range into a CIndex source range.
 static inline CXSourceRange translateSourceRange(ASTContext &Context,
                                                  SourceRange R) {
   return translateSourceRange(Context.getSourceManager(),
                               Context.getLangOptions(),
-                              R);
+                              CharSourceRange::getTokenRange(R));
 }
 
 static inline SourceLocation translateSourceLocation(CXSourceLocation L) {