temporal | 40ee551 | 2008-07-10 02:12:20 +0000 | [diff] [blame] | 1 | // Protocol Buffers - Google's data interchange format |
kenton@google.com | 24bf56f | 2008-09-24 20:31:01 +0000 | [diff] [blame] | 2 | // Copyright 2008 Google Inc. All rights reserved. |
temporal | 40ee551 | 2008-07-10 02:12:20 +0000 | [diff] [blame] | 3 | // http://code.google.com/p/protobuf/ |
| 4 | // |
kenton@google.com | 24bf56f | 2008-09-24 20:31:01 +0000 | [diff] [blame] | 5 | // Redistribution and use in source and binary forms, with or without |
| 6 | // modification, are permitted provided that the following conditions are |
| 7 | // met: |
temporal | 40ee551 | 2008-07-10 02:12:20 +0000 | [diff] [blame] | 8 | // |
kenton@google.com | 24bf56f | 2008-09-24 20:31:01 +0000 | [diff] [blame] | 9 | // * Redistributions of source code must retain the above copyright |
| 10 | // notice, this list of conditions and the following disclaimer. |
| 11 | // * Redistributions in binary form must reproduce the above |
| 12 | // copyright notice, this list of conditions and the following disclaimer |
| 13 | // in the documentation and/or other materials provided with the |
| 14 | // distribution. |
| 15 | // * Neither the name of Google Inc. nor the names of its |
| 16 | // contributors may be used to endorse or promote products derived from |
| 17 | // this software without specific prior written permission. |
temporal | 40ee551 | 2008-07-10 02:12:20 +0000 | [diff] [blame] | 18 | // |
kenton@google.com | 24bf56f | 2008-09-24 20:31:01 +0000 | [diff] [blame] | 19 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| 20 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| 21 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| 22 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
| 23 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| 24 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
| 25 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| 26 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| 27 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| 28 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| 29 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
temporal | 40ee551 | 2008-07-10 02:12:20 +0000 | [diff] [blame] | 30 | |
| 31 | // Author: kenton@google.com (Kenton Varda) |
| 32 | // Based on original Protocol Buffers design by |
| 33 | // Sanjay Ghemawat, Jeff Dean, and others. |
| 34 | // |
| 35 | // Class for parsing tokenized text from a ZeroCopyInputStream. |
| 36 | |
| 37 | #ifndef GOOGLE_PROTOBUF_IO_TOKENIZER_H__ |
| 38 | #define GOOGLE_PROTOBUF_IO_TOKENIZER_H__ |
| 39 | |
| 40 | #include <string> |
| 41 | #include <google/protobuf/stubs/common.h> |
| 42 | |
| 43 | namespace google { |
| 44 | namespace protobuf { |
| 45 | namespace io { |
| 46 | |
| 47 | class ZeroCopyInputStream; // zero_copy_stream.h |
| 48 | |
| 49 | // Defined in this file. |
| 50 | class ErrorCollector; |
| 51 | class Tokenizer; |
| 52 | |
| 53 | // Abstract interface for an object which collects the errors that occur |
| 54 | // during parsing. A typical implementation might simply print the errors |
| 55 | // to stdout. |
| 56 | class LIBPROTOBUF_EXPORT ErrorCollector { |
| 57 | public: |
| 58 | inline ErrorCollector() {} |
| 59 | virtual ~ErrorCollector(); |
| 60 | |
| 61 | // Indicates that there was an error in the input at the given line and |
| 62 | // column numbers. The numbers are zero-based, so you may want to add |
| 63 | // 1 to each before printing them. |
| 64 | virtual void AddError(int line, int column, const string& message) = 0; |
| 65 | |
kenton@google.com | fccb146 | 2009-12-18 02:11:36 +0000 | [diff] [blame] | 66 | // Indicates that there was a warning in the input at the given line and |
| 67 | // column numbers. The numbers are zero-based, so you may want to add |
| 68 | // 1 to each before printing them. |
| 69 | virtual void AddWarning(int line, int column, const string& message) { } |
| 70 | |
temporal | 40ee551 | 2008-07-10 02:12:20 +0000 | [diff] [blame] | 71 | private: |
| 72 | GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(ErrorCollector); |
| 73 | }; |
| 74 | |
| 75 | // This class converts a stream of raw text into a stream of tokens for |
| 76 | // the protocol definition parser to parse. The tokens recognized are |
| 77 | // similar to those that make up the C language; see the TokenType enum for |
| 78 | // precise descriptions. Whitespace and comments are skipped. By default, |
| 79 | // C- and C++-style comments are recognized, but other styles can be used by |
| 80 | // calling set_comment_style(). |
| 81 | class LIBPROTOBUF_EXPORT Tokenizer { |
| 82 | public: |
| 83 | // Construct a Tokenizer that reads and tokenizes text from the given |
| 84 | // input stream and writes errors to the given error_collector. |
| 85 | // The caller keeps ownership of input and error_collector. |
| 86 | Tokenizer(ZeroCopyInputStream* input, ErrorCollector* error_collector); |
| 87 | ~Tokenizer(); |
| 88 | |
| 89 | enum TokenType { |
| 90 | TYPE_START, // Next() has not yet been called. |
| 91 | TYPE_END, // End of input reached. "text" is empty. |
| 92 | |
| 93 | TYPE_IDENTIFIER, // A sequence of letters, digits, and underscores, not |
| 94 | // starting with a digit. It is an error for a number |
| 95 | // to be followed by an identifier with no space in |
| 96 | // between. |
| 97 | TYPE_INTEGER, // A sequence of digits representing an integer. Normally |
| 98 | // the digits are decimal, but a prefix of "0x" indicates |
| 99 | // a hex number and a leading zero indicates octal, just |
| 100 | // like with C numeric literals. A leading negative sign |
| 101 | // is NOT included in the token; it's up to the parser to |
| 102 | // interpret the unary minus operator on its own. |
| 103 | TYPE_FLOAT, // A floating point literal, with a fractional part and/or |
| 104 | // an exponent. Always in decimal. Again, never |
| 105 | // negative. |
| 106 | TYPE_STRING, // A quoted sequence of escaped characters. Either single |
| 107 | // or double quotes can be used, but they must match. |
| 108 | // A string literal cannot cross a line break. |
| 109 | TYPE_SYMBOL, // Any other printable character, like '!' or '+'. |
| 110 | // Symbols are always a single character, so "!+$%" is |
| 111 | // four tokens. |
| 112 | }; |
| 113 | |
| 114 | // Structure representing a token read from the token stream. |
| 115 | struct Token { |
| 116 | TokenType type; |
| 117 | string text; // The exact text of the token as it appeared in |
| 118 | // the input. e.g. tokens of TYPE_STRING will still |
| 119 | // be escaped and in quotes. |
| 120 | |
| 121 | // "line" and "column" specify the position of the first character of |
| 122 | // the token within the input stream. They are zero-based. |
| 123 | int line; |
| 124 | int column; |
liujisi@google.com | 33165fe | 2010-11-02 13:14:58 +0000 | [diff] [blame^] | 125 | int end_column; |
temporal | 40ee551 | 2008-07-10 02:12:20 +0000 | [diff] [blame] | 126 | }; |
| 127 | |
| 128 | // Get the current token. This is updated when Next() is called. Before |
| 129 | // the first call to Next(), current() has type TYPE_START and no contents. |
| 130 | const Token& current(); |
| 131 | |
liujisi@google.com | 33165fe | 2010-11-02 13:14:58 +0000 | [diff] [blame^] | 132 | // Return the previous token -- i.e. what current() returned before the |
| 133 | // previous call to Next(). |
| 134 | const Token& previous(); |
| 135 | |
temporal | 40ee551 | 2008-07-10 02:12:20 +0000 | [diff] [blame] | 136 | // Advance to the next token. Returns false if the end of the input is |
| 137 | // reached. |
| 138 | bool Next(); |
| 139 | |
| 140 | // Parse helpers --------------------------------------------------- |
| 141 | |
| 142 | // Parses a TYPE_FLOAT token. This never fails, so long as the text actually |
| 143 | // comes from a TYPE_FLOAT token parsed by Tokenizer. If it doesn't, the |
| 144 | // result is undefined (possibly an assert failure). |
| 145 | static double ParseFloat(const string& text); |
| 146 | |
| 147 | // Parses a TYPE_STRING token. This never fails, so long as the text actually |
| 148 | // comes from a TYPE_STRING token parsed by Tokenizer. If it doesn't, the |
| 149 | // result is undefined (possibly an assert failure). |
| 150 | static void ParseString(const string& text, string* output); |
| 151 | |
kenton@google.com | 26bd9ee | 2008-11-21 00:06:27 +0000 | [diff] [blame] | 152 | // Identical to ParseString, but appends to output. |
| 153 | static void ParseStringAppend(const string& text, string* output); |
| 154 | |
temporal | 40ee551 | 2008-07-10 02:12:20 +0000 | [diff] [blame] | 155 | // Parses a TYPE_INTEGER token. Returns false if the result would be |
| 156 | // greater than max_value. Otherwise, returns true and sets *output to the |
| 157 | // result. If the text is not from a Token of type TYPE_INTEGER originally |
| 158 | // parsed by a Tokenizer, the result is undefined (possibly an assert |
| 159 | // failure). |
| 160 | static bool ParseInteger(const string& text, uint64 max_value, |
| 161 | uint64* output); |
| 162 | |
| 163 | // Options --------------------------------------------------------- |
| 164 | |
| 165 | // Set true to allow floats to be suffixed with the letter 'f'. Tokens |
| 166 | // which would otherwise be integers but which have the 'f' suffix will be |
| 167 | // forced to be interpreted as floats. For all other purposes, the 'f' is |
| 168 | // ignored. |
| 169 | void set_allow_f_after_float(bool value) { allow_f_after_float_ = value; } |
| 170 | |
| 171 | // Valid values for set_comment_style(). |
| 172 | enum CommentStyle { |
| 173 | // Line comments begin with "//", block comments are delimited by "/*" and |
| 174 | // "*/". |
| 175 | CPP_COMMENT_STYLE, |
| 176 | // Line comments begin with "#". No way to write block comments. |
| 177 | SH_COMMENT_STYLE |
| 178 | }; |
| 179 | |
| 180 | // Sets the comment style. |
| 181 | void set_comment_style(CommentStyle style) { comment_style_ = style; } |
| 182 | |
| 183 | // ----------------------------------------------------------------- |
| 184 | private: |
| 185 | GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(Tokenizer); |
| 186 | |
| 187 | Token current_; // Returned by current(). |
liujisi@google.com | 33165fe | 2010-11-02 13:14:58 +0000 | [diff] [blame^] | 188 | Token previous_; // Returned by previous(). |
temporal | 40ee551 | 2008-07-10 02:12:20 +0000 | [diff] [blame] | 189 | |
| 190 | ZeroCopyInputStream* input_; |
| 191 | ErrorCollector* error_collector_; |
| 192 | |
| 193 | char current_char_; // == buffer_[buffer_pos_], updated by NextChar(). |
| 194 | const char* buffer_; // Current buffer returned from input_. |
| 195 | int buffer_size_; // Size of buffer_. |
| 196 | int buffer_pos_; // Current position within the buffer. |
| 197 | bool read_error_; // Did we previously encounter a read error? |
| 198 | |
| 199 | // Line and column number of current_char_ within the whole input stream. |
| 200 | int line_; |
| 201 | int column_; |
| 202 | |
| 203 | // Position in buffer_ where StartToken() was called. If the token |
| 204 | // started in the previous buffer, this is zero, and current_.text already |
| 205 | // contains the part of the token from the previous buffer. If not |
| 206 | // currently parsing a token, this is -1. |
| 207 | int token_start_; |
| 208 | |
| 209 | // Options. |
| 210 | bool allow_f_after_float_; |
| 211 | CommentStyle comment_style_; |
| 212 | |
| 213 | // Since we count columns we need to interpret tabs somehow. We'll take |
| 214 | // the standard 8-character definition for lack of any way to do better. |
| 215 | static const int kTabWidth = 8; |
| 216 | |
| 217 | // ----------------------------------------------------------------- |
| 218 | // Helper methods. |
| 219 | |
| 220 | // Consume this character and advance to the next one. |
| 221 | void NextChar(); |
| 222 | |
| 223 | // Read a new buffer from the input. |
| 224 | void Refresh(); |
| 225 | |
| 226 | // Called when the current character is the first character of a new |
| 227 | // token (not including whitespace or comments). |
| 228 | inline void StartToken(); |
| 229 | // Called when the current character is the first character after the |
| 230 | // end of the last token. After this returns, current_.text will |
| 231 | // contain all text consumed since StartToken() was called. |
| 232 | inline void EndToken(); |
| 233 | |
| 234 | // Convenience method to add an error at the current line and column. |
| 235 | void AddError(const string& message) { |
| 236 | error_collector_->AddError(line_, column_, message); |
| 237 | } |
| 238 | |
| 239 | // ----------------------------------------------------------------- |
| 240 | // The following four methods are used to consume tokens of specific |
| 241 | // types. They are actually used to consume all characters *after* |
| 242 | // the first, since the calling function consumes the first character |
| 243 | // in order to decide what kind of token is being read. |
| 244 | |
| 245 | // Read and consume a string, ending when the given delimiter is |
| 246 | // consumed. |
| 247 | void ConsumeString(char delimiter); |
| 248 | |
| 249 | // Read and consume a number, returning TYPE_FLOAT or TYPE_INTEGER |
| 250 | // depending on what was read. This needs to know if the first |
| 251 | // character was a zero in order to correctly recognize hex and octal |
| 252 | // numbers. |
| 253 | // It also needs to know if the first characted was a . to parse floating |
| 254 | // point correctly. |
| 255 | TokenType ConsumeNumber(bool started_with_zero, bool started_with_dot); |
| 256 | |
| 257 | // Consume the rest of a line. |
| 258 | void ConsumeLineComment(); |
| 259 | // Consume until "*/". |
| 260 | void ConsumeBlockComment(); |
| 261 | |
| 262 | // ----------------------------------------------------------------- |
| 263 | // These helper methods make the parsing code more readable. The |
| 264 | // "character classes" refered to are defined at the top of the .cc file. |
| 265 | // Basically it is a C++ class with one method: |
| 266 | // static bool InClass(char c); |
| 267 | // The method returns true if c is a member of this "class", like "Letter" |
| 268 | // or "Digit". |
| 269 | |
| 270 | // Returns true if the current character is of the given character |
| 271 | // class, but does not consume anything. |
| 272 | template<typename CharacterClass> |
| 273 | inline bool LookingAt(); |
| 274 | |
| 275 | // If the current character is in the given class, consume it and return |
| 276 | // true. Otherwise return false. |
| 277 | // e.g. TryConsumeOne<Letter>() |
| 278 | template<typename CharacterClass> |
| 279 | inline bool TryConsumeOne(); |
| 280 | |
| 281 | // Like above, but try to consume the specific character indicated. |
| 282 | inline bool TryConsume(char c); |
| 283 | |
| 284 | // Consume zero or more of the given character class. |
| 285 | template<typename CharacterClass> |
| 286 | inline void ConsumeZeroOrMore(); |
| 287 | |
| 288 | // Consume one or more of the given character class or log the given |
| 289 | // error message. |
| 290 | // e.g. ConsumeOneOrMore<Digit>("Expected digits."); |
| 291 | template<typename CharacterClass> |
| 292 | inline void ConsumeOneOrMore(const char* error); |
| 293 | }; |
| 294 | |
| 295 | // inline methods ==================================================== |
| 296 | inline const Tokenizer::Token& Tokenizer::current() { |
| 297 | return current_; |
| 298 | } |
| 299 | |
liujisi@google.com | 33165fe | 2010-11-02 13:14:58 +0000 | [diff] [blame^] | 300 | inline const Tokenizer::Token& Tokenizer::previous() { |
| 301 | return previous_; |
| 302 | } |
| 303 | |
kenton@google.com | 26bd9ee | 2008-11-21 00:06:27 +0000 | [diff] [blame] | 304 | inline void Tokenizer::ParseString(const string& text, string* output) { |
| 305 | output->clear(); |
| 306 | ParseStringAppend(text, output); |
| 307 | } |
| 308 | |
temporal | 40ee551 | 2008-07-10 02:12:20 +0000 | [diff] [blame] | 309 | } // namespace io |
| 310 | } // namespace protobuf |
| 311 | |
| 312 | } // namespace google |
| 313 | #endif // GOOGLE_PROTOBUF_IO_TOKENIZER_H__ |