src/ProtocolBuffers/TextTokenizer.cs

// Protocol Buffers - Google's data interchange format
// Copyright 2008 Google Inc.  All rights reserved.
// http://github.com/jskeet/dotnet-protobufs/
// Original C++/Java/Python code:
// http://code.google.com/p/protobuf/
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
//     * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
//     * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
//     * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
using System;
using System.Globalization;
using System.Text.RegularExpressions;

namespace Google.ProtocolBuffers {
  /// <summary>
  /// Represents a stream of tokens parsed from a string.
  /// </summary>
  internal sealed class TextTokenizer {
    private readonly string text;
    private string currentToken;

    /// <summary>
    /// The character index within the text to perform the next regex match at.
    /// </summary>
    private int matchPos = 0;

    /// <summary>
    /// The character index within the text at which the current token begins.
    /// </summary>
    private int pos = 0;

    /// <summary>
    /// The line number of the current token.
    /// </summary>
    private int line = 0;
    /// <summary>
    /// The column number of the current token.
    /// </summary>
    private int column = 0;

    /// <summary>
    /// The line number of the previous token.
    /// </summary>
    private int previousLine = 0;
    /// <summary>
    /// The column number of the previous token.
    /// </summary>
    private int previousColumn = 0;

#if SILVERLIGHT
    private const RegexOptions CompiledRegexWhereAvailable = RegexOptions.None;
#else
    private const RegexOptions CompiledRegexWhereAvailable = RegexOptions.Compiled;
#endif

    // Note: atomic groups used to mimic possessive quantifiers in Java in both of these regexes
    private static readonly Regex WhitespaceAndCommentPattern = new Regex("\\G(?>(\\s|(#.*$))+)", 
        CompiledRegexWhereAvailable | RegexOptions.Multiline);
    private static readonly Regex TokenPattern = new Regex(
      "\\G[a-zA-Z_](?>[0-9a-zA-Z_+-]*)|" +              // an identifier
      "\\G[0-9+-](?>[0-9a-zA-Z_.+-]*)|" +                  // a number
      "\\G\"(?>([^\"\\\n\\\\]|\\\\.)*)(\"|\\\\?$)|" +    // a double-quoted string
      "\\G\'(?>([^\"\\\n\\\\]|\\\\.)*)(\'|\\\\?$)",      // a single-quoted string
      RegexOptions.Compiled | RegexOptions.Multiline);

    private static readonly Regex DoubleInfinity = new Regex("^-?inf(inity)?$", CompiledRegexWhereAvailable | RegexOptions.IgnoreCase);
    private static readonly Regex FloatInfinity = new Regex("^-?inf(inity)?f?$", CompiledRegexWhereAvailable | RegexOptions.IgnoreCase);
    private static readonly Regex FloatNan = new Regex("^nanf?$", CompiledRegexWhereAvailable | RegexOptions.IgnoreCase);

    /** Construct a tokenizer that parses tokens from the given text. */
    public TextTokenizer(string text) {
      this.text = text;
      SkipWhitespace();
      NextToken();
    }

    /// <summary>
    /// Are we at the end of the input?
    /// </summary>
    public bool AtEnd {
      get { return currentToken.Length == 0; }
    }

    /// <summary>
    /// Advances to the next token.
    /// </summary>
    public void NextToken() {
      previousLine = line;
      previousColumn = column;

      // Advance the line counter to the current position.
      while (pos < matchPos) {
        if (text[pos] == '\n') {
          ++line;
          column = 0;
        } else {
          ++column;
        }
        ++pos;
      }

      // Match the next token.
      if (matchPos == text.Length) {
        // EOF
        currentToken = "";
      } else {
        Match match = TokenPattern.Match(text, matchPos);
        if (match.Success) {
          currentToken = match.Value;
          matchPos += match.Length;
        } else {
          // Take one character.
          currentToken = text[matchPos].ToString();
          matchPos++;
        }

        SkipWhitespace();
      }
    }

    /// <summary>
    /// Skip over any whitespace so that matchPos starts at the next token.
    /// </summary>
    private void SkipWhitespace() {
      Match match = WhitespaceAndCommentPattern.Match(text, matchPos);
      if (match.Success) {
        matchPos += match.Length;
      }
    }

    /// <summary>
    /// If the next token exactly matches the given token, consume it and return
    /// true. Otherwise, return false without doing anything.
    /// </summary>
    public bool TryConsume(string token) {
      if (currentToken == token) {
        NextToken();
        return true;
      }
      return false;
    }

    /*
     * If the next token exactly matches {@code token}, consume it.  Otherwise,
     * throw a {@link ParseException}.
     */
    /// <summary>
    /// If the next token exactly matches the specified one, consume it.
    /// Otherwise, throw a FormatException.
    /// </summary>
    /// <param name="token"></param>
    public void Consume(string token) {
      if (!TryConsume(token)) {
        throw CreateFormatException("Expected \"" + token + "\".");
      }
    }

    /// <summary>
    /// Returns true if the next token is an integer, but does not consume it.
    /// </summary>
    public bool LookingAtInteger() {
      if (currentToken.Length == 0) {
        return false;
      }

      char c = currentToken[0];
      return ('0' <= c && c <= '9') || c == '-' || c == '+';
    }

    /// <summary>
    /// If the next token is an identifier, consume it and return its value.
    /// Otherwise, throw a FormatException.
    /// </summary>
    public string ConsumeIdentifier() {
      foreach (char c in currentToken) {
        if (('a' <= c && c <= 'z') ||
            ('A' <= c && c <= 'Z') ||
            ('0' <= c && c <= '9') ||
            (c == '_') || (c == '.')) {
          // OK
        } else {
          throw CreateFormatException("Expected identifier.");
        }
      }

      string result = currentToken;
      NextToken();
      return result;
    }

    /// <summary>
    /// If the next token is a 32-bit signed integer, consume it and return its 
    /// value. Otherwise, throw a FormatException.
    /// </summary>
    public int ConsumeInt32()  {
      try {
        int result = TextFormat.ParseInt32(currentToken);
        NextToken();
        return result;
      } catch (FormatException e) {
        throw CreateIntegerParseException(e);
      }
    }

    /// <summary>
    /// If the next token is a 32-bit unsigned integer, consume it and return its
    /// value. Otherwise, throw a FormatException.
    /// </summary>
    public uint ConsumeUInt32() {
      try {
        uint result = TextFormat.ParseUInt32(currentToken);
        NextToken();
        return result;
      } catch (FormatException e) {
        throw CreateIntegerParseException(e);
      }
    }

    /// <summary>
    /// If the next token is a 64-bit signed integer, consume it and return its
    /// value. Otherwise, throw a FormatException.
    /// </summary>
    public long ConsumeInt64() {
      try {
        long result = TextFormat.ParseInt64(currentToken);
        NextToken();
        return result;
      } catch (FormatException e) {
        throw CreateIntegerParseException(e);
      }
    }

    /// <summary>
    /// If the next token is a 64-bit unsigned integer, consume it and return its
    /// value. Otherwise, throw a FormatException.
    /// </summary>
    public ulong ConsumeUInt64() {
      try {
        ulong result = TextFormat.ParseUInt64(currentToken);
        NextToken();
        return result;
      } catch (FormatException e) {
        throw CreateIntegerParseException(e);
      }
    }

    /// <summary>
    /// If the next token is a double, consume it and return its value.
    /// Otherwise, throw a FormatException.
    /// </summary>
    public double ConsumeDouble() {
      // We need to parse infinity and nan separately because
      // double.Parse() does not accept "inf", "infinity", or "nan".
      if (DoubleInfinity.IsMatch(currentToken)) {
        bool negative = currentToken.StartsWith("-");
        NextToken();
        return negative ? double.NegativeInfinity : double.PositiveInfinity;
      }
      if (currentToken.Equals("nan", StringComparison.InvariantCultureIgnoreCase)) {
        NextToken();
        return Double.NaN;
      }

      try {
        double result = double.Parse(currentToken, CultureInfo.InvariantCulture);
        NextToken();
        return result;
      } catch (FormatException e) {
        throw CreateFloatParseException(e);
      } catch (OverflowException e) {
        throw CreateFloatParseException(e);
      }
    }

    /// <summary>
    /// If the next token is a float, consume it and return its value.
    /// Otherwise, throw a FormatException.
    /// </summary>
    public float ConsumeFloat() {
      // We need to parse infinity and nan separately because
      // Float.parseFloat() does not accept "inf", "infinity", or "nan".
      if (FloatInfinity.IsMatch(currentToken)) {
        bool negative = currentToken.StartsWith("-");
        NextToken();
        return negative ? float.NegativeInfinity : float.PositiveInfinity;
      }
      if (FloatNan.IsMatch(currentToken)) {
        NextToken();
        return float.NaN;
      }

      if (currentToken.EndsWith("f")) {
        currentToken = currentToken.TrimEnd('f');
      }

      try {
        float result = float.Parse(currentToken, CultureInfo.InvariantCulture);
        NextToken();
        return result;
      } catch (FormatException e) {
        throw CreateFloatParseException(e);
      } catch (OverflowException e) {
        throw CreateFloatParseException(e);
      }
    }

    /// <summary>
    /// If the next token is a Boolean, consume it and return its value.
    /// Otherwise, throw a FormatException.    
    /// </summary>
    public bool ConsumeBoolean() {
      if (currentToken == "true") {
        NextToken();
        return true;
      } 
      if (currentToken == "false") {
        NextToken();
        return false;
      }
      throw CreateFormatException("Expected \"true\" or \"false\".");
    }

    /// <summary>
    /// If the next token is a string, consume it and return its (unescaped) value.
    /// Otherwise, throw a FormatException.
    /// </summary>
    public string ConsumeString() {
      return ConsumeByteString().ToStringUtf8();
    }

    /// <summary>
    /// If the next token is a string, consume it, unescape it as a
    /// ByteString and return it. Otherwise, throw a FormatException.
    /// </summary>
    public ByteString ConsumeByteString() {
      char quote = currentToken.Length > 0 ? currentToken[0] : '\0';
      if (quote != '\"' && quote != '\'') {
        throw CreateFormatException("Expected string.");
      }

      if (currentToken.Length < 2 ||
          currentToken[currentToken.Length-1] != quote) {
        throw CreateFormatException("String missing ending quote.");
      }

      try {
        string escaped = currentToken.Substring(1, currentToken.Length - 2);
        ByteString result = TextFormat.UnescapeBytes(escaped);
        NextToken();
        return result;
      } catch (FormatException e) {
        throw CreateFormatException(e.Message);
      }
    }

    /// <summary>
    /// Returns a format exception with the current line and column numbers
    /// in the description, suitable for throwing.
    /// </summary>
    public FormatException CreateFormatException(string description) {
      // Note:  People generally prefer one-based line and column numbers.
      return new FormatException((line + 1) + ":" + (column + 1) + ": " + description);
    }

    /// <summary>
    /// Returns a format exception with the line and column numbers of the
    /// previous token in the description, suitable for throwing.
    /// </summary>
    public FormatException CreateFormatExceptionPreviousToken(string description) {
      // Note:  People generally prefer one-based line and column numbers.
      return new FormatException((previousLine + 1) + ":" + (previousColumn + 1) + ": " + description);
    }

    /// <summary>
    /// Constructs an appropriate FormatException for the given existing exception
    /// when trying to parse an integer.
    /// </summary>
    private FormatException CreateIntegerParseException(FormatException e) {
      return CreateFormatException("Couldn't parse integer: " + e.Message);
    }

    /// <summary>
    /// Constructs an appropriate FormatException for the given existing exception
    /// when trying to parse a float or double.
    /// </summary>
    private FormatException CreateFloatParseException(Exception e) {
      return CreateFormatException("Couldn't parse number: " + e.Message);
    }
  }
}