src/scanner.cc

// Copyright 2010 the V8 project authors. All rights reserved.
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
//     * Redistributions of source code must retain the above copyright
//       notice, this list of conditions and the following disclaimer.
//     * Redistributions in binary form must reproduce the above
//       copyright notice, this list of conditions and the following
//       disclaimer in the documentation and/or other materials provided
//       with the distribution.
//     * Neither the name of Google Inc. nor the names of its
//       contributors may be used to endorse or promote products derived
//       from this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

#include "v8.h"

#include "ast.h"
#include "handles.h"
#include "scanner.h"
#include "unicode-inl.h"

namespace v8 {
namespace internal {

// ----------------------------------------------------------------------------
// UTF16Buffer

// CharacterStreamUTF16Buffer
CharacterStreamUTF16Buffer::CharacterStreamUTF16Buffer()
    : pushback_buffer_(0), last_(0), stream_(NULL) { }


void CharacterStreamUTF16Buffer::Initialize(Handle<String> data,
                                            unibrow::CharacterStream* input,
                                            int start_position,
                                            int end_position) {
  stream_ = input;
  if (start_position > 0) {
    SeekForward(start_position);
  }
  end_ = end_position != kNoEndPosition ? end_position : kMaxInt;
}


void CharacterStreamUTF16Buffer::PushBack(uc32 ch) {
  pushback_buffer()->Add(last_);
  last_ = ch;
  pos_--;
}


uc32 CharacterStreamUTF16Buffer::Advance() {
  ASSERT(end_ != kNoEndPosition);
  ASSERT(end_ >= 0);
  // NOTE: It is of importance to Persian / Farsi resources that we do
  // *not* strip format control characters in the scanner; see
  //
  //    https://bugzilla.mozilla.org/show_bug.cgi?id=274152
  //
  // So, even though ECMA-262, section 7.1, page 11, dictates that we
  // must remove Unicode format-control characters, we do not. This is
  // in line with how IE and SpiderMonkey handles it.
  if (!pushback_buffer()->is_empty()) {
    pos_++;
    return last_ = pushback_buffer()->RemoveLast();
  } else if (stream_->has_more() && pos_ < end_) {
    pos_++;
    uc32 next = stream_->GetNext();
    return last_ = next;
  } else {
    // Note: currently the following increment is necessary to avoid a
    // test-parser problem!
    pos_++;
    return last_ = static_cast<uc32>(-1);
  }
}


void CharacterStreamUTF16Buffer::SeekForward(int pos) {
  pos_ = pos;
  ASSERT(pushback_buffer()->is_empty());
  stream_->Seek(pos);
}


// ----------------------------------------------------------------------------
// Scanner::LiteralScope

Scanner::LiteralScope::LiteralScope(Scanner* self)
    : scanner_(self), complete_(false) {
  self->StartLiteral();
}


Scanner::LiteralScope::~LiteralScope() {
  if (!complete_) scanner_->DropLiteral();
}


void Scanner::LiteralScope::Complete() {
  scanner_->TerminateLiteral();
  complete_ = true;
}

// ----------------------------------------------------------------------------
// V8JavaScriptScanner

void V8JavaScriptScanner::Initialize(Handle<String> source,
                                     int literal_flags) {
  source_ = stream_initializer_.Init(source, NULL, 0, source->length());
  // Need to capture identifiers in order to recognize "get" and "set"
  // in object literals.
  literal_flags_ = literal_flags | kLiteralIdentifier;
  Init();
  // Skip initial whitespace allowing HTML comment ends just like
  // after a newline and scan first token.
  has_line_terminator_before_next_ = true;
  SkipWhiteSpace();
  Scan();
}


void V8JavaScriptScanner::Initialize(Handle<String> source,
                                     unibrow::CharacterStream* stream,
                                     int literal_flags) {
  source_ = stream_initializer_.Init(source, stream,
                                     0, UTF16Buffer::kNoEndPosition);
  literal_flags_ = literal_flags | kLiteralIdentifier;
  Init();
  // Skip initial whitespace allowing HTML comment ends just like
  // after a newline and scan first token.
  has_line_terminator_before_next_ = true;
  SkipWhiteSpace();
  Scan();
}


void V8JavaScriptScanner::Initialize(Handle<String> source,
                                     int start_position,
                                     int end_position,
                                     int literal_flags) {
  source_ = stream_initializer_.Init(source, NULL,
                                     start_position, end_position);
  literal_flags_ = literal_flags | kLiteralIdentifier;
  Init();
  // Skip initial whitespace allowing HTML comment ends just like
  // after a newline and scan first token.
  has_line_terminator_before_next_ = true;
  SkipWhiteSpace();
  Scan();
}


Token::Value V8JavaScriptScanner::NextCheckStack() {
  // BUG 1215673: Find a thread safe way to set a stack limit in
  // pre-parse mode. Otherwise, we cannot safely pre-parse from other
  // threads.
  StackLimitCheck check;
  if (check.HasOverflowed()) {
    stack_overflow_ = true;
    current_ = next_;
    next_.token = Token::ILLEGAL;
    return current_.token;
  } else {
    return Next();
  }
}


UTF16Buffer* StreamInitializer::Init(Handle<String> source,
                                     unibrow::CharacterStream* stream,
                                     int start_position,
                                     int end_position) {
  // Either initialize the scanner from a character stream or from a
  // string.
  ASSERT(source.is_null() || stream == NULL);

  // Initialize the source buffer.
  if (!source.is_null() && StringShape(*source).IsExternalTwoByte()) {
    two_byte_string_buffer_.Initialize(
        Handle<ExternalTwoByteString>::cast(source),
        start_position,
        end_position);
    return &two_byte_string_buffer_;
  } else if (!source.is_null() && StringShape(*source).IsExternalAscii()) {
    ascii_string_buffer_.Initialize(
        Handle<ExternalAsciiString>::cast(source),
        start_position,
        end_position);
    return &ascii_string_buffer_;
  } else {
    if (!source.is_null()) {
      safe_string_input_buffer_.Reset(source.location());
      stream = &safe_string_input_buffer_;
    }
    char_stream_buffer_.Initialize(source,
                                   stream,
                                   start_position,
                                   end_position);
    return &char_stream_buffer_;
  }
}

// ----------------------------------------------------------------------------
// JsonScanner

JsonScanner::JsonScanner() {}


void JsonScanner::Initialize(Handle<String> source) {
  source_ = stream_initializer_.Init(source, NULL, 0, source->length());
  Init();
  // Skip initial whitespace.
  SkipJsonWhiteSpace();
  // Preload first token as look-ahead.
  ScanJson();
}


Token::Value JsonScanner::Next() {
  // BUG 1215673: Find a thread safe way to set a stack limit in
  // pre-parse mode. Otherwise, we cannot safely pre-parse from other
  // threads.
  current_ = next_;
  // Check for stack-overflow before returning any tokens.
  StackLimitCheck check;
  if (check.HasOverflowed()) {
    stack_overflow_ = true;
    next_.token = Token::ILLEGAL;
  } else {
    ScanJson();
  }
  return current_.token;
}


bool JsonScanner::SkipJsonWhiteSpace() {
  int start_position = source_pos();
  // JSON WhiteSpace is tab, carrige-return, newline and space.
  while (c0_ == ' ' || c0_ == '\n' || c0_ == '\r' || c0_ == '\t') {
    Advance();
  }
  return source_pos() != start_position;
}


void JsonScanner::ScanJson() {
  next_.literal_chars = Vector<const char>();
  Token::Value token;
  do {
    // Remember the position of the next token
    next_.location.beg_pos = source_pos();
    switch (c0_) {
      case '\t':
      case '\r':
      case '\n':
      case ' ':
        Advance();
        token = Token::WHITESPACE;
        break;
      case '{':
        Advance();
        token = Token::LBRACE;
        break;
      case '}':
        Advance();
        token = Token::RBRACE;
        break;
      case '[':
        Advance();
        token = Token::LBRACK;
        break;
      case ']':
        Advance();
        token = Token::RBRACK;
        break;
      case ':':
        Advance();
        token = Token::COLON;
        break;
      case ',':
        Advance();
        token = Token::COMMA;
        break;
      case '"':
        token = ScanJsonString();
        break;
      case '-':
      case '0':
      case '1':
      case '2':
      case '3':
      case '4':
      case '5':
      case '6':
      case '7':
      case '8':
      case '9':
        token = ScanJsonNumber();
        break;
      case 't':
        token = ScanJsonIdentifier("true", Token::TRUE_LITERAL);
        break;
      case 'f':
        token = ScanJsonIdentifier("false", Token::FALSE_LITERAL);
        break;
      case 'n':
        token = ScanJsonIdentifier("null", Token::NULL_LITERAL);
        break;
      default:
        if (c0_ < 0) {
          Advance();
          token = Token::EOS;
        } else {
          Advance();
          token = Select(Token::ILLEGAL);
        }
    }
  } while (token == Token::WHITESPACE);

  next_.location.end_pos = source_pos();
  next_.token = token;
}


Token::Value JsonScanner::ScanJsonString() {
  ASSERT_EQ('"', c0_);
  Advance();
  LiteralScope literal(this);
  while (c0_ != '"' && c0_ > 0) {
    // Check for control character (0x00-0x1f) or unterminated string (<0).
    if (c0_ < 0x20) return Token::ILLEGAL;
    if (c0_ != '\\') {
      AddLiteralCharAdvance();
    } else {
      Advance();
      switch (c0_) {
        case '"':
        case '\\':
        case '/':
          AddLiteralChar(c0_);
          break;
        case 'b':
          AddLiteralChar('\x08');
          break;
        case 'f':
          AddLiteralChar('\x0c');
          break;
        case 'n':
          AddLiteralChar('\x0a');
          break;
        case 'r':
          AddLiteralChar('\x0d');
          break;
        case 't':
          AddLiteralChar('\x09');
          break;
        case 'u': {
          uc32 value = 0;
          for (int i = 0; i < 4; i++) {
            Advance();
            int digit = HexValue(c0_);
            if (digit < 0) {
              return Token::ILLEGAL;
            }
            value = value * 16 + digit;
          }
          AddLiteralChar(value);
          break;
        }
        default:
          return Token::ILLEGAL;
      }
      Advance();
    }
  }
  if (c0_ != '"') {
    return Token::ILLEGAL;
  }
  literal.Complete();
  Advance();
  return Token::STRING;
}


Token::Value JsonScanner::ScanJsonNumber() {
  LiteralScope literal(this);
  if (c0_ == '-') AddLiteralCharAdvance();
  if (c0_ == '0') {
    AddLiteralCharAdvance();
    // Prefix zero is only allowed if it's the only digit before
    // a decimal point or exponent.
    if ('0' <= c0_ && c0_ <= '9') return Token::ILLEGAL;
  } else {
    if (c0_ < '1' || c0_ > '9') return Token::ILLEGAL;
    do {
      AddLiteralCharAdvance();
    } while (c0_ >= '0' && c0_ <= '9');
  }
  if (c0_ == '.') {
    AddLiteralCharAdvance();
    if (c0_ < '0' || c0_ > '9') return Token::ILLEGAL;
    do {
      AddLiteralCharAdvance();
    } while (c0_ >= '0' && c0_ <= '9');
  }
  if (AsciiAlphaToLower(c0_) == 'e') {
    AddLiteralCharAdvance();
    if (c0_ == '-' || c0_ == '+') AddLiteralCharAdvance();
    if (c0_ < '0' || c0_ > '9') return Token::ILLEGAL;
    do {
      AddLiteralCharAdvance();
    } while (c0_ >= '0' && c0_ <= '9');
  }
  literal.Complete();
  return Token::NUMBER;
}


Token::Value JsonScanner::ScanJsonIdentifier(const char* text,
                                             Token::Value token) {
  LiteralScope literal(this);
  while (*text != '\0') {
    if (c0_ != *text) return Token::ILLEGAL;
    Advance();
    text++;
  }
  if (ScannerConstants::kIsIdentifierPart.get(c0_)) return Token::ILLEGAL;
  literal.Complete();
  return token;
}



} }  // namespace v8::internal