init/parser/tokenizer_test.cpp

// Copyright (C) 2015 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "tokenizer.h"

#include <errno.h>
#include <gtest/gtest.h>

#include <string>

namespace init {

#define SETUP_TEST(test_data)  \
  std::string data(test_data); \
  Tokenizer tokenizer(data);   \
  ASSERT_EQ(Tokenizer::TOK_START, tokenizer.current().type)

#define ASSERT_TEXT_TOKEN(test_text)              \
  ASSERT_TRUE(tokenizer.Next());                  \
  ASSERT_EQ(test_text, tokenizer.current().text); \
  ASSERT_EQ(Tokenizer::TOK_TEXT, tokenizer.current().type)

#define ASSERT_NEWLINE_TOKEN()   \
  ASSERT_TRUE(tokenizer.Next()); \
  ASSERT_EQ(Tokenizer::TOK_NEWLINE, tokenizer.current().type)

TEST(Tokenizer, Empty) {
  SETUP_TEST("");
  ASSERT_FALSE(tokenizer.Next());
}

TEST(Tokenizer, Simple) {
  SETUP_TEST("test");
  ASSERT_TEXT_TOKEN("test");
  ASSERT_FALSE(tokenizer.Next());
}

TEST(Tokenizer, LeadingWhiteSpace) {
  SETUP_TEST(" \t  \r  test");
  ASSERT_TEXT_TOKEN("test");
  ASSERT_FALSE(tokenizer.Next());
}

TEST(Tokenizer, TrailingWhiteSpace) {
  SETUP_TEST("test \t  \r  ");
  ASSERT_TEXT_TOKEN("test");
  ASSERT_FALSE(tokenizer.Next());
}

TEST(Tokenizer, WhiteSpace) {
  SETUP_TEST(" \t  \r  test \t  \r  ");
  ASSERT_TEXT_TOKEN("test");

  ASSERT_FALSE(tokenizer.Next());
}

TEST(Tokenizer, TwoTokens) {
  SETUP_TEST("  foo   bar ");
  ASSERT_TEXT_TOKEN("foo");
  ASSERT_TEXT_TOKEN("bar");

  ASSERT_FALSE(tokenizer.Next());
}

TEST(Tokenizer, MultiToken) {
  SETUP_TEST("one two three four five");
  ASSERT_TEXT_TOKEN("one");
  ASSERT_TEXT_TOKEN("two");
  ASSERT_TEXT_TOKEN("three");
  ASSERT_TEXT_TOKEN("four");
  ASSERT_TEXT_TOKEN("five");

  ASSERT_FALSE(tokenizer.Next());
}

TEST(Tokenizer, NewLine) {
  SETUP_TEST("\n");
  ASSERT_NEWLINE_TOKEN();

  ASSERT_FALSE(tokenizer.Next());
}

TEST(Tokenizer, TextNewLine) {
  SETUP_TEST("test\n");
  ASSERT_TEXT_TOKEN("test");
  ASSERT_NEWLINE_TOKEN();

  ASSERT_FALSE(tokenizer.Next());
}

TEST(Tokenizer, MultiTextNewLine) {
  SETUP_TEST("one\ntwo\nthree\n");
  ASSERT_TEXT_TOKEN("one");
  ASSERT_NEWLINE_TOKEN();
  ASSERT_TEXT_TOKEN("two");
  ASSERT_NEWLINE_TOKEN();
  ASSERT_TEXT_TOKEN("three");
  ASSERT_NEWLINE_TOKEN();

  ASSERT_FALSE(tokenizer.Next());
}

TEST(Tokenizer, MultiTextNewLineNoLineEnding) {
  SETUP_TEST("one\ntwo\nthree");
  ASSERT_TEXT_TOKEN("one");
  ASSERT_NEWLINE_TOKEN();
  ASSERT_TEXT_TOKEN("two");
  ASSERT_NEWLINE_TOKEN();
  ASSERT_TEXT_TOKEN("three");

  ASSERT_FALSE(tokenizer.Next());
}

TEST(Tokenizer, Comment) {
  SETUP_TEST("#test");
  ASSERT_FALSE(tokenizer.Next());
}

TEST(Tokenizer, CommentWhiteSpace) {
  SETUP_TEST(" \t  \r  #test \t  \r  ");
  ASSERT_FALSE(tokenizer.Next());
}

TEST(Tokenizer, CommentNewLine) {
  SETUP_TEST(" #test   \n");
  ASSERT_NEWLINE_TOKEN();
  ASSERT_FALSE(tokenizer.Next());
}

TEST(Tokenizer, CommentTwoNewLine) {
  SETUP_TEST(" #test   \n#test");
  ASSERT_NEWLINE_TOKEN();
  ASSERT_FALSE(tokenizer.Next());
}

TEST(Tokenizer, CommentWithText) {
  SETUP_TEST("foo bar #test");
  ASSERT_TEXT_TOKEN("foo");
  ASSERT_TEXT_TOKEN("bar");
  ASSERT_FALSE(tokenizer.Next());
}

TEST(Tokenizer, CommentWithTextNoSpace) {
  SETUP_TEST("foo bar#test");
  ASSERT_TEXT_TOKEN("foo");
  ASSERT_TEXT_TOKEN("bar");
  ASSERT_FALSE(tokenizer.Next());
}

TEST(Tokenizer, CommentWithTextLineFeed) {
  SETUP_TEST("foo bar #test\n");
  ASSERT_TEXT_TOKEN("foo");
  ASSERT_TEXT_TOKEN("bar");
  ASSERT_NEWLINE_TOKEN();
  ASSERT_FALSE(tokenizer.Next());
}

TEST(Tokenizer, CommentWithMultiTextLineFeed) {
  SETUP_TEST("#blah\nfoo bar #test\n#blah");
  ASSERT_NEWLINE_TOKEN();
  ASSERT_TEXT_TOKEN("foo");
  ASSERT_TEXT_TOKEN("bar");
  ASSERT_NEWLINE_TOKEN();
  ASSERT_FALSE(tokenizer.Next());
}

TEST(Tokenizer, SimpleEscaped) {
  SETUP_TEST("fo\\no bar");
  ASSERT_TEXT_TOKEN("fo\\no");
  ASSERT_TEXT_TOKEN("bar");
  ASSERT_FALSE(tokenizer.Next());
}

TEST(Tokenizer, EscapedLineContNoLineFeed) {
  SETUP_TEST("fo\\no bar \\ hello");
  ASSERT_TEXT_TOKEN("fo\\no");
  ASSERT_TEXT_TOKEN("bar");
  ASSERT_FALSE(tokenizer.Next());
}

TEST(Tokenizer, EscapedLineContLineFeed) {
  SETUP_TEST("fo\\no bar \\ hello\n");
  ASSERT_TEXT_TOKEN("fo\\no");
  ASSERT_TEXT_TOKEN("bar");
  ASSERT_FALSE(tokenizer.Next());
}

TEST(Tokenizer, EscapedLineCont) {
  SETUP_TEST("fo\\no bar \\\ntest");
  ASSERT_TEXT_TOKEN("fo\\no");
  ASSERT_TEXT_TOKEN("bar");
  ASSERT_TEXT_TOKEN("test");
  ASSERT_FALSE(tokenizer.Next());
}

TEST(Tokenizer, EscapedLineContWithBadChars) {
  SETUP_TEST("fo\\no bar \\bad bad bad\ntest");
  ASSERT_TEXT_TOKEN("fo\\no");
  ASSERT_TEXT_TOKEN("bar");
  ASSERT_TEXT_TOKEN("test");
  ASSERT_FALSE(tokenizer.Next());
}

TEST(Tokenizer, SimpleQuotes) {
  SETUP_TEST("foo \"single token\" bar");
  ASSERT_TEXT_TOKEN("foo");
  ASSERT_TEXT_TOKEN("single token");
  ASSERT_TEXT_TOKEN("bar");
  ASSERT_FALSE(tokenizer.Next());
}

TEST(Tokenizer, BadQuotes) {
  SETUP_TEST("foo \"single token");
  ASSERT_TEXT_TOKEN("foo");
  ASSERT_TEXT_TOKEN("single token");
  ASSERT_FALSE(tokenizer.Next());
}

}  // namespace init