feature-processor_test.cc

/*
 * Copyright (C) 2017 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include "feature-processor.h"

#include "model-executor.h"
#include "tensor-view.h"

#include "gmock/gmock.h"
#include "gtest/gtest.h"

namespace libtextclassifier2 {
namespace {

using testing::ElementsAreArray;
using testing::FloatEq;

flatbuffers::DetachedBuffer PackFeatureProcessorOptions(
    const FeatureProcessorOptionsT& options) {
  flatbuffers::FlatBufferBuilder builder;
  builder.Finish(CreateFeatureProcessorOptions(builder, &options));
  return builder.Release();
}

class TestingFeatureProcessor : public FeatureProcessor {
 public:
  using FeatureProcessor::CountIgnoredSpanBoundaryCodepoints;
  using FeatureProcessor::FeatureProcessor;
  using FeatureProcessor::ICUTokenize;
  using FeatureProcessor::IsCodepointInRanges;
  using FeatureProcessor::SpanToLabel;
  using FeatureProcessor::StripTokensFromOtherLines;
  using FeatureProcessor::supported_codepoint_ranges_;
  using FeatureProcessor::SupportedCodepointsRatio;
};

// EmbeddingExecutor that always returns features based on
class FakeEmbeddingExecutor : public EmbeddingExecutor {
 public:
  bool AddEmbedding(const TensorView<int>& sparse_features, float* dest,
                    int dest_size) override {
    TC_CHECK_GE(dest_size, 4);
    EXPECT_EQ(sparse_features.size(), 1);
    dest[0] = sparse_features.data()[0];
    dest[1] = sparse_features.data()[0];
    dest[2] = -sparse_features.data()[0];
    dest[3] = -sparse_features.data()[0];
    return true;
  }

 private:
  std::vector<float> storage_;
};

TEST(FeatureProcessorTest, SplitTokensOnSelectionBoundariesMiddle) {
  std::vector<Token> tokens{Token("Hělló", 0, 5),
                            Token("fěěbař@google.com", 6, 23),
                            Token("heře!", 24, 29)};

  internal::SplitTokensOnSelectionBoundaries({9, 12}, &tokens);

  // clang-format off
  EXPECT_THAT(tokens, ElementsAreArray(
                          {Token("Hělló", 0, 5),
                           Token("fěě", 6, 9),
                           Token("bař", 9, 12),
                           Token("@google.com", 12, 23),
                           Token("heře!", 24, 29)}));
  // clang-format on
}

TEST(FeatureProcessorTest, SplitTokensOnSelectionBoundariesBegin) {
  std::vector<Token> tokens{Token("Hělló", 0, 5),
                            Token("fěěbař@google.com", 6, 23),
                            Token("heře!", 24, 29)};

  internal::SplitTokensOnSelectionBoundaries({6, 12}, &tokens);

  // clang-format off
  EXPECT_THAT(tokens, ElementsAreArray(
                          {Token("Hělló", 0, 5),
                           Token("fěěbař", 6, 12),
                           Token("@google.com", 12, 23),
                           Token("heře!", 24, 29)}));
  // clang-format on
}

TEST(FeatureProcessorTest, SplitTokensOnSelectionBoundariesEnd) {
  std::vector<Token> tokens{Token("Hělló", 0, 5),
                            Token("fěěbař@google.com", 6, 23),
                            Token("heře!", 24, 29)};

  internal::SplitTokensOnSelectionBoundaries({9, 23}, &tokens);

  // clang-format off
  EXPECT_THAT(tokens, ElementsAreArray(
                          {Token("Hělló", 0, 5),
                           Token("fěě", 6, 9),
                           Token("bař@google.com", 9, 23),
                           Token("heře!", 24, 29)}));
  // clang-format on
}

TEST(FeatureProcessorTest, SplitTokensOnSelectionBoundariesWhole) {
  std::vector<Token> tokens{Token("Hělló", 0, 5),
                            Token("fěěbař@google.com", 6, 23),
                            Token("heře!", 24, 29)};

  internal::SplitTokensOnSelectionBoundaries({6, 23}, &tokens);

  // clang-format off
  EXPECT_THAT(tokens, ElementsAreArray(
                          {Token("Hělló", 0, 5),
                           Token("fěěbař@google.com", 6, 23),
                           Token("heře!", 24, 29)}));
  // clang-format on
}

TEST(FeatureProcessorTest, SplitTokensOnSelectionBoundariesCrossToken) {
  std::vector<Token> tokens{Token("Hělló", 0, 5),
                            Token("fěěbař@google.com", 6, 23),
                            Token("heře!", 24, 29)};

  internal::SplitTokensOnSelectionBoundaries({2, 9}, &tokens);

  // clang-format off
  EXPECT_THAT(tokens, ElementsAreArray(
                          {Token("Hě", 0, 2),
                           Token("lló", 2, 5),
                           Token("fěě", 6, 9),
                           Token("bař@google.com", 9, 23),
                           Token("heře!", 24, 29)}));
  // clang-format on
}

TEST(FeatureProcessorTest, KeepLineWithClickFirst) {
  FeatureProcessorOptionsT options;
  options.only_use_line_with_click = true;
  flatbuffers::DetachedBuffer options_fb = PackFeatureProcessorOptions(options);
  TestingFeatureProcessor feature_processor(
      flatbuffers::GetRoot<FeatureProcessorOptions>(options_fb.data()));

  const std::string context = "Fiřst Lině\nSěcond Lině\nThiřd Lině";
  const CodepointSpan span = {0, 5};
  // clang-format off
  std::vector<Token> tokens = {Token("Fiřst", 0, 5),
                               Token("Lině", 6, 10),
                               Token("Sěcond", 11, 17),
                               Token("Lině", 18, 22),
                               Token("Thiřd", 23, 28),
                               Token("Lině", 29, 33)};
  // clang-format on

  // Keeps the first line.
  feature_processor.StripTokensFromOtherLines(context, span, &tokens);
  EXPECT_THAT(tokens,
              ElementsAreArray({Token("Fiřst", 0, 5), Token("Lině", 6, 10)}));
}

TEST(FeatureProcessorTest, KeepLineWithClickSecond) {
  FeatureProcessorOptionsT options;
  options.only_use_line_with_click = true;
  flatbuffers::DetachedBuffer options_fb = PackFeatureProcessorOptions(options);
  TestingFeatureProcessor feature_processor(
      flatbuffers::GetRoot<FeatureProcessorOptions>(options_fb.data()));

  const std::string context = "Fiřst Lině\nSěcond Lině\nThiřd Lině";
  const CodepointSpan span = {18, 22};
  // clang-format off
  std::vector<Token> tokens = {Token("Fiřst", 0, 5),
                               Token("Lině", 6, 10),
                               Token("Sěcond", 11, 17),
                               Token("Lině", 18, 22),
                               Token("Thiřd", 23, 28),
                               Token("Lině", 29, 33)};
  // clang-format on

  // Keeps the first line.
  feature_processor.StripTokensFromOtherLines(context, span, &tokens);
  EXPECT_THAT(tokens, ElementsAreArray(
                          {Token("Sěcond", 11, 17), Token("Lině", 18, 22)}));
}

TEST(FeatureProcessorTest, KeepLineWithClickThird) {
  FeatureProcessorOptionsT options;
  options.only_use_line_with_click = true;
  flatbuffers::DetachedBuffer options_fb = PackFeatureProcessorOptions(options);
  TestingFeatureProcessor feature_processor(
      flatbuffers::GetRoot<FeatureProcessorOptions>(options_fb.data()));

  const std::string context = "Fiřst Lině\nSěcond Lině\nThiřd Lině";
  const CodepointSpan span = {24, 33};
  // clang-format off
  std::vector<Token> tokens = {Token("Fiřst", 0, 5),
                               Token("Lině", 6, 10),
                               Token("Sěcond", 11, 17),
                               Token("Lině", 18, 22),
                               Token("Thiřd", 23, 28),
                               Token("Lině", 29, 33)};
  // clang-format on

  // Keeps the first line.
  feature_processor.StripTokensFromOtherLines(context, span, &tokens);
  EXPECT_THAT(tokens, ElementsAreArray(
                          {Token("Thiřd", 23, 28), Token("Lině", 29, 33)}));
}

TEST(FeatureProcessorTest, KeepLineWithClickSecondWithPipe) {
  FeatureProcessorOptionsT options;
  options.only_use_line_with_click = true;
  flatbuffers::DetachedBuffer options_fb = PackFeatureProcessorOptions(options);
  TestingFeatureProcessor feature_processor(
      flatbuffers::GetRoot<FeatureProcessorOptions>(options_fb.data()));

  const std::string context = "Fiřst Lině|Sěcond Lině\nThiřd Lině";
  const CodepointSpan span = {18, 22};
  // clang-format off
  std::vector<Token> tokens = {Token("Fiřst", 0, 5),
                               Token("Lině", 6, 10),
                               Token("Sěcond", 11, 17),
                               Token("Lině", 18, 22),
                               Token("Thiřd", 23, 28),
                               Token("Lině", 29, 33)};
  // clang-format on

  // Keeps the first line.
  feature_processor.StripTokensFromOtherLines(context, span, &tokens);
  EXPECT_THAT(tokens, ElementsAreArray(
                          {Token("Sěcond", 11, 17), Token("Lině", 18, 22)}));
}

TEST(FeatureProcessorTest, KeepLineWithCrosslineClick) {
  FeatureProcessorOptionsT options;
  options.only_use_line_with_click = true;
  flatbuffers::DetachedBuffer options_fb = PackFeatureProcessorOptions(options);
  TestingFeatureProcessor feature_processor(
      flatbuffers::GetRoot<FeatureProcessorOptions>(options_fb.data()));

  const std::string context = "Fiřst Lině\nSěcond Lině\nThiřd Lině";
  const CodepointSpan span = {5, 23};
  // clang-format off
  std::vector<Token> tokens = {Token("Fiřst", 0, 5),
                               Token("Lině", 6, 10),
                               Token("Sěcond", 18, 23),
                               Token("Lině", 19, 23),
                               Token("Thiřd", 23, 28),
                               Token("Lině", 29, 33)};
  // clang-format on

  // Keeps the first line.
  feature_processor.StripTokensFromOtherLines(context, span, &tokens);
  EXPECT_THAT(tokens, ElementsAreArray(
                          {Token("Fiřst", 0, 5), Token("Lině", 6, 10),
                           Token("Sěcond", 18, 23), Token("Lině", 19, 23),
                           Token("Thiřd", 23, 28), Token("Lině", 29, 33)}));
}

TEST(FeatureProcessorTest, SpanToLabel) {
  FeatureProcessorOptionsT options;
  options.context_size = 1;
  options.max_selection_span = 1;
  options.snap_label_span_boundaries_to_containing_tokens = false;

  options.tokenization_codepoint_config.emplace_back(
      new TokenizationCodepointRangeT());
  auto& config = options.tokenization_codepoint_config.back();
  config->start = 32;
  config->end = 33;
  config->role = TokenizationCodepointRange_::Role_WHITESPACE_SEPARATOR;

  flatbuffers::DetachedBuffer options_fb = PackFeatureProcessorOptions(options);
  TestingFeatureProcessor feature_processor(
      flatbuffers::GetRoot<FeatureProcessorOptions>(options_fb.data()));
  std::vector<Token> tokens = feature_processor.Tokenize("one, two, three");
  ASSERT_EQ(3, tokens.size());
  int label;
  ASSERT_TRUE(feature_processor.SpanToLabel({5, 8}, tokens, &label));
  EXPECT_EQ(kInvalidLabel, label);
  ASSERT_TRUE(feature_processor.SpanToLabel({5, 9}, tokens, &label));
  EXPECT_NE(kInvalidLabel, label);
  TokenSpan token_span;
  feature_processor.LabelToTokenSpan(label, &token_span);
  EXPECT_EQ(0, token_span.first);
  EXPECT_EQ(0, token_span.second);

  // Reconfigure with snapping enabled.
  options.snap_label_span_boundaries_to_containing_tokens = true;
  flatbuffers::DetachedBuffer options2_fb =
      PackFeatureProcessorOptions(options);
  TestingFeatureProcessor feature_processor2(
      flatbuffers::GetRoot<FeatureProcessorOptions>(options2_fb.data()));
  int label2;
  ASSERT_TRUE(feature_processor2.SpanToLabel({5, 8}, tokens, &label2));
  EXPECT_EQ(label, label2);
  ASSERT_TRUE(feature_processor2.SpanToLabel({6, 9}, tokens, &label2));
  EXPECT_EQ(label, label2);
  ASSERT_TRUE(feature_processor2.SpanToLabel({5, 9}, tokens, &label2));
  EXPECT_EQ(label, label2);

  // Cross a token boundary.
  ASSERT_TRUE(feature_processor2.SpanToLabel({4, 9}, tokens, &label2));
  EXPECT_EQ(kInvalidLabel, label2);
  ASSERT_TRUE(feature_processor2.SpanToLabel({5, 10}, tokens, &label2));
  EXPECT_EQ(kInvalidLabel, label2);

  // Multiple tokens.
  options.context_size = 2;
  options.max_selection_span = 2;
  flatbuffers::DetachedBuffer options3_fb =
      PackFeatureProcessorOptions(options);
  TestingFeatureProcessor feature_processor3(
      flatbuffers::GetRoot<FeatureProcessorOptions>(options3_fb.data()));
  tokens = feature_processor3.Tokenize("zero, one, two, three, four");
  ASSERT_TRUE(feature_processor3.SpanToLabel({6, 15}, tokens, &label2));
  EXPECT_NE(kInvalidLabel, label2);
  feature_processor3.LabelToTokenSpan(label2, &token_span);
  EXPECT_EQ(1, token_span.first);
  EXPECT_EQ(0, token_span.second);

  int label3;
  ASSERT_TRUE(feature_processor3.SpanToLabel({6, 14}, tokens, &label3));
  EXPECT_EQ(label2, label3);
  ASSERT_TRUE(feature_processor3.SpanToLabel({6, 13}, tokens, &label3));
  EXPECT_EQ(label2, label3);
  ASSERT_TRUE(feature_processor3.SpanToLabel({7, 13}, tokens, &label3));
  EXPECT_EQ(label2, label3);
}

TEST(FeatureProcessorTest, SpanToLabelIgnoresPunctuation) {
  FeatureProcessorOptionsT options;
  options.context_size = 1;
  options.max_selection_span = 1;
  options.snap_label_span_boundaries_to_containing_tokens = false;

  options.tokenization_codepoint_config.emplace_back(
      new TokenizationCodepointRangeT());
  auto& config = options.tokenization_codepoint_config.back();
  config->start = 32;
  config->end = 33;
  config->role = TokenizationCodepointRange_::Role_WHITESPACE_SEPARATOR;

  flatbuffers::DetachedBuffer options_fb = PackFeatureProcessorOptions(options);
  TestingFeatureProcessor feature_processor(
      flatbuffers::GetRoot<FeatureProcessorOptions>(options_fb.data()));
  std::vector<Token> tokens = feature_processor.Tokenize("one, two, three");
  ASSERT_EQ(3, tokens.size());
  int label;
  ASSERT_TRUE(feature_processor.SpanToLabel({5, 8}, tokens, &label));
  EXPECT_EQ(kInvalidLabel, label);
  ASSERT_TRUE(feature_processor.SpanToLabel({5, 9}, tokens, &label));
  EXPECT_NE(kInvalidLabel, label);
  TokenSpan token_span;
  feature_processor.LabelToTokenSpan(label, &token_span);
  EXPECT_EQ(0, token_span.first);
  EXPECT_EQ(0, token_span.second);

  // Reconfigure with snapping enabled.
  options.snap_label_span_boundaries_to_containing_tokens = true;
  flatbuffers::DetachedBuffer options2_fb =
      PackFeatureProcessorOptions(options);
  TestingFeatureProcessor feature_processor2(
      flatbuffers::GetRoot<FeatureProcessorOptions>(options2_fb.data()));
  int label2;
  ASSERT_TRUE(feature_processor2.SpanToLabel({5, 8}, tokens, &label2));
  EXPECT_EQ(label, label2);
  ASSERT_TRUE(feature_processor2.SpanToLabel({6, 9}, tokens, &label2));
  EXPECT_EQ(label, label2);
  ASSERT_TRUE(feature_processor2.SpanToLabel({5, 9}, tokens, &label2));
  EXPECT_EQ(label, label2);

  // Cross a token boundary.
  ASSERT_TRUE(feature_processor2.SpanToLabel({4, 9}, tokens, &label2));
  EXPECT_EQ(kInvalidLabel, label2);
  ASSERT_TRUE(feature_processor2.SpanToLabel({5, 10}, tokens, &label2));
  EXPECT_EQ(kInvalidLabel, label2);

  // Multiple tokens.
  options.context_size = 2;
  options.max_selection_span = 2;
  flatbuffers::DetachedBuffer options3_fb =
      PackFeatureProcessorOptions(options);
  TestingFeatureProcessor feature_processor3(
      flatbuffers::GetRoot<FeatureProcessorOptions>(options3_fb.data()));
  tokens = feature_processor3.Tokenize("zero, one, two, three, four");
  ASSERT_TRUE(feature_processor3.SpanToLabel({6, 15}, tokens, &label2));
  EXPECT_NE(kInvalidLabel, label2);
  feature_processor3.LabelToTokenSpan(label2, &token_span);
  EXPECT_EQ(1, token_span.first);
  EXPECT_EQ(0, token_span.second);

  int label3;
  ASSERT_TRUE(feature_processor3.SpanToLabel({6, 14}, tokens, &label3));
  EXPECT_EQ(label2, label3);
  ASSERT_TRUE(feature_processor3.SpanToLabel({6, 13}, tokens, &label3));
  EXPECT_EQ(label2, label3);
  ASSERT_TRUE(feature_processor3.SpanToLabel({7, 13}, tokens, &label3));
  EXPECT_EQ(label2, label3);
}

TEST(FeatureProcessorTest, CenterTokenFromClick) {
  int token_index;

  // Exactly aligned indices.
  token_index = internal::CenterTokenFromClick(
      {6, 11},
      {Token("Hělló", 0, 5), Token("world", 6, 11), Token("heře!", 12, 17)});
  EXPECT_EQ(token_index, 1);

  // Click is contained in a token.
  token_index = internal::CenterTokenFromClick(
      {13, 17},
      {Token("Hělló", 0, 5), Token("world", 6, 11), Token("heře!", 12, 17)});
  EXPECT_EQ(token_index, 2);

  // Click spans two tokens.
  token_index = internal::CenterTokenFromClick(
      {6, 17},
      {Token("Hělló", 0, 5), Token("world", 6, 11), Token("heře!", 12, 17)});
  EXPECT_EQ(token_index, kInvalidIndex);
}

TEST(FeatureProcessorTest, CenterTokenFromMiddleOfSelection) {
  int token_index;

  // Selection of length 3. Exactly aligned indices.
  token_index = internal::CenterTokenFromMiddleOfSelection(
      {7, 27},
      {Token("Token1", 0, 6), Token("Token2", 7, 13), Token("Token3", 14, 20),
       Token("Token4", 21, 27), Token("Token5", 28, 34)});
  EXPECT_EQ(token_index, 2);

  // Selection of length 1 token. Exactly aligned indices.
  token_index = internal::CenterTokenFromMiddleOfSelection(
      {21, 27},
      {Token("Token1", 0, 6), Token("Token2", 7, 13), Token("Token3", 14, 20),
       Token("Token4", 21, 27), Token("Token5", 28, 34)});
  EXPECT_EQ(token_index, 3);

  // Selection marks sub-token range, with no tokens in it.
  token_index = internal::CenterTokenFromMiddleOfSelection(
      {29, 33},
      {Token("Token1", 0, 6), Token("Token2", 7, 13), Token("Token3", 14, 20),
       Token("Token4", 21, 27), Token("Token5", 28, 34)});
  EXPECT_EQ(token_index, kInvalidIndex);

  // Selection of length 2. Sub-token indices.
  token_index = internal::CenterTokenFromMiddleOfSelection(
      {3, 25},
      {Token("Token1", 0, 6), Token("Token2", 7, 13), Token("Token3", 14, 20),
       Token("Token4", 21, 27), Token("Token5", 28, 34)});
  EXPECT_EQ(token_index, 1);

  // Selection of length 1. Sub-token indices.
  token_index = internal::CenterTokenFromMiddleOfSelection(
      {22, 34},
      {Token("Token1", 0, 6), Token("Token2", 7, 13), Token("Token3", 14, 20),
       Token("Token4", 21, 27), Token("Token5", 28, 34)});
  EXPECT_EQ(token_index, 4);

  // Some invalid ones.
  token_index = internal::CenterTokenFromMiddleOfSelection({7, 27}, {});
  EXPECT_EQ(token_index, -1);
}

TEST(FeatureProcessorTest, SupportedCodepointsRatio) {
  FeatureProcessorOptionsT options;
  options.context_size = 2;
  options.max_selection_span = 2;
  options.snap_label_span_boundaries_to_containing_tokens = false;
  options.feature_version = 2;
  options.embedding_size = 4;
  options.bounds_sensitive_features.reset(
      new FeatureProcessorOptions_::BoundsSensitiveFeaturesT());
  options.bounds_sensitive_features->enabled = true;
  options.bounds_sensitive_features->num_tokens_before = 5;
  options.bounds_sensitive_features->num_tokens_inside_left = 3;
  options.bounds_sensitive_features->num_tokens_inside_right = 3;
  options.bounds_sensitive_features->num_tokens_after = 5;
  options.bounds_sensitive_features->include_inside_bag = true;
  options.bounds_sensitive_features->include_inside_length = true;

  options.tokenization_codepoint_config.emplace_back(
      new TokenizationCodepointRangeT());
  auto& config = options.tokenization_codepoint_config.back();
  config->start = 32;
  config->end = 33;
  config->role = TokenizationCodepointRange_::Role_WHITESPACE_SEPARATOR;

  {
    options.supported_codepoint_ranges.emplace_back(
        new FeatureProcessorOptions_::CodepointRangeT());
    auto& range = options.supported_codepoint_ranges.back();
    range->start = 0;
    range->end = 128;
  }

  {
    options.supported_codepoint_ranges.emplace_back(
        new FeatureProcessorOptions_::CodepointRangeT());
    auto& range = options.supported_codepoint_ranges.back();
    range->start = 10000;
    range->end = 10001;
  }

  {
    options.supported_codepoint_ranges.emplace_back(
        new FeatureProcessorOptions_::CodepointRangeT());
    auto& range = options.supported_codepoint_ranges.back();
    range->start = 20000;
    range->end = 30000;
  }

  flatbuffers::DetachedBuffer options_fb = PackFeatureProcessorOptions(options);
  TestingFeatureProcessor feature_processor(
      flatbuffers::GetRoot<FeatureProcessorOptions>(options_fb.data()));
  EXPECT_THAT(feature_processor.SupportedCodepointsRatio(
                  {0, 3}, feature_processor.Tokenize("aaa bbb ccc")),
              FloatEq(1.0));
  EXPECT_THAT(feature_processor.SupportedCodepointsRatio(
                  {0, 3}, feature_processor.Tokenize("aaa bbb ěěě")),
              FloatEq(2.0 / 3));
  EXPECT_THAT(feature_processor.SupportedCodepointsRatio(
                  {0, 3}, feature_processor.Tokenize("ěěě řřř ěěě")),
              FloatEq(0.0));
  EXPECT_FALSE(feature_processor.IsCodepointInRanges(
      -1, feature_processor.supported_codepoint_ranges_));
  EXPECT_TRUE(feature_processor.IsCodepointInRanges(
      0, feature_processor.supported_codepoint_ranges_));
  EXPECT_TRUE(feature_processor.IsCodepointInRanges(
      10, feature_processor.supported_codepoint_ranges_));
  EXPECT_TRUE(feature_processor.IsCodepointInRanges(
      127, feature_processor.supported_codepoint_ranges_));
  EXPECT_FALSE(feature_processor.IsCodepointInRanges(
      128, feature_processor.supported_codepoint_ranges_));
  EXPECT_FALSE(feature_processor.IsCodepointInRanges(
      9999, feature_processor.supported_codepoint_ranges_));
  EXPECT_TRUE(feature_processor.IsCodepointInRanges(
      10000, feature_processor.supported_codepoint_ranges_));
  EXPECT_FALSE(feature_processor.IsCodepointInRanges(
      10001, feature_processor.supported_codepoint_ranges_));
  EXPECT_TRUE(feature_processor.IsCodepointInRanges(
      25000, feature_processor.supported_codepoint_ranges_));

  std::unique_ptr<CachedFeatures> cached_features;

  FakeEmbeddingExecutor embedding_executor;

  const std::vector<Token> tokens = {Token("ěěě", 0, 3), Token("řřř", 4, 7),
                                     Token("eee", 8, 11)};

  options.min_supported_codepoint_ratio = 0.0;
  flatbuffers::DetachedBuffer options2_fb =
      PackFeatureProcessorOptions(options);
  TestingFeatureProcessor feature_processor2(
      flatbuffers::GetRoot<FeatureProcessorOptions>(options2_fb.data()));
  EXPECT_TRUE(feature_processor2.ExtractFeatures(
      tokens, {0, 3}, &embedding_executor,
      /*feature_vector_size=*/4, &cached_features));

  options.min_supported_codepoint_ratio = 0.2;
  flatbuffers::DetachedBuffer options3_fb =
      PackFeatureProcessorOptions(options);
  TestingFeatureProcessor feature_processor3(
      flatbuffers::GetRoot<FeatureProcessorOptions>(options3_fb.data()));
  EXPECT_TRUE(feature_processor3.ExtractFeatures(
      tokens, {0, 3}, &embedding_executor,
      /*feature_vector_size=*/4, &cached_features));

  options.min_supported_codepoint_ratio = 0.5;
  flatbuffers::DetachedBuffer options4_fb =
      PackFeatureProcessorOptions(options);
  TestingFeatureProcessor feature_processor4(
      flatbuffers::GetRoot<FeatureProcessorOptions>(options4_fb.data()));
  EXPECT_FALSE(feature_processor4.ExtractFeatures(
      tokens, {0, 3}, &embedding_executor,
      /*feature_vector_size=*/4, &cached_features));
}

TEST(FeatureProcessorTest, StripUnusedTokensWithNoRelativeClick) {
  std::vector<Token> tokens_orig{
      Token("0", 0, 0), Token("1", 0, 0), Token("2", 0, 0),  Token("3", 0, 0),
      Token("4", 0, 0), Token("5", 0, 0), Token("6", 0, 0),  Token("7", 0, 0),
      Token("8", 0, 0), Token("9", 0, 0), Token("10", 0, 0), Token("11", 0, 0),
      Token("12", 0, 0)};

  std::vector<Token> tokens;
  int click_index;

  // Try to click first token and see if it gets padded from left.
  tokens = tokens_orig;
  click_index = 0;
  internal::StripOrPadTokens({0, 0}, 2, &tokens, &click_index);
  // clang-format off
  EXPECT_EQ(tokens, std::vector<Token>({Token(),
                                        Token(),
                                        Token("0", 0, 0),
                                        Token("1", 0, 0),
                                        Token("2", 0, 0)}));
  // clang-format on
  EXPECT_EQ(click_index, 2);

  // When we click the second token nothing should get padded.
  tokens = tokens_orig;
  click_index = 2;
  internal::StripOrPadTokens({0, 0}, 2, &tokens, &click_index);
  // clang-format off
  EXPECT_EQ(tokens, std::vector<Token>({Token("0", 0, 0),
                                        Token("1", 0, 0),
                                        Token("2", 0, 0),
                                        Token("3", 0, 0),
                                        Token("4", 0, 0)}));
  // clang-format on
  EXPECT_EQ(click_index, 2);

  // When we click the last token tokens should get padded from the right.
  tokens = tokens_orig;
  click_index = 12;
  internal::StripOrPadTokens({0, 0}, 2, &tokens, &click_index);
  // clang-format off
  EXPECT_EQ(tokens, std::vector<Token>({Token("10", 0, 0),
                                        Token("11", 0, 0),
                                        Token("12", 0, 0),
                                        Token(),
                                        Token()}));
  // clang-format on
  EXPECT_EQ(click_index, 2);
}

TEST(FeatureProcessorTest, StripUnusedTokensWithRelativeClick) {
  std::vector<Token> tokens_orig{
      Token("0", 0, 0), Token("1", 0, 0), Token("2", 0, 0),  Token("3", 0, 0),
      Token("4", 0, 0), Token("5", 0, 0), Token("6", 0, 0),  Token("7", 0, 0),
      Token("8", 0, 0), Token("9", 0, 0), Token("10", 0, 0), Token("11", 0, 0),
      Token("12", 0, 0)};

  std::vector<Token> tokens;
  int click_index;

  // Try to click first token and see if it gets padded from left to maximum
  // context_size.
  tokens = tokens_orig;
  click_index = 0;
  internal::StripOrPadTokens({2, 3}, 2, &tokens, &click_index);
  // clang-format off
  EXPECT_EQ(tokens, std::vector<Token>({Token(),
                                        Token(),
                                        Token("0", 0, 0),
                                        Token("1", 0, 0),
                                        Token("2", 0, 0),
                                        Token("3", 0, 0),
                                        Token("4", 0, 0),
                                        Token("5", 0, 0)}));
  // clang-format on
  EXPECT_EQ(click_index, 2);

  // Clicking to the middle with enough context should not produce any padding.
  tokens = tokens_orig;
  click_index = 6;
  internal::StripOrPadTokens({3, 1}, 2, &tokens, &click_index);
  // clang-format off
  EXPECT_EQ(tokens, std::vector<Token>({Token("1", 0, 0),
                                        Token("2", 0, 0),
                                        Token("3", 0, 0),
                                        Token("4", 0, 0),
                                        Token("5", 0, 0),
                                        Token("6", 0, 0),
                                        Token("7", 0, 0),
                                        Token("8", 0, 0),
                                        Token("9", 0, 0)}));
  // clang-format on
  EXPECT_EQ(click_index, 5);

  // Clicking at the end should pad right to maximum context_size.
  tokens = tokens_orig;
  click_index = 11;
  internal::StripOrPadTokens({3, 1}, 2, &tokens, &click_index);
  // clang-format off
  EXPECT_EQ(tokens, std::vector<Token>({Token("6", 0, 0),
                                        Token("7", 0, 0),
                                        Token("8", 0, 0),
                                        Token("9", 0, 0),
                                        Token("10", 0, 0),
                                        Token("11", 0, 0),
                                        Token("12", 0, 0),
                                        Token(),
                                        Token()}));
  // clang-format on
  EXPECT_EQ(click_index, 5);
}

TEST(FeatureProcessorTest, InternalTokenizeOnScriptChange) {
  FeatureProcessorOptionsT options;
  options.tokenization_codepoint_config.emplace_back(
      new TokenizationCodepointRangeT());
  {
    auto& config = options.tokenization_codepoint_config.back();
    config->start = 0;
    config->end = 256;
    config->role = TokenizationCodepointRange_::Role_DEFAULT_ROLE;
    config->script_id = 1;
  }
  options.tokenize_on_script_change = false;

  flatbuffers::DetachedBuffer options_fb = PackFeatureProcessorOptions(options);
  TestingFeatureProcessor feature_processor(
      flatbuffers::GetRoot<FeatureProcessorOptions>(options_fb.data()));

  EXPECT_EQ(feature_processor.Tokenize("앨라배마123웹사이트"),
            std::vector<Token>({Token("앨라배마123웹사이트", 0, 11)}));

  options.tokenize_on_script_change = true;
  flatbuffers::DetachedBuffer options_fb2 =
      PackFeatureProcessorOptions(options);
  TestingFeatureProcessor feature_processor2(
      flatbuffers::GetRoot<FeatureProcessorOptions>(options_fb2.data()));

  EXPECT_EQ(feature_processor2.Tokenize("앨라배마123웹사이트"),
            std::vector<Token>({Token("앨라배마", 0, 4), Token("123", 4, 7),
                                Token("웹사이트", 7, 11)}));
}

#ifdef LIBTEXTCLASSIFIER_TEST_ICU
TEST(FeatureProcessorTest, ICUTokenize) {
  FeatureProcessorOptionsT options;
  options.tokenization_type = FeatureProcessorOptions_::TokenizationType_ICU;

  flatbuffers::DetachedBuffer options_fb = PackFeatureProcessorOptions(options);
  TestingFeatureProcessor feature_processor(
      flatbuffers::GetRoot<FeatureProcessorOptions>(options_fb.data()));
  std::vector<Token> tokens = feature_processor.Tokenize("พระบาทสมเด็จพระปรมิ");
  ASSERT_EQ(tokens,
            // clang-format off
            std::vector<Token>({Token("พระบาท", 0, 6),
                                Token("สมเด็จ", 6, 12),
                                Token("พระ", 12, 15),
                                Token("ปร", 15, 17),
                                Token("มิ", 17, 19)}));
  // clang-format on
}
#endif

#ifdef LIBTEXTCLASSIFIER_TEST_ICU
TEST(FeatureProcessorTest, ICUTokenizeWithWhitespaces) {
  FeatureProcessorOptionsT options;
  options.tokenization_type = FeatureProcessorOptions_::TokenizationType_ICU;
  options.icu_preserve_whitespace_tokens = true;

  flatbuffers::DetachedBuffer options_fb = PackFeatureProcessorOptions(options);
  TestingFeatureProcessor feature_processor(
      flatbuffers::GetRoot<FeatureProcessorOptions>(options_fb.data()));
  std::vector<Token> tokens =
      feature_processor.Tokenize("พระบาท สมเด็จ พระ ปร มิ");
  ASSERT_EQ(tokens,
            // clang-format off
            std::vector<Token>({Token("พระบาท", 0, 6),
                                Token(" ", 6, 7),
                                Token("สมเด็จ", 7, 13),
                                Token(" ", 13, 14),
                                Token("พระ", 14, 17),
                                Token(" ", 17, 18),
                                Token("ปร", 18, 20),
                                Token(" ", 20, 21),
                                Token("มิ", 21, 23)}));
  // clang-format on
}
#endif

#ifdef LIBTEXTCLASSIFIER_TEST_ICU
TEST(FeatureProcessorTest, MixedTokenize) {
  FeatureProcessorOptionsT options;
  options.tokenization_type = FeatureProcessorOptions_::TokenizationType_MIXED;

  options.tokenization_codepoint_config.emplace_back(
      new TokenizationCodepointRangeT());
  auto& config = options.tokenization_codepoint_config.back();
  config->start = 32;
  config->end = 33;
  config->role = TokenizationCodepointRange_::Role_WHITESPACE_SEPARATOR;

  {
    options.internal_tokenizer_codepoint_ranges.emplace_back(
        new FeatureProcessorOptions_::CodepointRangeT());
    auto& range = options.internal_tokenizer_codepoint_ranges.back();
    range->start = 0;
    range->end = 128;
  }

  {
    options.internal_tokenizer_codepoint_ranges.emplace_back(
        new FeatureProcessorOptions_::CodepointRangeT());
    auto& range = options.internal_tokenizer_codepoint_ranges.back();
    range->start = 128;
    range->end = 256;
  }

  {
    options.internal_tokenizer_codepoint_ranges.emplace_back(
        new FeatureProcessorOptions_::CodepointRangeT());
    auto& range = options.internal_tokenizer_codepoint_ranges.back();
    range->start = 256;
    range->end = 384;
  }

  {
    options.internal_tokenizer_codepoint_ranges.emplace_back(
        new FeatureProcessorOptions_::CodepointRangeT());
    auto& range = options.internal_tokenizer_codepoint_ranges.back();
    range->start = 384;
    range->end = 592;
  }

  flatbuffers::DetachedBuffer options_fb = PackFeatureProcessorOptions(options);
  TestingFeatureProcessor feature_processor(
      flatbuffers::GetRoot<FeatureProcessorOptions>(options_fb.data()));
  std::vector<Token> tokens = feature_processor.Tokenize(
      "こんにちはJapanese-ląnguagę text 世界 http://www.google.com/");
  ASSERT_EQ(tokens,
            // clang-format off
            std::vector<Token>({Token("こんにちは", 0, 5),
                                Token("Japanese-ląnguagę", 5, 22),
                                Token("text", 23, 27),
                                Token("世界", 28, 30),
                                Token("http://www.google.com/", 31, 53)}));
  // clang-format on
}
#endif

TEST(FeatureProcessorTest, IgnoredSpanBoundaryCodepoints) {
  FeatureProcessorOptionsT options;
  options.ignored_span_boundary_codepoints.push_back('.');
  options.ignored_span_boundary_codepoints.push_back(',');
  options.ignored_span_boundary_codepoints.push_back('[');
  options.ignored_span_boundary_codepoints.push_back(']');

  flatbuffers::DetachedBuffer options_fb = PackFeatureProcessorOptions(options);
  TestingFeatureProcessor feature_processor(
      flatbuffers::GetRoot<FeatureProcessorOptions>(options_fb.data()));

  const std::string text1_utf8 = "ěščř";
  const UnicodeText text1 = UTF8ToUnicodeText(text1_utf8, /*do_copy=*/false);
  EXPECT_EQ(feature_processor.CountIgnoredSpanBoundaryCodepoints(
                text1.begin(), text1.end(),
                /*count_from_beginning=*/true),
            0);
  EXPECT_EQ(feature_processor.CountIgnoredSpanBoundaryCodepoints(
                text1.begin(), text1.end(),
                /*count_from_beginning=*/false),
            0);

  const std::string text2_utf8 = ".,abčd";
  const UnicodeText text2 = UTF8ToUnicodeText(text2_utf8, /*do_copy=*/false);
  EXPECT_EQ(feature_processor.CountIgnoredSpanBoundaryCodepoints(
                text2.begin(), text2.end(),
                /*count_from_beginning=*/true),
            2);
  EXPECT_EQ(feature_processor.CountIgnoredSpanBoundaryCodepoints(
                text2.begin(), text2.end(),
                /*count_from_beginning=*/false),
            0);

  const std::string text3_utf8 = ".,abčd[]";
  const UnicodeText text3 = UTF8ToUnicodeText(text3_utf8, /*do_copy=*/false);
  EXPECT_EQ(feature_processor.CountIgnoredSpanBoundaryCodepoints(
                text3.begin(), text3.end(),
                /*count_from_beginning=*/true),
            2);
  EXPECT_EQ(feature_processor.CountIgnoredSpanBoundaryCodepoints(
                text3.begin(), text3.end(),
                /*count_from_beginning=*/false),
            2);

  const std::string text4_utf8 = "[abčd]";
  const UnicodeText text4 = UTF8ToUnicodeText(text4_utf8, /*do_copy=*/false);
  EXPECT_EQ(feature_processor.CountIgnoredSpanBoundaryCodepoints(
                text4.begin(), text4.end(),
                /*count_from_beginning=*/true),
            1);
  EXPECT_EQ(feature_processor.CountIgnoredSpanBoundaryCodepoints(
                text4.begin(), text4.end(),
                /*count_from_beginning=*/false),
            1);

  const std::string text5_utf8 = "";
  const UnicodeText text5 = UTF8ToUnicodeText(text5_utf8, /*do_copy=*/false);
  EXPECT_EQ(feature_processor.CountIgnoredSpanBoundaryCodepoints(
                text5.begin(), text5.end(),
                /*count_from_beginning=*/true),
            0);
  EXPECT_EQ(feature_processor.CountIgnoredSpanBoundaryCodepoints(
                text5.begin(), text5.end(),
                /*count_from_beginning=*/false),
            0);

  const std::string text6_utf8 = "012345ěščř";
  const UnicodeText text6 = UTF8ToUnicodeText(text6_utf8, /*do_copy=*/false);
  UnicodeText::const_iterator text6_begin = text6.begin();
  std::advance(text6_begin, 6);
  EXPECT_EQ(feature_processor.CountIgnoredSpanBoundaryCodepoints(
                text6_begin, text6.end(),
                /*count_from_beginning=*/true),
            0);
  EXPECT_EQ(feature_processor.CountIgnoredSpanBoundaryCodepoints(
                text6_begin, text6.end(),
                /*count_from_beginning=*/false),
            0);

  const std::string text7_utf8 = "012345.,ěščř";
  const UnicodeText text7 = UTF8ToUnicodeText(text7_utf8, /*do_copy=*/false);
  UnicodeText::const_iterator text7_begin = text7.begin();
  std::advance(text7_begin, 6);
  EXPECT_EQ(feature_processor.CountIgnoredSpanBoundaryCodepoints(
                text7_begin, text7.end(),
                /*count_from_beginning=*/true),
            2);
  UnicodeText::const_iterator text7_end = text7.begin();
  std::advance(text7_end, 8);
  EXPECT_EQ(feature_processor.CountIgnoredSpanBoundaryCodepoints(
                text7.begin(), text7_end,
                /*count_from_beginning=*/false),
            2);

  // Test not stripping.
  EXPECT_EQ(feature_processor.StripBoundaryCodepoints(
                "Hello [[[Wořld]] or not?", {0, 24}),
            std::make_pair(0, 24));
  // Test basic stripping.
  EXPECT_EQ(feature_processor.StripBoundaryCodepoints(
                "Hello [[[Wořld]] or not?", {6, 16}),
            std::make_pair(9, 14));
  // Test stripping when everything is stripped.
  EXPECT_EQ(
      feature_processor.StripBoundaryCodepoints("Hello [[[]] or not?", {6, 11}),
      std::make_pair(6, 6));
  // Test stripping empty string.
  EXPECT_EQ(feature_processor.StripBoundaryCodepoints("", {0, 0}),
            std::make_pair(0, 0));
}

TEST(FeatureProcessorTest, CodepointSpanToTokenSpan) {
  const std::vector<Token> tokens{Token("Hělló", 0, 5),
                                  Token("fěěbař@google.com", 6, 23),
                                  Token("heře!", 24, 29)};

  // Spans matching the tokens exactly.
  EXPECT_EQ(TokenSpan(0, 1), CodepointSpanToTokenSpan(tokens, {0, 5}));
  EXPECT_EQ(TokenSpan(1, 2), CodepointSpanToTokenSpan(tokens, {6, 23}));
  EXPECT_EQ(TokenSpan(2, 3), CodepointSpanToTokenSpan(tokens, {24, 29}));
  EXPECT_EQ(TokenSpan(0, 2), CodepointSpanToTokenSpan(tokens, {0, 23}));
  EXPECT_EQ(TokenSpan(1, 3), CodepointSpanToTokenSpan(tokens, {6, 29}));
  EXPECT_EQ(TokenSpan(0, 3), CodepointSpanToTokenSpan(tokens, {0, 29}));

  // Snapping to containing tokens has no effect.
  EXPECT_EQ(TokenSpan(0, 1), CodepointSpanToTokenSpan(tokens, {0, 5}, true));
  EXPECT_EQ(TokenSpan(1, 2), CodepointSpanToTokenSpan(tokens, {6, 23}, true));
  EXPECT_EQ(TokenSpan(2, 3), CodepointSpanToTokenSpan(tokens, {24, 29}, true));
  EXPECT_EQ(TokenSpan(0, 2), CodepointSpanToTokenSpan(tokens, {0, 23}, true));
  EXPECT_EQ(TokenSpan(1, 3), CodepointSpanToTokenSpan(tokens, {6, 29}, true));
  EXPECT_EQ(TokenSpan(0, 3), CodepointSpanToTokenSpan(tokens, {0, 29}, true));

  // Span boundaries inside tokens.
  EXPECT_EQ(TokenSpan(1, 2), CodepointSpanToTokenSpan(tokens, {1, 28}));
  EXPECT_EQ(TokenSpan(0, 3), CodepointSpanToTokenSpan(tokens, {1, 28}, true));

  // Tokens adjacent to the span, but not overlapping.
  EXPECT_EQ(TokenSpan(1, 2), CodepointSpanToTokenSpan(tokens, {5, 24}));
  EXPECT_EQ(TokenSpan(1, 2), CodepointSpanToTokenSpan(tokens, {5, 24}, true));
}

}  // namespace
}  // namespace libtextclassifier2