Export libtextclassifier to Android
Test: atest android.view.textclassifier.TextClassificationManagerTest
Change-Id: Id7a31dc60c8f6625ff8f2a9c85689e13b121a5a4
diff --git a/annotator/annotator.cc b/annotator/annotator.cc
index 562d58e..a8a7d8b 100644
--- a/annotator/annotator.cc
+++ b/annotator/annotator.cc
@@ -111,6 +111,7 @@
return classifier;
}
+
std::unique_ptr<Annotator> Annotator::FromScopedMmap(
std::unique_ptr<ScopedMmap>* mmap, const UniLib* unilib,
const CalendarLib* calendarlib) {
diff --git a/annotator/annotator_jni.cc b/annotator/annotator_jni.cc
index 57580fa..6907398 100644
--- a/annotator/annotator_jni.cc
+++ b/annotator/annotator_jni.cc
@@ -30,8 +30,8 @@
#include "utils/memory/mmap.h"
#include "utils/utf8/unilib.h"
-#ifdef LIBTEXTCLASSIFIER_UNILIB_JAVAICU
-#ifndef LIBTEXTCLASSIFIER_CALENDAR_JAVAICU
+#ifdef TC3_UNILIB_JAVAICU
+#ifndef TC3_CALENDAR_JAVAICU
#error Inconsistent usage of Java ICU components
#else
#define TC3_USE_JAVAICU
diff --git a/annotator/annotator_test.cc b/annotator/annotator_test.cc
index b6290d5..fbaf039 100644
--- a/annotator/annotator_test.cc
+++ b/annotator/annotator_test.cc
@@ -52,7 +52,7 @@
}
std::string GetModelPath() {
- return LIBTEXTCLASSIFIER_TEST_DATA_DIR;
+ return TC3_TEST_DATA_DIR;
}
class AnnotatorTest : public ::testing::TestWithParam<const char*> {
@@ -205,7 +205,7 @@
return result;
}
-#ifdef LIBTEXTCLASSIFIER_UNILIB_ICU
+#ifdef TC3_UNILIB_ICU
TEST_P(AnnotatorTest, ClassifyTextRegularExpression) {
const std::string test_model = ReadFile(GetModelPath() + GetParam());
std::unique_ptr<ModelT> unpacked_model = UnPackModel(test_model.c_str());
@@ -266,9 +266,9 @@
"www.google.com every today!|Call me at (800) 123-456 today.",
{51, 65})));
}
-#endif // LIBTEXTCLASSIFIER_UNILIB_ICU
+#endif // TC3_UNILIB_ICU
-#ifdef LIBTEXTCLASSIFIER_UNILIB_ICU
+#ifdef TC3_UNILIB_ICU
TEST_P(AnnotatorTest, SuggestSelectionRegularExpression) {
const std::string test_model = ReadFile(GetModelPath() + GetParam());
std::unique_ptr<ModelT> unpacked_model = UnPackModel(test_model.c_str());
@@ -308,9 +308,9 @@
EXPECT_EQ(classifier->SuggestSelection("cc: 4012 8888 8888 1881", {9, 14}),
std::make_pair(4, 23));
}
-#endif // LIBTEXTCLASSIFIER_UNILIB_ICU
+#endif // TC3_UNILIB_ICU
-#ifdef LIBTEXTCLASSIFIER_UNILIB_ICU
+#ifdef TC3_UNILIB_ICU
TEST_P(AnnotatorTest, SuggestSelectionRegularExpressionConflictsModelWins) {
const std::string test_model = ReadFile(GetModelPath() + GetParam());
std::unique_ptr<ModelT> unpacked_model = UnPackModel(test_model.c_str());
@@ -339,9 +339,9 @@
{55, 57}),
std::make_pair(26, 62));
}
-#endif // LIBTEXTCLASSIFIER_UNILIB_ICU
+#endif // TC3_UNILIB_ICU
-#ifdef LIBTEXTCLASSIFIER_UNILIB_ICU
+#ifdef TC3_UNILIB_ICU
TEST_P(AnnotatorTest, SuggestSelectionRegularExpressionConflictsRegexWins) {
const std::string test_model = ReadFile(GetModelPath() + GetParam());
std::unique_ptr<ModelT> unpacked_model = UnPackModel(test_model.c_str());
@@ -370,9 +370,9 @@
{55, 57}),
std::make_pair(55, 62));
}
-#endif // LIBTEXTCLASSIFIER_UNILIB_ICU
+#endif // TC3_UNILIB_ICU
-#ifdef LIBTEXTCLASSIFIER_UNILIB_ICU
+#ifdef TC3_UNILIB_ICU
TEST_P(AnnotatorTest, AnnotateRegex) {
const std::string test_model = ReadFile(GetModelPath() + GetParam());
std::unique_ptr<ModelT> unpacked_model = UnPackModel(test_model.c_str());
@@ -409,7 +409,7 @@
IsAnnotatedSpan(79, 91, "phone"),
IsAnnotatedSpan(107, 126, "payment_card")}));
}
-#endif // LIBTEXTCLASSIFIER_UNILIB_ICU
+#endif // TC3_UNILIB_ICU
TEST_P(AnnotatorTest, PhoneFiltering) {
std::unique_ptr<Annotator> classifier =
@@ -739,6 +739,7 @@
.empty());
}
+
TEST_P(AnnotatorTest, AnnotateSmallBatches) {
const std::string test_model = ReadFile(GetModelPath() + GetParam());
std::unique_ptr<ModelT> unpacked_model = UnPackModel(test_model.c_str());
@@ -768,7 +769,7 @@
EXPECT_TRUE(classifier->Annotate("853 225\n3556", options).empty());
}
-#ifdef LIBTEXTCLASSIFIER_UNILIB_ICU
+#ifdef TC3_UNILIB_ICU
TEST_P(AnnotatorTest, AnnotateFilteringDiscardAll) {
const std::string test_model = ReadFile(GetModelPath() + GetParam());
std::unique_ptr<ModelT> unpacked_model = UnPackModel(test_model.c_str());
@@ -791,7 +792,7 @@
EXPECT_EQ(classifier->Annotate(test_string).size(), 0);
}
-#endif // LIBTEXTCLASSIFIER_UNILIB_ICU
+#endif // TC3_UNILIB_ICU
TEST_P(AnnotatorTest, AnnotateFilteringKeepAll) {
const std::string test_model = ReadFile(GetModelPath() + GetParam());
@@ -873,7 +874,7 @@
}));
}
-#ifdef LIBTEXTCLASSIFIER_UNILIB_ICU
+#ifdef TC3_UNILIB_ICU
TEST_P(AnnotatorTest, AnnotateFilteredCollectionsSuppress) {
const std::string test_model = ReadFile(GetModelPath() + GetParam());
@@ -917,9 +918,9 @@
IsAnnotatedSpan(28, 55, "address"),
}));
}
-#endif // LIBTEXTCLASSIFIER_UNILIB_ICU
+#endif // TC3_UNILIB_ICU
-#ifdef LIBTEXTCLASSIFIER_CALENDAR_ICU
+#ifdef TC3_CALENDAR_ICU
TEST_P(AnnotatorTest, ClassifyTextDate) {
std::unique_ptr<Annotator> classifier =
Annotator::FromPath(GetModelPath() + GetParam());
@@ -968,9 +969,9 @@
EXPECT_EQ(result[0].datetime_parse_result.granularity,
DatetimeGranularity::GRANULARITY_DAY);
}
-#endif // LIBTEXTCLASSIFIER_UNILIB_ICU
+#endif // TC3_UNILIB_ICU
-#ifdef LIBTEXTCLASSIFIER_CALENDAR_ICU
+#ifdef TC3_CALENDAR_ICU
TEST_P(AnnotatorTest, ClassifyTextDatePriorities) {
std::unique_ptr<Annotator> classifier =
Annotator::FromPath(GetModelPath() + GetParam());
@@ -1001,9 +1002,9 @@
EXPECT_EQ(result[0].datetime_parse_result.granularity,
DatetimeGranularity::GRANULARITY_DAY);
}
-#endif // LIBTEXTCLASSIFIER_UNILIB_ICU
+#endif // TC3_UNILIB_ICU
-#ifdef LIBTEXTCLASSIFIER_CALENDAR_ICU
+#ifdef TC3_CALENDAR_ICU
TEST_P(AnnotatorTest, SuggestTextDateDisabled) {
const std::string test_model = ReadFile(GetModelPath() + GetParam());
std::unique_ptr<ModelT> unpacked_model = UnPackModel(test_model.c_str());
@@ -1027,7 +1028,7 @@
EXPECT_THAT(classifier->Annotate("january 1, 2017"),
ElementsAreArray({IsAnnotatedSpan(0, 15, "date")}));
}
-#endif // LIBTEXTCLASSIFIER_UNILIB_ICU
+#endif // TC3_UNILIB_ICU
class TestingAnnotator : public Annotator {
public:
@@ -1123,7 +1124,7 @@
EXPECT_THAT(chosen, ElementsAreArray({0, 2, 4}));
}
-#ifdef LIBTEXTCLASSIFIER_UNILIB_ICU
+#ifdef TC3_UNILIB_ICU
TEST_P(AnnotatorTest, LongInput) {
std::unique_ptr<Annotator> classifier =
Annotator::FromPath(GetModelPath() + GetParam(), &unilib_, &calendarlib_);
@@ -1152,9 +1153,9 @@
input_100k, {50000, 50000 + value_length})));
}
}
-#endif // LIBTEXTCLASSIFIER_UNILIB_ICU
+#endif // TC3_UNILIB_ICU
-#ifdef LIBTEXTCLASSIFIER_UNILIB_ICU
+#ifdef TC3_UNILIB_ICU
// These coarse tests are there only to make sure the execution happens in
// reasonable amount of time.
TEST_P(AnnotatorTest, LongInputNoResultCheck) {
@@ -1173,9 +1174,9 @@
classifier->ClassifyText(input_100k, {50000, 50000 + value_length});
}
}
-#endif // LIBTEXTCLASSIFIER_UNILIB_ICU
+#endif // TC3_UNILIB_ICU
-#ifdef LIBTEXTCLASSIFIER_UNILIB_ICU
+#ifdef TC3_UNILIB_ICU
TEST_P(AnnotatorTest, MaxTokenLength) {
const std::string test_model = ReadFile(GetModelPath() + GetParam());
std::unique_ptr<ModelT> unpacked_model = UnPackModel(test_model.c_str());
@@ -1210,9 +1211,9 @@
"I live at 350 Third Street, Cambridge.", {10, 37})),
"other");
}
-#endif // LIBTEXTCLASSIFIER_UNILIB_ICU
+#endif // TC3_UNILIB_ICU
-#ifdef LIBTEXTCLASSIFIER_UNILIB_ICU
+#ifdef TC3_UNILIB_ICU
TEST_P(AnnotatorTest, MinAddressTokenLength) {
const std::string test_model = ReadFile(GetModelPath() + GetParam());
std::unique_ptr<ModelT> unpacked_model = UnPackModel(test_model.c_str());
@@ -1247,7 +1248,7 @@
"I live at 350 Third Street, Cambridge.", {10, 37})),
"other");
}
-#endif // LIBTEXTCLASSIFIER_UNILIB_ICU
+#endif // TC3_UNILIB_ICU
} // namespace
} // namespace libtextclassifier3
diff --git a/annotator/datetime/parser_test.cc b/annotator/datetime/parser_test.cc
index efe7306..d46accf 100644
--- a/annotator/datetime/parser_test.cc
+++ b/annotator/datetime/parser_test.cc
@@ -34,7 +34,7 @@
namespace {
std::string GetModelPath() {
- return LIBTEXTCLASSIFIER_TEST_DATA_DIR;
+ return TC3_TEST_DATA_DIR;
}
std::string ReadFile(const std::string& file_name) {
diff --git a/annotator/feature-processor_test.cc b/annotator/feature-processor_test.cc
index 1788906..c9f0e0d 100644
--- a/annotator/feature-processor_test.cc
+++ b/annotator/feature-processor_test.cc
@@ -867,7 +867,7 @@
Token("웹사이트", 7, 11)}));
}
-#ifdef LIBTEXTCLASSIFIER_TEST_ICU
+#ifdef TC3_TEST_ICU
TEST_F(FeatureProcessorTest, ICUTokenize) {
FeatureProcessorOptionsT options;
options.tokenization_type = FeatureProcessorOptions_::TokenizationType_ICU;
@@ -889,7 +889,7 @@
}
#endif
-#ifdef LIBTEXTCLASSIFIER_TEST_ICU
+#ifdef TC3_TEST_ICU
TEST_F(FeatureProcessorTest, ICUTokenizeWithWhitespaces) {
FeatureProcessorOptionsT options;
options.tokenization_type = FeatureProcessorOptions_::TokenizationType_ICU;
@@ -917,7 +917,7 @@
}
#endif
-#ifdef LIBTEXTCLASSIFIER_TEST_ICU
+#ifdef TC3_TEST_ICU
TEST_F(FeatureProcessorTest, MixedTokenize) {
FeatureProcessorOptionsT options;
options.tokenization_type = FeatureProcessorOptions_::TokenizationType_MIXED;
diff --git a/annotator/model.fbs b/annotator/model.fbs
index a3d26f8..1f0a292 100755
--- a/annotator/model.fbs
+++ b/annotator/model.fbs
@@ -14,6 +14,8 @@
// limitations under the License.
//
+include "utils/intents/intent-config.fbs";
+
file_identifier "TC2 ";
// The possible model modes, represents a bit field.
@@ -129,59 +131,6 @@
GROUP_DUMMY2 = 13,
}
-// The type of variable to fetch.
-namespace libtextclassifier3;
-enum AndroidSimpleIntentGeneratorVariableType : int {
- INVALID_VARIABLE = 0,
-
- // The raw text that was classified.
- RAW_TEXT = 1,
-
- // Text as a URL with explicit protocol. If no protocol was specified, http
- // is prepended.
- URL_TEXT = 2,
-
- // The raw text, but URL encoded.
- URL_ENCODED_TEXT = 3,
-
- // For dates/times: the instant of the event in UTC millis.
- EVENT_TIME_MS_UTC = 4,
-
- // For dates/times: the start of the event in UTC millis.
- EVENT_START_MS_UTC = 5,
-
- // For dates/times: the end of the event in UTC millis.
- EVENT_END_MS_UTC = 6,
-
- // Name of the package that's running the classifier.
- PACKAGE_NAME = 7,
-}
-
-// Enumerates the possible extra types for the simple intent generator.
-namespace libtextclassifier3;
-enum AndroidSimpleIntentGeneratorExtraType : int {
- INVALID_EXTRA_TYPE = 0,
- STRING = 1,
- BOOL = 2,
- VARIABLE_AS_LONG = 3,
-}
-
-// Enumerates the possible condition types for the simple intent generator.
-namespace libtextclassifier3;
-enum AndroidSimpleIntentGeneratorConditionType : int {
- INVALID_CONDITION_TYPE = 0,
-
- // Queries the UserManager for the given boolean restriction. The condition
- // passes if the result is of getBoolean is false. The name of the
- // restriction to check is in the string_ field.
- USER_RESTRICTION_NOT_SET = 1,
-
- // Checks that the parsed event start time is at least a give number of
- // milliseconds in the future. (Only valid if there is a parsed event
- // time) The offset is stored in the int64_ field.
- EVENT_START_IN_FUTURE_MS = 2,
-}
-
namespace libtextclassifier3;
table CompressedBuffer {
buffer:[ubyte];
@@ -633,109 +582,4 @@
tokenize_on_script_change:bool = false;
}
-// Describes how intents for the various entity types should be generated on
-// Android. This is distributed through the model, but not used by
-// libtextclassifier yet - rather, it's passed to the calling Java code, which
-// implements the Intent generation logic.
-namespace libtextclassifier3;
-table AndroidIntentFactoryOptions {
- entity:[libtextclassifier3.AndroidIntentFactoryEntityOptions];
-}
-
-// Describes how intents should be generated for a particular entity type.
-namespace libtextclassifier3;
-table AndroidIntentFactoryEntityOptions {
- // The entity type as defined by one of the TextClassifier ENTITY_TYPE
- // constants. (e.g. "address", "phone", etc.)
- entity_type:string;
-
- // List of generators for all the different types of intents that should
- // be made available for the entity type.
- generator:[libtextclassifier3.AndroidIntentGeneratorOptions];
-}
-
-// Configures a single Android Intent generator.
-namespace libtextclassifier3;
-table AndroidIntentGeneratorOptions {
- // Strings for UI elements.
- strings:[libtextclassifier3.AndroidIntentGeneratorStrings];
-
- // Generator specific configuration.
- simple:libtextclassifier3.AndroidSimpleIntentGeneratorOptions;
-}
-
-// Language dependent configuration for an Android Intent generator.
-namespace libtextclassifier3;
-table AndroidIntentGeneratorStrings {
- // BCP 47 tag for the supported locale. Note that because of API level
- // restrictions, this must /not/ use wildcards. To e.g. match all English
- // locales, use only "en" and not "en_*". Reference the java.util.Locale
- // constructor for details.
- language_tag:string;
-
- // Title shown for the action (see RemoteAction.getTitle).
- title:string;
-
- // Description shown for the action (see
- // RemoteAction.getContentDescription).
- description:string;
-}
-
-// An extra to set on a simple intent generator Intent.
-namespace libtextclassifier3;
-table AndroidSimpleIntentGeneratorExtra {
- // The name of the extra to set.
- name:string;
-
- // The type of the extra to set.
- type:libtextclassifier3.AndroidSimpleIntentGeneratorExtraType;
-
- string_:string;
-
- bool_:bool;
- int32_:int;
-}
-
-// A condition that needs to be fulfilled for an Intent to get generated.
-namespace libtextclassifier3;
-table AndroidSimpleIntentGeneratorCondition {
- type:libtextclassifier3.AndroidSimpleIntentGeneratorConditionType;
-
- string_:string;
-
- int32_:int;
- int64_:long;
-}
-
-// Configures an intent generator where the logic is simple to be expressed with
-// basic rules - which covers the vast majority of use cases and is analogous
-// to Android Actions.
-// Most strings (action, data, type, ...) may contain variable references. To
-// use them, the generator must first declare all the variables it wishes to use
-// in the variables field. The values then become available as numbered
-// arguments (using the normal java.util.Formatter syntax) in the order they
-// were specified.
-namespace libtextclassifier3;
-table AndroidSimpleIntentGeneratorOptions {
- // The action to set on the Intent (see Intent.setAction). Supports variables.
- action:string;
-
- // The data to set on the Intent (see Intent.setData). Supports variables.
- data:string;
-
- // The type to set on the Intent (see Intent.setType). Supports variables.
- type:string;
-
- // The list of all the extras to add to the Intent.
- extra:[libtextclassifier3.AndroidSimpleIntentGeneratorExtra];
-
- // The list of all the variables that become available for substitution in
- // the action, data, type and extra strings. To e.g. set a field to the value
- // of the first variable, use "%0$s".
- variable:[libtextclassifier3.AndroidSimpleIntentGeneratorVariableType];
-
- // The list of all conditions that need to be fulfilled for Intent generation.
- condition:[libtextclassifier3.AndroidSimpleIntentGeneratorCondition];
-}
-
root_type libtextclassifier3.Model;
diff --git a/annotator/token-feature-extractor_test.cc b/annotator/token-feature-extractor_test.cc
index d669129..32383a9 100644
--- a/annotator/token-feature-extractor_test.cc
+++ b/annotator/token-feature-extractor_test.cc
@@ -233,7 +233,7 @@
EXPECT_THAT(dense_features, testing::ElementsAreArray({-1.0, -1.0}));
}
-#ifdef LIBTEXTCLASSIFIER_TEST_ICU
+#ifdef TC3_TEST_ICU
TEST_F(TokenFeatureExtractorTest, ICUCaseFeature) {
TokenFeatureExtractorOptions options;
options.num_buckets = 1000;
@@ -340,7 +340,7 @@
EXPECT_THAT(sparse_features, testing::ElementsAreArray(sparse_features2));
}
-#ifdef LIBTEXTCLASSIFIER_TEST_ICU
+#ifdef TC3_TEST_ICU
TEST_F(TokenFeatureExtractorTest, LowercaseUnicode) {
TokenFeatureExtractorOptions options;
options.num_buckets = 1000;
@@ -360,7 +360,7 @@
}
#endif
-#ifdef LIBTEXTCLASSIFIER_TEST_ICU
+#ifdef TC3_TEST_ICU
TEST_F(TokenFeatureExtractorTest, RegexFeatures) {
TokenFeatureExtractorOptions options;
options.num_buckets = 1000;