pw_tokenizer: Store tokens along with strings
- Instead of storing plain tokenized strings in the ELF, store tokenized
string entries with the token, string, and domain.
- Update the C++ tokenization code to remove length limitations. Since
the tokens are stored with the strings, the token never has to be
recalculated and there is no need for consistency between C++ and C
(which can only hash a fixed number of characters).
- Use a better unique name (__LINE__ and __COUNTER__) for tokenized
variables and sections.
- Change the default domain from "default" to "". This is a more obvious
default and takes less space in the ELF.
Change-Id: I74fba3be55c0df67a71ef22143fe4916803aa796
Reviewed-on: https://pigweed-review.googlesource.com/c/pigweed/pigweed/+/21980
Commit-Queue: Wyatt Hepler <hepler@google.com>
Reviewed-by: Keir Mierle <keir@google.com>
diff --git a/pw_tokenizer/BUILD b/pw_tokenizer/BUILD
index 020db7b..431aa4b 100644
--- a/pw_tokenizer/BUILD
+++ b/pw_tokenizer/BUILD
@@ -39,7 +39,7 @@
"tokenize.cc",
],
hdrs = [
- "public/pw_tokenizer/pw_tokenizer_65599_fixed_length_hash.h",
+ "public/pw_tokenizer/hash.h",
"public/pw_tokenizer/tokenize.h",
],
includes = ["public"],
@@ -150,8 +150,8 @@
pw_cc_test(
name = "argument_types_test",
srcs = [
- "argument_types_test_c.c",
"argument_types_test.cc",
+ "argument_types_test_c.c",
"pw_tokenizer_private/argument_types_test.h",
"tokenize_test_fakes.cc",
],
@@ -202,8 +202,8 @@
pw_cc_test(
name = "global_handlers_test",
srcs = [
- "global_handlers_test_c.c",
"global_handlers_test.cc",
+ "global_handlers_test_c.c",
"pw_tokenizer_private/tokenize_test.h",
],
deps = [
@@ -252,8 +252,8 @@
name = "tokenize_test",
srcs = [
"pw_tokenizer_private/tokenize_test.h",
- "tokenize_test_c.c",
"tokenize_test.cc",
+ "tokenize_test_c.c",
],
deps = [
":pw_tokenizer",