pw_tokenizer: Multiple tokens from one macro
In some cases it is desirable to have a macro that expands to multiple
tokenizer calls. Previously, the __LINE__ macro was used to name
intermediate variables for tokenizing; however, this doesn't work when a
macro expands to multiple tokenizations. This patch switches to the
__COUNTER__ macro which will work correctly even in the case of multiple
tokenizations in one macro expansion.
Change-Id: I35a02f2c04817c1bb0b1d085d37bb72531cda0d6
Reviewed-on: https://pigweed-review.googlesource.com/c/pigweed/pigweed/+/15440
Commit-Queue: Keir Mierle <keir@google.com>
Reviewed-by: Wyatt Hepler <hepler@google.com>
diff --git a/pw_tokenizer/tokenize_test.cc b/pw_tokenizer/tokenize_test.cc
index 18987fb..bc552bc 100644
--- a/pw_tokenizer/tokenize_test.cc
+++ b/pw_tokenizer/tokenize_test.cc
@@ -67,6 +67,25 @@
EXPECT_EQ(TestHash(">:-[]"), kGlobalToken);
}
+// Verify that we can tokenize multiple strings from one source line.
+#define THREE_FOR_ONE(first, second, third) \
+ [[maybe_unused]] constexpr uint32_t token_1 = \
+ PW_TOKENIZE_STRING_DOMAIN("ignored", first); \
+ [[maybe_unused]] constexpr uint32_t token_2 = \
+ PW_TOKENIZE_STRING_DOMAIN("ignored", second); \
+ [[maybe_unused]] constexpr uint32_t token_3 = \
+ PW_TOKENIZE_STRING_DOMAIN("ignored", third);
+
+TEST(TokenizeStringLiteral, MultipleTokenizationsInOneMacroExpansion) {
+ // This verifies that we can safely tokenize multiple times in a single macro
+ // expansion. This can be useful when for example a name and description are
+ // both tokenized after being passed into a macro.
+ //
+ // This test only verifies that this compiles correctly; it does not test
+ // that the tokenizations make it to the final token database.
+ THREE_FOR_ONE("hello", "yes", "something");
+}
+
class TokenizeToBuffer : public ::testing::Test {
public:
TokenizeToBuffer() : buffer_{} {}