Refactor information about tokens out into a new TokenKinds.def file. Use this
to share code a bit more, and fixes a diagnostic bug Uday pointed out where
parseCommaSeparatedList would print the wrong diagnostic when the end signifier
was not a ).
PiperOrigin-RevId: 202676858
diff --git a/lib/Parser/TokenKinds.def b/lib/Parser/TokenKinds.def
new file mode 100644
index 0000000..7eae470
--- /dev/null
+++ b/lib/Parser/TokenKinds.def
@@ -0,0 +1,97 @@
+//===- TokenKinds.def - MLIR Token Description ------------------*- C++ -*-===//
+//
+// Copyright 2019 The MLIR Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// =============================================================================
+//
+// This file is intended to be #include'd multiple times to extract information
+// about tokens for various clients in the lexer.
+//
+//===----------------------------------------------------------------------===//
+
+#if !defined(TOK_MARKER) && !defined(TOK_IDENTIFIER) && !defined(TOK_LITERAL)&&\
+ !defined(TOK_PUNCTUATION) && !defined(TOK_KEYWORD)
+# error Must define one of the TOK_ macros.
+#endif
+
+#ifndef TOK_MARKER
+#define TOK_MARKER(X)
+#endif
+#ifndef TOK_IDENTIFIER
+#define TOK_IDENTIFIER(NAME)
+#endif
+#ifndef TOK_LITERAL
+#define TOK_LITERAL(NAME)
+#endif
+#ifndef TOK_PUNCTUATION
+#define TOK_PUNCTUATION(NAME, SPELLING)
+#endif
+#ifndef TOK_KEYWORD
+#define TOK_KEYWORD(SPELLING)
+#endif
+
+
+// Markers
+TOK_MARKER(eof)
+TOK_MARKER(error)
+
+// Identifiers.
+TOK_IDENTIFIER(bare_identifier) // foo
+TOK_IDENTIFIER(at_identifier) // @foo
+TOK_IDENTIFIER(affine_map_identifier) // #foo
+// TODO: @@foo, etc.
+
+// Literals
+TOK_LITERAL(integer) // 42
+TOK_LITERAL(string) // "foo"
+
+// Punctuation.
+TOK_PUNCTUATION(arrow, "->")
+TOK_PUNCTUATION(colon, ":")
+TOK_PUNCTUATION(comma, ",")
+TOK_PUNCTUATION(question, "?")
+TOK_PUNCTUATION(questionquestion, "??")
+TOK_PUNCTUATION(l_paren, "(")
+TOK_PUNCTUATION(r_paren, ")")
+TOK_PUNCTUATION(l_brace, "{")
+TOK_PUNCTUATION(r_brace, "}")
+TOK_PUNCTUATION(less, "<")
+TOK_PUNCTUATION(greater, ">")
+// TODO: More punctuation.
+
+// Keywords. These turn "foo" into Token::kw_foo enums.
+TOK_KEYWORD(bf16)
+TOK_KEYWORD(br)
+TOK_KEYWORD(cfgfunc)
+TOK_KEYWORD(extfunc)
+TOK_KEYWORD(f16)
+TOK_KEYWORD(f32)
+TOK_KEYWORD(f64)
+TOK_KEYWORD(i1)
+TOK_KEYWORD(i16)
+TOK_KEYWORD(i32)
+TOK_KEYWORD(i64)
+TOK_KEYWORD(i8)
+TOK_KEYWORD(int)
+TOK_KEYWORD(memref)
+TOK_KEYWORD(mlfunc)
+TOK_KEYWORD(return)
+TOK_KEYWORD(tensor)
+TOK_KEYWORD(vector)
+
+#undef TOK_MARKER
+#undef TOK_IDENTIFIER
+#undef TOK_LITERAL
+#undef TOK_PUNCTUATION
+#undef TOK_KEYWORD