bpo-30455: Generate all token related code and docs from Grammar/Tokens. (GH-10370)
"Include/token.h", "Lib/token.py" (containing now some data moved from
"Lib/tokenize.py") and new files "Parser/token.c" (containing the code
moved from "Parser/tokenizer.c") and "Doc/library/token-list.inc" (included
in "Doc/library/token.rst") are now generated from "Grammar/Tokens" by
"Tools/scripts/generate_token.py". The script overwrites files only if
needed and can be used on the read-only sources tree.
"Lib/symbol.py" is now generated by "Tools/scripts/generate_symbol_py.py"
instead of been executable itself.
Added new make targets "regen-token" and "regen-symbol" which are now
dependencies of "regen-all".
The documentation contains now strings for operators and punctuation tokens.
diff --git a/Parser/token.c b/Parser/token.c
new file mode 100644
index 0000000..35519aa
--- /dev/null
+++ b/Parser/token.c
@@ -0,0 +1,233 @@
+/* Auto-generated by Tools/scripts/generate_token.py */
+
+#include "Python.h"
+#include "token.h"
+
+/* Token names */
+
+const char * const _PyParser_TokenNames[] = {
+ "ENDMARKER",
+ "NAME",
+ "NUMBER",
+ "STRING",
+ "NEWLINE",
+ "INDENT",
+ "DEDENT",
+ "LPAR",
+ "RPAR",
+ "LSQB",
+ "RSQB",
+ "COLON",
+ "COMMA",
+ "SEMI",
+ "PLUS",
+ "MINUS",
+ "STAR",
+ "SLASH",
+ "VBAR",
+ "AMPER",
+ "LESS",
+ "GREATER",
+ "EQUAL",
+ "DOT",
+ "PERCENT",
+ "LBRACE",
+ "RBRACE",
+ "EQEQUAL",
+ "NOTEQUAL",
+ "LESSEQUAL",
+ "GREATEREQUAL",
+ "TILDE",
+ "CIRCUMFLEX",
+ "LEFTSHIFT",
+ "RIGHTSHIFT",
+ "DOUBLESTAR",
+ "PLUSEQUAL",
+ "MINEQUAL",
+ "STAREQUAL",
+ "SLASHEQUAL",
+ "PERCENTEQUAL",
+ "AMPEREQUAL",
+ "VBAREQUAL",
+ "CIRCUMFLEXEQUAL",
+ "LEFTSHIFTEQUAL",
+ "RIGHTSHIFTEQUAL",
+ "DOUBLESTAREQUAL",
+ "DOUBLESLASH",
+ "DOUBLESLASHEQUAL",
+ "AT",
+ "ATEQUAL",
+ "RARROW",
+ "ELLIPSIS",
+ "OP",
+ "<ERRORTOKEN>",
+ "<COMMENT>",
+ "<NL>",
+ "<ENCODING>",
+ "<N_TOKENS>",
+};
+
+/* Return the token corresponding to a single character */
+
+int
+PyToken_OneChar(int c1)
+{
+ switch (c1) {
+ case '%': return PERCENT;
+ case '&': return AMPER;
+ case '(': return LPAR;
+ case ')': return RPAR;
+ case '*': return STAR;
+ case '+': return PLUS;
+ case ',': return COMMA;
+ case '-': return MINUS;
+ case '.': return DOT;
+ case '/': return SLASH;
+ case ':': return COLON;
+ case ';': return SEMI;
+ case '<': return LESS;
+ case '=': return EQUAL;
+ case '>': return GREATER;
+ case '@': return AT;
+ case '[': return LSQB;
+ case ']': return RSQB;
+ case '^': return CIRCUMFLEX;
+ case '{': return LBRACE;
+ case '|': return VBAR;
+ case '}': return RBRACE;
+ case '~': return TILDE;
+ }
+ return OP;
+}
+
+int
+PyToken_TwoChars(int c1, int c2)
+{
+ switch (c1) {
+ case '!':
+ switch (c2) {
+ case '=': return NOTEQUAL;
+ }
+ break;
+ case '%':
+ switch (c2) {
+ case '=': return PERCENTEQUAL;
+ }
+ break;
+ case '&':
+ switch (c2) {
+ case '=': return AMPEREQUAL;
+ }
+ break;
+ case '*':
+ switch (c2) {
+ case '*': return DOUBLESTAR;
+ case '=': return STAREQUAL;
+ }
+ break;
+ case '+':
+ switch (c2) {
+ case '=': return PLUSEQUAL;
+ }
+ break;
+ case '-':
+ switch (c2) {
+ case '=': return MINEQUAL;
+ case '>': return RARROW;
+ }
+ break;
+ case '/':
+ switch (c2) {
+ case '/': return DOUBLESLASH;
+ case '=': return SLASHEQUAL;
+ }
+ break;
+ case '<':
+ switch (c2) {
+ case '<': return LEFTSHIFT;
+ case '=': return LESSEQUAL;
+ case '>': return NOTEQUAL;
+ }
+ break;
+ case '=':
+ switch (c2) {
+ case '=': return EQEQUAL;
+ }
+ break;
+ case '>':
+ switch (c2) {
+ case '=': return GREATEREQUAL;
+ case '>': return RIGHTSHIFT;
+ }
+ break;
+ case '@':
+ switch (c2) {
+ case '=': return ATEQUAL;
+ }
+ break;
+ case '^':
+ switch (c2) {
+ case '=': return CIRCUMFLEXEQUAL;
+ }
+ break;
+ case '|':
+ switch (c2) {
+ case '=': return VBAREQUAL;
+ }
+ break;
+ }
+ return OP;
+}
+
+int
+PyToken_ThreeChars(int c1, int c2, int c3)
+{
+ switch (c1) {
+ case '*':
+ switch (c2) {
+ case '*':
+ switch (c3) {
+ case '=': return DOUBLESTAREQUAL;
+ }
+ break;
+ }
+ break;
+ case '.':
+ switch (c2) {
+ case '.':
+ switch (c3) {
+ case '.': return ELLIPSIS;
+ }
+ break;
+ }
+ break;
+ case '/':
+ switch (c2) {
+ case '/':
+ switch (c3) {
+ case '=': return DOUBLESLASHEQUAL;
+ }
+ break;
+ }
+ break;
+ case '<':
+ switch (c2) {
+ case '<':
+ switch (c3) {
+ case '=': return LEFTSHIFTEQUAL;
+ }
+ break;
+ }
+ break;
+ case '>':
+ switch (c2) {
+ case '>':
+ switch (c3) {
+ case '=': return RIGHTSHIFTEQUAL;
+ }
+ break;
+ }
+ break;
+ }
+ return OP;
+}