| Benjamin Peterson | 90f5ba5 | 2010-03-11 22:53:45 +0000 | [diff] [blame] | 1 | #! /usr/bin/env python3 |
| Martin v. Löwis | ef04c44 | 2008-03-19 05:04:44 +0000 | [diff] [blame] | 2 | |
| 3 | """Token constants (from "token.h").""" |
| 4 | |
| 5 | # Taken from Python (r53757) and modified to include some tokens |
| 6 | # originally monkeypatched in by pgen2.tokenize |
| 7 | |
| 8 | #--start constants-- |
| 9 | ENDMARKER = 0 |
| 10 | NAME = 1 |
| 11 | NUMBER = 2 |
| 12 | STRING = 3 |
| 13 | NEWLINE = 4 |
| 14 | INDENT = 5 |
| 15 | DEDENT = 6 |
| 16 | LPAR = 7 |
| 17 | RPAR = 8 |
| 18 | LSQB = 9 |
| 19 | RSQB = 10 |
| 20 | COLON = 11 |
| 21 | COMMA = 12 |
| 22 | SEMI = 13 |
| 23 | PLUS = 14 |
| 24 | MINUS = 15 |
| 25 | STAR = 16 |
| 26 | SLASH = 17 |
| 27 | VBAR = 18 |
| 28 | AMPER = 19 |
| 29 | LESS = 20 |
| 30 | GREATER = 21 |
| 31 | EQUAL = 22 |
| 32 | DOT = 23 |
| 33 | PERCENT = 24 |
| 34 | BACKQUOTE = 25 |
| 35 | LBRACE = 26 |
| 36 | RBRACE = 27 |
| 37 | EQEQUAL = 28 |
| 38 | NOTEQUAL = 29 |
| 39 | LESSEQUAL = 30 |
| 40 | GREATEREQUAL = 31 |
| 41 | TILDE = 32 |
| 42 | CIRCUMFLEX = 33 |
| 43 | LEFTSHIFT = 34 |
| 44 | RIGHTSHIFT = 35 |
| 45 | DOUBLESTAR = 36 |
| 46 | PLUSEQUAL = 37 |
| 47 | MINEQUAL = 38 |
| 48 | STAREQUAL = 39 |
| 49 | SLASHEQUAL = 40 |
| 50 | PERCENTEQUAL = 41 |
| 51 | AMPEREQUAL = 42 |
| 52 | VBAREQUAL = 43 |
| 53 | CIRCUMFLEXEQUAL = 44 |
| 54 | LEFTSHIFTEQUAL = 45 |
| 55 | RIGHTSHIFTEQUAL = 46 |
| 56 | DOUBLESTAREQUAL = 47 |
| 57 | DOUBLESLASH = 48 |
| 58 | DOUBLESLASHEQUAL = 49 |
| 59 | AT = 50 |
| Benjamin Peterson | 4ab92c8 | 2014-04-10 00:12:47 -0400 | [diff] [blame] | 60 | ATEQUAL = 51 |
| 61 | OP = 52 |
| 62 | COMMENT = 53 |
| 63 | NL = 54 |
| 64 | RARROW = 55 |
| 65 | ERRORTOKEN = 56 |
| 66 | N_TOKENS = 57 |
| Martin v. Löwis | ef04c44 | 2008-03-19 05:04:44 +0000 | [diff] [blame] | 67 | NT_OFFSET = 256 |
| 68 | #--end constants-- |
| 69 | |
| 70 | tok_name = {} |
| Martin v. Löwis | 8a5f8ca | 2008-03-19 05:33:36 +0000 | [diff] [blame] | 71 | for _name, _value in list(globals().items()): |
| Martin v. Löwis | ef04c44 | 2008-03-19 05:04:44 +0000 | [diff] [blame] | 72 | if type(_value) is type(0): |
| 73 | tok_name[_value] = _name |
| 74 | |
| 75 | |
| 76 | def ISTERMINAL(x): |
| 77 | return x < NT_OFFSET |
| 78 | |
| 79 | def ISNONTERMINAL(x): |
| 80 | return x >= NT_OFFSET |
| 81 | |
| 82 | def ISEOF(x): |
| 83 | return x == ENDMARKER |