| ENDMARKER | |
| NAME | |
| NUMBER | |
| STRING | |
| NEWLINE | |
| INDENT | |
| DEDENT | |
| LPAR '(' | |
| RPAR ')' | |
| LSQB '[' | |
| RSQB ']' | |
| COLON ':' | |
| COMMA ',' | |
| SEMI ';' | |
| PLUS '+' | |
| MINUS '-' | |
| STAR '*' | |
| SLASH '/' | |
| VBAR '|' | |
| AMPER '&' | |
| LESS '<' | |
| GREATER '>' | |
| EQUAL '=' | |
| DOT '.' | |
| PERCENT '%' | |
| LBRACE '{' | |
| RBRACE '}' | |
| EQEQUAL '==' | |
| NOTEQUAL '!=' | |
| LESSEQUAL '<=' | |
| GREATEREQUAL '>=' | |
| TILDE '~' | |
| CIRCUMFLEX '^' | |
| LEFTSHIFT '<<' | |
| RIGHTSHIFT '>>' | |
| DOUBLESTAR '**' | |
| PLUSEQUAL '+=' | |
| MINEQUAL '-=' | |
| STAREQUAL '*=' | |
| SLASHEQUAL '/=' | |
| PERCENTEQUAL '%=' | |
| AMPEREQUAL '&=' | |
| VBAREQUAL '|=' | |
| CIRCUMFLEXEQUAL '^=' | |
| LEFTSHIFTEQUAL '<<=' | |
| RIGHTSHIFTEQUAL '>>=' | |
| DOUBLESTAREQUAL '**=' | |
| DOUBLESLASH '//' | |
| DOUBLESLASHEQUAL '//=' | |
| AT '@' | |
| ATEQUAL '@=' | |
| RARROW '->' | |
| ELLIPSIS '...' | |
| COLONEQUAL ':=' | |
| OP | |
| AWAIT | |
| ASYNC | |
| TYPE_IGNORE | |
| TYPE_COMMENT | |
| ERRORTOKEN | |
| # These aren't used by the C tokenizer but are needed for tokenize.py | |
| COMMENT | |
| NL | |
| ENCODING |