ENDMARKER | |
NAME | |
NUMBER | |
STRING | |
NEWLINE | |
INDENT | |
DEDENT | |
LPAR '(' | |
RPAR ')' | |
LSQB '[' | |
RSQB ']' | |
COLON ':' | |
COMMA ',' | |
SEMI ';' | |
PLUS '+' | |
MINUS '-' | |
STAR '*' | |
SLASH '/' | |
VBAR '|' | |
AMPER '&' | |
LESS '<' | |
GREATER '>' | |
EQUAL '=' | |
DOT '.' | |
PERCENT '%' | |
LBRACE '{' | |
RBRACE '}' | |
EQEQUAL '==' | |
NOTEQUAL '!=' | |
LESSEQUAL '<=' | |
GREATEREQUAL '>=' | |
TILDE '~' | |
CIRCUMFLEX '^' | |
LEFTSHIFT '<<' | |
RIGHTSHIFT '>>' | |
DOUBLESTAR '**' | |
PLUSEQUAL '+=' | |
MINEQUAL '-=' | |
STAREQUAL '*=' | |
SLASHEQUAL '/=' | |
PERCENTEQUAL '%=' | |
AMPEREQUAL '&=' | |
VBAREQUAL '|=' | |
CIRCUMFLEXEQUAL '^=' | |
LEFTSHIFTEQUAL '<<=' | |
RIGHTSHIFTEQUAL '>>=' | |
DOUBLESTAREQUAL '**=' | |
DOUBLESLASH '//' | |
DOUBLESLASHEQUAL '//=' | |
AT '@' | |
ATEQUAL '@=' | |
RARROW '->' | |
ELLIPSIS '...' | |
COLONEQUAL ':=' | |
OP | |
TYPE_IGNORE | |
TYPE_COMMENT | |
ERRORTOKEN | |
# These aren't used by the C tokenizer but are needed for tokenize.py | |
COMMENT | |
NL | |
ENCODING |