Guido van Rossum | a330996 | 1993-07-28 09:05:47 +0000 | [diff] [blame] | 1 | #ifndef Py_TOKENIZER_H |
| 2 | #define Py_TOKENIZER_H |
| 3 | #ifdef __cplusplus |
| 4 | extern "C" { |
| 5 | #endif |
| 6 | |
Martin v. Löwis | 00f1e3f | 2002-08-04 17:29:52 +0000 | [diff] [blame] | 7 | #include "object.h" |
Guido van Rossum | f70e43a | 1991-02-19 12:39:46 +0000 | [diff] [blame] | 8 | |
Guido van Rossum | 85a5fbb | 1990-10-14 12:07:46 +0000 | [diff] [blame] | 9 | /* Tokenizer interface */ |
| 10 | |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 11 | #include "token.h" /* For token types */ |
Guido van Rossum | 85a5fbb | 1990-10-14 12:07:46 +0000 | [diff] [blame] | 12 | |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 13 | #define MAXINDENT 100 /* Max indentation level */ |
Serhiy Storchaka | 94cf308 | 2018-12-17 17:34:14 +0200 | [diff] [blame] | 14 | #define MAXLEVEL 200 /* Max parentheses level */ |
Guido van Rossum | 85a5fbb | 1990-10-14 12:07:46 +0000 | [diff] [blame] | 15 | |
Neil Schemenauer | 3f993c3 | 2007-09-21 20:50:26 +0000 | [diff] [blame] | 16 | enum decoding_state { |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 17 | STATE_INIT, |
Pablo Galindo | 261a452 | 2021-03-28 23:48:05 +0100 | [diff] [blame] | 18 | STATE_SEEK_CODING, |
| 19 | STATE_NORMAL |
Neil Schemenauer | 3f993c3 | 2007-09-21 20:50:26 +0000 | [diff] [blame] | 20 | }; |
| 21 | |
Miss Islington (bot) | 1fb6b9e | 2021-05-22 15:23:26 -0700 | [diff] [blame] | 22 | enum interactive_underflow_t { |
| 23 | /* Normal mode of operation: return a new token when asked in interactie mode */ |
| 24 | IUNDERFLOW_NORMAL, |
| 25 | /* Forcefully return ENDMARKER when asked for a new token in interactive mode. This |
Christian Clauss | dcfbe4f | 2021-10-07 16:31:33 +0200 | [diff] [blame] | 26 | * can be used to prevent the tokenizer to prompt the user for new tokens */ |
Miss Islington (bot) | 1fb6b9e | 2021-05-22 15:23:26 -0700 | [diff] [blame] | 27 | IUNDERFLOW_STOP, |
| 28 | }; |
| 29 | |
Guido van Rossum | 85a5fbb | 1990-10-14 12:07:46 +0000 | [diff] [blame] | 30 | /* Tokenizer state */ |
| 31 | struct tok_state { |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 32 | /* Input state; buf <= cur <= inp <= end */ |
| 33 | /* NB an entire line is held in the buffer */ |
| 34 | char *buf; /* Input buffer, or NULL; malloc'ed if fp != NULL */ |
| 35 | char *cur; /* Next character in buffer */ |
| 36 | char *inp; /* End of data in buffer */ |
Pablo Galindo | cd8dcbc | 2021-03-14 04:38:40 +0100 | [diff] [blame] | 37 | int fp_interactive; /* If the file descriptor is interactive */ |
| 38 | char *interactive_src_start; /* The start of the source parsed so far in interactive mode */ |
| 39 | char *interactive_src_end; /* The end of the source parsed so far in interactive mode */ |
Andy Lester | 384f3c5 | 2020-02-27 20:44:52 -0600 | [diff] [blame] | 40 | const char *end; /* End of input buffer if buf != NULL */ |
| 41 | const char *start; /* Start of current token if not NULL */ |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 42 | int done; /* E_OK normally, E_EOF at EOF, otherwise error code */ |
| 43 | /* NB If done != E_OK, cur must be == inp!!! */ |
| 44 | FILE *fp; /* Rest of input; NULL if tokenizing a string */ |
| 45 | int tabsize; /* Tab spacing */ |
| 46 | int indent; /* Current indentation index */ |
| 47 | int indstack[MAXINDENT]; /* Stack of indents */ |
| 48 | int atbol; /* Nonzero if at begin of new line */ |
| 49 | int pendin; /* Pending indents (if > 0) or dedents (if < 0) */ |
Serhiy Storchaka | c679227 | 2013-10-19 21:03:34 +0300 | [diff] [blame] | 50 | const char *prompt, *nextprompt; /* For interactive prompting */ |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 51 | int lineno; /* Current line number */ |
Anthony Sottile | 995d9b9 | 2019-01-12 20:05:13 -0800 | [diff] [blame] | 52 | int first_lineno; /* First line of a single line or multi line string |
| 53 | expression (cf. issue 16806) */ |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 54 | int level; /* () [] {} Parentheses nesting level */ |
| 55 | /* Used to allow free continuations inside them */ |
Serhiy Storchaka | 94cf308 | 2018-12-17 17:34:14 +0200 | [diff] [blame] | 56 | char parenstack[MAXLEVEL]; |
| 57 | int parenlinenostack[MAXLEVEL]; |
Pablo Galindo | d6d6371 | 2021-01-19 23:59:33 +0000 | [diff] [blame] | 58 | int parencolstack[MAXLEVEL]; |
Victor Stinner | 7f2fee3 | 2011-04-05 00:39:01 +0200 | [diff] [blame] | 59 | PyObject *filename; |
Serhiy Storchaka | 94cf308 | 2018-12-17 17:34:14 +0200 | [diff] [blame] | 60 | /* Stuff for checking on different tab sizes */ |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 61 | int altindstack[MAXINDENT]; /* Stack of alternate indents */ |
| 62 | /* Stuff for PEP 0263 */ |
| 63 | enum decoding_state decoding_state; |
| 64 | int decoding_erred; /* whether erred in decoding */ |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 65 | char *encoding; /* Source encoding. */ |
| 66 | int cont_line; /* whether we are in a continuation line. */ |
| 67 | const char* line_start; /* pointer to start of current line */ |
Anthony Sottile | 995d9b9 | 2019-01-12 20:05:13 -0800 | [diff] [blame] | 68 | const char* multi_line_start; /* pointer to start of first line of |
| 69 | a single line or multi line string |
| 70 | expression (cf. issue 16806) */ |
Victor Stinner | 22a351a | 2010-10-14 12:04:34 +0000 | [diff] [blame] | 71 | PyObject *decoding_readline; /* open(...).readline */ |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 72 | PyObject *decoding_buffer; |
Antoine Pitrou | f95a1b3 | 2010-05-09 15:52:27 +0000 | [diff] [blame] | 73 | const char* enc; /* Encoding for the current str. */ |
Pablo Galindo Salgado | 07cf66f | 2021-11-21 04:15:22 +0000 | [diff] [blame] | 74 | char* str; /* Source string being tokenized (if tokenizing from a string)*/ |
Andy Lester | 384f3c5 | 2020-02-27 20:44:52 -0600 | [diff] [blame] | 75 | char* input; /* Tokenizer's newline translated copy of the string. */ |
Guido van Rossum | dcfcd14 | 2019-01-31 03:40:27 -0800 | [diff] [blame] | 76 | |
| 77 | int type_comments; /* Whether to look for type comments */ |
Guido van Rossum | 495da29 | 2019-03-07 12:38:08 -0800 | [diff] [blame] | 78 | |
| 79 | /* async/await related fields (still needed depending on feature_version) */ |
| 80 | int async_hacks; /* =1 if async/await aren't always keywords */ |
| 81 | int async_def; /* =1 if tokens are inside an 'async def' body. */ |
| 82 | int async_def_indent; /* Indentation level of the outermost 'async def'. */ |
| 83 | int async_def_nl; /* =1 if the outermost 'async def' had at least one |
| 84 | NEWLINE token after it. */ |
Miss Islington (bot) | 1fb6b9e | 2021-05-22 15:23:26 -0700 | [diff] [blame] | 85 | /* How to proceed when asked for a new token in interactive mode */ |
Serhiy Storchaka | 93242d7 | 2021-10-03 20:03:49 +0300 | [diff] [blame] | 86 | enum interactive_underflow_t interactive_underflow; |
Guido van Rossum | 85a5fbb | 1990-10-14 12:07:46 +0000 | [diff] [blame] | 87 | }; |
| 88 | |
Benjamin Peterson | aeaa592 | 2009-11-13 00:17:59 +0000 | [diff] [blame] | 89 | extern struct tok_state *PyTokenizer_FromString(const char *, int); |
| 90 | extern struct tok_state *PyTokenizer_FromUTF8(const char *, int); |
Serhiy Storchaka | c679227 | 2013-10-19 21:03:34 +0300 | [diff] [blame] | 91 | extern struct tok_state *PyTokenizer_FromFile(FILE *, const char*, |
| 92 | const char *, const char *); |
Tim Peters | dbd9ba6 | 2000-07-09 03:09:57 +0000 | [diff] [blame] | 93 | extern void PyTokenizer_Free(struct tok_state *); |
Andy Lester | 384f3c5 | 2020-02-27 20:44:52 -0600 | [diff] [blame] | 94 | extern int PyTokenizer_Get(struct tok_state *, const char **, const char **); |
Guido van Rossum | a330996 | 1993-07-28 09:05:47 +0000 | [diff] [blame] | 95 | |
Pablo Galindo | f2cf1e3 | 2019-04-13 17:05:14 +0100 | [diff] [blame] | 96 | #define tok_dump _Py_tok_dump |
| 97 | |
Guido van Rossum | a330996 | 1993-07-28 09:05:47 +0000 | [diff] [blame] | 98 | #ifdef __cplusplus |
| 99 | } |
| 100 | #endif |
| 101 | #endif /* !Py_TOKENIZER_H */ |