Guido van Rossum | a330996 | 1993-07-28 09:05:47 +0000 | [diff] [blame] | 1 | #ifndef Py_TOKENIZER_H |
| 2 | #define Py_TOKENIZER_H |
| 3 | #ifdef __cplusplus |
| 4 | extern "C" { |
| 5 | #endif |
| 6 | |
Martin v. Löwis | 00f1e3f | 2002-08-04 17:29:52 +0000 | [diff] [blame] | 7 | #include "object.h" |
Guido van Rossum | f70e43a | 1991-02-19 12:39:46 +0000 | [diff] [blame] | 8 | |
Guido van Rossum | 85a5fbb | 1990-10-14 12:07:46 +0000 | [diff] [blame] | 9 | /* Tokenizer interface */ |
| 10 | |
Antoine Pitrou | 7f14f0d | 2010-05-09 16:14:21 +0000 | [diff] [blame] | 11 | #include "token.h" /* For token types */ |
Guido van Rossum | 85a5fbb | 1990-10-14 12:07:46 +0000 | [diff] [blame] | 12 | |
Antoine Pitrou | 7f14f0d | 2010-05-09 16:14:21 +0000 | [diff] [blame] | 13 | #define MAXINDENT 100 /* Max indentation level */ |
Guido van Rossum | 85a5fbb | 1990-10-14 12:07:46 +0000 | [diff] [blame] | 14 | |
Neil Schemenauer | 3f993c3 | 2007-09-21 20:50:26 +0000 | [diff] [blame] | 15 | enum decoding_state { |
Antoine Pitrou | 7f14f0d | 2010-05-09 16:14:21 +0000 | [diff] [blame] | 16 | STATE_INIT, |
| 17 | STATE_RAW, |
Georg Brandl | f55aa80 | 2010-11-26 08:59:40 +0000 | [diff] [blame] | 18 | STATE_NORMAL /* have a codec associated with input */ |
Neil Schemenauer | 3f993c3 | 2007-09-21 20:50:26 +0000 | [diff] [blame] | 19 | }; |
| 20 | |
Guido van Rossum | 85a5fbb | 1990-10-14 12:07:46 +0000 | [diff] [blame] | 21 | /* Tokenizer state */ |
| 22 | struct tok_state { |
Antoine Pitrou | 7f14f0d | 2010-05-09 16:14:21 +0000 | [diff] [blame] | 23 | /* Input state; buf <= cur <= inp <= end */ |
| 24 | /* NB an entire line is held in the buffer */ |
| 25 | char *buf; /* Input buffer, or NULL; malloc'ed if fp != NULL */ |
| 26 | char *cur; /* Next character in buffer */ |
| 27 | char *inp; /* End of data in buffer */ |
| 28 | char *end; /* End of input buffer if buf != NULL */ |
| 29 | char *start; /* Start of current token if not NULL */ |
| 30 | int done; /* E_OK normally, E_EOF at EOF, otherwise error code */ |
| 31 | /* NB If done != E_OK, cur must be == inp!!! */ |
| 32 | FILE *fp; /* Rest of input; NULL if tokenizing a string */ |
| 33 | int tabsize; /* Tab spacing */ |
| 34 | int indent; /* Current indentation index */ |
| 35 | int indstack[MAXINDENT]; /* Stack of indents */ |
| 36 | int atbol; /* Nonzero if at begin of new line */ |
| 37 | int pendin; /* Pending indents (if > 0) or dedents (if < 0) */ |
| 38 | char *prompt, *nextprompt; /* For interactive prompting */ |
| 39 | int lineno; /* Current line number */ |
| 40 | int level; /* () [] {} Parentheses nesting level */ |
| 41 | /* Used to allow free continuations inside them */ |
| 42 | /* Stuff for checking on different tab sizes */ |
Victor Stinner | 15244f7 | 2010-10-19 01:22:07 +0000 | [diff] [blame] | 43 | const char *filename; /* encoded to the filesystem encoding */ |
Antoine Pitrou | 7f14f0d | 2010-05-09 16:14:21 +0000 | [diff] [blame] | 44 | int altwarning; /* Issue warning if alternate tabs don't match */ |
| 45 | int alterror; /* Issue error if alternate tabs don't match */ |
| 46 | int alttabsize; /* Alternate tab spacing */ |
| 47 | int altindstack[MAXINDENT]; /* Stack of alternate indents */ |
| 48 | /* Stuff for PEP 0263 */ |
| 49 | enum decoding_state decoding_state; |
| 50 | int decoding_erred; /* whether erred in decoding */ |
| 51 | int read_coding_spec; /* whether 'coding:...' has been read */ |
| 52 | char *encoding; /* Source encoding. */ |
| 53 | int cont_line; /* whether we are in a continuation line. */ |
| 54 | const char* line_start; /* pointer to start of current line */ |
Martin v. Löwis | 1ee99d3 | 2002-08-04 20:10:29 +0000 | [diff] [blame] | 55 | #ifndef PGEN |
Antoine Pitrou | 7f14f0d | 2010-05-09 16:14:21 +0000 | [diff] [blame] | 56 | PyObject *decoding_readline; /* codecs.open(...).readline */ |
| 57 | PyObject *decoding_buffer; |
Martin v. Löwis | 1ee99d3 | 2002-08-04 20:10:29 +0000 | [diff] [blame] | 58 | #endif |
Antoine Pitrou | 7f14f0d | 2010-05-09 16:14:21 +0000 | [diff] [blame] | 59 | const char* enc; /* Encoding for the current str. */ |
| 60 | const char* str; |
Guido van Rossum | 85a5fbb | 1990-10-14 12:07:46 +0000 | [diff] [blame] | 61 | }; |
| 62 | |
Martin v. Löwis | 95292d6 | 2002-12-11 14:04:59 +0000 | [diff] [blame] | 63 | extern struct tok_state *PyTokenizer_FromString(const char *); |
Benjamin Peterson | f5b5224 | 2009-03-02 23:31:26 +0000 | [diff] [blame] | 64 | extern struct tok_state *PyTokenizer_FromUTF8(const char *); |
Martin v. Löwis | 85bcc66 | 2007-09-04 09:18:06 +0000 | [diff] [blame] | 65 | extern struct tok_state *PyTokenizer_FromFile(FILE *, char*, |
Antoine Pitrou | 7f14f0d | 2010-05-09 16:14:21 +0000 | [diff] [blame] | 66 | char *, char *); |
Tim Peters | dbd9ba6 | 2000-07-09 03:09:57 +0000 | [diff] [blame] | 67 | extern void PyTokenizer_Free(struct tok_state *); |
| 68 | extern int PyTokenizer_Get(struct tok_state *, char **, char **); |
Antoine Pitrou | 7f14f0d | 2010-05-09 16:14:21 +0000 | [diff] [blame] | 69 | extern char * PyTokenizer_RestoreEncoding(struct tok_state* tok, |
| 70 | int len, int *offset); |
Guido van Rossum | 40d20bc | 2007-10-22 00:09:51 +0000 | [diff] [blame] | 71 | extern char * PyTokenizer_FindEncoding(int); |
Guido van Rossum | a330996 | 1993-07-28 09:05:47 +0000 | [diff] [blame] | 72 | |
| 73 | #ifdef __cplusplus |
| 74 | } |
| 75 | #endif |
| 76 | #endif /* !Py_TOKENIZER_H */ |