| Guido van Rossum | 85a5fbb | 1990-10-14 12:07:46 +0000 | [diff] [blame^] | 1 | /* Tokenizer interface */ |
| 2 | |
| 3 | #include "token.h" /* For token types */ |
| 4 | |
| 5 | #define MAXINDENT 100 /* Max indentation level */ |
| 6 | |
| 7 | /* Tokenizer state */ |
| 8 | struct tok_state { |
| 9 | /* Input state; buf <= cur <= inp <= end */ |
| 10 | /* NB an entire token must fit in the buffer */ |
| 11 | char *buf; /* Input buffer */ |
| 12 | char *cur; /* Next character in buffer */ |
| 13 | char *inp; /* End of data in buffer */ |
| 14 | char *end; /* End of input buffer */ |
| 15 | int done; /* 0 normally, 1 at EOF, -1 after error */ |
| 16 | FILE *fp; /* Rest of input; NULL if tokenizing a string */ |
| 17 | int tabsize; /* Tab spacing */ |
| 18 | int indent; /* Current indentation index */ |
| 19 | int indstack[MAXINDENT]; /* Stack of indents */ |
| 20 | int atbol; /* Nonzero if at begin of new line */ |
| 21 | int pendin; /* Pending indents (if > 0) or dedents (if < 0) */ |
| 22 | char *prompt, *nextprompt; /* For interactive prompting */ |
| 23 | int lineno; /* Current line number */ |
| 24 | }; |
| 25 | |
| 26 | extern struct tok_state *tok_setups PROTO((char *)); |
| 27 | extern struct tok_state *tok_setupf PROTO((FILE *, char *ps1, char *ps2)); |
| 28 | extern void tok_free PROTO((struct tok_state *)); |
| 29 | extern int tok_get PROTO((struct tok_state *, char **, char **)); |