Guido van Rossum | a330996 | 1993-07-28 09:05:47 +0000 | [diff] [blame] | 1 | #ifndef Py_TOKENIZER_H |
| 2 | #define Py_TOKENIZER_H |
| 3 | #ifdef __cplusplus |
| 4 | extern "C" { |
| 5 | #endif |
| 6 | |
Guido van Rossum | f70e43a | 1991-02-19 12:39:46 +0000 | [diff] [blame] | 7 | /*********************************************************** |
Guido van Rossum | fd71b9e | 2000-06-30 23:50:40 +0000 | [diff] [blame] | 8 | Copyright (c) 2000, BeOpen.com. |
| 9 | Copyright (c) 1995-2000, Corporation for National Research Initiatives. |
| 10 | Copyright (c) 1990-1995, Stichting Mathematisch Centrum. |
| 11 | All rights reserved. |
Guido van Rossum | f70e43a | 1991-02-19 12:39:46 +0000 | [diff] [blame] | 12 | |
Guido van Rossum | fd71b9e | 2000-06-30 23:50:40 +0000 | [diff] [blame] | 13 | See the file "Misc/COPYRIGHT" for information on usage and |
| 14 | redistribution of this file, and for a DISCLAIMER OF ALL WARRANTIES. |
Guido van Rossum | f70e43a | 1991-02-19 12:39:46 +0000 | [diff] [blame] | 15 | ******************************************************************/ |
| 16 | |
Guido van Rossum | 85a5fbb | 1990-10-14 12:07:46 +0000 | [diff] [blame] | 17 | /* Tokenizer interface */ |
| 18 | |
| 19 | #include "token.h" /* For token types */ |
| 20 | |
| 21 | #define MAXINDENT 100 /* Max indentation level */ |
| 22 | |
| 23 | /* Tokenizer state */ |
| 24 | struct tok_state { |
| 25 | /* Input state; buf <= cur <= inp <= end */ |
Guido van Rossum | 6ac258d | 1993-05-12 08:24:20 +0000 | [diff] [blame] | 26 | /* NB an entire line is held in the buffer */ |
| 27 | char *buf; /* Input buffer, or NULL; malloc'ed if fp != NULL */ |
Guido van Rossum | 85a5fbb | 1990-10-14 12:07:46 +0000 | [diff] [blame] | 28 | char *cur; /* Next character in buffer */ |
| 29 | char *inp; /* End of data in buffer */ |
Guido van Rossum | 6ac258d | 1993-05-12 08:24:20 +0000 | [diff] [blame] | 30 | char *end; /* End of input buffer if buf != NULL */ |
Guido van Rossum | 1d5735e | 1994-08-30 08:27:36 +0000 | [diff] [blame] | 31 | char *start; /* Start of current token if not NULL */ |
Guido van Rossum | 6ac258d | 1993-05-12 08:24:20 +0000 | [diff] [blame] | 32 | int done; /* E_OK normally, E_EOF at EOF, otherwise error code */ |
| 33 | /* NB If done != E_OK, cur must be == inp!!! */ |
Guido van Rossum | 85a5fbb | 1990-10-14 12:07:46 +0000 | [diff] [blame] | 34 | FILE *fp; /* Rest of input; NULL if tokenizing a string */ |
| 35 | int tabsize; /* Tab spacing */ |
| 36 | int indent; /* Current indentation index */ |
| 37 | int indstack[MAXINDENT]; /* Stack of indents */ |
| 38 | int atbol; /* Nonzero if at begin of new line */ |
| 39 | int pendin; /* Pending indents (if > 0) or dedents (if < 0) */ |
| 40 | char *prompt, *nextprompt; /* For interactive prompting */ |
| 41 | int lineno; /* Current line number */ |
Guido van Rossum | a849b83 | 1993-05-12 11:35:44 +0000 | [diff] [blame] | 42 | int level; /* () [] {} Parentheses nesting level */ |
| 43 | /* Used to allow free continuations inside them */ |
Guido van Rossum | 926f13a | 1998-04-09 21:38:06 +0000 | [diff] [blame] | 44 | /* Stuff for checking on different tab sizes */ |
| 45 | char *filename; /* For error messages */ |
| 46 | int altwarning; /* Issue warning if alternate tabs don't match */ |
| 47 | int alterror; /* Issue error if alternate tabs don't match */ |
| 48 | int alttabsize; /* Alternate tab spacing */ |
| 49 | int altindstack[MAXINDENT]; /* Stack of alternate indents */ |
Guido van Rossum | 85a5fbb | 1990-10-14 12:07:46 +0000 | [diff] [blame] | 50 | }; |
| 51 | |
Tim Peters | dbd9ba6 | 2000-07-09 03:09:57 +0000 | [diff] [blame] | 52 | extern struct tok_state *PyTokenizer_FromString(char *); |
| 53 | extern struct tok_state *PyTokenizer_FromFile(FILE *, char *, char *); |
| 54 | extern void PyTokenizer_Free(struct tok_state *); |
| 55 | extern int PyTokenizer_Get(struct tok_state *, char **, char **); |
Guido van Rossum | a330996 | 1993-07-28 09:05:47 +0000 | [diff] [blame] | 56 | |
| 57 | #ifdef __cplusplus |
| 58 | } |
| 59 | #endif |
| 60 | #endif /* !Py_TOKENIZER_H */ |