Guido van Rossum | f70e43a | 1991-02-19 12:39:46 +0000 | [diff] [blame] | 1 | /*********************************************************** |
Guido van Rossum | 5113f5f | 1992-04-05 14:20:22 +0000 | [diff] [blame] | 2 | Copyright 1991, 1992 by Stichting Mathematisch Centrum, Amsterdam, The |
Guido van Rossum | f70e43a | 1991-02-19 12:39:46 +0000 | [diff] [blame] | 3 | Netherlands. |
| 4 | |
| 5 | All Rights Reserved |
| 6 | |
| 7 | Permission to use, copy, modify, and distribute this software and its |
| 8 | documentation for any purpose and without fee is hereby granted, |
| 9 | provided that the above copyright notice appear in all copies and that |
| 10 | both that copyright notice and this permission notice appear in |
| 11 | supporting documentation, and that the names of Stichting Mathematisch |
| 12 | Centrum or CWI not be used in advertising or publicity pertaining to |
| 13 | distribution of the software without specific, written prior permission. |
| 14 | |
| 15 | STICHTING MATHEMATISCH CENTRUM DISCLAIMS ALL WARRANTIES WITH REGARD TO |
| 16 | THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND |
| 17 | FITNESS, IN NO EVENT SHALL STICHTING MATHEMATISCH CENTRUM BE LIABLE |
| 18 | FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES |
| 19 | WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN |
| 20 | ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT |
| 21 | OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. |
| 22 | |
| 23 | ******************************************************************/ |
| 24 | |
Guido van Rossum | 85a5fbb | 1990-10-14 12:07:46 +0000 | [diff] [blame] | 25 | /* Tokenizer interface */ |
| 26 | |
| 27 | #include "token.h" /* For token types */ |
| 28 | |
| 29 | #define MAXINDENT 100 /* Max indentation level */ |
| 30 | |
| 31 | /* Tokenizer state */ |
| 32 | struct tok_state { |
| 33 | /* Input state; buf <= cur <= inp <= end */ |
| 34 | /* NB an entire token must fit in the buffer */ |
| 35 | char *buf; /* Input buffer */ |
| 36 | char *cur; /* Next character in buffer */ |
| 37 | char *inp; /* End of data in buffer */ |
| 38 | char *end; /* End of input buffer */ |
| 39 | int done; /* 0 normally, 1 at EOF, -1 after error */ |
| 40 | FILE *fp; /* Rest of input; NULL if tokenizing a string */ |
| 41 | int tabsize; /* Tab spacing */ |
| 42 | int indent; /* Current indentation index */ |
| 43 | int indstack[MAXINDENT]; /* Stack of indents */ |
| 44 | int atbol; /* Nonzero if at begin of new line */ |
| 45 | int pendin; /* Pending indents (if > 0) or dedents (if < 0) */ |
| 46 | char *prompt, *nextprompt; /* For interactive prompting */ |
| 47 | int lineno; /* Current line number */ |
| 48 | }; |
| 49 | |
| 50 | extern struct tok_state *tok_setups PROTO((char *)); |
| 51 | extern struct tok_state *tok_setupf PROTO((FILE *, char *ps1, char *ps2)); |
| 52 | extern void tok_free PROTO((struct tok_state *)); |
| 53 | extern int tok_get PROTO((struct tok_state *, char **, char **)); |