blob: d5b36647b6f96d96e04adda62811ff11a2db7e6b [file] [log] [blame]
Guido van Rossuma3309961993-07-28 09:05:47 +00001#ifndef Py_TOKENIZER_H
2#define Py_TOKENIZER_H
3#ifdef __cplusplus
4extern "C" {
5#endif
6
Guido van Rossumf70e43a1991-02-19 12:39:46 +00007/***********************************************************
Guido van Rossumb9f8d6e1995-01-04 19:08:09 +00008Copyright 1991-1995 by Stichting Mathematisch Centrum, Amsterdam,
9The Netherlands.
Guido van Rossumf70e43a1991-02-19 12:39:46 +000010
11 All Rights Reserved
12
13Permission to use, copy, modify, and distribute this software and its
14documentation for any purpose and without fee is hereby granted,
15provided that the above copyright notice appear in all copies and that
16both that copyright notice and this permission notice appear in
17supporting documentation, and that the names of Stichting Mathematisch
18Centrum or CWI not be used in advertising or publicity pertaining to
19distribution of the software without specific, written prior permission.
20
21STICHTING MATHEMATISCH CENTRUM DISCLAIMS ALL WARRANTIES WITH REGARD TO
22THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
23FITNESS, IN NO EVENT SHALL STICHTING MATHEMATISCH CENTRUM BE LIABLE
24FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
25WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
26ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT
27OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
28
29******************************************************************/
30
Guido van Rossum85a5fbb1990-10-14 12:07:46 +000031/* Tokenizer interface */
32
33#include "token.h" /* For token types */
34
35#define MAXINDENT 100 /* Max indentation level */
36
37/* Tokenizer state */
38struct tok_state {
39 /* Input state; buf <= cur <= inp <= end */
Guido van Rossum6ac258d1993-05-12 08:24:20 +000040 /* NB an entire line is held in the buffer */
41 char *buf; /* Input buffer, or NULL; malloc'ed if fp != NULL */
Guido van Rossum85a5fbb1990-10-14 12:07:46 +000042 char *cur; /* Next character in buffer */
43 char *inp; /* End of data in buffer */
Guido van Rossum6ac258d1993-05-12 08:24:20 +000044 char *end; /* End of input buffer if buf != NULL */
Guido van Rossum1d5735e1994-08-30 08:27:36 +000045 char *start; /* Start of current token if not NULL */
Guido van Rossum6ac258d1993-05-12 08:24:20 +000046 int done; /* E_OK normally, E_EOF at EOF, otherwise error code */
47 /* NB If done != E_OK, cur must be == inp!!! */
Guido van Rossum85a5fbb1990-10-14 12:07:46 +000048 FILE *fp; /* Rest of input; NULL if tokenizing a string */
49 int tabsize; /* Tab spacing */
50 int indent; /* Current indentation index */
51 int indstack[MAXINDENT]; /* Stack of indents */
52 int atbol; /* Nonzero if at begin of new line */
53 int pendin; /* Pending indents (if > 0) or dedents (if < 0) */
54 char *prompt, *nextprompt; /* For interactive prompting */
55 int lineno; /* Current line number */
Guido van Rossuma849b831993-05-12 11:35:44 +000056 int level; /* () [] {} Parentheses nesting level */
57 /* Used to allow free continuations inside them */
Guido van Rossum85a5fbb1990-10-14 12:07:46 +000058};
59
60extern struct tok_state *tok_setups PROTO((char *));
Guido van Rossum248a50c1993-12-20 12:53:10 +000061extern struct tok_state *tok_setupf PROTO((FILE *, char *, char *));
Guido van Rossum85a5fbb1990-10-14 12:07:46 +000062extern void tok_free PROTO((struct tok_state *));
63extern int tok_get PROTO((struct tok_state *, char **, char **));
Guido van Rossuma3309961993-07-28 09:05:47 +000064
65#ifdef __cplusplus
66}
67#endif
68#endif /* !Py_TOKENIZER_H */