blob: a47b537f554b5ab7f4104589b76fde56b0c598e1 [file] [log] [blame]
Guido van Rossuma3309961993-07-28 09:05:47 +00001#ifndef Py_TOKENIZER_H
2#define Py_TOKENIZER_H
3#ifdef __cplusplus
4extern "C" {
5#endif
6
Guido van Rossumf70e43a1991-02-19 12:39:46 +00007/***********************************************************
Guido van Rossumb9f8d6e1995-01-04 19:08:09 +00008Copyright 1991-1995 by Stichting Mathematisch Centrum, Amsterdam,
9The Netherlands.
Guido van Rossumf70e43a1991-02-19 12:39:46 +000010
11 All Rights Reserved
12
Guido van Rossumd266eb41996-10-25 14:44:06 +000013Permission to use, copy, modify, and distribute this software and its
14documentation for any purpose and without fee is hereby granted,
Guido van Rossumf70e43a1991-02-19 12:39:46 +000015provided that the above copyright notice appear in all copies and that
Guido van Rossumd266eb41996-10-25 14:44:06 +000016both that copyright notice and this permission notice appear in
Guido van Rossumf70e43a1991-02-19 12:39:46 +000017supporting documentation, and that the names of Stichting Mathematisch
Guido van Rossumd266eb41996-10-25 14:44:06 +000018Centrum or CWI or Corporation for National Research Initiatives or
19CNRI not be used in advertising or publicity pertaining to
20distribution of the software without specific, written prior
21permission.
Guido van Rossumf70e43a1991-02-19 12:39:46 +000022
Guido van Rossumd266eb41996-10-25 14:44:06 +000023While CWI is the initial source for this software, a modified version
24is made available by the Corporation for National Research Initiatives
25(CNRI) at the Internet address ftp://ftp.python.org.
26
27STICHTING MATHEMATISCH CENTRUM AND CNRI DISCLAIM ALL WARRANTIES WITH
28REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF
29MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL STICHTING MATHEMATISCH
30CENTRUM OR CNRI BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL
31DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR
32PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
33TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
34PERFORMANCE OF THIS SOFTWARE.
Guido van Rossumf70e43a1991-02-19 12:39:46 +000035
36******************************************************************/
37
Guido van Rossum85a5fbb1990-10-14 12:07:46 +000038/* Tokenizer interface */
39
40#include "token.h" /* For token types */
41
42#define MAXINDENT 100 /* Max indentation level */
43
44/* Tokenizer state */
45struct tok_state {
46 /* Input state; buf <= cur <= inp <= end */
Guido van Rossum6ac258d1993-05-12 08:24:20 +000047 /* NB an entire line is held in the buffer */
48 char *buf; /* Input buffer, or NULL; malloc'ed if fp != NULL */
Guido van Rossum85a5fbb1990-10-14 12:07:46 +000049 char *cur; /* Next character in buffer */
50 char *inp; /* End of data in buffer */
Guido van Rossum6ac258d1993-05-12 08:24:20 +000051 char *end; /* End of input buffer if buf != NULL */
Guido van Rossum1d5735e1994-08-30 08:27:36 +000052 char *start; /* Start of current token if not NULL */
Guido van Rossum6ac258d1993-05-12 08:24:20 +000053 int done; /* E_OK normally, E_EOF at EOF, otherwise error code */
54 /* NB If done != E_OK, cur must be == inp!!! */
Guido van Rossum85a5fbb1990-10-14 12:07:46 +000055 FILE *fp; /* Rest of input; NULL if tokenizing a string */
56 int tabsize; /* Tab spacing */
57 int indent; /* Current indentation index */
58 int indstack[MAXINDENT]; /* Stack of indents */
59 int atbol; /* Nonzero if at begin of new line */
60 int pendin; /* Pending indents (if > 0) or dedents (if < 0) */
61 char *prompt, *nextprompt; /* For interactive prompting */
62 int lineno; /* Current line number */
Guido van Rossuma849b831993-05-12 11:35:44 +000063 int level; /* () [] {} Parentheses nesting level */
64 /* Used to allow free continuations inside them */
Guido van Rossum926f13a1998-04-09 21:38:06 +000065 /* Stuff for checking on different tab sizes */
66 char *filename; /* For error messages */
67 int altwarning; /* Issue warning if alternate tabs don't match */
68 int alterror; /* Issue error if alternate tabs don't match */
69 int alttabsize; /* Alternate tab spacing */
70 int altindstack[MAXINDENT]; /* Stack of alternate indents */
Guido van Rossum85a5fbb1990-10-14 12:07:46 +000071};
72
Guido van Rossum86bea461997-04-29 21:03:06 +000073extern struct tok_state *PyTokenizer_FromString Py_PROTO((char *));
74extern struct tok_state *PyTokenizer_FromFile
75 Py_PROTO((FILE *, char *, char *));
76extern void PyTokenizer_Free Py_PROTO((struct tok_state *));
77extern int PyTokenizer_Get Py_PROTO((struct tok_state *, char **, char **));
Guido van Rossuma3309961993-07-28 09:05:47 +000078
79#ifdef __cplusplus
80}
81#endif
82#endif /* !Py_TOKENIZER_H */