blob: 2be3bf2e3158999a4ba3b23c99f768c54b9f9e14 [file] [log] [blame]
Guido van Rossuma3309961993-07-28 09:05:47 +00001#ifndef Py_TOKENIZER_H
2#define Py_TOKENIZER_H
3#ifdef __cplusplus
4extern "C" {
5#endif
6
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +00007#include "object.h"
Guido van Rossumf70e43a1991-02-19 12:39:46 +00008
Guido van Rossum85a5fbb1990-10-14 12:07:46 +00009/* Tokenizer interface */
10
Antoine Pitrouf95a1b32010-05-09 15:52:27 +000011#include "token.h" /* For token types */
Guido van Rossum85a5fbb1990-10-14 12:07:46 +000012
Antoine Pitrouf95a1b32010-05-09 15:52:27 +000013#define MAXINDENT 100 /* Max indentation level */
Guido van Rossum85a5fbb1990-10-14 12:07:46 +000014
Neil Schemenauer3f993c32007-09-21 20:50:26 +000015enum decoding_state {
Antoine Pitrouf95a1b32010-05-09 15:52:27 +000016 STATE_INIT,
17 STATE_RAW,
Georg Brandl2b15bd82010-10-29 04:54:13 +000018 STATE_NORMAL /* have a codec associated with input */
Neil Schemenauer3f993c32007-09-21 20:50:26 +000019};
20
Guido van Rossum85a5fbb1990-10-14 12:07:46 +000021/* Tokenizer state */
22struct tok_state {
Antoine Pitrouf95a1b32010-05-09 15:52:27 +000023 /* Input state; buf <= cur <= inp <= end */
24 /* NB an entire line is held in the buffer */
25 char *buf; /* Input buffer, or NULL; malloc'ed if fp != NULL */
26 char *cur; /* Next character in buffer */
27 char *inp; /* End of data in buffer */
28 char *end; /* End of input buffer if buf != NULL */
29 char *start; /* Start of current token if not NULL */
30 int done; /* E_OK normally, E_EOF at EOF, otherwise error code */
31 /* NB If done != E_OK, cur must be == inp!!! */
32 FILE *fp; /* Rest of input; NULL if tokenizing a string */
33 int tabsize; /* Tab spacing */
34 int indent; /* Current indentation index */
35 int indstack[MAXINDENT]; /* Stack of indents */
36 int atbol; /* Nonzero if at begin of new line */
37 int pendin; /* Pending indents (if > 0) or dedents (if < 0) */
38 char *prompt, *nextprompt; /* For interactive prompting */
39 int lineno; /* Current line number */
40 int level; /* () [] {} Parentheses nesting level */
41 /* Used to allow free continuations inside them */
42 /* Stuff for checking on different tab sizes */
Victor Stinner4c7c8c32010-10-16 13:14:10 +000043 const char *filename; /* encoded to the filesystem encoding */
Antoine Pitrouf95a1b32010-05-09 15:52:27 +000044 int altwarning; /* Issue warning if alternate tabs don't match */
45 int alterror; /* Issue error if alternate tabs don't match */
46 int alttabsize; /* Alternate tab spacing */
47 int altindstack[MAXINDENT]; /* Stack of alternate indents */
48 /* Stuff for PEP 0263 */
49 enum decoding_state decoding_state;
50 int decoding_erred; /* whether erred in decoding */
51 int read_coding_spec; /* whether 'coding:...' has been read */
52 char *encoding; /* Source encoding. */
53 int cont_line; /* whether we are in a continuation line. */
54 const char* line_start; /* pointer to start of current line */
Martin v. Löwis1ee99d32002-08-04 20:10:29 +000055#ifndef PGEN
Victor Stinner22a351a2010-10-14 12:04:34 +000056 PyObject *decoding_readline; /* open(...).readline */
Antoine Pitrouf95a1b32010-05-09 15:52:27 +000057 PyObject *decoding_buffer;
Martin v. Löwis1ee99d32002-08-04 20:10:29 +000058#endif
Antoine Pitrouf95a1b32010-05-09 15:52:27 +000059 const char* enc; /* Encoding for the current str. */
60 const char* str;
61 const char* input; /* Tokenizer's newline translated copy of the string. */
Guido van Rossum85a5fbb1990-10-14 12:07:46 +000062};
63
Benjamin Petersonaeaa5922009-11-13 00:17:59 +000064extern struct tok_state *PyTokenizer_FromString(const char *, int);
65extern struct tok_state *PyTokenizer_FromUTF8(const char *, int);
Martin v. Löwis85bcc662007-09-04 09:18:06 +000066extern struct tok_state *PyTokenizer_FromFile(FILE *, char*,
Antoine Pitrouf95a1b32010-05-09 15:52:27 +000067 char *, char *);
Tim Petersdbd9ba62000-07-09 03:09:57 +000068extern void PyTokenizer_Free(struct tok_state *);
69extern int PyTokenizer_Get(struct tok_state *, char **, char **);
Antoine Pitrouf95a1b32010-05-09 15:52:27 +000070extern char * PyTokenizer_RestoreEncoding(struct tok_state* tok,
71 int len, int *offset);
Guido van Rossum40d20bc2007-10-22 00:09:51 +000072extern char * PyTokenizer_FindEncoding(int);
Guido van Rossuma3309961993-07-28 09:05:47 +000073
74#ifdef __cplusplus
75}
76#endif
77#endif /* !Py_TOKENIZER_H */