blob: 28dcea1085a38c50f9bf624966ad4be0d29d5aea [file] [log] [blame]
Guido van Rossumf70e43a1991-02-19 12:39:46 +00001
Guido van Rossum85a5fbb1990-10-14 12:07:46 +00002/* Tokenizer implementation */
3
Jack Jansen7b8c7542002-04-14 20:12:41 +00004#include "Python.h"
Guido van Rossum3f5da241990-12-20 15:06:42 +00005#include "pgenheaders.h"
6
Guido van Rossum85a5fbb1990-10-14 12:07:46 +00007#include <ctype.h>
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +00008#include <assert.h>
Guido van Rossum85a5fbb1990-10-14 12:07:46 +00009
Guido van Rossum85a5fbb1990-10-14 12:07:46 +000010#include "tokenizer.h"
11#include "errcode.h"
12
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +000013#ifndef PGEN
14#include "unicodeobject.h"
Christian Heimes2c9c7a52008-05-26 13:42:13 +000015#include "bytesobject.h"
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +000016#include "fileobject.h"
17#include "codecs.h"
18#include "abstract.h"
19#endif /* PGEN */
20
Martin v. Löwis5b222132007-06-10 09:51:05 +000021#define is_potential_identifier_start(c) (\
Antoine Pitrouf95a1b32010-05-09 15:52:27 +000022 (c >= 'a' && c <= 'z')\
23 || (c >= 'A' && c <= 'Z')\
24 || c == '_'\
25 || (c >= 128))
Martin v. Löwis5b222132007-06-10 09:51:05 +000026
27#define is_potential_identifier_char(c) (\
Antoine Pitrouf95a1b32010-05-09 15:52:27 +000028 (c >= 'a' && c <= 'z')\
29 || (c >= 'A' && c <= 'Z')\
30 || (c >= '0' && c <= '9')\
31 || c == '_'\
32 || (c >= 128))
Martin v. Löwis5b222132007-06-10 09:51:05 +000033
Martin v. Löwis566f6af2002-10-26 14:39:10 +000034extern char *PyOS_Readline(FILE *, FILE *, char *);
Guido van Rossumf4b1a641994-08-29 12:43:07 +000035/* Return malloc'ed string including trailing \n;
36 empty malloc'ed string for EOF;
37 NULL if interrupted */
38
Guido van Rossum4fe87291992-02-26 15:24:44 +000039/* Don't ever change this -- it would break the portability of Python code */
Guido van Rossum85a5fbb1990-10-14 12:07:46 +000040#define TABSIZE 8
Guido van Rossum85a5fbb1990-10-14 12:07:46 +000041
Guido van Rossum3f5da241990-12-20 15:06:42 +000042/* Forward */
Tim Petersdbd9ba62000-07-09 03:09:57 +000043static struct tok_state *tok_new(void);
44static int tok_nextc(struct tok_state *tok);
45static void tok_backup(struct tok_state *tok, int c);
Guido van Rossum3f5da241990-12-20 15:06:42 +000046
Brett Cannond5ec98c2007-10-20 02:54:14 +000047
Guido van Rossum85a5fbb1990-10-14 12:07:46 +000048/* Token names */
49
Guido van Rossum86bea461997-04-29 21:03:06 +000050char *_PyParser_TokenNames[] = {
Antoine Pitrouf95a1b32010-05-09 15:52:27 +000051 "ENDMARKER",
52 "NAME",
53 "NUMBER",
54 "STRING",
55 "NEWLINE",
56 "INDENT",
57 "DEDENT",
58 "LPAR",
59 "RPAR",
60 "LSQB",
61 "RSQB",
62 "COLON",
63 "COMMA",
64 "SEMI",
65 "PLUS",
66 "MINUS",
67 "STAR",
68 "SLASH",
69 "VBAR",
70 "AMPER",
71 "LESS",
72 "GREATER",
73 "EQUAL",
74 "DOT",
75 "PERCENT",
76 "LBRACE",
77 "RBRACE",
78 "EQEQUAL",
79 "NOTEQUAL",
80 "LESSEQUAL",
81 "GREATEREQUAL",
82 "TILDE",
83 "CIRCUMFLEX",
84 "LEFTSHIFT",
85 "RIGHTSHIFT",
86 "DOUBLESTAR",
87 "PLUSEQUAL",
88 "MINEQUAL",
89 "STAREQUAL",
90 "SLASHEQUAL",
91 "PERCENTEQUAL",
92 "AMPEREQUAL",
93 "VBAREQUAL",
94 "CIRCUMFLEXEQUAL",
95 "LEFTSHIFTEQUAL",
96 "RIGHTSHIFTEQUAL",
97 "DOUBLESTAREQUAL",
98 "DOUBLESLASH",
99 "DOUBLESLASHEQUAL",
100 "AT",
101 "RARROW",
102 "ELLIPSIS",
103 /* This table must match the #defines in token.h! */
104 "OP",
105 "<ERRORTOKEN>",
106 "<N_TOKENS>"
Guido van Rossum85a5fbb1990-10-14 12:07:46 +0000107};
108
109
110/* Create and initialize a new tok_state structure */
111
112static struct tok_state *
Thomas Wouters23c9e002000-07-22 19:20:54 +0000113tok_new(void)
Guido van Rossum85a5fbb1990-10-14 12:07:46 +0000114{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000115 struct tok_state *tok = (struct tok_state *)PyMem_MALLOC(
116 sizeof(struct tok_state));
117 if (tok == NULL)
118 return NULL;
119 tok->buf = tok->cur = tok->end = tok->inp = tok->start = NULL;
120 tok->done = E_OK;
121 tok->fp = NULL;
122 tok->input = NULL;
123 tok->tabsize = TABSIZE;
124 tok->indent = 0;
125 tok->indstack[0] = 0;
126 tok->atbol = 1;
127 tok->pendin = 0;
128 tok->prompt = tok->nextprompt = NULL;
129 tok->lineno = 0;
130 tok->level = 0;
131 tok->filename = NULL;
132 tok->altwarning = 1;
133 tok->alterror = 1;
134 tok->alttabsize = 1;
135 tok->altindstack[0] = 0;
136 tok->decoding_state = STATE_INIT;
137 tok->decoding_erred = 0;
138 tok->read_coding_spec = 0;
139 tok->enc = NULL;
140 tok->encoding = NULL;
141 tok->cont_line = 0;
Martin v. Löwis1ee99d32002-08-04 20:10:29 +0000142#ifndef PGEN
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000143 tok->decoding_readline = NULL;
144 tok->decoding_buffer = NULL;
Martin v. Löwis1ee99d32002-08-04 20:10:29 +0000145#endif
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000146 return tok;
Guido van Rossum85a5fbb1990-10-14 12:07:46 +0000147}
148
Benjamin Petersonaeaa5922009-11-13 00:17:59 +0000149static char *
150new_string(const char *s, Py_ssize_t len)
151{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000152 char* result = (char *)PyMem_MALLOC(len + 1);
153 if (result != NULL) {
154 memcpy(result, s, len);
155 result[len] = '\0';
156 }
157 return result;
Benjamin Petersonaeaa5922009-11-13 00:17:59 +0000158}
159
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +0000160#ifdef PGEN
161
162static char *
163decoding_fgets(char *s, int size, struct tok_state *tok)
164{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000165 return fgets(s, size, tok->fp);
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +0000166}
167
168static int
169decoding_feof(struct tok_state *tok)
170{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000171 return feof(tok->fp);
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +0000172}
173
Benjamin Petersonaeaa5922009-11-13 00:17:59 +0000174static char *
175decode_str(const char *str, int exec_input, struct tok_state *tok)
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +0000176{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000177 return new_string(str, strlen(str));
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +0000178}
179
180#else /* PGEN */
181
182static char *
183error_ret(struct tok_state *tok) /* XXX */
184{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000185 tok->decoding_erred = 1;
186 if (tok->fp != NULL && tok->buf != NULL) /* see PyTokenizer_Free */
187 PyMem_FREE(tok->buf);
188 tok->buf = NULL;
189 return NULL; /* as if it were EOF */
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +0000190}
191
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +0000192
193static char *
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000194get_normal_name(char *s) /* for utf-8 and latin-1 */
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +0000195{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000196 char buf[13];
197 int i;
198 for (i = 0; i < 12; i++) {
199 int c = s[i];
200 if (c == '\0')
201 break;
202 else if (c == '_')
203 buf[i] = '-';
204 else
205 buf[i] = tolower(c);
206 }
207 buf[i] = '\0';
208 if (strcmp(buf, "utf-8") == 0 ||
209 strncmp(buf, "utf-8-", 6) == 0)
210 return "utf-8";
211 else if (strcmp(buf, "latin-1") == 0 ||
212 strcmp(buf, "iso-8859-1") == 0 ||
213 strcmp(buf, "iso-latin-1") == 0 ||
214 strncmp(buf, "latin-1-", 8) == 0 ||
215 strncmp(buf, "iso-8859-1-", 11) == 0 ||
216 strncmp(buf, "iso-latin-1-", 12) == 0)
217 return "iso-8859-1";
218 else
219 return s;
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +0000220}
221
222/* Return the coding spec in S, or NULL if none is found. */
223
224static char *
Martin v. Löwis18e16552006-02-15 17:27:45 +0000225get_coding_spec(const char *s, Py_ssize_t size)
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +0000226{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000227 Py_ssize_t i;
228 /* Coding spec must be in a comment, and that comment must be
229 * the only statement on the source code line. */
230 for (i = 0; i < size - 6; i++) {
231 if (s[i] == '#')
232 break;
233 if (s[i] != ' ' && s[i] != '\t' && s[i] != '\014')
234 return NULL;
235 }
236 for (; i < size - 6; i++) { /* XXX inefficient search */
237 const char* t = s + i;
238 if (strncmp(t, "coding", 6) == 0) {
239 const char* begin = NULL;
240 t += 6;
241 if (t[0] != ':' && t[0] != '=')
242 continue;
243 do {
244 t++;
245 } while (t[0] == '\x20' || t[0] == '\t');
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +0000246
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000247 begin = t;
248 while (Py_ISALNUM(t[0]) ||
249 t[0] == '-' || t[0] == '_' || t[0] == '.')
250 t++;
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +0000251
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000252 if (begin < t) {
253 char* r = new_string(begin, t - begin);
254 char* q = get_normal_name(r);
255 if (r != q) {
256 PyMem_FREE(r);
257 r = new_string(q, strlen(q));
258 }
259 return r;
260 }
261 }
262 }
263 return NULL;
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +0000264}
265
266/* Check whether the line contains a coding spec. If it does,
267 invoke the set_readline function for the new encoding.
268 This function receives the tok_state and the new encoding.
269 Return 1 on success, 0 on failure. */
270
271static int
Martin v. Löwis18e16552006-02-15 17:27:45 +0000272check_coding_spec(const char* line, Py_ssize_t size, struct tok_state *tok,
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000273 int set_readline(struct tok_state *, const char *))
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +0000274{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000275 char * cs;
276 int r = 1;
Tim Peters17db21f2002-09-03 15:39:58 +0000277
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000278 if (tok->cont_line)
279 /* It's a continuation line, so it can't be a coding spec. */
280 return 1;
281 cs = get_coding_spec(line, size);
282 if (cs != NULL) {
283 tok->read_coding_spec = 1;
284 if (tok->encoding == NULL) {
285 assert(tok->decoding_state == STATE_RAW);
286 if (strcmp(cs, "utf-8") == 0) {
287 tok->encoding = cs;
288 } else {
289 r = set_readline(tok, cs);
290 if (r) {
291 tok->encoding = cs;
292 tok->decoding_state = STATE_NORMAL;
293 }
294 else
295 PyMem_FREE(cs);
296 }
297 } else { /* then, compare cs with BOM */
298 r = (strcmp(tok->encoding, cs) == 0);
299 PyMem_FREE(cs);
300 }
301 }
302 if (!r) {
303 cs = tok->encoding;
304 if (!cs)
305 cs = "with BOM";
306 PyErr_Format(PyExc_SyntaxError, "encoding problem: %s", cs);
307 }
308 return r;
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +0000309}
310
311/* See whether the file starts with a BOM. If it does,
312 invoke the set_readline function with the new encoding.
313 Return 1 on success, 0 on failure. */
314
315static int
316check_bom(int get_char(struct tok_state *),
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000317 void unget_char(int, struct tok_state *),
318 int set_readline(struct tok_state *, const char *),
319 struct tok_state *tok)
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +0000320{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000321 int ch1, ch2, ch3;
322 ch1 = get_char(tok);
323 tok->decoding_state = STATE_RAW;
324 if (ch1 == EOF) {
325 return 1;
326 } else if (ch1 == 0xEF) {
327 ch2 = get_char(tok);
328 if (ch2 != 0xBB) {
329 unget_char(ch2, tok);
330 unget_char(ch1, tok);
331 return 1;
332 }
333 ch3 = get_char(tok);
334 if (ch3 != 0xBF) {
335 unget_char(ch3, tok);
336 unget_char(ch2, tok);
337 unget_char(ch1, tok);
338 return 1;
339 }
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +0000340#if 0
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000341 /* Disable support for UTF-16 BOMs until a decision
342 is made whether this needs to be supported. */
343 } else if (ch1 == 0xFE) {
344 ch2 = get_char(tok);
345 if (ch2 != 0xFF) {
346 unget_char(ch2, tok);
347 unget_char(ch1, tok);
348 return 1;
349 }
350 if (!set_readline(tok, "utf-16-be"))
351 return 0;
352 tok->decoding_state = STATE_NORMAL;
353 } else if (ch1 == 0xFF) {
354 ch2 = get_char(tok);
355 if (ch2 != 0xFE) {
356 unget_char(ch2, tok);
357 unget_char(ch1, tok);
358 return 1;
359 }
360 if (!set_readline(tok, "utf-16-le"))
361 return 0;
362 tok->decoding_state = STATE_NORMAL;
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +0000363#endif
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000364 } else {
365 unget_char(ch1, tok);
366 return 1;
367 }
368 if (tok->encoding != NULL)
369 PyMem_FREE(tok->encoding);
370 tok->encoding = new_string("utf-8", 5); /* resulting is in utf-8 */
371 /* No need to set_readline: input is already utf-8 */
372 return 1;
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +0000373}
374
375/* Read a line of text from TOK into S, using the stream in TOK.
Walter Dörwaldc1f5fff2005-07-12 21:53:43 +0000376 Return NULL on failure, else S.
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000377
Walter Dörwaldc1f5fff2005-07-12 21:53:43 +0000378 On entry, tok->decoding_buffer will be one of:
379 1) NULL: need to call tok->decoding_readline to get a new line
380 2) PyUnicodeObject *: decoding_feof has called tok->decoding_readline and
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000381 stored the result in tok->decoding_buffer
Christian Heimes9c4756e2008-05-26 13:22:05 +0000382 3) PyByteArrayObject *: previous call to fp_readl did not have enough room
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000383 (in the s buffer) to copy entire contents of the line read
384 by tok->decoding_readline. tok->decoding_buffer has the overflow.
385 In this case, fp_readl is called in a loop (with an expanded buffer)
386 until the buffer ends with a '\n' (or until the end of the file is
387 reached): see tok_nextc and its calls to decoding_fgets.
Walter Dörwaldc1f5fff2005-07-12 21:53:43 +0000388*/
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +0000389
390static char *
391fp_readl(char *s, int size, struct tok_state *tok)
392{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000393 PyObject* bufobj;
394 const char *buf;
395 Py_ssize_t buflen;
Walter Dörwaldc1f5fff2005-07-12 21:53:43 +0000396
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000397 /* Ask for one less byte so we can terminate it */
398 assert(size > 0);
399 size--;
Walter Dörwaldc1f5fff2005-07-12 21:53:43 +0000400
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000401 if (tok->decoding_buffer) {
402 bufobj = tok->decoding_buffer;
403 Py_INCREF(bufobj);
404 }
405 else
406 {
407 bufobj = PyObject_CallObject(tok->decoding_readline, NULL);
408 if (bufobj == NULL)
409 goto error;
410 }
411 if (PyUnicode_CheckExact(bufobj))
412 {
413 buf = _PyUnicode_AsStringAndSize(bufobj, &buflen);
414 if (buf == NULL) {
415 goto error;
416 }
417 }
418 else
419 {
420 buf = PyByteArray_AsString(bufobj);
421 if (buf == NULL) {
422 goto error;
423 }
424 buflen = PyByteArray_GET_SIZE(bufobj);
425 }
Amaury Forgeot d'Arc65f9ace2007-11-15 23:19:43 +0000426
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000427 Py_XDECREF(tok->decoding_buffer);
428 if (buflen > size) {
429 /* Too many chars, the rest goes into tok->decoding_buffer */
430 tok->decoding_buffer = PyByteArray_FromStringAndSize(buf+size,
431 buflen-size);
432 if (tok->decoding_buffer == NULL)
433 goto error;
434 buflen = size;
435 }
436 else
437 tok->decoding_buffer = NULL;
Amaury Forgeot d'Arc65f9ace2007-11-15 23:19:43 +0000438
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000439 memcpy(s, buf, buflen);
440 s[buflen] = '\0';
441 if (buflen == 0) /* EOF */
442 s = NULL;
443 Py_DECREF(bufobj);
444 return s;
Neal Norwitz41eaedd2007-08-12 00:03:22 +0000445
446error:
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000447 Py_XDECREF(bufobj);
448 return error_ret(tok);
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +0000449}
450
451/* Set the readline function for TOK to a StreamReader's
452 readline function. The StreamReader is named ENC.
453
454 This function is called from check_bom and check_coding_spec.
455
456 ENC is usually identical to the future value of tok->encoding,
457 except for the (currently unsupported) case of UTF-16.
458
459 Return 1 on success, 0 on failure. */
460
461static int
462fp_setreadl(struct tok_state *tok, const char* enc)
463{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000464 PyObject *readline = NULL, *stream = NULL, *io = NULL;
Victor Stinner22a351a2010-10-14 12:04:34 +0000465 int fd;
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +0000466
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000467 io = PyImport_ImportModuleNoBlock("io");
468 if (io == NULL)
469 goto cleanup;
Guido van Rossum9cbfffd2007-06-07 00:54:15 +0000470
Victor Stinner22a351a2010-10-14 12:04:34 +0000471 fd = fileno(tok->fp);
472 if (lseek(fd, 0, SEEK_SET) == (off_t)-1) {
473 PyErr_SetFromErrnoWithFilename(PyExc_OSError, NULL);
474 goto cleanup;
475 }
476
477 stream = PyObject_CallMethod(io, "open", "isisOOO",
478 fd, "r", -1, enc, Py_None, Py_None, Py_False);
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000479 if (stream == NULL)
480 goto cleanup;
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +0000481
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000482 Py_XDECREF(tok->decoding_readline);
483 readline = PyObject_GetAttrString(stream, "readline");
484 tok->decoding_readline = readline;
Guido van Rossum9cbfffd2007-06-07 00:54:15 +0000485
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000486 /* The file has been reopened; parsing will restart from
487 * the beginning of the file, we have to reset the line number.
488 * But this function has been called from inside tok_nextc() which
489 * will increment lineno before it returns. So we set it -1 so that
490 * the next call to tok_nextc() will start with tok->lineno == 0.
491 */
492 tok->lineno = -1;
Amaury Forgeot d'Arccf8016a2008-10-09 23:37:48 +0000493
Guido van Rossum9cbfffd2007-06-07 00:54:15 +0000494 cleanup:
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000495 Py_XDECREF(stream);
496 Py_XDECREF(io);
497 return readline != NULL;
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +0000498}
499
500/* Fetch the next byte from TOK. */
501
502static int fp_getc(struct tok_state *tok) {
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000503 return getc(tok->fp);
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +0000504}
505
506/* Unfetch the last byte back into TOK. */
507
508static void fp_ungetc(int c, struct tok_state *tok) {
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000509 ungetc(c, tok->fp);
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +0000510}
511
Martin v. Löwis447d33e2007-07-29 18:10:01 +0000512/* Check whether the characters at s start a valid
513 UTF-8 sequence. Return the number of characters forming
514 the sequence if yes, 0 if not. */
515static int valid_utf8(const unsigned char* s)
516{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000517 int expected = 0;
518 int length;
519 if (*s < 0x80)
520 /* single-byte code */
521 return 1;
522 if (*s < 0xc0)
523 /* following byte */
524 return 0;
525 if (*s < 0xE0)
526 expected = 1;
527 else if (*s < 0xF0)
528 expected = 2;
529 else if (*s < 0xF8)
530 expected = 3;
531 else
532 return 0;
533 length = expected + 1;
534 for (; expected; expected--)
535 if (s[expected] < 0x80 || s[expected] >= 0xC0)
536 return 0;
537 return length;
Martin v. Löwis447d33e2007-07-29 18:10:01 +0000538}
539
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +0000540/* Read a line of input from TOK. Determine encoding
541 if necessary. */
542
543static char *
544decoding_fgets(char *s, int size, struct tok_state *tok)
545{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000546 char *line = NULL;
547 int badchar = 0;
548 for (;;) {
549 if (tok->decoding_state == STATE_NORMAL) {
550 /* We already have a codec associated with
551 this input. */
552 line = fp_readl(s, size, tok);
553 break;
554 } else if (tok->decoding_state == STATE_RAW) {
555 /* We want a 'raw' read. */
556 line = Py_UniversalNewlineFgets(s, size,
557 tok->fp, NULL);
558 break;
559 } else {
560 /* We have not yet determined the encoding.
561 If an encoding is found, use the file-pointer
562 reader functions from now on. */
563 if (!check_bom(fp_getc, fp_ungetc, fp_setreadl, tok))
564 return error_ret(tok);
565 assert(tok->decoding_state != STATE_INIT);
566 }
567 }
568 if (line != NULL && tok->lineno < 2 && !tok->read_coding_spec) {
569 if (!check_coding_spec(line, strlen(line), tok, fp_setreadl)) {
570 return error_ret(tok);
571 }
572 }
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +0000573#ifndef PGEN
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000574 /* The default encoding is UTF-8, so make sure we don't have any
575 non-UTF-8 sequences in it. */
576 if (line && !tok->encoding) {
577 unsigned char *c;
578 int length;
579 for (c = (unsigned char *)line; *c; c += length)
580 if (!(length = valid_utf8(c))) {
581 badchar = *c;
582 break;
583 }
584 }
585 if (badchar) {
586 /* Need to add 1 to the line number, since this line
587 has not been counted, yet. */
588 PyErr_Format(PyExc_SyntaxError,
589 "Non-UTF-8 code starting with '\\x%.2x' "
590 "in file %.200s on line %i, "
591 "but no encoding declared; "
592 "see http://python.org/dev/peps/pep-0263/ for details",
593 badchar, tok->filename, tok->lineno + 1);
594 return error_ret(tok);
595 }
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +0000596#endif
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000597 return line;
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +0000598}
599
600static int
601decoding_feof(struct tok_state *tok)
602{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000603 if (tok->decoding_state != STATE_NORMAL) {
604 return feof(tok->fp);
605 } else {
606 PyObject* buf = tok->decoding_buffer;
607 if (buf == NULL) {
608 buf = PyObject_CallObject(tok->decoding_readline, NULL);
609 if (buf == NULL) {
610 error_ret(tok);
611 return 1;
612 } else {
613 tok->decoding_buffer = buf;
614 }
615 }
616 return PyObject_Length(buf) == 0;
617 }
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +0000618}
619
620/* Fetch a byte from TOK, using the string buffer. */
621
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000622static int
623buf_getc(struct tok_state *tok) {
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000624 return Py_CHARMASK(*tok->str++);
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +0000625}
626
627/* Unfetch a byte from TOK, using the string buffer. */
628
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000629static void
630buf_ungetc(int c, struct tok_state *tok) {
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000631 tok->str--;
632 assert(Py_CHARMASK(*tok->str) == c); /* tok->cur may point to read-only segment */
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +0000633}
634
635/* Set the readline function for TOK to ENC. For the string-based
636 tokenizer, this means to just record the encoding. */
637
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000638static int
639buf_setreadl(struct tok_state *tok, const char* enc) {
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000640 tok->enc = enc;
641 return 1;
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +0000642}
643
644/* Return a UTF-8 encoding Python string object from the
645 C byte string STR, which is encoded with ENC. */
646
647static PyObject *
648translate_into_utf8(const char* str, const char* enc) {
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000649 PyObject *utf8;
650 PyObject* buf = PyUnicode_Decode(str, strlen(str), enc, NULL);
651 if (buf == NULL)
652 return NULL;
653 utf8 = PyUnicode_AsUTF8String(buf);
654 Py_DECREF(buf);
655 return utf8;
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +0000656}
657
Benjamin Petersonaeaa5922009-11-13 00:17:59 +0000658
659static char *
660translate_newlines(const char *s, int exec_input, struct tok_state *tok) {
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000661 int skip_next_lf = 0, needed_length = strlen(s) + 2, final_length;
662 char *buf, *current;
663 char c = '\0';
664 buf = PyMem_MALLOC(needed_length);
665 if (buf == NULL) {
666 tok->done = E_NOMEM;
667 return NULL;
668 }
669 for (current = buf; *s; s++, current++) {
670 c = *s;
671 if (skip_next_lf) {
672 skip_next_lf = 0;
673 if (c == '\n') {
674 c = *++s;
675 if (!c)
676 break;
677 }
678 }
679 if (c == '\r') {
680 skip_next_lf = 1;
681 c = '\n';
682 }
683 *current = c;
684 }
685 /* If this is exec input, add a newline to the end of the string if
686 there isn't one already. */
687 if (exec_input && c != '\n') {
688 *current = '\n';
689 current++;
690 }
691 *current = '\0';
692 final_length = current - buf + 1;
693 if (final_length < needed_length && final_length)
694 /* should never fail */
695 buf = PyMem_REALLOC(buf, final_length);
696 return buf;
Benjamin Petersonaeaa5922009-11-13 00:17:59 +0000697}
698
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +0000699/* Decode a byte string STR for use as the buffer of TOK.
700 Look for encoding declarations inside STR, and record them
701 inside TOK. */
702
703static const char *
Benjamin Petersonaeaa5922009-11-13 00:17:59 +0000704decode_str(const char *input, int single, struct tok_state *tok)
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +0000705{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000706 PyObject* utf8 = NULL;
707 const char *str;
708 const char *s;
709 const char *newl[2] = {NULL, NULL};
710 int lineno = 0;
711 tok->input = str = translate_newlines(input, single, tok);
712 if (str == NULL)
713 return NULL;
714 tok->enc = NULL;
715 tok->str = str;
716 if (!check_bom(buf_getc, buf_ungetc, buf_setreadl, tok))
717 return error_ret(tok);
718 str = tok->str; /* string after BOM if any */
719 assert(str);
720 if (tok->enc != NULL) {
721 utf8 = translate_into_utf8(str, tok->enc);
722 if (utf8 == NULL)
723 return error_ret(tok);
724 str = PyBytes_AsString(utf8);
725 }
726 for (s = str;; s++) {
727 if (*s == '\0') break;
728 else if (*s == '\n') {
729 assert(lineno < 2);
730 newl[lineno] = s;
731 lineno++;
732 if (lineno == 2) break;
733 }
734 }
735 tok->enc = NULL;
736 /* need to check line 1 and 2 separately since check_coding_spec
737 assumes a single line as input */
738 if (newl[0]) {
739 if (!check_coding_spec(str, newl[0] - str, tok, buf_setreadl))
740 return error_ret(tok);
741 if (tok->enc == NULL && newl[1]) {
742 if (!check_coding_spec(newl[0]+1, newl[1] - newl[0],
743 tok, buf_setreadl))
744 return error_ret(tok);
745 }
746 }
747 if (tok->enc != NULL) {
748 assert(utf8 == NULL);
749 utf8 = translate_into_utf8(str, tok->enc);
750 if (utf8 == NULL)
751 return error_ret(tok);
752 str = PyBytes_AS_STRING(utf8);
753 }
754 assert(tok->decoding_buffer == NULL);
755 tok->decoding_buffer = utf8; /* CAUTION */
756 return str;
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +0000757}
758
759#endif /* PGEN */
Guido van Rossum85a5fbb1990-10-14 12:07:46 +0000760
761/* Set up tokenizer for string */
762
763struct tok_state *
Benjamin Petersonaeaa5922009-11-13 00:17:59 +0000764PyTokenizer_FromString(const char *str, int exec_input)
Guido van Rossum85a5fbb1990-10-14 12:07:46 +0000765{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000766 struct tok_state *tok = tok_new();
767 if (tok == NULL)
768 return NULL;
769 str = (char *)decode_str(str, exec_input, tok);
770 if (str == NULL) {
771 PyTokenizer_Free(tok);
772 return NULL;
773 }
Neal Norwitzdee2fd52005-11-16 05:12:59 +0000774
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000775 /* XXX: constify members. */
776 tok->buf = tok->cur = tok->end = tok->inp = (char*)str;
777 return tok;
Guido van Rossum85a5fbb1990-10-14 12:07:46 +0000778}
779
Benjamin Petersonf5b52242009-03-02 23:31:26 +0000780struct tok_state *
Benjamin Petersonaeaa5922009-11-13 00:17:59 +0000781PyTokenizer_FromUTF8(const char *str, int exec_input)
Benjamin Petersonf5b52242009-03-02 23:31:26 +0000782{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000783 struct tok_state *tok = tok_new();
784 if (tok == NULL)
785 return NULL;
Benjamin Petersonaeaa5922009-11-13 00:17:59 +0000786#ifndef PGEN
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000787 tok->input = str = translate_newlines(str, exec_input, tok);
Benjamin Petersonaeaa5922009-11-13 00:17:59 +0000788#endif
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000789 if (str == NULL) {
790 PyTokenizer_Free(tok);
791 return NULL;
792 }
793 tok->decoding_state = STATE_RAW;
794 tok->read_coding_spec = 1;
795 tok->enc = NULL;
796 tok->str = str;
797 tok->encoding = (char *)PyMem_MALLOC(6);
798 if (!tok->encoding) {
799 PyTokenizer_Free(tok);
800 return NULL;
801 }
802 strcpy(tok->encoding, "utf-8");
Benjamin Petersonf5b52242009-03-02 23:31:26 +0000803
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000804 /* XXX: constify members. */
805 tok->buf = tok->cur = tok->end = tok->inp = (char*)str;
806 return tok;
Benjamin Petersonf5b52242009-03-02 23:31:26 +0000807}
808
Guido van Rossum8c11a5c1991-07-27 21:42:56 +0000809/* Set up tokenizer for file */
Guido van Rossum85a5fbb1990-10-14 12:07:46 +0000810
811struct tok_state *
Martin v. Löwis85bcc662007-09-04 09:18:06 +0000812PyTokenizer_FromFile(FILE *fp, char* enc, char *ps1, char *ps2)
Guido van Rossum85a5fbb1990-10-14 12:07:46 +0000813{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000814 struct tok_state *tok = tok_new();
815 if (tok == NULL)
816 return NULL;
817 if ((tok->buf = (char *)PyMem_MALLOC(BUFSIZ)) == NULL) {
818 PyTokenizer_Free(tok);
819 return NULL;
820 }
821 tok->cur = tok->inp = tok->buf;
822 tok->end = tok->buf + BUFSIZ;
823 tok->fp = fp;
824 tok->prompt = ps1;
825 tok->nextprompt = ps2;
826 if (enc != NULL) {
827 /* Must copy encoding declaration since it
828 gets copied into the parse tree. */
829 tok->encoding = PyMem_MALLOC(strlen(enc)+1);
830 if (!tok->encoding) {
831 PyTokenizer_Free(tok);
832 return NULL;
833 }
834 strcpy(tok->encoding, enc);
835 tok->decoding_state = STATE_NORMAL;
836 }
837 return tok;
Guido van Rossum85a5fbb1990-10-14 12:07:46 +0000838}
839
840
841/* Free a tok_state structure */
842
843void
Thomas Wouters23c9e002000-07-22 19:20:54 +0000844PyTokenizer_Free(struct tok_state *tok)
Guido van Rossum85a5fbb1990-10-14 12:07:46 +0000845{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000846 if (tok->encoding != NULL)
847 PyMem_FREE(tok->encoding);
Martin v. Löwis1ee99d32002-08-04 20:10:29 +0000848#ifndef PGEN
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000849 Py_XDECREF(tok->decoding_readline);
850 Py_XDECREF(tok->decoding_buffer);
Martin v. Löwis1ee99d32002-08-04 20:10:29 +0000851#endif
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000852 if (tok->fp != NULL && tok->buf != NULL)
853 PyMem_FREE(tok->buf);
854 if (tok->input)
855 PyMem_FREE((char *)tok->input);
856 PyMem_FREE(tok);
Guido van Rossum85a5fbb1990-10-14 12:07:46 +0000857}
858
Guido van Rossum85a5fbb1990-10-14 12:07:46 +0000859/* Get next char, updating state; error code goes into tok->done */
860
861static int
Thomas Wouters23c9e002000-07-22 19:20:54 +0000862tok_nextc(register struct tok_state *tok)
Guido van Rossum85a5fbb1990-10-14 12:07:46 +0000863{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000864 for (;;) {
865 if (tok->cur != tok->inp) {
866 return Py_CHARMASK(*tok->cur++); /* Fast path */
867 }
868 if (tok->done != E_OK)
869 return EOF;
870 if (tok->fp == NULL) {
871 char *end = strchr(tok->inp, '\n');
872 if (end != NULL)
873 end++;
874 else {
875 end = strchr(tok->inp, '\0');
876 if (end == tok->inp) {
877 tok->done = E_EOF;
878 return EOF;
879 }
880 }
881 if (tok->start == NULL)
882 tok->buf = tok->cur;
883 tok->line_start = tok->cur;
884 tok->lineno++;
885 tok->inp = end;
886 return Py_CHARMASK(*tok->cur++);
887 }
888 if (tok->prompt != NULL) {
889 char *newtok = PyOS_Readline(stdin, stdout, tok->prompt);
Martin v. Löwis85bcc662007-09-04 09:18:06 +0000890#ifndef PGEN
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000891 if (tok->encoding && newtok && *newtok) {
892 /* Recode to UTF-8 */
893 Py_ssize_t buflen;
894 const char* buf;
895 PyObject *u = translate_into_utf8(newtok, tok->encoding);
896 PyMem_FREE(newtok);
897 if (!u) {
898 tok->done = E_DECODE;
899 return EOF;
900 }
901 buflen = PyBytes_GET_SIZE(u);
902 buf = PyBytes_AS_STRING(u);
903 if (!buf) {
904 Py_DECREF(u);
905 tok->done = E_DECODE;
906 return EOF;
907 }
908 newtok = PyMem_MALLOC(buflen+1);
909 strcpy(newtok, buf);
910 Py_DECREF(u);
911 }
Martin v. Löwis85bcc662007-09-04 09:18:06 +0000912#endif
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000913 if (tok->nextprompt != NULL)
914 tok->prompt = tok->nextprompt;
915 if (newtok == NULL)
916 tok->done = E_INTR;
917 else if (*newtok == '\0') {
918 PyMem_FREE(newtok);
919 tok->done = E_EOF;
920 }
921 else if (tok->start != NULL) {
922 size_t start = tok->start - tok->buf;
923 size_t oldlen = tok->cur - tok->buf;
924 size_t newlen = oldlen + strlen(newtok);
925 char *buf = tok->buf;
926 buf = (char *)PyMem_REALLOC(buf, newlen+1);
927 tok->lineno++;
928 if (buf == NULL) {
929 PyMem_FREE(tok->buf);
930 tok->buf = NULL;
931 PyMem_FREE(newtok);
932 tok->done = E_NOMEM;
933 return EOF;
934 }
935 tok->buf = buf;
936 tok->cur = tok->buf + oldlen;
937 tok->line_start = tok->cur;
938 strcpy(tok->buf + oldlen, newtok);
939 PyMem_FREE(newtok);
940 tok->inp = tok->buf + newlen;
941 tok->end = tok->inp + 1;
942 tok->start = tok->buf + start;
943 }
944 else {
945 tok->lineno++;
946 if (tok->buf != NULL)
947 PyMem_FREE(tok->buf);
948 tok->buf = newtok;
949 tok->line_start = tok->buf;
950 tok->cur = tok->buf;
951 tok->line_start = tok->buf;
952 tok->inp = strchr(tok->buf, '\0');
953 tok->end = tok->inp + 1;
954 }
955 }
956 else {
957 int done = 0;
958 Py_ssize_t cur = 0;
959 char *pt;
960 if (tok->start == NULL) {
961 if (tok->buf == NULL) {
962 tok->buf = (char *)
963 PyMem_MALLOC(BUFSIZ);
964 if (tok->buf == NULL) {
965 tok->done = E_NOMEM;
966 return EOF;
967 }
968 tok->end = tok->buf + BUFSIZ;
969 }
970 if (decoding_fgets(tok->buf, (int)(tok->end - tok->buf),
971 tok) == NULL) {
972 tok->done = E_EOF;
973 done = 1;
974 }
975 else {
976 tok->done = E_OK;
977 tok->inp = strchr(tok->buf, '\0');
978 done = tok->inp[-1] == '\n';
979 }
980 }
981 else {
982 cur = tok->cur - tok->buf;
983 if (decoding_feof(tok)) {
984 tok->done = E_EOF;
985 done = 1;
986 }
987 else
988 tok->done = E_OK;
989 }
990 tok->lineno++;
991 /* Read until '\n' or EOF */
992 while (!done) {
993 Py_ssize_t curstart = tok->start == NULL ? -1 :
994 tok->start - tok->buf;
995 Py_ssize_t curvalid = tok->inp - tok->buf;
996 Py_ssize_t newsize = curvalid + BUFSIZ;
997 char *newbuf = tok->buf;
998 newbuf = (char *)PyMem_REALLOC(newbuf,
999 newsize);
1000 if (newbuf == NULL) {
1001 tok->done = E_NOMEM;
1002 tok->cur = tok->inp;
1003 return EOF;
1004 }
1005 tok->buf = newbuf;
1006 tok->inp = tok->buf + curvalid;
1007 tok->end = tok->buf + newsize;
1008 tok->start = curstart < 0 ? NULL :
1009 tok->buf + curstart;
1010 if (decoding_fgets(tok->inp,
1011 (int)(tok->end - tok->inp),
1012 tok) == NULL) {
1013 /* Break out early on decoding
1014 errors, as tok->buf will be NULL
1015 */
1016 if (tok->decoding_erred)
1017 return EOF;
1018 /* Last line does not end in \n,
1019 fake one */
1020 strcpy(tok->inp, "\n");
1021 }
1022 tok->inp = strchr(tok->inp, '\0');
1023 done = tok->inp[-1] == '\n';
1024 }
1025 if (tok->buf != NULL) {
1026 tok->cur = tok->buf + cur;
1027 tok->line_start = tok->cur;
1028 /* replace "\r\n" with "\n" */
1029 /* For Mac leave the \r, giving a syntax error */
1030 pt = tok->inp - 2;
1031 if (pt >= tok->buf && *pt == '\r') {
1032 *pt++ = '\n';
1033 *pt = '\0';
1034 tok->inp = pt;
1035 }
1036 }
1037 }
1038 if (tok->done != E_OK) {
1039 if (tok->prompt != NULL)
1040 PySys_WriteStderr("\n");
1041 tok->cur = tok->inp;
1042 return EOF;
1043 }
1044 }
1045 /*NOTREACHED*/
Guido van Rossum85a5fbb1990-10-14 12:07:46 +00001046}
1047
1048
1049/* Back-up one character */
1050
1051static void
Thomas Wouters23c9e002000-07-22 19:20:54 +00001052tok_backup(register struct tok_state *tok, register int c)
Guido van Rossum85a5fbb1990-10-14 12:07:46 +00001053{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001054 if (c != EOF) {
1055 if (--tok->cur < tok->buf)
1056 Py_FatalError("tok_backup: beginning of buffer");
1057 if (*tok->cur != c)
1058 *tok->cur = c;
1059 }
Guido van Rossum85a5fbb1990-10-14 12:07:46 +00001060}
1061
1062
1063/* Return the token corresponding to a single character */
1064
1065int
Thomas Wouters23c9e002000-07-22 19:20:54 +00001066PyToken_OneChar(int c)
Guido van Rossum85a5fbb1990-10-14 12:07:46 +00001067{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001068 switch (c) {
1069 case '(': return LPAR;
1070 case ')': return RPAR;
1071 case '[': return LSQB;
1072 case ']': return RSQB;
1073 case ':': return COLON;
1074 case ',': return COMMA;
1075 case ';': return SEMI;
1076 case '+': return PLUS;
1077 case '-': return MINUS;
1078 case '*': return STAR;
1079 case '/': return SLASH;
1080 case '|': return VBAR;
1081 case '&': return AMPER;
1082 case '<': return LESS;
1083 case '>': return GREATER;
1084 case '=': return EQUAL;
1085 case '.': return DOT;
1086 case '%': return PERCENT;
1087 case '{': return LBRACE;
1088 case '}': return RBRACE;
1089 case '^': return CIRCUMFLEX;
1090 case '~': return TILDE;
1091 case '@': return AT;
1092 default: return OP;
1093 }
Guido van Rossum85a5fbb1990-10-14 12:07:46 +00001094}
1095
1096
Guido van Rossumfbab9051991-10-20 20:25:03 +00001097int
Thomas Wouters23c9e002000-07-22 19:20:54 +00001098PyToken_TwoChars(int c1, int c2)
Guido van Rossumfbab9051991-10-20 20:25:03 +00001099{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001100 switch (c1) {
1101 case '=':
1102 switch (c2) {
1103 case '=': return EQEQUAL;
1104 }
1105 break;
1106 case '!':
1107 switch (c2) {
1108 case '=': return NOTEQUAL;
1109 }
1110 break;
1111 case '<':
1112 switch (c2) {
1113 case '>': return NOTEQUAL;
1114 case '=': return LESSEQUAL;
1115 case '<': return LEFTSHIFT;
1116 }
1117 break;
1118 case '>':
1119 switch (c2) {
1120 case '=': return GREATEREQUAL;
1121 case '>': return RIGHTSHIFT;
1122 }
1123 break;
1124 case '+':
1125 switch (c2) {
1126 case '=': return PLUSEQUAL;
1127 }
1128 break;
1129 case '-':
1130 switch (c2) {
1131 case '=': return MINEQUAL;
1132 case '>': return RARROW;
1133 }
1134 break;
1135 case '*':
1136 switch (c2) {
1137 case '*': return DOUBLESTAR;
1138 case '=': return STAREQUAL;
1139 }
1140 break;
1141 case '/':
1142 switch (c2) {
1143 case '/': return DOUBLESLASH;
1144 case '=': return SLASHEQUAL;
1145 }
1146 break;
1147 case '|':
1148 switch (c2) {
1149 case '=': return VBAREQUAL;
1150 }
1151 break;
1152 case '%':
1153 switch (c2) {
1154 case '=': return PERCENTEQUAL;
1155 }
1156 break;
1157 case '&':
1158 switch (c2) {
1159 case '=': return AMPEREQUAL;
1160 }
1161 break;
1162 case '^':
1163 switch (c2) {
1164 case '=': return CIRCUMFLEXEQUAL;
1165 }
1166 break;
1167 }
1168 return OP;
Guido van Rossumfbab9051991-10-20 20:25:03 +00001169}
1170
Thomas Wouters434d0822000-08-24 20:11:32 +00001171int
1172PyToken_ThreeChars(int c1, int c2, int c3)
1173{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001174 switch (c1) {
1175 case '<':
1176 switch (c2) {
1177 case '<':
1178 switch (c3) {
1179 case '=':
1180 return LEFTSHIFTEQUAL;
1181 }
1182 break;
1183 }
1184 break;
1185 case '>':
1186 switch (c2) {
1187 case '>':
1188 switch (c3) {
1189 case '=':
1190 return RIGHTSHIFTEQUAL;
1191 }
1192 break;
1193 }
1194 break;
1195 case '*':
1196 switch (c2) {
1197 case '*':
1198 switch (c3) {
1199 case '=':
1200 return DOUBLESTAREQUAL;
1201 }
1202 break;
1203 }
1204 break;
1205 case '/':
1206 switch (c2) {
1207 case '/':
1208 switch (c3) {
1209 case '=':
1210 return DOUBLESLASHEQUAL;
1211 }
1212 break;
1213 }
1214 break;
1215 case '.':
1216 switch (c2) {
Georg Brandldde00282007-03-18 19:01:53 +00001217 case '.':
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001218 switch (c3) {
1219 case '.':
1220 return ELLIPSIS;
1221 }
1222 break;
1223 }
1224 break;
1225 }
1226 return OP;
Thomas Wouters434d0822000-08-24 20:11:32 +00001227}
Guido van Rossumfbab9051991-10-20 20:25:03 +00001228
Guido van Rossum926f13a1998-04-09 21:38:06 +00001229static int
Thomas Wouters23c9e002000-07-22 19:20:54 +00001230indenterror(struct tok_state *tok)
Guido van Rossum926f13a1998-04-09 21:38:06 +00001231{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001232 if (tok->alterror) {
1233 tok->done = E_TABSPACE;
1234 tok->cur = tok->inp;
1235 return 1;
1236 }
1237 if (tok->altwarning) {
1238 PySys_WriteStderr("%s: inconsistent use of tabs and spaces "
1239 "in indentation\n", tok->filename);
1240 tok->altwarning = 0;
1241 }
1242 return 0;
Guido van Rossum926f13a1998-04-09 21:38:06 +00001243}
1244
Martin v. Löwis47383402007-08-15 07:32:56 +00001245#ifdef PGEN
Victor Stinner52f6dd72010-03-12 14:45:56 +00001246#define verify_identifier(tok) 1
Martin v. Löwis47383402007-08-15 07:32:56 +00001247#else
1248/* Verify that the identifier follows PEP 3131. */
1249static int
Victor Stinner52f6dd72010-03-12 14:45:56 +00001250verify_identifier(struct tok_state *tok)
Martin v. Löwis47383402007-08-15 07:32:56 +00001251{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001252 PyObject *s;
1253 int result;
1254 s = PyUnicode_DecodeUTF8(tok->start, tok->cur - tok->start, NULL);
1255 if (s == NULL) {
1256 if (PyErr_ExceptionMatches(PyExc_UnicodeDecodeError)) {
1257 PyErr_Clear();
1258 tok->done = E_IDENTIFIER;
1259 } else {
1260 tok->done = E_ERROR;
1261 }
1262 return 0;
1263 }
1264 result = PyUnicode_IsIdentifier(s);
1265 Py_DECREF(s);
1266 if (result == 0)
1267 tok->done = E_IDENTIFIER;
1268 return result;
Martin v. Löwis47383402007-08-15 07:32:56 +00001269}
1270#endif
Guido van Rossum926f13a1998-04-09 21:38:06 +00001271
Guido van Rossum85a5fbb1990-10-14 12:07:46 +00001272/* Get next token, after space stripping etc. */
1273
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +00001274static int
1275tok_get(register struct tok_state *tok, char **p_start, char **p_end)
Guido van Rossum85a5fbb1990-10-14 12:07:46 +00001276{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001277 register int c;
1278 int blankline, nonascii;
Guido van Rossum8c11a5c1991-07-27 21:42:56 +00001279
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001280 *p_start = *p_end = NULL;
Guido van Rossum8c11a5c1991-07-27 21:42:56 +00001281 nextline:
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001282 tok->start = NULL;
1283 blankline = 0;
Guido van Rossum8c11a5c1991-07-27 21:42:56 +00001284
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001285 /* Get indentation level */
1286 if (tok->atbol) {
1287 register int col = 0;
1288 register int altcol = 0;
1289 tok->atbol = 0;
1290 for (;;) {
1291 c = tok_nextc(tok);
1292 if (c == ' ')
1293 col++, altcol++;
1294 else if (c == '\t') {
1295 col = (col/tok->tabsize + 1) * tok->tabsize;
1296 altcol = (altcol/tok->alttabsize + 1)
1297 * tok->alttabsize;
1298 }
1299 else if (c == '\014') /* Control-L (formfeed) */
1300 col = altcol = 0; /* For Emacs users */
1301 else
1302 break;
1303 }
1304 tok_backup(tok, c);
1305 if (c == '#' || c == '\n') {
1306 /* Lines with only whitespace and/or comments
1307 shouldn't affect the indentation and are
1308 not passed to the parser as NEWLINE tokens,
1309 except *totally* empty lines in interactive
1310 mode, which signal the end of a command group. */
1311 if (col == 0 && c == '\n' && tok->prompt != NULL)
1312 blankline = 0; /* Let it through */
1313 else
1314 blankline = 1; /* Ignore completely */
1315 /* We can't jump back right here since we still
1316 may need to skip to the end of a comment */
1317 }
1318 if (!blankline && tok->level == 0) {
1319 if (col == tok->indstack[tok->indent]) {
1320 /* No change */
1321 if (altcol != tok->altindstack[tok->indent]) {
1322 if (indenterror(tok))
1323 return ERRORTOKEN;
1324 }
1325 }
1326 else if (col > tok->indstack[tok->indent]) {
1327 /* Indent -- always one */
1328 if (tok->indent+1 >= MAXINDENT) {
1329 tok->done = E_TOODEEP;
1330 tok->cur = tok->inp;
1331 return ERRORTOKEN;
1332 }
1333 if (altcol <= tok->altindstack[tok->indent]) {
1334 if (indenterror(tok))
1335 return ERRORTOKEN;
1336 }
1337 tok->pendin++;
1338 tok->indstack[++tok->indent] = col;
1339 tok->altindstack[tok->indent] = altcol;
1340 }
1341 else /* col < tok->indstack[tok->indent] */ {
1342 /* Dedent -- any number, must be consistent */
1343 while (tok->indent > 0 &&
1344 col < tok->indstack[tok->indent]) {
1345 tok->pendin--;
1346 tok->indent--;
1347 }
1348 if (col != tok->indstack[tok->indent]) {
1349 tok->done = E_DEDENT;
1350 tok->cur = tok->inp;
1351 return ERRORTOKEN;
1352 }
1353 if (altcol != tok->altindstack[tok->indent]) {
1354 if (indenterror(tok))
1355 return ERRORTOKEN;
1356 }
1357 }
1358 }
1359 }
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001360
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001361 tok->start = tok->cur;
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001362
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001363 /* Return pending indents/dedents */
1364 if (tok->pendin != 0) {
1365 if (tok->pendin < 0) {
1366 tok->pendin++;
1367 return DEDENT;
1368 }
1369 else {
1370 tok->pendin--;
1371 return INDENT;
1372 }
1373 }
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001374
Guido van Rossum85a5fbb1990-10-14 12:07:46 +00001375 again:
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001376 tok->start = NULL;
1377 /* Skip spaces */
1378 do {
1379 c = tok_nextc(tok);
1380 } while (c == ' ' || c == '\t' || c == '\014');
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001381
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001382 /* Set start of current token */
1383 tok->start = tok->cur - 1;
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001384
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001385 /* Skip comment */
1386 if (c == '#')
1387 while (c != EOF && c != '\n')
1388 c = tok_nextc(tok);
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001389
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001390 /* Check for EOF and errors now */
1391 if (c == EOF) {
1392 return tok->done == E_EOF ? ENDMARKER : ERRORTOKEN;
1393 }
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001394
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001395 /* Identifier (most frequent token!) */
1396 nonascii = 0;
1397 if (is_potential_identifier_start(c)) {
1398 /* Process b"", r"" and br"" */
1399 if (c == 'b' || c == 'B') {
1400 c = tok_nextc(tok);
1401 if (c == '"' || c == '\'')
1402 goto letter_quote;
1403 }
1404 if (c == 'r' || c == 'R') {
1405 c = tok_nextc(tok);
1406 if (c == '"' || c == '\'')
1407 goto letter_quote;
1408 }
1409 while (is_potential_identifier_char(c)) {
1410 if (c >= 128)
1411 nonascii = 1;
1412 c = tok_nextc(tok);
1413 }
1414 tok_backup(tok, c);
1415 if (nonascii &&
1416 !verify_identifier(tok)) {
1417 tok->done = E_IDENTIFIER;
1418 return ERRORTOKEN;
1419 }
1420 *p_start = tok->start;
1421 *p_end = tok->cur;
1422 return NAME;
1423 }
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001424
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001425 /* Newline */
1426 if (c == '\n') {
1427 tok->atbol = 1;
1428 if (blankline || tok->level > 0)
1429 goto nextline;
1430 *p_start = tok->start;
1431 *p_end = tok->cur - 1; /* Leave '\n' out of the string */
1432 tok->cont_line = 0;
1433 return NEWLINE;
1434 }
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001435
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001436 /* Period or number starting with period? */
1437 if (c == '.') {
1438 c = tok_nextc(tok);
1439 if (isdigit(c)) {
1440 goto fraction;
1441 } else if (c == '.') {
1442 c = tok_nextc(tok);
1443 if (c == '.') {
1444 *p_start = tok->start;
1445 *p_end = tok->cur;
1446 return ELLIPSIS;
1447 } else {
1448 tok_backup(tok, c);
1449 }
1450 tok_backup(tok, '.');
1451 } else {
1452 tok_backup(tok, c);
1453 }
1454 *p_start = tok->start;
1455 *p_end = tok->cur;
1456 return DOT;
1457 }
Guido van Rossumf595fde1996-01-12 01:31:58 +00001458
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001459 /* Number */
1460 if (isdigit(c)) {
1461 if (c == '0') {
1462 /* Hex, octal or binary -- maybe. */
1463 c = tok_nextc(tok);
1464 if (c == '.')
1465 goto fraction;
1466 if (c == 'j' || c == 'J')
1467 goto imaginary;
1468 if (c == 'x' || c == 'X') {
Georg Brandlfceab5a2008-01-19 20:08:23 +00001469
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001470 /* Hex */
1471 c = tok_nextc(tok);
1472 if (!isxdigit(c)) {
1473 tok->done = E_TOKEN;
1474 tok_backup(tok, c);
1475 return ERRORTOKEN;
1476 }
1477 do {
1478 c = tok_nextc(tok);
1479 } while (isxdigit(c));
1480 }
1481 else if (c == 'o' || c == 'O') {
1482 /* Octal */
1483 c = tok_nextc(tok);
1484 if (c < '0' || c >= '8') {
1485 tok->done = E_TOKEN;
1486 tok_backup(tok, c);
1487 return ERRORTOKEN;
1488 }
1489 do {
1490 c = tok_nextc(tok);
1491 } while ('0' <= c && c < '8');
1492 }
1493 else if (c == 'b' || c == 'B') {
1494 /* Binary */
1495 c = tok_nextc(tok);
1496 if (c != '0' && c != '1') {
1497 tok->done = E_TOKEN;
1498 tok_backup(tok, c);
1499 return ERRORTOKEN;
1500 }
1501 do {
1502 c = tok_nextc(tok);
1503 } while (c == '0' || c == '1');
1504 }
1505 else {
1506 int nonzero = 0;
1507 /* maybe old-style octal; c is first char of it */
1508 /* in any case, allow '0' as a literal */
1509 while (c == '0')
1510 c = tok_nextc(tok);
1511 while (isdigit(c)) {
1512 nonzero = 1;
1513 c = tok_nextc(tok);
1514 }
1515 if (c == '.')
1516 goto fraction;
1517 else if (c == 'e' || c == 'E')
1518 goto exponent;
1519 else if (c == 'j' || c == 'J')
1520 goto imaginary;
1521 else if (nonzero) {
1522 tok->done = E_TOKEN;
1523 tok_backup(tok, c);
1524 return ERRORTOKEN;
1525 }
1526 }
1527 }
1528 else {
1529 /* Decimal */
1530 do {
1531 c = tok_nextc(tok);
1532 } while (isdigit(c));
1533 {
1534 /* Accept floating point numbers. */
1535 if (c == '.') {
1536 fraction:
1537 /* Fraction */
1538 do {
1539 c = tok_nextc(tok);
1540 } while (isdigit(c));
1541 }
1542 if (c == 'e' || c == 'E') {
1543 exponent:
1544 /* Exponent part */
1545 c = tok_nextc(tok);
1546 if (c == '+' || c == '-')
1547 c = tok_nextc(tok);
1548 if (!isdigit(c)) {
1549 tok->done = E_TOKEN;
1550 tok_backup(tok, c);
1551 return ERRORTOKEN;
1552 }
1553 do {
1554 c = tok_nextc(tok);
1555 } while (isdigit(c));
1556 }
1557 if (c == 'j' || c == 'J')
1558 /* Imaginary part */
1559 imaginary:
1560 c = tok_nextc(tok);
1561 }
1562 }
1563 tok_backup(tok, c);
1564 *p_start = tok->start;
1565 *p_end = tok->cur;
1566 return NUMBER;
1567 }
Guido van Rossum24dacb31997-04-06 03:46:20 +00001568
1569 letter_quote:
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001570 /* String */
1571 if (c == '\'' || c == '"') {
1572 int quote = c;
1573 int quote_size = 1; /* 1 or 3 */
1574 int end_quote_size = 0;
Guido van Rossumcf171a72007-11-16 00:51:45 +00001575
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001576 /* Find the quote size and start of string */
1577 c = tok_nextc(tok);
1578 if (c == quote) {
1579 c = tok_nextc(tok);
1580 if (c == quote)
1581 quote_size = 3;
1582 else
1583 end_quote_size = 1; /* empty string found */
1584 }
1585 if (c != quote)
1586 tok_backup(tok, c);
Guido van Rossumcf171a72007-11-16 00:51:45 +00001587
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001588 /* Get rest of string */
1589 while (end_quote_size != quote_size) {
1590 c = tok_nextc(tok);
1591 if (c == EOF) {
1592 if (quote_size == 3)
1593 tok->done = E_EOFS;
1594 else
1595 tok->done = E_EOLS;
1596 tok->cur = tok->inp;
1597 return ERRORTOKEN;
1598 }
1599 if (quote_size == 1 && c == '\n') {
1600 tok->done = E_EOLS;
1601 tok->cur = tok->inp;
1602 return ERRORTOKEN;
1603 }
1604 if (c == quote)
1605 end_quote_size += 1;
1606 else {
1607 end_quote_size = 0;
1608 if (c == '\\')
1609 c = tok_nextc(tok); /* skip escaped char */
1610 }
1611 }
Guido van Rossumcf171a72007-11-16 00:51:45 +00001612
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001613 *p_start = tok->start;
1614 *p_end = tok->cur;
1615 return STRING;
1616 }
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001617
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001618 /* Line continuation */
1619 if (c == '\\') {
1620 c = tok_nextc(tok);
1621 if (c != '\n') {
1622 tok->done = E_LINECONT;
1623 tok->cur = tok->inp;
1624 return ERRORTOKEN;
1625 }
1626 tok->cont_line = 1;
1627 goto again; /* Read next line */
1628 }
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001629
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001630 /* Check for two-character token */
1631 {
1632 int c2 = tok_nextc(tok);
1633 int token = PyToken_TwoChars(c, c2);
1634 if (token != OP) {
1635 int c3 = tok_nextc(tok);
1636 int token3 = PyToken_ThreeChars(c, c2, c3);
1637 if (token3 != OP) {
1638 token = token3;
1639 } else {
1640 tok_backup(tok, c3);
1641 }
1642 *p_start = tok->start;
1643 *p_end = tok->cur;
1644 return token;
1645 }
1646 tok_backup(tok, c2);
1647 }
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001648
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001649 /* Keep track of parentheses nesting level */
1650 switch (c) {
1651 case '(':
1652 case '[':
1653 case '{':
1654 tok->level++;
1655 break;
1656 case ')':
1657 case ']':
1658 case '}':
1659 tok->level--;
1660 break;
1661 }
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001662
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001663 /* Punctuation character */
1664 *p_start = tok->start;
1665 *p_end = tok->cur;
1666 return PyToken_OneChar(c);
Guido van Rossum85a5fbb1990-10-14 12:07:46 +00001667}
1668
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +00001669int
1670PyTokenizer_Get(struct tok_state *tok, char **p_start, char **p_end)
1671{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001672 int result = tok_get(tok, p_start, p_end);
1673 if (tok->decoding_erred) {
1674 result = ERRORTOKEN;
1675 tok->done = E_DECODE;
1676 }
1677 return result;
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +00001678}
Guido van Rossum85a5fbb1990-10-14 12:07:46 +00001679
Guido van Rossum40d20bc2007-10-22 00:09:51 +00001680/* Get -*- encoding -*- from a Python file.
Guido van Rossumce3a72a2007-10-19 23:16:50 +00001681
1682 PyTokenizer_FindEncoding returns NULL when it can't find the encoding in
Guido van Rossumcf171a72007-11-16 00:51:45 +00001683 the first or second line of the file (in which case the encoding
Brett Cannone4539892007-10-20 03:46:49 +00001684 should be assumed to be PyUnicode_GetDefaultEncoding()).
1685
Guido van Rossum40d20bc2007-10-22 00:09:51 +00001686 The char * returned is malloc'ed via PyMem_MALLOC() and thus must be freed
1687 by the caller.
Guido van Rossumce3a72a2007-10-19 23:16:50 +00001688*/
1689char *
Guido van Rossum40d20bc2007-10-22 00:09:51 +00001690PyTokenizer_FindEncoding(int fd)
1691{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001692 struct tok_state *tok;
1693 FILE *fp;
1694 char *p_start =NULL , *p_end =NULL , *encoding = NULL;
Guido van Rossumce3a72a2007-10-19 23:16:50 +00001695
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001696 fd = dup(fd);
1697 if (fd < 0) {
1698 return NULL;
1699 }
1700 fp = fdopen(fd, "r");
1701 if (fp == NULL) {
1702 return NULL;
1703 }
1704 tok = PyTokenizer_FromFile(fp, NULL, NULL, NULL);
1705 if (tok == NULL) {
1706 fclose(fp);
1707 return NULL;
1708 }
1709 while (tok->lineno < 2 && tok->done == E_OK) {
1710 PyTokenizer_Get(tok, &p_start, &p_end);
1711 }
1712 fclose(fp);
1713 if (tok->encoding) {
1714 encoding = (char *)PyMem_MALLOC(strlen(tok->encoding) + 1);
1715 if (encoding)
1716 strcpy(encoding, tok->encoding);
1717 }
1718 PyTokenizer_Free(tok);
1719 return encoding;
Guido van Rossumce3a72a2007-10-19 23:16:50 +00001720}
Thomas Wouters89d996e2007-09-08 17:39:28 +00001721
Guido van Rossum408027e1996-12-30 16:17:54 +00001722#ifdef Py_DEBUG
Guido van Rossum85a5fbb1990-10-14 12:07:46 +00001723
1724void
Thomas Wouters23c9e002000-07-22 19:20:54 +00001725tok_dump(int type, char *start, char *end)
Guido van Rossum85a5fbb1990-10-14 12:07:46 +00001726{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001727 printf("%s", _PyParser_TokenNames[type]);
1728 if (type == NAME || type == NUMBER || type == STRING || type == OP)
1729 printf("(%.*s)", (int)(end - start), start);
Guido van Rossum85a5fbb1990-10-14 12:07:46 +00001730}
1731
1732#endif