blob: c2ec659fed888f769febaa302b90a4d63d009e97 [file] [log] [blame]
Guido van Rossumf70e43a1991-02-19 12:39:46 +00001
Guido van Rossum85a5fbb1990-10-14 12:07:46 +00002/* Tokenizer implementation */
3
Jack Jansen7b8c7542002-04-14 20:12:41 +00004#include "Python.h"
Guido van Rossum3f5da241990-12-20 15:06:42 +00005
Guido van Rossum85a5fbb1990-10-14 12:07:46 +00006#include <ctype.h>
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +00007#include <assert.h>
Guido van Rossum85a5fbb1990-10-14 12:07:46 +00008
Guido van Rossum85a5fbb1990-10-14 12:07:46 +00009#include "tokenizer.h"
10#include "errcode.h"
11
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +000012#include "unicodeobject.h"
Christian Heimes2c9c7a52008-05-26 13:42:13 +000013#include "bytesobject.h"
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +000014#include "fileobject.h"
15#include "codecs.h"
16#include "abstract.h"
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +000017
Victor Stinnerf2ddc6a2017-11-17 01:25:47 -080018/* Alternate tab spacing */
19#define ALTTABSIZE 1
20
Martin v. Löwis5b222132007-06-10 09:51:05 +000021#define is_potential_identifier_start(c) (\
Antoine Pitrouf95a1b32010-05-09 15:52:27 +000022 (c >= 'a' && c <= 'z')\
23 || (c >= 'A' && c <= 'Z')\
24 || c == '_'\
25 || (c >= 128))
Martin v. Löwis5b222132007-06-10 09:51:05 +000026
27#define is_potential_identifier_char(c) (\
Antoine Pitrouf95a1b32010-05-09 15:52:27 +000028 (c >= 'a' && c <= 'z')\
29 || (c >= 'A' && c <= 'Z')\
30 || (c >= '0' && c <= '9')\
31 || c == '_'\
32 || (c >= 128))
Martin v. Löwis5b222132007-06-10 09:51:05 +000033
Serhiy Storchakac6792272013-10-19 21:03:34 +030034extern char *PyOS_Readline(FILE *, FILE *, const char *);
Guido van Rossumf4b1a641994-08-29 12:43:07 +000035/* Return malloc'ed string including trailing \n;
36 empty malloc'ed string for EOF;
37 NULL if interrupted */
38
Guido van Rossum4fe87291992-02-26 15:24:44 +000039/* Don't ever change this -- it would break the portability of Python code */
Guido van Rossum85a5fbb1990-10-14 12:07:46 +000040#define TABSIZE 8
Guido van Rossum85a5fbb1990-10-14 12:07:46 +000041
Guido van Rossum3f5da241990-12-20 15:06:42 +000042/* Forward */
Tim Petersdbd9ba62000-07-09 03:09:57 +000043static struct tok_state *tok_new(void);
44static int tok_nextc(struct tok_state *tok);
45static void tok_backup(struct tok_state *tok, int c);
Guido van Rossum3f5da241990-12-20 15:06:42 +000046
Brett Cannond5ec98c2007-10-20 02:54:14 +000047
Guido van Rossumdcfcd142019-01-31 03:40:27 -080048/* Spaces in this constant are treated as "zero or more spaces or tabs" when
49 tokenizing. */
50static const char* type_comment_prefix = "# type: ";
51
Guido van Rossum85a5fbb1990-10-14 12:07:46 +000052/* Create and initialize a new tok_state structure */
53
54static struct tok_state *
Thomas Wouters23c9e002000-07-22 19:20:54 +000055tok_new(void)
Guido van Rossum85a5fbb1990-10-14 12:07:46 +000056{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +000057 struct tok_state *tok = (struct tok_state *)PyMem_MALLOC(
58 sizeof(struct tok_state));
59 if (tok == NULL)
60 return NULL;
61 tok->buf = tok->cur = tok->end = tok->inp = tok->start = NULL;
62 tok->done = E_OK;
63 tok->fp = NULL;
64 tok->input = NULL;
65 tok->tabsize = TABSIZE;
66 tok->indent = 0;
67 tok->indstack[0] = 0;
Yury Selivanov75445082015-05-11 22:57:16 -040068
Antoine Pitrouf95a1b32010-05-09 15:52:27 +000069 tok->atbol = 1;
70 tok->pendin = 0;
71 tok->prompt = tok->nextprompt = NULL;
72 tok->lineno = 0;
73 tok->level = 0;
Antoine Pitrouf95a1b32010-05-09 15:52:27 +000074 tok->altindstack[0] = 0;
75 tok->decoding_state = STATE_INIT;
76 tok->decoding_erred = 0;
77 tok->read_coding_spec = 0;
78 tok->enc = NULL;
79 tok->encoding = NULL;
80 tok->cont_line = 0;
Victor Stinner7f2fee32011-04-05 00:39:01 +020081 tok->filename = NULL;
Antoine Pitrouf95a1b32010-05-09 15:52:27 +000082 tok->decoding_readline = NULL;
83 tok->decoding_buffer = NULL;
Guido van Rossumdcfcd142019-01-31 03:40:27 -080084 tok->type_comments = 0;
Yury Selivanov96ec9342015-07-23 15:01:58 +030085
Guido van Rossum495da292019-03-07 12:38:08 -080086 tok->async_hacks = 0;
87 tok->async_def = 0;
88 tok->async_def_indent = 0;
89 tok->async_def_nl = 0;
90
Antoine Pitrouf95a1b32010-05-09 15:52:27 +000091 return tok;
Guido van Rossum85a5fbb1990-10-14 12:07:46 +000092}
93
Benjamin Petersonaeaa5922009-11-13 00:17:59 +000094static char *
Benjamin Peterson2dbfd882013-07-15 19:15:34 -070095new_string(const char *s, Py_ssize_t len, struct tok_state *tok)
Benjamin Petersonaeaa5922009-11-13 00:17:59 +000096{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +000097 char* result = (char *)PyMem_MALLOC(len + 1);
Benjamin Peterson2dbfd882013-07-15 19:15:34 -070098 if (!result) {
99 tok->done = E_NOMEM;
100 return NULL;
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000101 }
Benjamin Peterson2dbfd882013-07-15 19:15:34 -0700102 memcpy(result, s, len);
103 result[len] = '\0';
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000104 return result;
Benjamin Petersonaeaa5922009-11-13 00:17:59 +0000105}
106
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +0000107static char *
108error_ret(struct tok_state *tok) /* XXX */
109{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000110 tok->decoding_erred = 1;
111 if (tok->fp != NULL && tok->buf != NULL) /* see PyTokenizer_Free */
112 PyMem_FREE(tok->buf);
Serhiy Storchaka0d441112015-11-14 15:10:35 +0200113 tok->buf = tok->cur = tok->end = tok->inp = tok->start = NULL;
114 tok->done = E_DECODE;
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000115 return NULL; /* as if it were EOF */
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +0000116}
117
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +0000118
Serhiy Storchakaef1585e2015-12-25 20:01:53 +0200119static const char *
120get_normal_name(const char *s) /* for utf-8 and latin-1 */
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +0000121{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000122 char buf[13];
123 int i;
124 for (i = 0; i < 12; i++) {
125 int c = s[i];
126 if (c == '\0')
127 break;
128 else if (c == '_')
129 buf[i] = '-';
130 else
131 buf[i] = tolower(c);
132 }
133 buf[i] = '\0';
134 if (strcmp(buf, "utf-8") == 0 ||
135 strncmp(buf, "utf-8-", 6) == 0)
136 return "utf-8";
137 else if (strcmp(buf, "latin-1") == 0 ||
138 strcmp(buf, "iso-8859-1") == 0 ||
139 strcmp(buf, "iso-latin-1") == 0 ||
140 strncmp(buf, "latin-1-", 8) == 0 ||
141 strncmp(buf, "iso-8859-1-", 11) == 0 ||
142 strncmp(buf, "iso-latin-1-", 12) == 0)
143 return "iso-8859-1";
144 else
145 return s;
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +0000146}
147
148/* Return the coding spec in S, or NULL if none is found. */
149
Benjamin Peterson2dbfd882013-07-15 19:15:34 -0700150static int
151get_coding_spec(const char *s, char **spec, Py_ssize_t size, struct tok_state *tok)
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +0000152{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000153 Py_ssize_t i;
Benjamin Peterson2dbfd882013-07-15 19:15:34 -0700154 *spec = NULL;
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000155 /* Coding spec must be in a comment, and that comment must be
156 * the only statement on the source code line. */
157 for (i = 0; i < size - 6; i++) {
158 if (s[i] == '#')
159 break;
160 if (s[i] != ' ' && s[i] != '\t' && s[i] != '\014')
Benjamin Peterson2dbfd882013-07-15 19:15:34 -0700161 return 1;
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000162 }
163 for (; i < size - 6; i++) { /* XXX inefficient search */
164 const char* t = s + i;
165 if (strncmp(t, "coding", 6) == 0) {
166 const char* begin = NULL;
167 t += 6;
168 if (t[0] != ':' && t[0] != '=')
169 continue;
170 do {
171 t++;
172 } while (t[0] == '\x20' || t[0] == '\t');
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +0000173
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000174 begin = t;
175 while (Py_ISALNUM(t[0]) ||
176 t[0] == '-' || t[0] == '_' || t[0] == '.')
177 t++;
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +0000178
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000179 if (begin < t) {
Benjamin Peterson2dbfd882013-07-15 19:15:34 -0700180 char* r = new_string(begin, t - begin, tok);
Serhiy Storchakaef1585e2015-12-25 20:01:53 +0200181 const char* q;
Benjamin Peterson2dbfd882013-07-15 19:15:34 -0700182 if (!r)
183 return 0;
Benjamin Peterson265fba42013-07-15 20:50:22 -0700184 q = get_normal_name(r);
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000185 if (r != q) {
186 PyMem_FREE(r);
Benjamin Peterson2dbfd882013-07-15 19:15:34 -0700187 r = new_string(q, strlen(q), tok);
188 if (!r)
189 return 0;
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000190 }
Benjamin Peterson2dbfd882013-07-15 19:15:34 -0700191 *spec = r;
Serhiy Storchakae431d3c2016-03-20 23:36:29 +0200192 break;
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000193 }
194 }
195 }
Benjamin Peterson2dbfd882013-07-15 19:15:34 -0700196 return 1;
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +0000197}
198
199/* Check whether the line contains a coding spec. If it does,
200 invoke the set_readline function for the new encoding.
201 This function receives the tok_state and the new encoding.
202 Return 1 on success, 0 on failure. */
203
204static int
Martin v. Löwis18e16552006-02-15 17:27:45 +0000205check_coding_spec(const char* line, Py_ssize_t size, struct tok_state *tok,
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000206 int set_readline(struct tok_state *, const char *))
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +0000207{
Benjamin Peterson2dbfd882013-07-15 19:15:34 -0700208 char *cs;
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000209 int r = 1;
Tim Peters17db21f2002-09-03 15:39:58 +0000210
Serhiy Storchaka768c16c2014-01-09 18:36:09 +0200211 if (tok->cont_line) {
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000212 /* It's a continuation line, so it can't be a coding spec. */
Serhiy Storchaka768c16c2014-01-09 18:36:09 +0200213 tok->read_coding_spec = 1;
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000214 return 1;
Serhiy Storchaka768c16c2014-01-09 18:36:09 +0200215 }
Benjamin Peterson2dbfd882013-07-15 19:15:34 -0700216 if (!get_coding_spec(line, &cs, size, tok))
217 return 0;
Serhiy Storchaka768c16c2014-01-09 18:36:09 +0200218 if (!cs) {
219 Py_ssize_t i;
220 for (i = 0; i < size; i++) {
221 if (line[i] == '#' || line[i] == '\n' || line[i] == '\r')
222 break;
223 if (line[i] != ' ' && line[i] != '\t' && line[i] != '\014') {
224 /* Stop checking coding spec after a line containing
225 * anything except a comment. */
226 tok->read_coding_spec = 1;
227 break;
228 }
229 }
Benjamin Peterson2dbfd882013-07-15 19:15:34 -0700230 return 1;
Serhiy Storchaka768c16c2014-01-09 18:36:09 +0200231 }
Benjamin Peterson2dbfd882013-07-15 19:15:34 -0700232 tok->read_coding_spec = 1;
233 if (tok->encoding == NULL) {
234 assert(tok->decoding_state == STATE_RAW);
235 if (strcmp(cs, "utf-8") == 0) {
236 tok->encoding = cs;
237 } else {
238 r = set_readline(tok, cs);
239 if (r) {
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000240 tok->encoding = cs;
Benjamin Peterson2dbfd882013-07-15 19:15:34 -0700241 tok->decoding_state = STATE_NORMAL;
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000242 }
Benjamin Peterson2dbfd882013-07-15 19:15:34 -0700243 else {
Serhiy Storchaka3af14aa2013-06-09 16:51:52 +0300244 PyErr_Format(PyExc_SyntaxError,
Benjamin Peterson2dbfd882013-07-15 19:15:34 -0700245 "encoding problem: %s", cs);
246 PyMem_FREE(cs);
247 }
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000248 }
Benjamin Peterson2dbfd882013-07-15 19:15:34 -0700249 } else { /* then, compare cs with BOM */
250 r = (strcmp(tok->encoding, cs) == 0);
251 if (!r)
252 PyErr_Format(PyExc_SyntaxError,
253 "encoding problem: %s with BOM", cs);
254 PyMem_FREE(cs);
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000255 }
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000256 return r;
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +0000257}
258
259/* See whether the file starts with a BOM. If it does,
260 invoke the set_readline function with the new encoding.
261 Return 1 on success, 0 on failure. */
262
263static int
264check_bom(int get_char(struct tok_state *),
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000265 void unget_char(int, struct tok_state *),
266 int set_readline(struct tok_state *, const char *),
267 struct tok_state *tok)
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +0000268{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000269 int ch1, ch2, ch3;
270 ch1 = get_char(tok);
271 tok->decoding_state = STATE_RAW;
272 if (ch1 == EOF) {
273 return 1;
274 } else if (ch1 == 0xEF) {
275 ch2 = get_char(tok);
276 if (ch2 != 0xBB) {
277 unget_char(ch2, tok);
278 unget_char(ch1, tok);
279 return 1;
280 }
281 ch3 = get_char(tok);
282 if (ch3 != 0xBF) {
283 unget_char(ch3, tok);
284 unget_char(ch2, tok);
285 unget_char(ch1, tok);
286 return 1;
287 }
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +0000288#if 0
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000289 /* Disable support for UTF-16 BOMs until a decision
290 is made whether this needs to be supported. */
291 } else if (ch1 == 0xFE) {
292 ch2 = get_char(tok);
293 if (ch2 != 0xFF) {
294 unget_char(ch2, tok);
295 unget_char(ch1, tok);
296 return 1;
297 }
298 if (!set_readline(tok, "utf-16-be"))
299 return 0;
300 tok->decoding_state = STATE_NORMAL;
301 } else if (ch1 == 0xFF) {
302 ch2 = get_char(tok);
303 if (ch2 != 0xFE) {
304 unget_char(ch2, tok);
305 unget_char(ch1, tok);
306 return 1;
307 }
308 if (!set_readline(tok, "utf-16-le"))
309 return 0;
310 tok->decoding_state = STATE_NORMAL;
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +0000311#endif
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000312 } else {
313 unget_char(ch1, tok);
314 return 1;
315 }
316 if (tok->encoding != NULL)
317 PyMem_FREE(tok->encoding);
Benjamin Peterson2dbfd882013-07-15 19:15:34 -0700318 tok->encoding = new_string("utf-8", 5, tok);
319 if (!tok->encoding)
320 return 0;
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000321 /* No need to set_readline: input is already utf-8 */
322 return 1;
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +0000323}
324
325/* Read a line of text from TOK into S, using the stream in TOK.
Walter Dörwaldc1f5fff2005-07-12 21:53:43 +0000326 Return NULL on failure, else S.
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000327
Walter Dörwaldc1f5fff2005-07-12 21:53:43 +0000328 On entry, tok->decoding_buffer will be one of:
329 1) NULL: need to call tok->decoding_readline to get a new line
330 2) PyUnicodeObject *: decoding_feof has called tok->decoding_readline and
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000331 stored the result in tok->decoding_buffer
Christian Heimes9c4756e2008-05-26 13:22:05 +0000332 3) PyByteArrayObject *: previous call to fp_readl did not have enough room
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000333 (in the s buffer) to copy entire contents of the line read
334 by tok->decoding_readline. tok->decoding_buffer has the overflow.
335 In this case, fp_readl is called in a loop (with an expanded buffer)
336 until the buffer ends with a '\n' (or until the end of the file is
337 reached): see tok_nextc and its calls to decoding_fgets.
Walter Dörwaldc1f5fff2005-07-12 21:53:43 +0000338*/
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +0000339
340static char *
341fp_readl(char *s, int size, struct tok_state *tok)
342{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000343 PyObject* bufobj;
344 const char *buf;
345 Py_ssize_t buflen;
Walter Dörwaldc1f5fff2005-07-12 21:53:43 +0000346
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000347 /* Ask for one less byte so we can terminate it */
348 assert(size > 0);
349 size--;
Walter Dörwaldc1f5fff2005-07-12 21:53:43 +0000350
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000351 if (tok->decoding_buffer) {
352 bufobj = tok->decoding_buffer;
353 Py_INCREF(bufobj);
354 }
355 else
356 {
Victor Stinnera5ed5f02016-12-06 18:45:50 +0100357 bufobj = _PyObject_CallNoArg(tok->decoding_readline);
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000358 if (bufobj == NULL)
359 goto error;
360 }
361 if (PyUnicode_CheckExact(bufobj))
362 {
Serhiy Storchaka06515832016-11-20 09:13:07 +0200363 buf = PyUnicode_AsUTF8AndSize(bufobj, &buflen);
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000364 if (buf == NULL) {
365 goto error;
366 }
367 }
368 else
369 {
370 buf = PyByteArray_AsString(bufobj);
371 if (buf == NULL) {
372 goto error;
373 }
374 buflen = PyByteArray_GET_SIZE(bufobj);
375 }
Amaury Forgeot d'Arc65f9ace2007-11-15 23:19:43 +0000376
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000377 Py_XDECREF(tok->decoding_buffer);
378 if (buflen > size) {
379 /* Too many chars, the rest goes into tok->decoding_buffer */
380 tok->decoding_buffer = PyByteArray_FromStringAndSize(buf+size,
381 buflen-size);
382 if (tok->decoding_buffer == NULL)
383 goto error;
384 buflen = size;
385 }
386 else
387 tok->decoding_buffer = NULL;
Amaury Forgeot d'Arc65f9ace2007-11-15 23:19:43 +0000388
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000389 memcpy(s, buf, buflen);
390 s[buflen] = '\0';
391 if (buflen == 0) /* EOF */
392 s = NULL;
393 Py_DECREF(bufobj);
394 return s;
Neal Norwitz41eaedd2007-08-12 00:03:22 +0000395
396error:
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000397 Py_XDECREF(bufobj);
398 return error_ret(tok);
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +0000399}
400
401/* Set the readline function for TOK to a StreamReader's
402 readline function. The StreamReader is named ENC.
403
404 This function is called from check_bom and check_coding_spec.
405
406 ENC is usually identical to the future value of tok->encoding,
407 except for the (currently unsupported) case of UTF-16.
408
409 Return 1 on success, 0 on failure. */
410
411static int
412fp_setreadl(struct tok_state *tok, const char* enc)
413{
Benjamin Peterson35ee9482016-09-12 22:06:58 -0700414 PyObject *readline, *io, *stream;
Martin v. Löwisbd928fe2011-10-14 10:20:37 +0200415 _Py_IDENTIFIER(open);
416 _Py_IDENTIFIER(readline);
Victor Stinner22a351a2010-10-14 12:04:34 +0000417 int fd;
Serhiy Storchaka768c16c2014-01-09 18:36:09 +0200418 long pos;
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +0000419
Victor Stinner22a351a2010-10-14 12:04:34 +0000420 fd = fileno(tok->fp);
Serhiy Storchaka768c16c2014-01-09 18:36:09 +0200421 /* Due to buffering the file offset for fd can be different from the file
Martin v. Löwis815b41b2014-02-28 15:27:29 +0100422 * position of tok->fp. If tok->fp was opened in text mode on Windows,
423 * its file position counts CRLF as one char and can't be directly mapped
424 * to the file offset for fd. Instead we step back one byte and read to
425 * the end of line.*/
Serhiy Storchaka768c16c2014-01-09 18:36:09 +0200426 pos = ftell(tok->fp);
Martin v. Löwis815b41b2014-02-28 15:27:29 +0100427 if (pos == -1 ||
428 lseek(fd, (off_t)(pos > 0 ? pos - 1 : pos), SEEK_SET) == (off_t)-1) {
Victor Stinner22a351a2010-10-14 12:04:34 +0000429 PyErr_SetFromErrnoWithFilename(PyExc_OSError, NULL);
Benjamin Peterson35ee9482016-09-12 22:06:58 -0700430 return 0;
Victor Stinner22a351a2010-10-14 12:04:34 +0000431 }
432
Benjamin Peterson35ee9482016-09-12 22:06:58 -0700433 io = PyImport_ImportModuleNoBlock("io");
434 if (io == NULL)
435 return 0;
436
Martin v. Löwisafe55bb2011-10-09 10:38:36 +0200437 stream = _PyObject_CallMethodId(io, &PyId_open, "isisOOO",
Victor Stinner22a351a2010-10-14 12:04:34 +0000438 fd, "r", -1, enc, Py_None, Py_None, Py_False);
Benjamin Peterson35ee9482016-09-12 22:06:58 -0700439 Py_DECREF(io);
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000440 if (stream == NULL)
Benjamin Peterson35ee9482016-09-12 22:06:58 -0700441 return 0;
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +0000442
Martin v. Löwis1ee1b6f2011-10-10 18:11:30 +0200443 readline = _PyObject_GetAttrId(stream, &PyId_readline);
Benjamin Peterson35ee9482016-09-12 22:06:58 -0700444 Py_DECREF(stream);
445 if (readline == NULL)
446 return 0;
Serhiy Storchaka48842712016-04-06 09:45:48 +0300447 Py_XSETREF(tok->decoding_readline, readline);
Benjamin Peterson35ee9482016-09-12 22:06:58 -0700448
Martin v. Löwis815b41b2014-02-28 15:27:29 +0100449 if (pos > 0) {
Victor Stinnera5ed5f02016-12-06 18:45:50 +0100450 PyObject *bufobj = _PyObject_CallNoArg(readline);
Benjamin Peterson35ee9482016-09-12 22:06:58 -0700451 if (bufobj == NULL)
452 return 0;
453 Py_DECREF(bufobj);
Martin v. Löwis815b41b2014-02-28 15:27:29 +0100454 }
Guido van Rossum9cbfffd2007-06-07 00:54:15 +0000455
Benjamin Peterson35ee9482016-09-12 22:06:58 -0700456 return 1;
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +0000457}
458
459/* Fetch the next byte from TOK. */
460
461static int fp_getc(struct tok_state *tok) {
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000462 return getc(tok->fp);
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +0000463}
464
465/* Unfetch the last byte back into TOK. */
466
467static void fp_ungetc(int c, struct tok_state *tok) {
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000468 ungetc(c, tok->fp);
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +0000469}
470
Martin v. Löwis447d33e2007-07-29 18:10:01 +0000471/* Check whether the characters at s start a valid
472 UTF-8 sequence. Return the number of characters forming
473 the sequence if yes, 0 if not. */
474static int valid_utf8(const unsigned char* s)
475{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000476 int expected = 0;
477 int length;
478 if (*s < 0x80)
479 /* single-byte code */
480 return 1;
481 if (*s < 0xc0)
482 /* following byte */
483 return 0;
484 if (*s < 0xE0)
485 expected = 1;
486 else if (*s < 0xF0)
487 expected = 2;
488 else if (*s < 0xF8)
489 expected = 3;
490 else
491 return 0;
492 length = expected + 1;
493 for (; expected; expected--)
494 if (s[expected] < 0x80 || s[expected] >= 0xC0)
495 return 0;
496 return length;
Martin v. Löwis447d33e2007-07-29 18:10:01 +0000497}
498
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +0000499/* Read a line of input from TOK. Determine encoding
500 if necessary. */
501
502static char *
503decoding_fgets(char *s, int size, struct tok_state *tok)
504{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000505 char *line = NULL;
506 int badchar = 0;
507 for (;;) {
508 if (tok->decoding_state == STATE_NORMAL) {
509 /* We already have a codec associated with
510 this input. */
511 line = fp_readl(s, size, tok);
512 break;
513 } else if (tok->decoding_state == STATE_RAW) {
514 /* We want a 'raw' read. */
515 line = Py_UniversalNewlineFgets(s, size,
516 tok->fp, NULL);
517 break;
518 } else {
519 /* We have not yet determined the encoding.
520 If an encoding is found, use the file-pointer
521 reader functions from now on. */
522 if (!check_bom(fp_getc, fp_ungetc, fp_setreadl, tok))
523 return error_ret(tok);
524 assert(tok->decoding_state != STATE_INIT);
525 }
526 }
527 if (line != NULL && tok->lineno < 2 && !tok->read_coding_spec) {
528 if (!check_coding_spec(line, strlen(line), tok, fp_setreadl)) {
529 return error_ret(tok);
530 }
531 }
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000532 /* The default encoding is UTF-8, so make sure we don't have any
533 non-UTF-8 sequences in it. */
534 if (line && !tok->encoding) {
535 unsigned char *c;
536 int length;
537 for (c = (unsigned char *)line; *c; c += length)
538 if (!(length = valid_utf8(c))) {
539 badchar = *c;
540 break;
541 }
542 }
543 if (badchar) {
544 /* Need to add 1 to the line number, since this line
545 has not been counted, yet. */
Jesus Ceac1935d22011-04-25 04:03:58 +0200546 PyErr_Format(PyExc_SyntaxError,
547 "Non-UTF-8 code starting with '\\x%.2x' "
548 "in file %U on line %i, "
549 "but no encoding declared; "
550 "see http://python.org/dev/peps/pep-0263/ for details",
551 badchar, tok->filename, tok->lineno + 1);
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000552 return error_ret(tok);
553 }
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000554 return line;
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +0000555}
556
557static int
558decoding_feof(struct tok_state *tok)
559{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000560 if (tok->decoding_state != STATE_NORMAL) {
561 return feof(tok->fp);
562 } else {
563 PyObject* buf = tok->decoding_buffer;
564 if (buf == NULL) {
Victor Stinnera5ed5f02016-12-06 18:45:50 +0100565 buf = _PyObject_CallNoArg(tok->decoding_readline);
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000566 if (buf == NULL) {
567 error_ret(tok);
568 return 1;
569 } else {
570 tok->decoding_buffer = buf;
571 }
572 }
573 return PyObject_Length(buf) == 0;
574 }
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +0000575}
576
577/* Fetch a byte from TOK, using the string buffer. */
578
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000579static int
580buf_getc(struct tok_state *tok) {
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000581 return Py_CHARMASK(*tok->str++);
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +0000582}
583
584/* Unfetch a byte from TOK, using the string buffer. */
585
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000586static void
587buf_ungetc(int c, struct tok_state *tok) {
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000588 tok->str--;
589 assert(Py_CHARMASK(*tok->str) == c); /* tok->cur may point to read-only segment */
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +0000590}
591
592/* Set the readline function for TOK to ENC. For the string-based
593 tokenizer, this means to just record the encoding. */
594
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000595static int
596buf_setreadl(struct tok_state *tok, const char* enc) {
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000597 tok->enc = enc;
598 return 1;
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +0000599}
600
601/* Return a UTF-8 encoding Python string object from the
602 C byte string STR, which is encoded with ENC. */
603
604static PyObject *
605translate_into_utf8(const char* str, const char* enc) {
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000606 PyObject *utf8;
607 PyObject* buf = PyUnicode_Decode(str, strlen(str), enc, NULL);
608 if (buf == NULL)
609 return NULL;
610 utf8 = PyUnicode_AsUTF8String(buf);
611 Py_DECREF(buf);
612 return utf8;
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +0000613}
614
Benjamin Petersonaeaa5922009-11-13 00:17:59 +0000615
616static char *
617translate_newlines(const char *s, int exec_input, struct tok_state *tok) {
Victor Stinner79697732013-06-05 00:44:00 +0200618 int skip_next_lf = 0;
619 size_t needed_length = strlen(s) + 2, final_length;
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000620 char *buf, *current;
621 char c = '\0';
622 buf = PyMem_MALLOC(needed_length);
623 if (buf == NULL) {
624 tok->done = E_NOMEM;
625 return NULL;
626 }
627 for (current = buf; *s; s++, current++) {
628 c = *s;
629 if (skip_next_lf) {
630 skip_next_lf = 0;
631 if (c == '\n') {
632 c = *++s;
633 if (!c)
634 break;
635 }
636 }
637 if (c == '\r') {
638 skip_next_lf = 1;
639 c = '\n';
640 }
641 *current = c;
642 }
643 /* If this is exec input, add a newline to the end of the string if
644 there isn't one already. */
645 if (exec_input && c != '\n') {
646 *current = '\n';
647 current++;
648 }
649 *current = '\0';
650 final_length = current - buf + 1;
Pablo Galindocb90c892019-03-19 17:17:58 +0000651 if (final_length < needed_length && final_length) {
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000652 /* should never fail */
Pablo Galindocb90c892019-03-19 17:17:58 +0000653 char* result = PyMem_REALLOC(buf, final_length);
654 if (result == NULL) {
655 PyMem_FREE(buf);
656 }
657 buf = result;
658 }
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000659 return buf;
Benjamin Petersonaeaa5922009-11-13 00:17:59 +0000660}
661
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +0000662/* Decode a byte string STR for use as the buffer of TOK.
663 Look for encoding declarations inside STR, and record them
664 inside TOK. */
665
666static const char *
Benjamin Petersonaeaa5922009-11-13 00:17:59 +0000667decode_str(const char *input, int single, struct tok_state *tok)
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +0000668{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000669 PyObject* utf8 = NULL;
670 const char *str;
671 const char *s;
672 const char *newl[2] = {NULL, NULL};
673 int lineno = 0;
674 tok->input = str = translate_newlines(input, single, tok);
675 if (str == NULL)
676 return NULL;
677 tok->enc = NULL;
678 tok->str = str;
679 if (!check_bom(buf_getc, buf_ungetc, buf_setreadl, tok))
680 return error_ret(tok);
681 str = tok->str; /* string after BOM if any */
682 assert(str);
683 if (tok->enc != NULL) {
684 utf8 = translate_into_utf8(str, tok->enc);
685 if (utf8 == NULL)
686 return error_ret(tok);
687 str = PyBytes_AsString(utf8);
688 }
689 for (s = str;; s++) {
690 if (*s == '\0') break;
691 else if (*s == '\n') {
692 assert(lineno < 2);
693 newl[lineno] = s;
694 lineno++;
695 if (lineno == 2) break;
696 }
697 }
698 tok->enc = NULL;
699 /* need to check line 1 and 2 separately since check_coding_spec
700 assumes a single line as input */
701 if (newl[0]) {
702 if (!check_coding_spec(str, newl[0] - str, tok, buf_setreadl))
703 return error_ret(tok);
Serhiy Storchaka768c16c2014-01-09 18:36:09 +0200704 if (tok->enc == NULL && !tok->read_coding_spec && newl[1]) {
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000705 if (!check_coding_spec(newl[0]+1, newl[1] - newl[0],
706 tok, buf_setreadl))
707 return error_ret(tok);
708 }
709 }
710 if (tok->enc != NULL) {
711 assert(utf8 == NULL);
712 utf8 = translate_into_utf8(str, tok->enc);
713 if (utf8 == NULL)
714 return error_ret(tok);
715 str = PyBytes_AS_STRING(utf8);
716 }
717 assert(tok->decoding_buffer == NULL);
718 tok->decoding_buffer = utf8; /* CAUTION */
719 return str;
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +0000720}
721
Guido van Rossum85a5fbb1990-10-14 12:07:46 +0000722/* Set up tokenizer for string */
723
724struct tok_state *
Benjamin Petersonaeaa5922009-11-13 00:17:59 +0000725PyTokenizer_FromString(const char *str, int exec_input)
Guido van Rossum85a5fbb1990-10-14 12:07:46 +0000726{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000727 struct tok_state *tok = tok_new();
728 if (tok == NULL)
729 return NULL;
Serhiy Storchakac6792272013-10-19 21:03:34 +0300730 str = decode_str(str, exec_input, tok);
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000731 if (str == NULL) {
732 PyTokenizer_Free(tok);
733 return NULL;
734 }
Neal Norwitzdee2fd52005-11-16 05:12:59 +0000735
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000736 /* XXX: constify members. */
737 tok->buf = tok->cur = tok->end = tok->inp = (char*)str;
738 return tok;
Guido van Rossum85a5fbb1990-10-14 12:07:46 +0000739}
740
Benjamin Petersonf5b52242009-03-02 23:31:26 +0000741struct tok_state *
Benjamin Petersonaeaa5922009-11-13 00:17:59 +0000742PyTokenizer_FromUTF8(const char *str, int exec_input)
Benjamin Petersonf5b52242009-03-02 23:31:26 +0000743{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000744 struct tok_state *tok = tok_new();
745 if (tok == NULL)
746 return NULL;
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000747 tok->input = str = translate_newlines(str, exec_input, tok);
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000748 if (str == NULL) {
749 PyTokenizer_Free(tok);
750 return NULL;
751 }
752 tok->decoding_state = STATE_RAW;
753 tok->read_coding_spec = 1;
754 tok->enc = NULL;
755 tok->str = str;
756 tok->encoding = (char *)PyMem_MALLOC(6);
757 if (!tok->encoding) {
758 PyTokenizer_Free(tok);
759 return NULL;
760 }
761 strcpy(tok->encoding, "utf-8");
Benjamin Petersonf5b52242009-03-02 23:31:26 +0000762
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000763 /* XXX: constify members. */
764 tok->buf = tok->cur = tok->end = tok->inp = (char*)str;
765 return tok;
Benjamin Petersonf5b52242009-03-02 23:31:26 +0000766}
767
Guido van Rossum8c11a5c1991-07-27 21:42:56 +0000768/* Set up tokenizer for file */
Guido van Rossum85a5fbb1990-10-14 12:07:46 +0000769
770struct tok_state *
Serhiy Storchakac6792272013-10-19 21:03:34 +0300771PyTokenizer_FromFile(FILE *fp, const char* enc,
772 const char *ps1, const char *ps2)
Guido van Rossum85a5fbb1990-10-14 12:07:46 +0000773{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000774 struct tok_state *tok = tok_new();
775 if (tok == NULL)
776 return NULL;
777 if ((tok->buf = (char *)PyMem_MALLOC(BUFSIZ)) == NULL) {
778 PyTokenizer_Free(tok);
779 return NULL;
780 }
781 tok->cur = tok->inp = tok->buf;
782 tok->end = tok->buf + BUFSIZ;
783 tok->fp = fp;
784 tok->prompt = ps1;
785 tok->nextprompt = ps2;
786 if (enc != NULL) {
787 /* Must copy encoding declaration since it
788 gets copied into the parse tree. */
789 tok->encoding = PyMem_MALLOC(strlen(enc)+1);
790 if (!tok->encoding) {
791 PyTokenizer_Free(tok);
792 return NULL;
793 }
794 strcpy(tok->encoding, enc);
795 tok->decoding_state = STATE_NORMAL;
796 }
797 return tok;
Guido van Rossum85a5fbb1990-10-14 12:07:46 +0000798}
799
800
801/* Free a tok_state structure */
802
803void
Thomas Wouters23c9e002000-07-22 19:20:54 +0000804PyTokenizer_Free(struct tok_state *tok)
Guido van Rossum85a5fbb1990-10-14 12:07:46 +0000805{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000806 if (tok->encoding != NULL)
807 PyMem_FREE(tok->encoding);
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000808 Py_XDECREF(tok->decoding_readline);
809 Py_XDECREF(tok->decoding_buffer);
Victor Stinner7f2fee32011-04-05 00:39:01 +0200810 Py_XDECREF(tok->filename);
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000811 if (tok->fp != NULL && tok->buf != NULL)
812 PyMem_FREE(tok->buf);
813 if (tok->input)
814 PyMem_FREE((char *)tok->input);
815 PyMem_FREE(tok);
Guido van Rossum85a5fbb1990-10-14 12:07:46 +0000816}
817
Guido van Rossum85a5fbb1990-10-14 12:07:46 +0000818/* Get next char, updating state; error code goes into tok->done */
819
820static int
Antoine Pitrou9ed5f272013-08-13 20:18:52 +0200821tok_nextc(struct tok_state *tok)
Guido van Rossum85a5fbb1990-10-14 12:07:46 +0000822{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000823 for (;;) {
824 if (tok->cur != tok->inp) {
825 return Py_CHARMASK(*tok->cur++); /* Fast path */
826 }
827 if (tok->done != E_OK)
828 return EOF;
829 if (tok->fp == NULL) {
830 char *end = strchr(tok->inp, '\n');
831 if (end != NULL)
832 end++;
833 else {
834 end = strchr(tok->inp, '\0');
835 if (end == tok->inp) {
836 tok->done = E_EOF;
837 return EOF;
838 }
839 }
840 if (tok->start == NULL)
841 tok->buf = tok->cur;
842 tok->line_start = tok->cur;
843 tok->lineno++;
844 tok->inp = end;
845 return Py_CHARMASK(*tok->cur++);
846 }
847 if (tok->prompt != NULL) {
848 char *newtok = PyOS_Readline(stdin, stdout, tok->prompt);
Victor Stinner89e34362011-01-07 18:47:22 +0000849 if (newtok != NULL) {
850 char *translated = translate_newlines(newtok, 0, tok);
851 PyMem_FREE(newtok);
852 if (translated == NULL)
853 return EOF;
854 newtok = translated;
855 }
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000856 if (tok->encoding && newtok && *newtok) {
857 /* Recode to UTF-8 */
858 Py_ssize_t buflen;
859 const char* buf;
860 PyObject *u = translate_into_utf8(newtok, tok->encoding);
861 PyMem_FREE(newtok);
862 if (!u) {
863 tok->done = E_DECODE;
864 return EOF;
865 }
866 buflen = PyBytes_GET_SIZE(u);
867 buf = PyBytes_AS_STRING(u);
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000868 newtok = PyMem_MALLOC(buflen+1);
Zackery Spytz4c49da02018-12-07 03:11:30 -0700869 if (newtok == NULL) {
870 Py_DECREF(u);
871 tok->done = E_NOMEM;
872 return EOF;
873 }
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000874 strcpy(newtok, buf);
875 Py_DECREF(u);
876 }
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000877 if (tok->nextprompt != NULL)
878 tok->prompt = tok->nextprompt;
879 if (newtok == NULL)
880 tok->done = E_INTR;
881 else if (*newtok == '\0') {
882 PyMem_FREE(newtok);
883 tok->done = E_EOF;
884 }
885 else if (tok->start != NULL) {
886 size_t start = tok->start - tok->buf;
887 size_t oldlen = tok->cur - tok->buf;
888 size_t newlen = oldlen + strlen(newtok);
889 char *buf = tok->buf;
890 buf = (char *)PyMem_REALLOC(buf, newlen+1);
891 tok->lineno++;
892 if (buf == NULL) {
893 PyMem_FREE(tok->buf);
894 tok->buf = NULL;
895 PyMem_FREE(newtok);
896 tok->done = E_NOMEM;
897 return EOF;
898 }
899 tok->buf = buf;
900 tok->cur = tok->buf + oldlen;
901 tok->line_start = tok->cur;
902 strcpy(tok->buf + oldlen, newtok);
903 PyMem_FREE(newtok);
904 tok->inp = tok->buf + newlen;
905 tok->end = tok->inp + 1;
906 tok->start = tok->buf + start;
907 }
908 else {
909 tok->lineno++;
910 if (tok->buf != NULL)
911 PyMem_FREE(tok->buf);
912 tok->buf = newtok;
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000913 tok->cur = tok->buf;
914 tok->line_start = tok->buf;
915 tok->inp = strchr(tok->buf, '\0');
916 tok->end = tok->inp + 1;
917 }
918 }
919 else {
920 int done = 0;
921 Py_ssize_t cur = 0;
922 char *pt;
923 if (tok->start == NULL) {
924 if (tok->buf == NULL) {
925 tok->buf = (char *)
926 PyMem_MALLOC(BUFSIZ);
927 if (tok->buf == NULL) {
928 tok->done = E_NOMEM;
929 return EOF;
930 }
931 tok->end = tok->buf + BUFSIZ;
932 }
933 if (decoding_fgets(tok->buf, (int)(tok->end - tok->buf),
934 tok) == NULL) {
Serhiy Storchaka0d441112015-11-14 15:10:35 +0200935 if (!tok->decoding_erred)
936 tok->done = E_EOF;
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000937 done = 1;
938 }
939 else {
940 tok->done = E_OK;
941 tok->inp = strchr(tok->buf, '\0');
Benjamin Peterson26d998c2016-09-18 23:41:11 -0700942 done = tok->inp == tok->buf || tok->inp[-1] == '\n';
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000943 }
944 }
945 else {
946 cur = tok->cur - tok->buf;
947 if (decoding_feof(tok)) {
948 tok->done = E_EOF;
949 done = 1;
950 }
951 else
952 tok->done = E_OK;
953 }
954 tok->lineno++;
955 /* Read until '\n' or EOF */
956 while (!done) {
957 Py_ssize_t curstart = tok->start == NULL ? -1 :
958 tok->start - tok->buf;
959 Py_ssize_t curvalid = tok->inp - tok->buf;
960 Py_ssize_t newsize = curvalid + BUFSIZ;
961 char *newbuf = tok->buf;
962 newbuf = (char *)PyMem_REALLOC(newbuf,
963 newsize);
964 if (newbuf == NULL) {
965 tok->done = E_NOMEM;
966 tok->cur = tok->inp;
967 return EOF;
968 }
969 tok->buf = newbuf;
Serhiy Storchaka0d441112015-11-14 15:10:35 +0200970 tok->cur = tok->buf + cur;
971 tok->line_start = tok->cur;
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000972 tok->inp = tok->buf + curvalid;
973 tok->end = tok->buf + newsize;
974 tok->start = curstart < 0 ? NULL :
975 tok->buf + curstart;
976 if (decoding_fgets(tok->inp,
977 (int)(tok->end - tok->inp),
978 tok) == NULL) {
979 /* Break out early on decoding
980 errors, as tok->buf will be NULL
981 */
982 if (tok->decoding_erred)
983 return EOF;
984 /* Last line does not end in \n,
985 fake one */
Anthony Sottileabea73b2019-05-18 11:27:17 -0700986 if (tok->inp[-1] != '\n')
987 strcpy(tok->inp, "\n");
Antoine Pitrouf95a1b32010-05-09 15:52:27 +0000988 }
989 tok->inp = strchr(tok->inp, '\0');
990 done = tok->inp[-1] == '\n';
991 }
992 if (tok->buf != NULL) {
993 tok->cur = tok->buf + cur;
994 tok->line_start = tok->cur;
995 /* replace "\r\n" with "\n" */
996 /* For Mac leave the \r, giving a syntax error */
997 pt = tok->inp - 2;
998 if (pt >= tok->buf && *pt == '\r') {
999 *pt++ = '\n';
1000 *pt = '\0';
1001 tok->inp = pt;
1002 }
1003 }
1004 }
1005 if (tok->done != E_OK) {
1006 if (tok->prompt != NULL)
1007 PySys_WriteStderr("\n");
1008 tok->cur = tok->inp;
1009 return EOF;
1010 }
1011 }
1012 /*NOTREACHED*/
Guido van Rossum85a5fbb1990-10-14 12:07:46 +00001013}
1014
1015
1016/* Back-up one character */
1017
1018static void
Antoine Pitrou9ed5f272013-08-13 20:18:52 +02001019tok_backup(struct tok_state *tok, int c)
Guido van Rossum85a5fbb1990-10-14 12:07:46 +00001020{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001021 if (c != EOF) {
1022 if (--tok->cur < tok->buf)
1023 Py_FatalError("tok_backup: beginning of buffer");
1024 if (*tok->cur != c)
1025 *tok->cur = c;
1026 }
Guido van Rossum85a5fbb1990-10-14 12:07:46 +00001027}
1028
1029
Guido van Rossum926f13a1998-04-09 21:38:06 +00001030static int
Serhiy Storchakacf7303e2018-07-09 15:09:35 +03001031syntaxerror(struct tok_state *tok, const char *format, ...)
1032{
Serhiy Storchakacf7303e2018-07-09 15:09:35 +03001033 va_list vargs;
1034#ifdef HAVE_STDARG_PROTOTYPES
1035 va_start(vargs, format);
1036#else
1037 va_start(vargs);
1038#endif
1039 PyErr_FormatV(PyExc_SyntaxError, format, vargs);
1040 va_end(vargs);
1041 PyErr_SyntaxLocationObject(tok->filename,
1042 tok->lineno,
Victor Stinnerc8846162018-07-21 03:36:06 +02001043 (int)(tok->cur - tok->line_start));
Serhiy Storchakacf7303e2018-07-09 15:09:35 +03001044 tok->done = E_ERROR;
Serhiy Storchakacf7303e2018-07-09 15:09:35 +03001045 return ERRORTOKEN;
1046}
1047
1048static int
Thomas Wouters23c9e002000-07-22 19:20:54 +00001049indenterror(struct tok_state *tok)
Guido van Rossum926f13a1998-04-09 21:38:06 +00001050{
Victor Stinnerf2ddc6a2017-11-17 01:25:47 -08001051 tok->done = E_TABSPACE;
1052 tok->cur = tok->inp;
1053 return ERRORTOKEN;
Guido van Rossum926f13a1998-04-09 21:38:06 +00001054}
1055
Martin v. Löwisd63a3b82011-09-28 07:41:54 +02001056/* Verify that the identifier follows PEP 3131.
1057 All identifier strings are guaranteed to be "ready" unicode objects.
1058 */
Martin v. Löwis47383402007-08-15 07:32:56 +00001059static int
Victor Stinner52f6dd72010-03-12 14:45:56 +00001060verify_identifier(struct tok_state *tok)
Martin v. Löwis47383402007-08-15 07:32:56 +00001061{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001062 PyObject *s;
1063 int result;
Benjamin Petersond73aca72015-04-21 12:05:19 -04001064 if (tok->decoding_erred)
1065 return 0;
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001066 s = PyUnicode_DecodeUTF8(tok->start, tok->cur - tok->start, NULL);
Zackery Spytz5061a742018-09-10 00:27:31 -06001067 if (s == NULL) {
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001068 if (PyErr_ExceptionMatches(PyExc_UnicodeDecodeError)) {
1069 PyErr_Clear();
1070 tok->done = E_IDENTIFIER;
1071 } else {
1072 tok->done = E_ERROR;
1073 }
1074 return 0;
1075 }
1076 result = PyUnicode_IsIdentifier(s);
1077 Py_DECREF(s);
1078 if (result == 0)
1079 tok->done = E_IDENTIFIER;
1080 return result;
Martin v. Löwis47383402007-08-15 07:32:56 +00001081}
Guido van Rossum926f13a1998-04-09 21:38:06 +00001082
Brett Cannona721aba2016-09-09 14:57:09 -07001083static int
1084tok_decimal_tail(struct tok_state *tok)
1085{
1086 int c;
1087
1088 while (1) {
1089 do {
1090 c = tok_nextc(tok);
1091 } while (isdigit(c));
1092 if (c != '_') {
1093 break;
1094 }
1095 c = tok_nextc(tok);
1096 if (!isdigit(c)) {
Brett Cannona721aba2016-09-09 14:57:09 -07001097 tok_backup(tok, c);
Serhiy Storchakacf7303e2018-07-09 15:09:35 +03001098 syntaxerror(tok, "invalid decimal literal");
Brett Cannona721aba2016-09-09 14:57:09 -07001099 return 0;
1100 }
1101 }
1102 return c;
1103}
1104
Guido van Rossum85a5fbb1990-10-14 12:07:46 +00001105/* Get next token, after space stripping etc. */
1106
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +00001107static int
Antoine Pitrou9ed5f272013-08-13 20:18:52 +02001108tok_get(struct tok_state *tok, char **p_start, char **p_end)
Guido van Rossum85a5fbb1990-10-14 12:07:46 +00001109{
Antoine Pitrou9ed5f272013-08-13 20:18:52 +02001110 int c;
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001111 int blankline, nonascii;
Guido van Rossum8c11a5c1991-07-27 21:42:56 +00001112
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001113 *p_start = *p_end = NULL;
Guido van Rossum8c11a5c1991-07-27 21:42:56 +00001114 nextline:
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001115 tok->start = NULL;
1116 blankline = 0;
Guido van Rossum8c11a5c1991-07-27 21:42:56 +00001117
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001118 /* Get indentation level */
1119 if (tok->atbol) {
Antoine Pitrou9ed5f272013-08-13 20:18:52 +02001120 int col = 0;
1121 int altcol = 0;
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001122 tok->atbol = 0;
1123 for (;;) {
1124 c = tok_nextc(tok);
Brett Cannona721aba2016-09-09 14:57:09 -07001125 if (c == ' ') {
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001126 col++, altcol++;
Brett Cannona721aba2016-09-09 14:57:09 -07001127 }
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001128 else if (c == '\t') {
Victor Stinnerf2ddc6a2017-11-17 01:25:47 -08001129 col = (col / tok->tabsize + 1) * tok->tabsize;
1130 altcol = (altcol / ALTTABSIZE + 1) * ALTTABSIZE;
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001131 }
Brett Cannona721aba2016-09-09 14:57:09 -07001132 else if (c == '\014') {/* Control-L (formfeed) */
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001133 col = altcol = 0; /* For Emacs users */
Brett Cannona721aba2016-09-09 14:57:09 -07001134 }
1135 else {
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001136 break;
Brett Cannona721aba2016-09-09 14:57:09 -07001137 }
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001138 }
1139 tok_backup(tok, c);
1140 if (c == '#' || c == '\n') {
1141 /* Lines with only whitespace and/or comments
1142 shouldn't affect the indentation and are
1143 not passed to the parser as NEWLINE tokens,
1144 except *totally* empty lines in interactive
1145 mode, which signal the end of a command group. */
Brett Cannona721aba2016-09-09 14:57:09 -07001146 if (col == 0 && c == '\n' && tok->prompt != NULL) {
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001147 blankline = 0; /* Let it through */
Brett Cannona721aba2016-09-09 14:57:09 -07001148 }
1149 else {
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001150 blankline = 1; /* Ignore completely */
Brett Cannona721aba2016-09-09 14:57:09 -07001151 }
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001152 /* We can't jump back right here since we still
1153 may need to skip to the end of a comment */
1154 }
1155 if (!blankline && tok->level == 0) {
1156 if (col == tok->indstack[tok->indent]) {
1157 /* No change */
1158 if (altcol != tok->altindstack[tok->indent]) {
Victor Stinnerf2ddc6a2017-11-17 01:25:47 -08001159 return indenterror(tok);
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001160 }
1161 }
1162 else if (col > tok->indstack[tok->indent]) {
1163 /* Indent -- always one */
1164 if (tok->indent+1 >= MAXINDENT) {
1165 tok->done = E_TOODEEP;
1166 tok->cur = tok->inp;
1167 return ERRORTOKEN;
1168 }
1169 if (altcol <= tok->altindstack[tok->indent]) {
Victor Stinnerf2ddc6a2017-11-17 01:25:47 -08001170 return indenterror(tok);
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001171 }
1172 tok->pendin++;
1173 tok->indstack[++tok->indent] = col;
1174 tok->altindstack[tok->indent] = altcol;
1175 }
1176 else /* col < tok->indstack[tok->indent] */ {
1177 /* Dedent -- any number, must be consistent */
1178 while (tok->indent > 0 &&
1179 col < tok->indstack[tok->indent]) {
1180 tok->pendin--;
1181 tok->indent--;
1182 }
1183 if (col != tok->indstack[tok->indent]) {
1184 tok->done = E_DEDENT;
1185 tok->cur = tok->inp;
1186 return ERRORTOKEN;
1187 }
1188 if (altcol != tok->altindstack[tok->indent]) {
Victor Stinnerf2ddc6a2017-11-17 01:25:47 -08001189 return indenterror(tok);
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001190 }
1191 }
1192 }
1193 }
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001194
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001195 tok->start = tok->cur;
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001196
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001197 /* Return pending indents/dedents */
1198 if (tok->pendin != 0) {
1199 if (tok->pendin < 0) {
1200 tok->pendin++;
1201 return DEDENT;
1202 }
1203 else {
1204 tok->pendin--;
1205 return INDENT;
1206 }
1207 }
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001208
Guido van Rossum495da292019-03-07 12:38:08 -08001209 /* Peek ahead at the next character */
1210 c = tok_nextc(tok);
1211 tok_backup(tok, c);
1212 /* Check if we are closing an async function */
1213 if (tok->async_def
1214 && !blankline
1215 /* Due to some implementation artifacts of type comments,
1216 * a TYPE_COMMENT at the start of a function won't set an
1217 * indentation level and it will produce a NEWLINE after it.
1218 * To avoid spuriously ending an async function due to this,
1219 * wait until we have some non-newline char in front of us. */
1220 && c != '\n'
1221 && tok->level == 0
1222 /* There was a NEWLINE after ASYNC DEF,
1223 so we're past the signature. */
1224 && tok->async_def_nl
1225 /* Current indentation level is less than where
1226 the async function was defined */
1227 && tok->async_def_indent >= tok->indent)
1228 {
1229 tok->async_def = 0;
1230 tok->async_def_indent = 0;
1231 tok->async_def_nl = 0;
1232 }
1233
Guido van Rossum85a5fbb1990-10-14 12:07:46 +00001234 again:
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001235 tok->start = NULL;
1236 /* Skip spaces */
1237 do {
1238 c = tok_nextc(tok);
1239 } while (c == ' ' || c == '\t' || c == '\014');
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001240
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001241 /* Set start of current token */
1242 tok->start = tok->cur - 1;
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001243
Guido van Rossumdcfcd142019-01-31 03:40:27 -08001244 /* Skip comment, unless it's a type comment */
Brett Cannona721aba2016-09-09 14:57:09 -07001245 if (c == '#') {
Guido van Rossumdcfcd142019-01-31 03:40:27 -08001246 const char *prefix, *p, *type_start;
1247
Brett Cannona721aba2016-09-09 14:57:09 -07001248 while (c != EOF && c != '\n') {
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001249 c = tok_nextc(tok);
Brett Cannona721aba2016-09-09 14:57:09 -07001250 }
Guido van Rossumdcfcd142019-01-31 03:40:27 -08001251
1252 if (tok->type_comments) {
1253 p = tok->start;
1254 prefix = type_comment_prefix;
1255 while (*prefix && p < tok->cur) {
1256 if (*prefix == ' ') {
1257 while (*p == ' ' || *p == '\t') {
1258 p++;
1259 }
1260 } else if (*prefix == *p) {
1261 p++;
1262 } else {
1263 break;
1264 }
1265
1266 prefix++;
1267 }
1268
1269 /* This is a type comment if we matched all of type_comment_prefix. */
1270 if (!*prefix) {
1271 int is_type_ignore = 1;
Michael J. Sullivan933e1502019-05-22 07:54:20 -07001272 const char *ignore_end = p + 6;
Guido van Rossumdcfcd142019-01-31 03:40:27 -08001273 tok_backup(tok, c); /* don't eat the newline or EOF */
1274
1275 type_start = p;
1276
Michael J. Sullivand8320ec2019-05-11 11:17:24 -07001277 /* A TYPE_IGNORE is "type: ignore" followed by the end of the token
Michael J. Sullivand8a82e22019-05-22 13:43:37 -07001278 * or anything ASCII and non-alphanumeric. */
Michael J. Sullivand8320ec2019-05-11 11:17:24 -07001279 is_type_ignore = (
Michael J. Sullivan933e1502019-05-22 07:54:20 -07001280 tok->cur >= ignore_end && memcmp(p, "ignore", 6) == 0
Michael J. Sullivand8a82e22019-05-22 13:43:37 -07001281 && !(tok->cur > ignore_end
1282 && ((unsigned char)ignore_end[0] >= 128 || Py_ISALNUM(ignore_end[0]))));
Guido van Rossumdcfcd142019-01-31 03:40:27 -08001283
1284 if (is_type_ignore) {
Michael J. Sullivan933e1502019-05-22 07:54:20 -07001285 *p_start = (char *) ignore_end;
1286 *p_end = tok->cur;
1287
Guido van Rossumdcfcd142019-01-31 03:40:27 -08001288 /* If this type ignore is the only thing on the line, consume the newline also. */
1289 if (blankline) {
1290 tok_nextc(tok);
1291 tok->atbol = 1;
1292 }
1293 return TYPE_IGNORE;
1294 } else {
1295 *p_start = (char *) type_start; /* after type_comment_prefix */
1296 *p_end = tok->cur;
1297 return TYPE_COMMENT;
1298 }
1299 }
1300 }
Brett Cannona721aba2016-09-09 14:57:09 -07001301 }
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001302
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001303 /* Check for EOF and errors now */
1304 if (c == EOF) {
1305 return tok->done == E_EOF ? ENDMARKER : ERRORTOKEN;
1306 }
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001307
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001308 /* Identifier (most frequent token!) */
1309 nonascii = 0;
1310 if (is_potential_identifier_start(c)) {
Berker Peksag6f805622017-02-05 04:32:39 +03001311 /* Process the various legal combinations of b"", r"", u"", and f"". */
Eric V. Smith235a6f02015-09-19 14:51:32 -04001312 int saw_b = 0, saw_r = 0, saw_u = 0, saw_f = 0;
Antoine Pitrou3a5d4cb2012-01-12 22:46:19 +01001313 while (1) {
Eric V. Smith235a6f02015-09-19 14:51:32 -04001314 if (!(saw_b || saw_u || saw_f) && (c == 'b' || c == 'B'))
Antoine Pitrou3a5d4cb2012-01-12 22:46:19 +01001315 saw_b = 1;
Armin Ronacher6ecf77b2012-03-04 12:04:06 +00001316 /* Since this is a backwards compatibility support literal we don't
1317 want to support it in arbitrary order like byte literals. */
Brett Cannona721aba2016-09-09 14:57:09 -07001318 else if (!(saw_b || saw_u || saw_r || saw_f)
1319 && (c == 'u'|| c == 'U')) {
Armin Ronacher6ecf77b2012-03-04 12:04:06 +00001320 saw_u = 1;
Brett Cannona721aba2016-09-09 14:57:09 -07001321 }
Christian Heimes0b3847d2012-06-20 11:17:58 +02001322 /* ur"" and ru"" are not supported */
Brett Cannona721aba2016-09-09 14:57:09 -07001323 else if (!(saw_r || saw_u) && (c == 'r' || c == 'R')) {
Antoine Pitrou3a5d4cb2012-01-12 22:46:19 +01001324 saw_r = 1;
Brett Cannona721aba2016-09-09 14:57:09 -07001325 }
1326 else if (!(saw_f || saw_b || saw_u) && (c == 'f' || c == 'F')) {
Eric V. Smith235a6f02015-09-19 14:51:32 -04001327 saw_f = 1;
Brett Cannona721aba2016-09-09 14:57:09 -07001328 }
1329 else {
Antoine Pitrou3a5d4cb2012-01-12 22:46:19 +01001330 break;
Brett Cannona721aba2016-09-09 14:57:09 -07001331 }
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001332 c = tok_nextc(tok);
Brett Cannona721aba2016-09-09 14:57:09 -07001333 if (c == '"' || c == '\'') {
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001334 goto letter_quote;
Brett Cannona721aba2016-09-09 14:57:09 -07001335 }
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001336 }
1337 while (is_potential_identifier_char(c)) {
Brett Cannona721aba2016-09-09 14:57:09 -07001338 if (c >= 128) {
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001339 nonascii = 1;
Brett Cannona721aba2016-09-09 14:57:09 -07001340 }
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001341 c = tok_nextc(tok);
1342 }
1343 tok_backup(tok, c);
Brett Cannona721aba2016-09-09 14:57:09 -07001344 if (nonascii && !verify_identifier(tok)) {
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001345 return ERRORTOKEN;
Brett Cannona721aba2016-09-09 14:57:09 -07001346 }
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001347 *p_start = tok->start;
1348 *p_end = tok->cur;
Yury Selivanov75445082015-05-11 22:57:16 -04001349
Guido van Rossum495da292019-03-07 12:38:08 -08001350 /* async/await parsing block. */
1351 if (tok->cur - tok->start == 5 && tok->start[0] == 'a') {
1352 /* May be an 'async' or 'await' token. For Python 3.7 or
1353 later we recognize them unconditionally. For Python
1354 3.5 or 3.6 we recognize 'async' in front of 'def', and
1355 either one inside of 'async def'. (Technically we
1356 shouldn't recognize these at all for 3.4 or earlier,
1357 but there's no *valid* Python 3.4 code that would be
1358 rejected, and async functions will be rejected in a
1359 later phase.) */
1360 if (!tok->async_hacks || tok->async_def) {
1361 /* Always recognize the keywords. */
1362 if (memcmp(tok->start, "async", 5) == 0) {
1363 return ASYNC;
1364 }
1365 if (memcmp(tok->start, "await", 5) == 0) {
1366 return AWAIT;
1367 }
1368 }
1369 else if (memcmp(tok->start, "async", 5) == 0) {
1370 /* The current token is 'async'.
1371 Look ahead one token to see if that is 'def'. */
1372
1373 struct tok_state ahead_tok;
1374 char *ahead_tok_start = NULL, *ahead_tok_end = NULL;
1375 int ahead_tok_kind;
1376
1377 memcpy(&ahead_tok, tok, sizeof(ahead_tok));
1378 ahead_tok_kind = tok_get(&ahead_tok, &ahead_tok_start,
1379 &ahead_tok_end);
1380
1381 if (ahead_tok_kind == NAME
1382 && ahead_tok.cur - ahead_tok.start == 3
1383 && memcmp(ahead_tok.start, "def", 3) == 0)
1384 {
1385 /* The next token is going to be 'def', so instead of
1386 returning a plain NAME token, return ASYNC. */
1387 tok->async_def_indent = tok->indent;
1388 tok->async_def = 1;
1389 return ASYNC;
1390 }
1391 }
1392 }
1393
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001394 return NAME;
1395 }
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001396
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001397 /* Newline */
1398 if (c == '\n') {
1399 tok->atbol = 1;
Brett Cannona721aba2016-09-09 14:57:09 -07001400 if (blankline || tok->level > 0) {
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001401 goto nextline;
Brett Cannona721aba2016-09-09 14:57:09 -07001402 }
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001403 *p_start = tok->start;
1404 *p_end = tok->cur - 1; /* Leave '\n' out of the string */
1405 tok->cont_line = 0;
Guido van Rossum495da292019-03-07 12:38:08 -08001406 if (tok->async_def) {
1407 /* We're somewhere inside an 'async def' function, and
1408 we've encountered a NEWLINE after its signature. */
1409 tok->async_def_nl = 1;
1410 }
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001411 return NEWLINE;
1412 }
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001413
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001414 /* Period or number starting with period? */
1415 if (c == '.') {
1416 c = tok_nextc(tok);
1417 if (isdigit(c)) {
1418 goto fraction;
1419 } else if (c == '.') {
1420 c = tok_nextc(tok);
1421 if (c == '.') {
1422 *p_start = tok->start;
1423 *p_end = tok->cur;
1424 return ELLIPSIS;
Brett Cannona721aba2016-09-09 14:57:09 -07001425 }
1426 else {
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001427 tok_backup(tok, c);
1428 }
1429 tok_backup(tok, '.');
Brett Cannona721aba2016-09-09 14:57:09 -07001430 }
1431 else {
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001432 tok_backup(tok, c);
1433 }
1434 *p_start = tok->start;
1435 *p_end = tok->cur;
1436 return DOT;
1437 }
Guido van Rossumf595fde1996-01-12 01:31:58 +00001438
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001439 /* Number */
1440 if (isdigit(c)) {
1441 if (c == '0') {
1442 /* Hex, octal or binary -- maybe. */
1443 c = tok_nextc(tok);
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001444 if (c == 'x' || c == 'X') {
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001445 /* Hex */
1446 c = tok_nextc(tok);
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001447 do {
Brett Cannona721aba2016-09-09 14:57:09 -07001448 if (c == '_') {
1449 c = tok_nextc(tok);
1450 }
1451 if (!isxdigit(c)) {
Brett Cannona721aba2016-09-09 14:57:09 -07001452 tok_backup(tok, c);
Serhiy Storchakacf7303e2018-07-09 15:09:35 +03001453 return syntaxerror(tok, "invalid hexadecimal literal");
Brett Cannona721aba2016-09-09 14:57:09 -07001454 }
1455 do {
1456 c = tok_nextc(tok);
1457 } while (isxdigit(c));
1458 } while (c == '_');
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001459 }
1460 else if (c == 'o' || c == 'O') {
1461 /* Octal */
1462 c = tok_nextc(tok);
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001463 do {
Brett Cannona721aba2016-09-09 14:57:09 -07001464 if (c == '_') {
1465 c = tok_nextc(tok);
1466 }
1467 if (c < '0' || c >= '8') {
Brett Cannona721aba2016-09-09 14:57:09 -07001468 tok_backup(tok, c);
Serhiy Storchakacf7303e2018-07-09 15:09:35 +03001469 if (isdigit(c)) {
1470 return syntaxerror(tok,
1471 "invalid digit '%c' in octal literal", c);
1472 }
1473 else {
1474 return syntaxerror(tok, "invalid octal literal");
1475 }
Brett Cannona721aba2016-09-09 14:57:09 -07001476 }
1477 do {
1478 c = tok_nextc(tok);
1479 } while ('0' <= c && c < '8');
1480 } while (c == '_');
Serhiy Storchakacf7303e2018-07-09 15:09:35 +03001481 if (isdigit(c)) {
1482 return syntaxerror(tok,
1483 "invalid digit '%c' in octal literal", c);
1484 }
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001485 }
1486 else if (c == 'b' || c == 'B') {
1487 /* Binary */
1488 c = tok_nextc(tok);
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001489 do {
Brett Cannona721aba2016-09-09 14:57:09 -07001490 if (c == '_') {
1491 c = tok_nextc(tok);
1492 }
1493 if (c != '0' && c != '1') {
Brett Cannona721aba2016-09-09 14:57:09 -07001494 tok_backup(tok, c);
Serhiy Storchakacf7303e2018-07-09 15:09:35 +03001495 if (isdigit(c)) {
1496 return syntaxerror(tok,
1497 "invalid digit '%c' in binary literal", c);
1498 }
1499 else {
1500 return syntaxerror(tok, "invalid binary literal");
1501 }
Brett Cannona721aba2016-09-09 14:57:09 -07001502 }
1503 do {
1504 c = tok_nextc(tok);
1505 } while (c == '0' || c == '1');
1506 } while (c == '_');
Serhiy Storchakacf7303e2018-07-09 15:09:35 +03001507 if (isdigit(c)) {
1508 return syntaxerror(tok,
1509 "invalid digit '%c' in binary literal", c);
1510 }
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001511 }
1512 else {
1513 int nonzero = 0;
1514 /* maybe old-style octal; c is first char of it */
1515 /* in any case, allow '0' as a literal */
Brett Cannona721aba2016-09-09 14:57:09 -07001516 while (1) {
1517 if (c == '_') {
1518 c = tok_nextc(tok);
1519 if (!isdigit(c)) {
Brett Cannona721aba2016-09-09 14:57:09 -07001520 tok_backup(tok, c);
Serhiy Storchakacf7303e2018-07-09 15:09:35 +03001521 return syntaxerror(tok, "invalid decimal literal");
Brett Cannona721aba2016-09-09 14:57:09 -07001522 }
1523 }
1524 if (c != '0') {
1525 break;
1526 }
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001527 c = tok_nextc(tok);
1528 }
Brett Cannona721aba2016-09-09 14:57:09 -07001529 if (isdigit(c)) {
1530 nonzero = 1;
1531 c = tok_decimal_tail(tok);
1532 if (c == 0) {
1533 return ERRORTOKEN;
1534 }
1535 }
1536 if (c == '.') {
1537 c = tok_nextc(tok);
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001538 goto fraction;
Brett Cannona721aba2016-09-09 14:57:09 -07001539 }
1540 else if (c == 'e' || c == 'E') {
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001541 goto exponent;
Brett Cannona721aba2016-09-09 14:57:09 -07001542 }
1543 else if (c == 'j' || c == 'J') {
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001544 goto imaginary;
Brett Cannona721aba2016-09-09 14:57:09 -07001545 }
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001546 else if (nonzero) {
Brett Cannona721aba2016-09-09 14:57:09 -07001547 /* Old-style octal: now disallowed. */
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001548 tok_backup(tok, c);
Serhiy Storchakacf7303e2018-07-09 15:09:35 +03001549 return syntaxerror(tok,
1550 "leading zeros in decimal integer "
1551 "literals are not permitted; "
1552 "use an 0o prefix for octal integers");
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001553 }
1554 }
1555 }
1556 else {
1557 /* Decimal */
Brett Cannona721aba2016-09-09 14:57:09 -07001558 c = tok_decimal_tail(tok);
1559 if (c == 0) {
1560 return ERRORTOKEN;
1561 }
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001562 {
1563 /* Accept floating point numbers. */
1564 if (c == '.') {
Brett Cannona721aba2016-09-09 14:57:09 -07001565 c = tok_nextc(tok);
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001566 fraction:
1567 /* Fraction */
Brett Cannona721aba2016-09-09 14:57:09 -07001568 if (isdigit(c)) {
1569 c = tok_decimal_tail(tok);
1570 if (c == 0) {
1571 return ERRORTOKEN;
1572 }
1573 }
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001574 }
1575 if (c == 'e' || c == 'E') {
Benjamin Petersonc4161622014-06-07 12:36:39 -07001576 int e;
1577 exponent:
1578 e = c;
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001579 /* Exponent part */
1580 c = tok_nextc(tok);
Benjamin Petersonc4161622014-06-07 12:36:39 -07001581 if (c == '+' || c == '-') {
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001582 c = tok_nextc(tok);
Benjamin Petersonc4161622014-06-07 12:36:39 -07001583 if (!isdigit(c)) {
Benjamin Petersonc4161622014-06-07 12:36:39 -07001584 tok_backup(tok, c);
Serhiy Storchakacf7303e2018-07-09 15:09:35 +03001585 return syntaxerror(tok, "invalid decimal literal");
Benjamin Petersonc4161622014-06-07 12:36:39 -07001586 }
1587 } else if (!isdigit(c)) {
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001588 tok_backup(tok, c);
Benjamin Petersonc4161622014-06-07 12:36:39 -07001589 tok_backup(tok, e);
1590 *p_start = tok->start;
1591 *p_end = tok->cur;
1592 return NUMBER;
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001593 }
Brett Cannona721aba2016-09-09 14:57:09 -07001594 c = tok_decimal_tail(tok);
1595 if (c == 0) {
1596 return ERRORTOKEN;
1597 }
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001598 }
Brett Cannona721aba2016-09-09 14:57:09 -07001599 if (c == 'j' || c == 'J') {
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001600 /* Imaginary part */
1601 imaginary:
1602 c = tok_nextc(tok);
Brett Cannona721aba2016-09-09 14:57:09 -07001603 }
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001604 }
1605 }
1606 tok_backup(tok, c);
1607 *p_start = tok->start;
1608 *p_end = tok->cur;
1609 return NUMBER;
1610 }
Guido van Rossum24dacb31997-04-06 03:46:20 +00001611
1612 letter_quote:
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001613 /* String */
1614 if (c == '\'' || c == '"') {
1615 int quote = c;
1616 int quote_size = 1; /* 1 or 3 */
1617 int end_quote_size = 0;
Guido van Rossumcf171a72007-11-16 00:51:45 +00001618
Anthony Sottile995d9b92019-01-12 20:05:13 -08001619 /* Nodes of type STRING, especially multi line strings
1620 must be handled differently in order to get both
1621 the starting line number and the column offset right.
1622 (cf. issue 16806) */
1623 tok->first_lineno = tok->lineno;
1624 tok->multi_line_start = tok->line_start;
1625
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001626 /* Find the quote size and start of string */
1627 c = tok_nextc(tok);
1628 if (c == quote) {
1629 c = tok_nextc(tok);
Brett Cannona721aba2016-09-09 14:57:09 -07001630 if (c == quote) {
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001631 quote_size = 3;
Brett Cannona721aba2016-09-09 14:57:09 -07001632 }
1633 else {
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001634 end_quote_size = 1; /* empty string found */
Brett Cannona721aba2016-09-09 14:57:09 -07001635 }
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001636 }
Brett Cannona721aba2016-09-09 14:57:09 -07001637 if (c != quote) {
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001638 tok_backup(tok, c);
Brett Cannona721aba2016-09-09 14:57:09 -07001639 }
Guido van Rossumcf171a72007-11-16 00:51:45 +00001640
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001641 /* Get rest of string */
1642 while (end_quote_size != quote_size) {
1643 c = tok_nextc(tok);
1644 if (c == EOF) {
Brett Cannona721aba2016-09-09 14:57:09 -07001645 if (quote_size == 3) {
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001646 tok->done = E_EOFS;
Brett Cannona721aba2016-09-09 14:57:09 -07001647 }
1648 else {
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001649 tok->done = E_EOLS;
Brett Cannona721aba2016-09-09 14:57:09 -07001650 }
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001651 tok->cur = tok->inp;
1652 return ERRORTOKEN;
1653 }
1654 if (quote_size == 1 && c == '\n') {
1655 tok->done = E_EOLS;
1656 tok->cur = tok->inp;
1657 return ERRORTOKEN;
1658 }
Brett Cannona721aba2016-09-09 14:57:09 -07001659 if (c == quote) {
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001660 end_quote_size += 1;
Brett Cannona721aba2016-09-09 14:57:09 -07001661 }
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001662 else {
1663 end_quote_size = 0;
Brett Cannona721aba2016-09-09 14:57:09 -07001664 if (c == '\\') {
Christian Heimesc6cc23d2016-09-09 00:09:45 +02001665 tok_nextc(tok); /* skip escaped char */
Brett Cannona721aba2016-09-09 14:57:09 -07001666 }
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001667 }
1668 }
Guido van Rossumcf171a72007-11-16 00:51:45 +00001669
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001670 *p_start = tok->start;
1671 *p_end = tok->cur;
1672 return STRING;
1673 }
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001674
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001675 /* Line continuation */
1676 if (c == '\\') {
1677 c = tok_nextc(tok);
1678 if (c != '\n') {
1679 tok->done = E_LINECONT;
1680 tok->cur = tok->inp;
1681 return ERRORTOKEN;
1682 }
Anthony Sottileabea73b2019-05-18 11:27:17 -07001683 c = tok_nextc(tok);
1684 if (c == EOF) {
1685 tok->done = E_EOF;
1686 tok->cur = tok->inp;
1687 return ERRORTOKEN;
1688 } else {
1689 tok_backup(tok, c);
1690 }
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001691 tok->cont_line = 1;
1692 goto again; /* Read next line */
1693 }
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001694
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001695 /* Check for two-character token */
1696 {
1697 int c2 = tok_nextc(tok);
1698 int token = PyToken_TwoChars(c, c2);
1699 if (token != OP) {
1700 int c3 = tok_nextc(tok);
1701 int token3 = PyToken_ThreeChars(c, c2, c3);
1702 if (token3 != OP) {
1703 token = token3;
Brett Cannona721aba2016-09-09 14:57:09 -07001704 }
1705 else {
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001706 tok_backup(tok, c3);
1707 }
1708 *p_start = tok->start;
1709 *p_end = tok->cur;
1710 return token;
1711 }
1712 tok_backup(tok, c2);
1713 }
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001714
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001715 /* Keep track of parentheses nesting level */
1716 switch (c) {
1717 case '(':
1718 case '[':
1719 case '{':
Serhiy Storchaka94cf3082018-12-17 17:34:14 +02001720 if (tok->level >= MAXLEVEL) {
1721 return syntaxerror(tok, "too many nested parentheses");
1722 }
1723 tok->parenstack[tok->level] = c;
1724 tok->parenlinenostack[tok->level] = tok->lineno;
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001725 tok->level++;
1726 break;
1727 case ')':
1728 case ']':
1729 case '}':
Serhiy Storchaka94cf3082018-12-17 17:34:14 +02001730 if (!tok->level) {
1731 return syntaxerror(tok, "unmatched '%c'", c);
1732 }
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001733 tok->level--;
Serhiy Storchaka94cf3082018-12-17 17:34:14 +02001734 int opening = tok->parenstack[tok->level];
1735 if (!((opening == '(' && c == ')') ||
1736 (opening == '[' && c == ']') ||
1737 (opening == '{' && c == '}')))
1738 {
1739 if (tok->parenlinenostack[tok->level] != tok->lineno) {
1740 return syntaxerror(tok,
1741 "closing parenthesis '%c' does not match "
1742 "opening parenthesis '%c' on line %d",
1743 c, opening, tok->parenlinenostack[tok->level]);
1744 }
1745 else {
1746 return syntaxerror(tok,
1747 "closing parenthesis '%c' does not match "
1748 "opening parenthesis '%c'",
1749 c, opening);
1750 }
1751 }
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001752 break;
1753 }
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001754
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001755 /* Punctuation character */
1756 *p_start = tok->start;
1757 *p_end = tok->cur;
1758 return PyToken_OneChar(c);
Guido van Rossum85a5fbb1990-10-14 12:07:46 +00001759}
1760
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +00001761int
1762PyTokenizer_Get(struct tok_state *tok, char **p_start, char **p_end)
1763{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001764 int result = tok_get(tok, p_start, p_end);
1765 if (tok->decoding_erred) {
1766 result = ERRORTOKEN;
1767 tok->done = E_DECODE;
1768 }
1769 return result;
Martin v. Löwis00f1e3f2002-08-04 17:29:52 +00001770}
Guido van Rossum85a5fbb1990-10-14 12:07:46 +00001771
Victor Stinnerfe7c5b52011-04-05 01:48:03 +02001772/* Get the encoding of a Python file. Check for the coding cookie and check if
1773 the file starts with a BOM.
Guido van Rossumce3a72a2007-10-19 23:16:50 +00001774
Victor Stinnerfe7c5b52011-04-05 01:48:03 +02001775 PyTokenizer_FindEncodingFilename() returns NULL when it can't find the
1776 encoding in the first or second line of the file (in which case the encoding
1777 should be assumed to be UTF-8).
Brett Cannone4539892007-10-20 03:46:49 +00001778
Victor Stinnerfe7c5b52011-04-05 01:48:03 +02001779 The char* returned is malloc'ed via PyMem_MALLOC() and thus must be freed
1780 by the caller. */
1781
Guido van Rossumce3a72a2007-10-19 23:16:50 +00001782char *
Victor Stinnerfe7c5b52011-04-05 01:48:03 +02001783PyTokenizer_FindEncodingFilename(int fd, PyObject *filename)
Guido van Rossum40d20bc2007-10-22 00:09:51 +00001784{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001785 struct tok_state *tok;
1786 FILE *fp;
1787 char *p_start =NULL , *p_end =NULL , *encoding = NULL;
Guido van Rossumce3a72a2007-10-19 23:16:50 +00001788
Victor Stinnerdaf45552013-08-28 00:53:59 +02001789 fd = _Py_dup(fd);
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001790 if (fd < 0) {
1791 return NULL;
1792 }
Victor Stinnerdaf45552013-08-28 00:53:59 +02001793
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001794 fp = fdopen(fd, "r");
1795 if (fp == NULL) {
1796 return NULL;
1797 }
1798 tok = PyTokenizer_FromFile(fp, NULL, NULL, NULL);
1799 if (tok == NULL) {
1800 fclose(fp);
1801 return NULL;
1802 }
Victor Stinnerfe7c5b52011-04-05 01:48:03 +02001803 if (filename != NULL) {
1804 Py_INCREF(filename);
1805 tok->filename = filename;
1806 }
1807 else {
1808 tok->filename = PyUnicode_FromString("<string>");
1809 if (tok->filename == NULL) {
1810 fclose(fp);
1811 PyTokenizer_Free(tok);
1812 return encoding;
1813 }
1814 }
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001815 while (tok->lineno < 2 && tok->done == E_OK) {
1816 PyTokenizer_Get(tok, &p_start, &p_end);
1817 }
1818 fclose(fp);
1819 if (tok->encoding) {
1820 encoding = (char *)PyMem_MALLOC(strlen(tok->encoding) + 1);
1821 if (encoding)
1822 strcpy(encoding, tok->encoding);
1823 }
1824 PyTokenizer_Free(tok);
1825 return encoding;
Guido van Rossumce3a72a2007-10-19 23:16:50 +00001826}
Thomas Wouters89d996e2007-09-08 17:39:28 +00001827
Victor Stinnerfe7c5b52011-04-05 01:48:03 +02001828char *
1829PyTokenizer_FindEncoding(int fd)
1830{
1831 return PyTokenizer_FindEncodingFilename(fd, NULL);
1832}
1833
Guido van Rossum408027e1996-12-30 16:17:54 +00001834#ifdef Py_DEBUG
Guido van Rossum85a5fbb1990-10-14 12:07:46 +00001835
1836void
Thomas Wouters23c9e002000-07-22 19:20:54 +00001837tok_dump(int type, char *start, char *end)
Guido van Rossum85a5fbb1990-10-14 12:07:46 +00001838{
Antoine Pitrouf95a1b32010-05-09 15:52:27 +00001839 printf("%s", _PyParser_TokenNames[type]);
1840 if (type == NAME || type == NUMBER || type == STRING || type == OP)
1841 printf("(%.*s)", (int)(end - start), start);
Guido van Rossum85a5fbb1990-10-14 12:07:46 +00001842}
1843
1844#endif