commit d73aca769f1f6eebb46faa9161cbebe806db3659
author Benjamin Peterson <benjamin@python.org> Tue Apr 21 12:05:19 2015 -0400
committer Benjamin Peterson <benjamin@python.org> Tue Apr 21 12:05:19 2015 -0400
tree d257c03a998c63a1a89b9e36bb820c43b1beb0db
parent 6de708fd46207f6af67d4c0e8902f0d56ea4495c

do not call into python api if an exception is set (#24022)

Parser/tokenizer.c

diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
index 22accd1..5e041ea 100644
--- a/Parser/tokenizer.c
+++ b/Parser/tokenizer.c
@@ -1301,6 +1301,8 @@
 {
     PyObject *s;
     int result;
+    if (tok->decoding_erred)
+        return 0;
     s = PyUnicode_DecodeUTF8(tok->start, tok->cur - tok->start, NULL);
     if (s == NULL || PyUnicode_READY(s) == -1) {
         if (PyErr_ExceptionMatches(PyExc_UnicodeDecodeError)) {
@@ -1469,11 +1471,8 @@
             c = tok_nextc(tok);
         }
         tok_backup(tok, c);
-        if (nonascii &&
-            !verify_identifier(tok)) {
-            tok->done = E_IDENTIFIER;
+        if (nonascii && !verify_identifier(tok))
             return ERRORTOKEN;
-        }
         *p_start = tok->start;
         *p_end = tok->cur;
         return NAME;