Implement PEP 393.
diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
index f4d7e3f..a0a3a67 100644
--- a/Parser/tokenizer.c
+++ b/Parser/tokenizer.c
@@ -1258,14 +1258,16 @@
#ifdef PGEN
#define verify_identifier(tok) 1
#else
-/* Verify that the identifier follows PEP 3131. */
+/* Verify that the identifier follows PEP 3131.
+ All identifier strings are guaranteed to be "ready" unicode objects.
+ */
static int
verify_identifier(struct tok_state *tok)
{
PyObject *s;
int result;
s = PyUnicode_DecodeUTF8(tok->start, tok->cur - tok->start, NULL);
- if (s == NULL) {
+ if (s == NULL || PyUnicode_READY(s) == -1) {
if (PyErr_ExceptionMatches(PyExc_UnicodeDecodeError)) {
PyErr_Clear();
tok->done = E_IDENTIFIER;