Implement PEP 3131. Add isidentifier to str.
diff --git a/Python/ast.c b/Python/ast.c
index 5426c02..c13d093 100644
--- a/Python/ast.c
+++ b/Python/ast.c
@@ -47,8 +47,27 @@
#define COMP_SETCOMP 2
static identifier
-new_identifier(const char* n, PyArena *arena) {
+new_identifier(const char* n, PyArena *arena)
+{
PyObject* id = PyUnicode_DecodeUTF8(n, strlen(n), NULL);
+ Py_UNICODE *u = PyUnicode_AS_UNICODE(id);
+ /* Check whether there are non-ASCII characters in the
+ identifier; if so, normalize to NFKC. */
+ for (; *u; u++) {
+ if (*u >= 128) {
+ PyObject *m = PyImport_ImportModule("unicodedata");
+ PyObject *id2;
+ if (!m)
+ return NULL;
+ id2 = PyObject_CallMethod(m, "normalize", "sO", "NFKC", id);
+ Py_DECREF(m);
+ if (!id2)
+ return NULL;
+ Py_DECREF(id);
+ id = id2;
+ break;
+ }
+ }
PyUnicode_InternInPlace(&id);
PyArena_AddPyObject(arena, id);
return id;