Make identifiers str (not str8) objects throughout.
This affects the parser, various object implementations,
and all places that put identifiers into C string literals.
In testing, a number of crashes occurred as code would
fail when the recursion limit was reached (such as the
Unicode interning dictionary having key/value pairs where
key is not value). To solve these, I added an overflowed
flag, which allows for 50 more recursions after the
limit was reached and the exception was raised, and
a recursion_critical flag, which indicates that recursion
absolutely must be allowed, i.e. that a certain call
must not cause a stack overflow exception.
There are still some places where both str and str8 are
accepted as identifiers; these should eventually be
removed.
diff --git a/Python/ast.c b/Python/ast.c
index e0bd18e..b34411b 100644
--- a/Python/ast.c
+++ b/Python/ast.c
@@ -48,7 +48,8 @@
static identifier
new_identifier(const char* n, PyArena *arena) {
- PyObject* id = PyString_InternFromString(n);
+ PyObject* id = PyUnicode_DecodeUTF8(n, strlen(n), NULL);
+ PyUnicode_InternInPlace(&id);
PyArena_AddPyObject(arena, id);
return id;
}
@@ -334,12 +335,10 @@
static int
forbidden_name(expr_ty e, const node *n)
{
- const char *id;
const char **p;
- assert(PyString_Check(e->v.Name.id));
- id = PyString_AS_STRING(e->v.Name.id);
+ assert(PyUnicode_Check(e->v.Name.id));
for (p = FORBIDDEN; *p; p++) {
- if (strcmp(*p, id) == 0) {
+ if (PyUnicode_CompareWithASCIIString(e->v.Name.id, *p) == 0) {
ast_error(n, "assignment to keyword");
return 1;
}
@@ -375,7 +374,7 @@
switch (e->kind) {
case Attribute_kind:
if (ctx == Store &&
- !strcmp(PyString_AS_STRING(e->v.Attribute.attr), "None")) {
+ !PyUnicode_CompareWithASCIIString(e->v.Attribute.attr, "None")) {
return ast_error(n, "assignment to None");
}
e->v.Attribute.ctx = ctx;
@@ -2235,6 +2234,7 @@
int i;
size_t len;
char *s;
+ PyObject *uni;
len = 0;
for (i = 0; i < NCH(n); i += 2)
@@ -2255,13 +2255,20 @@
}
--s;
*s = '\0';
- PyString_InternInPlace(&str);
+ uni = PyUnicode_DecodeUTF8(PyString_AS_STRING(str),
+ PyString_GET_SIZE(str),
+ NULL);
+ Py_DECREF(str);
+ if (!uni)
+ return NULL;
+ str = uni;
+ PyUnicode_InternInPlace(&str);
PyArena_AddPyObject(c->c_arena, str);
return alias(str, NULL, c->c_arena);
}
break;
case STAR:
- str = PyString_InternFromString("*");
+ str = PyUnicode_InternFromString("*");
PyArena_AddPyObject(c->c_arena, str);
return alias(str, NULL, c->c_arena);
default: