Add interning of unicode strings by copying the functionality from
stringobject.c.
Intern "True" and "False" in bool_repr() again as it was in the
8bit string era.
diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c
index 999b166..854310b 100644
--- a/Objects/unicodeobject.c
+++ b/Objects/unicodeobject.c
@@ -92,6 +92,16 @@
extern "C" {
#endif
+/* This dictionary holds all interned unicode strings. Note that references
+ to strings in this dictionary are *not* counted in the string's ob_refcnt.
+ When the interned string reaches a refcnt of 0 the string deallocation
+ function will delete the reference from this dictionary.
+
+ Another way to look at this is that to say that the actual reference
+ count of a string is: s->ob_refcnt + (s->ob_sstate?2:0)
+*/
+static PyObject *interned;
+
/* Free list for Unicode objects */
static PyUnicodeObject *unicode_freelist;
static int unicode_freelist_size;
@@ -276,6 +286,7 @@
unicode->str[length] = 0;
unicode->length = length;
unicode->hash = -1;
+ unicode->state = 0;
unicode->defenc = NULL;
return unicode;
@@ -288,6 +299,25 @@
static
void unicode_dealloc(register PyUnicodeObject *unicode)
{
+ switch (PyUnicode_CHECK_INTERNED(unicode)) {
+ case SSTATE_NOT_INTERNED:
+ break;
+
+ case SSTATE_INTERNED_MORTAL:
+ /* revive dead object temporarily for DelItem */
+ unicode->ob_refcnt = 3;
+ if (PyDict_DelItem(interned, (PyObject *)unicode) != 0)
+ Py_FatalError(
+ "deletion of interned unicode string failed");
+ break;
+
+ case SSTATE_INTERNED_IMMORTAL:
+ Py_FatalError("Immortal interned unicode string died.");
+
+ default:
+ Py_FatalError("Inconsistent interned unicode string state.");
+ }
+
if (PyUnicode_CheckExact(unicode) &&
unicode_freelist_size < MAX_UNICODE_FREELIST_SIZE) {
/* Keep-Alive optimization */
@@ -8564,6 +8594,115 @@
unicode_freelist_size = 0;
}
+void
+PyUnicode_InternInPlace(PyObject **p)
+{
+ register PyUnicodeObject *s = (PyUnicodeObject *)(*p);
+ PyObject *t;
+ if (s == NULL || !PyUnicode_Check(s))
+ Py_FatalError(
+ "PyUnicode_InternInPlace: unicode strings only please!");
+ /* If it's a subclass, we don't really know what putting
+ it in the interned dict might do. */
+ if (!PyUnicode_CheckExact(s))
+ return;
+ if (PyUnicode_CHECK_INTERNED(s))
+ return;
+ if (interned == NULL) {
+ interned = PyDict_New();
+ if (interned == NULL) {
+ PyErr_Clear(); /* Don't leave an exception */
+ return;
+ }
+ }
+ t = PyDict_GetItem(interned, (PyObject *)s);
+ if (t) {
+ Py_INCREF(t);
+ Py_DECREF(*p);
+ *p = t;
+ return;
+ }
+
+ if (PyDict_SetItem(interned, (PyObject *)s, (PyObject *)s) < 0) {
+ PyErr_Clear();
+ return;
+ }
+ /* The two references in interned are not counted by refcnt.
+ The deallocator will take care of this */
+ s->ob_refcnt -= 2;
+ PyUnicode_CHECK_INTERNED(s) = SSTATE_INTERNED_MORTAL;
+}
+
+void
+PyUnicode_InternImmortal(PyObject **p)
+{
+ PyUnicode_InternInPlace(p);
+ if (PyUnicode_CHECK_INTERNED(*p) != SSTATE_INTERNED_IMMORTAL) {
+ PyUnicode_CHECK_INTERNED(*p) = SSTATE_INTERNED_IMMORTAL;
+ Py_INCREF(*p);
+ }
+}
+
+PyObject *
+PyUnicode_InternFromString(const char *cp)
+{
+ PyObject *s = PyUnicode_FromString(cp);
+ if (s == NULL)
+ return NULL;
+ PyUnicode_InternInPlace(&s);
+ return s;
+}
+
+void _Py_ReleaseInternedUnicodeStrings(void)
+{
+ PyObject *keys;
+ PyUnicodeObject *s;
+ Py_ssize_t i, n;
+ Py_ssize_t immortal_size = 0, mortal_size = 0;
+
+ if (interned == NULL || !PyDict_Check(interned))
+ return;
+ keys = PyDict_Keys(interned);
+ if (keys == NULL || !PyList_Check(keys)) {
+ PyErr_Clear();
+ return;
+ }
+
+ /* Since _Py_ReleaseInternedUnicodeStrings() is intended to help a leak
+ detector, interned unicode strings are not forcibly deallocated;
+ rather, we give them their stolen references back, and then clear
+ and DECREF the interned dict. */
+
+ n = PyList_GET_SIZE(keys);
+ fprintf(stderr, "releasing %" PY_FORMAT_SIZE_T "d interned strings\n",
+ n);
+ for (i = 0; i < n; i++) {
+ s = (PyUnicodeObject *) PyList_GET_ITEM(keys, i);
+ switch (s->state) {
+ case SSTATE_NOT_INTERNED:
+ /* XXX Shouldn't happen */
+ break;
+ case SSTATE_INTERNED_IMMORTAL:
+ s->ob_refcnt += 1;
+ immortal_size += s->length;
+ break;
+ case SSTATE_INTERNED_MORTAL:
+ s->ob_refcnt += 2;
+ mortal_size += s->length;
+ break;
+ default:
+ Py_FatalError("Inconsistent interned string state.");
+ }
+ s->state = SSTATE_NOT_INTERNED;
+ }
+ fprintf(stderr, "total size of all interned strings: "
+ "%" PY_FORMAT_SIZE_T "d/%" PY_FORMAT_SIZE_T "d "
+ "mortal/immortal\n", mortal_size, immortal_size);
+ Py_DECREF(keys);
+ PyDict_Clear(interned);
+ Py_DECREF(interned);
+ interned = NULL;
+}
/********************* Unicode Iterator **************************/