Implemented Martin's suggestion to clear the free lists during the garbage collection of the highest generation.
diff --git a/Doc/c-api/method.rst b/Doc/c-api/method.rst
index f20f14c..c104f89 100644
--- a/Doc/c-api/method.rst
+++ b/Doc/c-api/method.rst
@@ -63,3 +63,10 @@
.. cfunction:: PyObject* PyMethod_GET_SELF(PyObject *meth)
Macro version of :cfunc:`PyMethod_Self` which avoids error checking.
+
+
+.. cfunction:: int PyMethod_ClearFreeList(void)
+
+ Clear the free list. Return the total number of freed items.
+
+ .. versionadded:: 2.6
diff --git a/Doc/c-api/tuple.rst b/Doc/c-api/tuple.rst
index b25adb3..1fcf5fc 100644
--- a/Doc/c-api/tuple.rst
+++ b/Doc/c-api/tuple.rst
@@ -115,3 +115,10 @@
.. versionchanged:: 2.2
Removed unused third parameter, *last_is_sticky*.
+
+
+.. cfunction:: int PyMethod_ClearFreeList(void)
+
+ Clear the free list. Return the total number of freed items.
+
+ .. versionadded:: 2.6
diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst
index 7c570ab..5e5d1a2 100644
--- a/Doc/c-api/unicode.rst
+++ b/Doc/c-api/unicode.rst
@@ -89,6 +89,13 @@
Return a pointer to the internal buffer of the object. *o* has to be a
:ctype:`PyUnicodeObject` (not checked).
+
+.. cfunction:: int PyUnicode_ClearFreeList(void)
+
+ Clear the free list. Return the total number of freed items.
+
+ .. versionadded:: 2.6
+
Unicode provides many different character properties. The most often needed ones
are available through these macros which are mapped to C functions depending on
the Python configuration.
diff --git a/Include/classobject.h b/Include/classobject.h
index 8f8db7d..118dd09 100644
--- a/Include/classobject.h
+++ b/Include/classobject.h
@@ -74,6 +74,7 @@
PyAPI_FUNC(int) PyClass_IsSubclass(PyObject *, PyObject *);
+PyAPI_FUNC(int) PyMethod_ClearFreeList(void);
#ifdef __cplusplus
}
diff --git a/Include/frameobject.h b/Include/frameobject.h
index 794f651..297b66d 100644
--- a/Include/frameobject.h
+++ b/Include/frameobject.h
@@ -75,6 +75,8 @@
PyAPI_FUNC(void) PyFrame_LocalsToFast(PyFrameObject *, int);
PyAPI_FUNC(void) PyFrame_FastToLocals(PyFrameObject *);
+PyAPI_FUNC(int) PyFrame_ClearFreeList(void);
+
#ifdef __cplusplus
}
#endif
diff --git a/Include/methodobject.h b/Include/methodobject.h
index 810a69c..6e160b6 100644
--- a/Include/methodobject.h
+++ b/Include/methodobject.h
@@ -85,6 +85,8 @@
PyObject *m_module; /* The __module__ attribute, can be anything */
} PyCFunctionObject;
+PyAPI_FUNC(int) PyCFunction_ClearFreeList(void);
+
#ifdef __cplusplus
}
#endif
diff --git a/Include/tupleobject.h b/Include/tupleobject.h
index 1b51b1b..58479ee 100644
--- a/Include/tupleobject.h
+++ b/Include/tupleobject.h
@@ -52,6 +52,8 @@
/* Macro, *only* to be used to fill in brand new tuples */
#define PyTuple_SET_ITEM(op, i, v) (((PyTupleObject *)(op))->ob_item[i] = v)
+PyAPI_FUNC(int) PyTuple_ClearFreeList(void);
+
#ifdef __cplusplus
}
#endif
diff --git a/Include/unicodeobject.h b/Include/unicodeobject.h
index 8809b34..d7ed0fa 100644
--- a/Include/unicodeobject.h
+++ b/Include/unicodeobject.h
@@ -209,6 +209,7 @@
# define _PyUnicode_AsDefaultEncodedString _PyUnicodeUCS2_AsDefaultEncodedString
# define _PyUnicode_Fini _PyUnicodeUCS2_Fini
# define _PyUnicode_Init _PyUnicodeUCS2_Init
+# define PyUnicode_ClearFreeList PyUnicodeUCS2_ClearFreelist
# define _PyUnicode_IsAlpha _PyUnicodeUCS2_IsAlpha
# define _PyUnicode_IsDecimalDigit _PyUnicodeUCS2_IsDecimalDigit
# define _PyUnicode_IsDigit _PyUnicodeUCS2_IsDigit
@@ -295,6 +296,7 @@
# define _PyUnicode_AsDefaultEncodedString _PyUnicodeUCS4_AsDefaultEncodedString
# define _PyUnicode_Fini _PyUnicodeUCS4_Fini
# define _PyUnicode_Init _PyUnicodeUCS4_Init
+# define PyUnicode_ClearFreeList PyUnicodeUCS2_ClearFreelist
# define _PyUnicode_IsAlpha _PyUnicodeUCS4_IsAlpha
# define _PyUnicode_IsDecimalDigit _PyUnicodeUCS4_IsDecimalDigit
# define _PyUnicode_IsDigit _PyUnicodeUCS4_IsDigit
@@ -403,6 +405,8 @@
extern "C" {
#endif
+PyAPI_FUNC(int) PyUnicode_ClearFreeList(void);
+
/* --- Unicode Type ------------------------------------------------------- */
typedef struct {
diff --git a/Misc/NEWS b/Misc/NEWS
index fe4b89f..134ac7e 100644
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -12,6 +12,10 @@
Core and builtins
-----------------
+- Clear all free list during a gc.collect() of the highest generation in order
+ to allow pymalloc to free more arenas. Python may give back memory to the
+ OS earlier.
+
- Issue #2045: Fix an infinite recursion triggered when printing a subclass of
collections.defaultdict, if its default_factory is set to a bound method.
diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c
index 522cc89..4f8c85f 100644
--- a/Modules/gcmodule.c
+++ b/Modules/gcmodule.c
@@ -19,6 +19,7 @@
*/
#include "Python.h"
+#include "frameobject.h" /* for PyFrame_ClearFreeList */
/* Get an object's GC head */
#define AS_GC(o) ((PyGC_Head *)(o)-1)
@@ -722,6 +723,21 @@
}
}
+/* Clear all free lists
+ * All free lists are cleared during the collection of the highest generation.
+ * Allocated items in the free list may keep a pymalloc arena occupied.
+ * Clearing the free lists may give back memory to the OS earlier.
+ */
+static void
+clear_freelists(void)
+{
+ (void)PyMethod_ClearFreeList();
+ (void)PyFrame_ClearFreeList();
+ (void)PyCFunction_ClearFreeList();
+ (void)PyTuple_ClearFreeList();
+ (void)PyUnicode_ClearFreeList();
+}
+
/* This is the main function. Read this to understand how the
* collection process works. */
static Py_ssize_t
@@ -874,6 +890,12 @@
*/
(void)handle_finalizers(&finalizers, old);
+ /* Clear free list only during the collection of the higest
+ * generation */
+ if (generation == NUM_GENERATIONS-1) {
+ clear_freelists();
+ }
+
if (PyErr_Occurred()) {
if (gc_str == NULL)
gc_str = PyString_FromString("garbage collection");
diff --git a/Objects/classobject.c b/Objects/classobject.c
index 759027b..6c77153 100644
--- a/Objects/classobject.c
+++ b/Objects/classobject.c
@@ -2626,9 +2626,11 @@
/* Clear out the free list */
-void
-PyMethod_Fini(void)
+int
+PyMethod_ClearFreeList(void)
{
+ int freelist_size = numfree;
+
while (free_list) {
PyMethodObject *im = free_list;
free_list = (PyMethodObject *)(im->im_self);
@@ -2636,4 +2638,11 @@
numfree--;
}
assert(numfree == 0);
+ return freelist_size;
+}
+
+void
+PyMethod_Fini(void)
+{
+ (void)PyMethod_ClearFreeList();
}
diff --git a/Objects/frameobject.c b/Objects/frameobject.c
index df9c2e0..025431e 100644
--- a/Objects/frameobject.c
+++ b/Objects/frameobject.c
@@ -889,10 +889,11 @@
}
/* Clear out the free list */
-
-void
-PyFrame_Fini(void)
+int
+PyFrame_ClearFreeList(void)
{
+ int freelist_size = numfree;
+
while (free_list != NULL) {
PyFrameObject *f = free_list;
free_list = free_list->f_back;
@@ -900,6 +901,13 @@
--numfree;
}
assert(numfree == 0);
+ return freelist_size;
+}
+
+void
+PyFrame_Fini(void)
+{
+ (void)PyFrame_ClearFreeList();
Py_XDECREF(builtin_object);
builtin_object = NULL;
}
diff --git a/Objects/methodobject.c b/Objects/methodobject.c
index 0d9cf1c..d661c47 100644
--- a/Objects/methodobject.c
+++ b/Objects/methodobject.c
@@ -353,9 +353,11 @@
/* Clear out the free list */
-void
-PyCFunction_Fini(void)
+int
+PyCFunction_ClearFreeList(void)
{
+ int freelist_size = numfree;
+
while (free_list) {
PyCFunctionObject *v = free_list;
free_list = (PyCFunctionObject *)(v->m_self);
@@ -363,6 +365,13 @@
numfree--;
}
assert(numfree == 0);
+ return freelist_size;
+}
+
+void
+PyCFunction_Fini(void)
+{
+ (void)PyCFunction_ClearFreeList();
}
/* PyCFunction_New() is now just a macro that calls PyCFunction_NewEx(),
diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c
index dfd2a16..e9cb3ef 100644
--- a/Objects/tupleobject.c
+++ b/Objects/tupleobject.c
@@ -832,19 +832,18 @@
return 0;
}
-void
-PyTuple_Fini(void)
+int
+PyTuple_ClearFreeList(void)
{
+ int freelist_size = 0;
#if PyTuple_MAXSAVESIZE > 0
int i;
-
- Py_XDECREF(free_list[0]);
- free_list[0] = NULL;
-
for (i = 1; i < PyTuple_MAXSAVESIZE; i++) {
PyTupleObject *p, *q;
p = free_list[i];
+ freelist_size += numfree[i];
free_list[i] = NULL;
+ numfree[i] = 0;
while (p) {
q = p;
p = (PyTupleObject *)(p->ob_item[0]);
@@ -852,6 +851,20 @@
}
}
#endif
+ return freelist_size;
+}
+
+void
+PyTuple_Fini(void)
+{
+#if PyTuple_MAXSAVESIZE > 0
+ /* empty tuples are used all over the place and applications may
+ * rely on the fact that an empty tuple is a singleton. */
+ Py_XDECREF(free_list[0]);
+ free_list[0] = NULL;
+
+ (void)PyTuple_ClearFreeList();
+#endif
}
/*********************** Tuple Iterator **************************/
diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c
index 74b4796..3e15f53 100644
--- a/Objects/unicodeobject.c
+++ b/Objects/unicodeobject.c
@@ -8853,10 +8853,29 @@
/* Finalize the Unicode implementation */
+int
+PyUnicode_ClearFreeList(void)
+{
+ int freelist_size = numfree;
+ PyUnicodeObject *u;
+
+ for (u = free_list; u != NULL;) {
+ PyUnicodeObject *v = u;
+ u = *(PyUnicodeObject **)u;
+ if (v->str)
+ PyMem_DEL(v->str);
+ Py_XDECREF(v->defenc);
+ PyObject_Del(v);
+ numfree--;
+ }
+ free_list = NULL;
+ assert(numfree == 0);
+ return freelist_size;
+}
+
void
_PyUnicode_Fini(void)
{
- PyUnicodeObject *u;
int i;
Py_XDECREF(unicode_empty);
@@ -8868,17 +8887,7 @@
unicode_latin1[i] = NULL;
}
}
-
- for (u = free_list; u != NULL;) {
- PyUnicodeObject *v = u;
- u = *(PyUnicodeObject **)u;
- if (v->str)
- PyMem_DEL(v->str);
- Py_XDECREF(v->defenc);
- PyObject_Del(v);
- }
- free_list = NULL;
- numfree = 0;
+ (void)PyUnicode_ClearFreeList();
}
#ifdef __cplusplus