Issue #3329: Add new APIs to customize memory allocators
* Add a new PyMemAllocators structure
* New functions:
- PyMem_RawMalloc(), PyMem_RawRealloc(), PyMem_RawFree(): GIL-free memory
allocator functions
- PyMem_GetRawAllocators(), PyMem_SetRawAllocators()
- PyMem_GetAllocators(), PyMem_SetAllocators()
- PyMem_SetupDebugHooks()
- _PyObject_GetArenaAllocators(), _PyObject_SetArenaAllocators()
* Add unit test for PyMem_Malloc(0) and PyObject_Malloc(0)
* Add unit test for new get/set allocators functions
* PyObject_Malloc() now falls back on PyMem_Malloc() instead of malloc() if
size is bigger than SMALL_REQUEST_THRESHOLD, and PyObject_Realloc() falls
back on PyMem_Realloc() instead of realloc()
* PyMem_Malloc() and PyMem_Realloc() now always call malloc() and realloc(),
instead of calling PyObject_Malloc() and PyObject_Realloc() in debug mode
diff --git a/Include/objimpl.h b/Include/objimpl.h
index 8f36360..740c46f 100644
--- a/Include/objimpl.h
+++ b/Include/objimpl.h
@@ -94,9 +94,9 @@
the object gets initialized via PyObject_{Init, InitVar} after obtaining
the raw memory.
*/
-PyAPI_FUNC(void *) PyObject_Malloc(size_t);
-PyAPI_FUNC(void *) PyObject_Realloc(void *, size_t);
-PyAPI_FUNC(void) PyObject_Free(void *);
+PyAPI_FUNC(void *) PyObject_Malloc(size_t size);
+PyAPI_FUNC(void *) PyObject_Realloc(void *ptr, size_t new_size);
+PyAPI_FUNC(void) PyObject_Free(void *ptr);
/* This function returns the number of allocated memory blocks, regardless of size */
PyAPI_FUNC(Py_ssize_t) _Py_GetAllocatedBlocks(void);
@@ -106,41 +106,46 @@
#ifndef Py_LIMITED_API
PyAPI_FUNC(void) _PyObject_DebugMallocStats(FILE *out);
#endif /* #ifndef Py_LIMITED_API */
-#ifdef PYMALLOC_DEBUG /* WITH_PYMALLOC && PYMALLOC_DEBUG */
-PyAPI_FUNC(void *) _PyObject_DebugMalloc(size_t nbytes);
-PyAPI_FUNC(void *) _PyObject_DebugRealloc(void *p, size_t nbytes);
-PyAPI_FUNC(void) _PyObject_DebugFree(void *p);
-PyAPI_FUNC(void) _PyObject_DebugDumpAddress(const void *p);
-PyAPI_FUNC(void) _PyObject_DebugCheckAddress(const void *p);
-PyAPI_FUNC(void *) _PyObject_DebugMallocApi(char api, size_t nbytes);
-PyAPI_FUNC(void *) _PyObject_DebugReallocApi(char api, void *p, size_t nbytes);
-PyAPI_FUNC(void) _PyObject_DebugFreeApi(char api, void *p);
-PyAPI_FUNC(void) _PyObject_DebugCheckAddressApi(char api, const void *p);
-PyAPI_FUNC(void *) _PyMem_DebugMalloc(size_t nbytes);
-PyAPI_FUNC(void *) _PyMem_DebugRealloc(void *p, size_t nbytes);
-PyAPI_FUNC(void) _PyMem_DebugFree(void *p);
-#define PyObject_MALLOC _PyObject_DebugMalloc
-#define PyObject_Malloc _PyObject_DebugMalloc
-#define PyObject_REALLOC _PyObject_DebugRealloc
-#define PyObject_Realloc _PyObject_DebugRealloc
-#define PyObject_FREE _PyObject_DebugFree
-#define PyObject_Free _PyObject_DebugFree
+#endif
-#else /* WITH_PYMALLOC && ! PYMALLOC_DEBUG */
+/* Macros */
#define PyObject_MALLOC PyObject_Malloc
#define PyObject_REALLOC PyObject_Realloc
#define PyObject_FREE PyObject_Free
-#endif
-
-#else /* ! WITH_PYMALLOC */
-#define PyObject_MALLOC PyMem_MALLOC
-#define PyObject_REALLOC PyMem_REALLOC
-#define PyObject_FREE PyMem_FREE
-
-#endif /* WITH_PYMALLOC */
-
#define PyObject_Del PyObject_Free
-#define PyObject_DEL PyObject_FREE
+#define PyObject_DEL PyObject_Free
+
+/* Get internal functions of PyObject_Malloc(), PyObject_Realloc() and
+ PyObject_Free(). *ctx_p is an arbitrary user value. */
+PyAPI_FUNC(void) PyObject_GetAllocators(PyMemAllocators *allocators);
+
+/* Set internal functions of PyObject_Malloc(), PyObject_Realloc() and PyObject_Free().
+ ctx is an arbitrary user value.
+
+ malloc(ctx, 0) and realloc(ctx, ptr, 0) must not return NULL: it would be
+ treated as an error.
+
+ PyMem_SetupDebugHooks() should be called to reinstall debug hooks if new
+ functions do no call original functions anymore. */
+PyAPI_FUNC(void) PyObject_SetAllocators(PyMemAllocators *allocators);
+
+/* Get internal functions allocating and deallocating arenas for
+ PyObject_Malloc(), PyObject_Realloc() and PyObject_Free().
+ *ctx_p is an arbitrary user value. */
+PyAPI_FUNC(void) _PyObject_GetArenaAllocators(
+ void **ctx_p,
+ void* (**malloc_p) (void *ctx, size_t size),
+ void (**free_p) (void *ctx, void *ptr, size_t size)
+ );
+
+/* Get internal functions allocating and deallocating arenas for
+ PyObject_Malloc(), PyObject_Realloc() and PyObject_Free().
+ ctx is an arbitrary user value. */
+PyAPI_FUNC(void) _PyObject_SetArenaAllocators(
+ void *ctx,
+ void* (*malloc) (void *ctx, size_t size),
+ void (*free) (void *ctx, void *ptr, size_t size)
+ );
/*
* Generic object allocator interface
diff --git a/Include/pymem.h b/Include/pymem.h
index 10b5bea..34d9318 100644
--- a/Include/pymem.h
+++ b/Include/pymem.h
@@ -11,6 +11,40 @@
extern "C" {
#endif
+typedef struct {
+ /* user context passed as the first argument to the 3 functions */
+ void *ctx;
+
+ /* allocate memory */
+ void* (*malloc) (void *ctx, size_t size);
+
+ /* allocate memory or resize a memory buffer */
+ void* (*realloc) (void *ctx, void *ptr, size_t new_size);
+
+ /* release memory */
+ void (*free) (void *ctx, void *ptr);
+} PyMemAllocators;
+
+/* Raw memory allocators, system functions: malloc(), realloc(), free().
+
+ These functions are thread-safe, the GIL does not need to be held. */
+
+/* Get internal functions of PyMem_RawMalloc(), PyMem_RawRealloc() and
+ PyMem_RawFree(). *ctx_p is an arbitrary user value. */
+PyAPI_FUNC(void) PyMem_GetRawAllocators(PyMemAllocators *allocators);
+
+/* Set internal functions of PyMem_RawMalloc(), PyMem_RawRealloc() and
+ PyMem_RawFree(). ctx is an arbitrary user value.
+
+ PyMem_SetupDebugHooks() should be called to reinstall debug hooks if new
+ functions do no call original functions anymore. */
+PyAPI_FUNC(void) PyMem_SetRawAllocators(PyMemAllocators *allocators);
+
+PyAPI_FUNC(void *) PyMem_RawMalloc(size_t size);
+PyAPI_FUNC(void *) PyMem_RawRealloc(void *ptr, size_t new_size);
+PyAPI_FUNC(void) PyMem_RawFree(void *ptr);
+
+
/* BEWARE:
Each interface exports both functions and macros. Extension modules should
@@ -49,21 +83,11 @@
performed on failure (no exception is set, no warning is printed, etc).
*/
-PyAPI_FUNC(void *) PyMem_Malloc(size_t);
-PyAPI_FUNC(void *) PyMem_Realloc(void *, size_t);
-PyAPI_FUNC(void) PyMem_Free(void *);
-
-/* Starting from Python 1.6, the wrappers Py_{Malloc,Realloc,Free} are
- no longer supported. They used to call PyErr_NoMemory() on failure. */
+PyAPI_FUNC(void *) PyMem_Malloc(size_t size);
+PyAPI_FUNC(void *) PyMem_Realloc(void *ptr, size_t new_size);
+PyAPI_FUNC(void) PyMem_Free(void *ptr);
/* Macros. */
-#ifdef PYMALLOC_DEBUG
-/* Redirect all memory operations to Python's debugging allocator. */
-#define PyMem_MALLOC _PyMem_DebugMalloc
-#define PyMem_REALLOC _PyMem_DebugRealloc
-#define PyMem_FREE _PyMem_DebugFree
-
-#else /* ! PYMALLOC_DEBUG */
/* PyMem_MALLOC(0) means malloc(1). Some systems would return NULL
for malloc(0), which would be treated as an error. Some platforms
@@ -71,13 +95,9 @@
pymalloc. To solve these problems, allocate an extra byte. */
/* Returns NULL to indicate error if a negative size or size larger than
Py_ssize_t can represent is supplied. Helps prevents security holes. */
-#define PyMem_MALLOC(n) ((size_t)(n) > (size_t)PY_SSIZE_T_MAX ? NULL \
- : malloc((n) ? (n) : 1))
-#define PyMem_REALLOC(p, n) ((size_t)(n) > (size_t)PY_SSIZE_T_MAX ? NULL \
- : realloc((p), (n) ? (n) : 1))
-#define PyMem_FREE free
-
-#endif /* PYMALLOC_DEBUG */
+#define PyMem_MALLOC(n) PyMem_Malloc(n)
+#define PyMem_REALLOC(p, n) PyMem_Realloc(p, n)
+#define PyMem_FREE(p) PyMem_Free(p)
/*
* Type-oriented memory interface
@@ -115,6 +135,37 @@
#define PyMem_Del PyMem_Free
#define PyMem_DEL PyMem_FREE
+/* Get internal functions of PyMem_Malloc(), PyMem_Realloc()
+ and PyMem_Free() */
+PyAPI_FUNC(void) PyMem_GetAllocators(PyMemAllocators *allocators);
+
+/* Set internal functions of PyMem_Malloc(), PyMem_Realloc() and PyMem_Free().
+
+ malloc(ctx, 0) and realloc(ctx, ptr, 0) must not return NULL: it would be
+ treated as an error.
+
+ PyMem_SetupDebugHooks() should be called to reinstall debug hooks if new
+ functions do no call original functions anymore. */
+PyAPI_FUNC(void) PyMem_SetAllocators(PyMemAllocators *allocators);
+
+/* Setup hooks to detect bugs in the following Python memory allocator
+ functions:
+
+ - PyMem_RawMalloc(), PyMem_RawRealloc(), PyMem_RawFree()
+ - PyMem_Malloc(), PyMem_Realloc(), PyMem_Free()
+ - PyObject_Malloc(), PyObject_Realloc() and PyObject_Free()
+
+ Newly allocated memory is filled with the byte 0xCB, freed memory is filled
+ with the byte 0xDB. Additionnal checks:
+
+ - detect API violations, ex: PyObject_Free() called on a buffer allocated
+ by PyMem_Malloc()
+ - detect write before the start of the buffer (buffer underflow)
+ - detect write after the end of the buffer (buffer overflow)
+
+ The function does nothing if Python is not compiled is debug mode. */
+PyAPI_FUNC(void) PyMem_SetupDebugHooks(void);
+
#ifdef __cplusplus
}
#endif