Implement Valgrind support, redzones, and quarantine.
Implement Valgrind support, as well as the redzone and quarantine
features, which help Valgrind detect memory errors. Redzones are only
implemented for small objects because the changes necessary to support
redzones around large and huge objects are complicated by in-place
reallocation, to the point that it isn't clear that the maintenance
burden is worth the incremental improvement to Valgrind support.
Merge arena_salloc() and arena_salloc_demote().
Refactor i[v]salloc() to expose the 'demote' option.
diff --git a/INSTALL b/INSTALL
index 8a825df..a5942ec 100644
--- a/INSTALL
+++ b/INSTALL
@@ -113,8 +113,12 @@
mmap(2).
--disable-fill
- Disable support for junk/zero filling of memory. See the "opt.junk"/
- "opt.zero" option documentation for usage details.
+ Disable support for junk/zero filling of memory, quarantine, and redzones.
+ See the "opt.junk", "opt.zero", "opt.quarantine", and "opt.redzone" option
+ documentation for usage details.
+
+--disable-valgrind
+ Disable support for Valgrind.
--disable-experimental
Disable support for the experimental API (*allocm()).
diff --git a/Makefile.in b/Makefile.in
index 7df4fc6..8aa9425 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -49,9 +49,9 @@
@srcroot@src/chunk_dss.c @srcroot@src/chunk_mmap.c \
@srcroot@src/ckh.c @srcroot@src/ctl.c @srcroot@src/extent.c \
@srcroot@src/hash.c @srcroot@src/huge.c @srcroot@src/mb.c \
- @srcroot@src/mutex.c @srcroot@src/prof.c @srcroot@src/rtree.c \
- @srcroot@src/stats.c @srcroot@src/tcache.c @srcroot@src/util.c \
- @srcroot@src/tsd.c
+ @srcroot@src/mutex.c @srcroot@src/prof.c @srcroot@src/quarantine.c \
+ @srcroot@src/rtree.c @srcroot@src/stats.c @srcroot@src/tcache.c \
+ @srcroot@src/util.c @srcroot@src/tsd.c
ifeq (macho, @abi@)
CSRCS += @srcroot@src/zone.c
endif
diff --git a/configure.ac b/configure.ac
index 8e94b5c..a272ecd 100644
--- a/configure.ac
+++ b/configure.ac
@@ -685,7 +685,8 @@
dnl Support the junk/zero filling option by default.
AC_ARG_ENABLE([fill],
- [AS_HELP_STRING([--disable-fill], [Disable support for junk/zero filling])],
+ [AS_HELP_STRING([--disable-fill],
+ [Disable support for junk/zero filling, quarantine, and redzones])],
[if test "x$enable_fill" = "xno" ; then
enable_fill="0"
else
@@ -727,6 +728,38 @@
fi
AC_SUBST([enable_utrace])
+dnl Support Valgrind by default.
+AC_ARG_ENABLE([valgrind],
+ [AS_HELP_STRING([--disable-valgrind], [Disable support for Valgrind])],
+[if test "x$enable_valgrind" = "xno" ; then
+ enable_valgrind="0"
+else
+ enable_valgrind="1"
+fi
+],
+[enable_valgrind="1"]
+)
+if test "x$enable_valgrind" = "x1" ; then
+ JE_COMPILABLE([valgrind], [
+#include <valgrind/valgrind.h>
+#include <valgrind/memcheck.h>
+
+#if defined(__VALGRIND_MAJOR__) && defined(__VALGRIND_MINOR__) \
+ && (__VALGRIND_MAJOR__ > 3 || (__VALGRIND_MAJOR__ == 3 && \
+ __VALGRIND_MINOR__ >= 6))
+#else
+# error "Incompatible Valgrind version"
+#endif
+], [], [je_cv_valgrind])
+ if test "x${je_cv_valgrind}" = "xno" ; then
+ enable_valgrind="0"
+ fi
+ if test "x$enable_valgrind" = "x1" ; then
+ AC_DEFINE([JEMALLOC_VALGRIND], [ ])
+ fi
+fi
+AC_SUBST([enable_valgrind])
+
dnl Do not support the xmalloc option by default.
AC_ARG_ENABLE([xmalloc],
[AS_HELP_STRING([--enable-xmalloc], [Support xmalloc option])],
@@ -1088,8 +1121,9 @@
AC_MSG_RESULT([prof-gcc : ${enable_prof_gcc}])
AC_MSG_RESULT([tcache : ${enable_tcache}])
AC_MSG_RESULT([fill : ${enable_fill}])
-AC_MSG_RESULT([xmalloc : ${enable_xmalloc}])
AC_MSG_RESULT([utrace : ${enable_utrace}])
+AC_MSG_RESULT([valgrind : ${enable_valgrind}])
+AC_MSG_RESULT([xmalloc : ${enable_xmalloc}])
AC_MSG_RESULT([dss : ${enable_dss}])
AC_MSG_RESULT([lazy_lock : ${enable_lazy_lock}])
AC_MSG_RESULT([tls : ${enable_tls}])
diff --git a/doc/jemalloc.xml.in b/doc/jemalloc.xml.in
index 8ae8262..a47c763 100644
--- a/doc/jemalloc.xml.in
+++ b/doc/jemalloc.xml.in
@@ -722,6 +722,16 @@
<varlistentry>
<term>
+ <mallctl>config.valgrind</mallctl>
+ (<type>bool</type>)
+ <literal>r-</literal>
+ </term>
+ <listitem><para><option>--enable-valgrind</option> was specified during
+ build configuration.</para></listitem>
+ </varlistentry>
+
+ <varlistentry>
+ <term>
<mallctl>config.xmalloc</mallctl>
(<type>bool</type>)
<literal>r-</literal>
@@ -819,6 +829,47 @@
configuration, in which case it is enabled by default.</para></listitem>
</varlistentry>
+ <varlistentry id="opt.quarantine">
+ <term>
+ <mallctl>opt.quarantine</mallctl>
+ (<type>size_t</type>)
+ <literal>r-</literal>
+ [<option>--enable-fill</option>]
+ </term>
+ <listitem><para>Per thread quarantine size in bytes. If non-zero, each
+ thread maintains a FIFO object quarantine that stores up to the
+ specified number of bytes of memory. The quarantined memory is not
+ freed until it is released from quarantine, though it is immediately
+ junk-filled if the <link
+ linkend="opt.junk"><mallctl>opt.junk</mallctl></link> option is
+ enabled. This feature is of particular use in combination with <ulink
+ url="http://http://valgrind.org/">Valgrind</ulink>, which can detect
+ attempts to access quarantined objects. This is intended for debugging
+ and will impact performance negatively. The default quarantine size is
+ 0.</para></listitem>
+ </varlistentry>
+
+ <varlistentry id="opt.redzone">
+ <term>
+ <mallctl>opt.redzone</mallctl>
+ (<type>bool</type>)
+ <literal>r-</literal>
+ [<option>--enable-fill</option>]
+ </term>
+ <listitem><para>Redzones enabled/disabled. If enabled, small
+ allocations have redzones before and after them. Furthermore, if the
+ <link linkend="opt.junk"><mallctl>opt.junk</mallctl></link> option is
+ enabled, the redzones are checked for corruption during deallocation.
+ However, the primary intended purpose of this feature is to be used in
+ combination with <ulink
+ url="http://http://valgrind.org/">Valgrind</ulink>, which needs
+ redzones in order to do effective buffer overflow/underflow detection.
+ This option is intended for debugging and will impact performance
+ negatively. This option is disabled by default unless
+ <option>--enable-debug</option> is specified during configuration, in
+ which case it is enabled by default.</para></listitem>
+ </varlistentry>
+
<varlistentry id="opt.zero">
<term>
<mallctl>opt.zero</mallctl>
@@ -849,6 +900,25 @@
is disabled by default.</para></listitem>
</varlistentry>
+ <varlistentry id="opt.valgrind">
+ <term>
+ <mallctl>opt.valgrind</mallctl>
+ (<type>bool</type>)
+ <literal>r-</literal>
+ [<option>--enable-valgrind</option>]
+ </term>
+ <listitem><para><ulink
+ url="http://http://valgrind.org/">Valgrind</ulink> support
+ enabled/disabled. If enabled, several other options are automatically
+ modified during options processing to work well with Valgrind: <link
+ linkend="opt.junk"><mallctl>opt.junk</mallctl></link> and <link
+ linkend="opt.zero"><mallctl>opt.zero</mallctl></link> are set to false,
+ <link linkend="opt.quarantine"><mallctl>opt.quarantine</mallctl></link>
+ is set to 16 MiB, and <link
+ linkend="opt.redzone"><mallctl>opt.redzone</mallctl></link> is set to
+ true. This option is disabled by default.</para></listitem>
+ </varlistentry>
+
<varlistentry id="opt.xmalloc">
<term>
<mallctl>opt.xmalloc</mallctl>
@@ -1764,10 +1834,11 @@
<para>This implementation does not provide much detail about the problems
it detects, because the performance impact for storing such information
- would be prohibitive. There are a number of allocator implementations
- available on the Internet which focus on detecting and pinpointing problems
- by trading performance for extra sanity checks and detailed
- diagnostics.</para>
+ would be prohibitive. However, jemalloc does integrate with the most
+ excellent <ulink url="http://http://valgrind.org/">Valgrind</ulink> tool if
+ the <option>--enable-valgrind</option> configuration option is enabled and
+ the <link linkend="opt.valgrind"><mallctl>opt.valgrind</mallctl></link>
+ option is enabled.</para>
</refsect1>
<refsect1 id="diagnostic_messages">
<title>DIAGNOSTIC MESSAGES</title>
diff --git a/include/jemalloc/internal/arena.h b/include/jemalloc/internal/arena.h
index d25a2b1..f52fac4 100644
--- a/include/jemalloc/internal/arena.h
+++ b/include/jemalloc/internal/arena.h
@@ -16,7 +16,7 @@
* constraint is relaxed (ignored) for runs that are so small that the
* per-region overhead is greater than:
*
- * (RUN_MAX_OVRHD / (reg_size << (3+RUN_BFP))
+ * (RUN_MAX_OVRHD / (reg_interval << (3+RUN_BFP))
*/
#define RUN_BFP 12
/* \/ Implicit binary fixed point. */
@@ -28,6 +28,12 @@
#define RUN_MAXREGS (1U << LG_RUN_MAXREGS)
/*
+ * Minimum redzone size. Redzones may be larger than this if necessary to
+ * preserve region alignment.
+ */
+#define REDZONE_MINSIZE 16
+
+/*
* The minimum ratio of active:dirty pages per arena is computed as:
*
* (nactive >> opt_lg_dirty_mult) >= ndirty
@@ -192,11 +198,50 @@
* Read-only information associated with each element of arena_t's bins array
* is stored separately, partly to reduce memory usage (only one copy, rather
* than one per arena), but mainly to avoid false cacheline sharing.
+ *
+ * Each run has the following layout:
+ *
+ * /--------------------\
+ * | arena_run_t header |
+ * | ... |
+ * bitmap_offset | bitmap |
+ * | ... |
+ * ctx0_offset | ctx map |
+ * | ... |
+ * |--------------------|
+ * | redzone |
+ * reg0_offset | region 0 |
+ * | redzone |
+ * |--------------------| \
+ * | redzone | |
+ * | region 1 | > reg_interval
+ * | redzone | /
+ * |--------------------|
+ * | ... |
+ * | ... |
+ * | ... |
+ * |--------------------|
+ * | redzone |
+ * | region nregs-1 |
+ * | redzone |
+ * |--------------------|
+ * | alignment pad? |
+ * \--------------------/
+ *
+ * reg_interval has at least the same minimum alignment as reg_size; this
+ * preserves the alignment constraint that sa2u() depends on. Alignment pad is
+ * either 0 or redzone_size; it is present only if needed to align reg0_offset.
*/
struct arena_bin_info_s {
/* Size of regions in a run for this bin's size class. */
size_t reg_size;
+ /* Redzone size. */
+ size_t redzone_size;
+
+ /* Interval between regions (reg_size + (redzone_size << 1)). */
+ size_t reg_interval;
+
/* Total size of a run for this bin's size class. */
size_t run_size;
@@ -357,13 +402,15 @@
void arena_prof_accum(arena_t *arena, uint64_t accumbytes);
void arena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin,
size_t binind, uint64_t prof_accumbytes);
+void arena_alloc_junk_small(void *ptr, arena_bin_info_t *bin_info,
+ bool zero);
+void arena_dalloc_junk_small(void *ptr, arena_bin_info_t *bin_info);
void *arena_malloc_small(arena_t *arena, size_t size, bool zero);
void *arena_malloc_large(arena_t *arena, size_t size, bool zero);
void *arena_palloc(arena_t *arena, size_t size, size_t alloc_size,
size_t alignment, bool zero);
-size_t arena_salloc(const void *ptr);
+size_t arena_salloc(const void *ptr, bool demote);
void arena_prof_promoted(const void *ptr, size_t size);
-size_t arena_salloc_demote(const void *ptr);
void arena_dalloc_bin(arena_t *arena, arena_chunk_t *chunk, void *ptr,
arena_chunk_map_t *mapelm);
void arena_dalloc_large(arena_t *arena, arena_chunk_t *chunk, void *ptr);
@@ -408,7 +455,7 @@
arena_run_regind(arena_run_t *run, arena_bin_info_t *bin_info, const void *ptr)
{
unsigned shift, diff, regind;
- size_t size;
+ size_t interval;
/*
* Freeing a pointer lower than region zero can cause assertion
@@ -425,12 +472,12 @@
bin_info->reg0_offset);
/* Rescale (factor powers of 2 out of the numerator and denominator). */
- size = bin_info->reg_size;
- shift = ffs(size) - 1;
+ interval = bin_info->reg_interval;
+ shift = ffs(interval) - 1;
diff >>= shift;
- size >>= shift;
+ interval >>= shift;
- if (size == 1) {
+ if (interval == 1) {
/* The divisor was a power of 2. */
regind = diff;
} else {
@@ -442,7 +489,7 @@
*
* becomes
*
- * (X * size_invs[D - 3]) >> SIZE_INV_SHIFT
+ * (X * interval_invs[D - 3]) >> SIZE_INV_SHIFT
*
* We can omit the first three elements, because we never
* divide by 0, and 1 and 2 are both powers of two, which are
@@ -450,7 +497,7 @@
*/
#define SIZE_INV_SHIFT ((sizeof(unsigned) << 3) - LG_RUN_MAXREGS)
#define SIZE_INV(s) (((1U << SIZE_INV_SHIFT) / (s)) + 1)
- static const unsigned size_invs[] = {
+ static const unsigned interval_invs[] = {
SIZE_INV(3),
SIZE_INV(4), SIZE_INV(5), SIZE_INV(6), SIZE_INV(7),
SIZE_INV(8), SIZE_INV(9), SIZE_INV(10), SIZE_INV(11),
@@ -461,14 +508,16 @@
SIZE_INV(28), SIZE_INV(29), SIZE_INV(30), SIZE_INV(31)
};
- if (size <= ((sizeof(size_invs) / sizeof(unsigned)) + 2))
- regind = (diff * size_invs[size - 3]) >> SIZE_INV_SHIFT;
- else
- regind = diff / size;
+ if (interval <= ((sizeof(interval_invs) / sizeof(unsigned)) +
+ 2)) {
+ regind = (diff * interval_invs[interval - 3]) >>
+ SIZE_INV_SHIFT;
+ } else
+ regind = diff / interval;
#undef SIZE_INV
#undef SIZE_INV_SHIFT
}
- assert(diff == regind * size);
+ assert(diff == regind * interval);
assert(regind < bin_info->nregs);
return (regind);
@@ -610,7 +659,7 @@
&arena_bin_info[binind];
assert(((uintptr_t)ptr - ((uintptr_t)run +
(uintptr_t)bin_info->reg0_offset)) %
- bin_info->reg_size == 0);
+ bin_info->reg_interval == 0);
}
malloc_mutex_lock(&bin->lock);
arena_dalloc_bin(arena, chunk, ptr, mapelm);
diff --git a/include/jemalloc/internal/jemalloc_internal.h.in b/include/jemalloc/internal/jemalloc_internal.h.in
index 66dd357..a16e5e2 100644
--- a/include/jemalloc/internal/jemalloc_internal.h.in
+++ b/include/jemalloc/internal/jemalloc_internal.h.in
@@ -40,6 +40,11 @@
#include <sys/ktrace.h>
#endif
+#ifdef JEMALLOC_VALGRIND
+#include <valgrind/valgrind.h>
+#include <valgrind/memcheck.h>
+#endif
+
#include "jemalloc/internal/private_namespace.h"
#ifdef JEMALLOC_CC_SILENCE
@@ -125,6 +130,13 @@
false
#endif
;
+static const bool config_valgrind =
+#ifdef JEMALLOC_VALGRIND
+ true
+#else
+ false
+#endif
+ ;
static const bool config_xmalloc =
#ifdef JEMALLOC_XMALLOC
true
@@ -281,6 +293,77 @@
#define PAGE_CEILING(s) \
(((s) + PAGE_MASK) & ~PAGE_MASK)
+#ifdef JEMALLOC_VALGRIND
+/*
+ * The JEMALLOC_VALGRIND_*() macros must be macros rather than functions
+ * so that when Valgrind reports errors, there are no extra stack frames
+ * in the backtraces.
+ *
+ * The size that is reported to valgrind must be consistent through a chain of
+ * malloc..realloc..realloc calls. Request size isn't recorded anywhere in
+ * jemalloc, so it is critical that all callers of these macros provide usize
+ * rather than request size. As a result, buffer overflow detection is
+ * technically weakened for the standard API, though it is generally accepted
+ * practice to consider any extra bytes reported by malloc_usable_size() as
+ * usable space.
+ */
+#define JEMALLOC_VALGRIND_MALLOC(cond, ptr, usize, zero) do { \
+ if (config_valgrind && opt_valgrind && cond) \
+ VALGRIND_MALLOCLIKE_BLOCK(ptr, usize, p2rz(ptr), zero); \
+} while (0)
+#define JEMALLOC_VALGRIND_REALLOC(ptr, usize, old_ptr, old_usize, \
+ old_rzsize, zero) do { \
+ if (config_valgrind && opt_valgrind) { \
+ size_t rzsize = p2rz(ptr); \
+ \
+ if (ptr == old_ptr) { \
+ VALGRIND_RESIZEINPLACE_BLOCK(ptr, old_usize, \
+ usize, rzsize); \
+ if (zero && old_usize < usize) { \
+ VALGRIND_MAKE_MEM_DEFINED( \
+ (void *)((uintptr_t)ptr + \
+ old_usize), usize - old_usize); \
+ } \
+ } else { \
+ if (old_ptr != NULL) { \
+ VALGRIND_FREELIKE_BLOCK(old_ptr, \
+ old_rzsize); \
+ } \
+ if (ptr != NULL) { \
+ size_t copy_size = (old_usize < usize) \
+ ? old_usize : usize; \
+ size_t tail_size = usize - copy_size; \
+ VALGRIND_MALLOCLIKE_BLOCK(ptr, usize, \
+ rzsize, false); \
+ if (copy_size > 0) { \
+ VALGRIND_MAKE_MEM_DEFINED(ptr, \
+ copy_size); \
+ } \
+ if (zero && tail_size > 0) { \
+ VALGRIND_MAKE_MEM_DEFINED( \
+ (void *)((uintptr_t)ptr + \
+ copy_size), tail_size); \
+ } \
+ } \
+ } \
+ } \
+} while (0)
+#define JEMALLOC_VALGRIND_FREE(ptr, rzsize) do { \
+ if (config_valgrind && opt_valgrind) \
+ VALGRIND_FREELIKE_BLOCK(ptr, rzsize); \
+} while (0)
+#else
+#define VALGRIND_MALLOCLIKE_BLOCK(addr, sizeB, rzB, is_zeroed)
+#define VALGRIND_RESIZEINPLACE_BLOCK(addr, oldSizeB, newSizeB, rzB)
+#define VALGRIND_FREELIKE_BLOCK(addr, rzB)
+#define VALGRIND_MAKE_MEM_UNDEFINED(_qzz_addr, _qzz_len)
+#define VALGRIND_MAKE_MEM_DEFINED(_qzz_addr, _qzz_len)
+#define JEMALLOC_VALGRIND_MALLOC(cond, ptr, usize, zero)
+#define JEMALLOC_VALGRIND_REALLOC(ptr, usize, old_ptr, old_usize, \
+ old_rzsize, zero)
+#define JEMALLOC_VALGRIND_FREE(ptr, rzsize)
+#endif
+
#include "jemalloc/internal/util.h"
#include "jemalloc/internal/atomic.h"
#include "jemalloc/internal/prng.h"
@@ -300,6 +383,7 @@
#include "jemalloc/internal/rtree.h"
#include "jemalloc/internal/tcache.h"
#include "jemalloc/internal/hash.h"
+#include "jemalloc/internal/quarantine.h"
#include "jemalloc/internal/prof.h"
#undef JEMALLOC_H_TYPES
@@ -325,6 +409,7 @@
#include "jemalloc/internal/rtree.h"
#include "jemalloc/internal/tcache.h"
#include "jemalloc/internal/hash.h"
+#include "jemalloc/internal/quarantine.h"
#include "jemalloc/internal/prof.h"
typedef struct {
@@ -343,7 +428,10 @@
extern bool opt_abort;
extern bool opt_junk;
+extern size_t opt_quarantine;
+extern bool opt_redzone;
extern bool opt_utrace;
+extern bool opt_valgrind;
extern bool opt_xmalloc;
extern bool opt_zero;
extern size_t opt_narenas;
@@ -385,6 +473,7 @@
#include "jemalloc/internal/rtree.h"
#include "jemalloc/internal/tcache.h"
#include "jemalloc/internal/hash.h"
+#include "jemalloc/internal/quarantine.h"
#include "jemalloc/internal/prof.h"
#undef JEMALLOC_H_EXTERNS
@@ -550,14 +639,18 @@
#include "jemalloc/internal/tcache.h"
#include "jemalloc/internal/arena.h"
#include "jemalloc/internal/hash.h"
+#include "jemalloc/internal/quarantine.h"
#ifndef JEMALLOC_ENABLE_INLINE
void *imalloc(size_t size);
void *icalloc(size_t size);
void *ipalloc(size_t usize, size_t alignment, bool zero);
-size_t isalloc(const void *ptr);
-size_t ivsalloc(const void *ptr);
+size_t isalloc(const void *ptr, bool demote);
+size_t ivsalloc(const void *ptr, bool demote);
+size_t u2rz(size_t usize);
+size_t p2rz(const void *ptr);
void idalloc(void *ptr);
+void iqalloc(void *ptr);
void *iralloc(void *ptr, size_t size, size_t extra, size_t alignment,
bool zero, bool no_move);
malloc_tsd_protos(JEMALLOC_ATTR(unused), thread_allocated, thread_allocated_t)
@@ -621,21 +714,25 @@
return (ret);
}
+/*
+ * Typical usage:
+ * void *ptr = [...]
+ * size_t sz = isalloc(ptr, config_prof);
+ */
JEMALLOC_INLINE size_t
-isalloc(const void *ptr)
+isalloc(const void *ptr, bool demote)
{
size_t ret;
arena_chunk_t *chunk;
assert(ptr != NULL);
+ /* Demotion only makes sense if config_prof is true. */
+ assert(config_prof || demote == false);
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
if (chunk != ptr) {
/* Region. */
- if (config_prof)
- ret = arena_salloc_demote(ptr);
- else
- ret = arena_salloc(ptr);
+ ret = arena_salloc(ptr, demote);
} else
ret = huge_salloc(ptr);
@@ -643,14 +740,36 @@
}
JEMALLOC_INLINE size_t
-ivsalloc(const void *ptr)
+ivsalloc(const void *ptr, bool demote)
{
/* Return 0 if ptr is not within a chunk managed by jemalloc. */
if (rtree_get(chunks_rtree, (uintptr_t)CHUNK_ADDR2BASE(ptr)) == NULL)
return (0);
- return (isalloc(ptr));
+ return (isalloc(ptr, demote));
+}
+
+JEMALLOC_INLINE size_t
+u2rz(size_t usize)
+{
+ size_t ret;
+
+ if (usize <= SMALL_MAXCLASS) {
+ size_t binind = SMALL_SIZE2BIN(usize);
+ ret = arena_bin_info[binind].redzone_size;
+ } else
+ ret = 0;
+
+ return (ret);
+}
+
+JEMALLOC_INLINE size_t
+p2rz(const void *ptr)
+{
+ size_t usize = isalloc(ptr, false);
+
+ return (u2rz(usize));
}
JEMALLOC_INLINE void
@@ -667,6 +786,16 @@
huge_dalloc(ptr, true);
}
+JEMALLOC_INLINE void
+iqalloc(void *ptr)
+{
+
+ if (config_fill && opt_quarantine)
+ quarantine(ptr);
+ else
+ idalloc(ptr);
+}
+
JEMALLOC_INLINE void *
iralloc(void *ptr, size_t size, size_t extra, size_t alignment, bool zero,
bool no_move)
@@ -677,14 +806,14 @@
assert(ptr != NULL);
assert(size != 0);
- oldsize = isalloc(ptr);
+ oldsize = isalloc(ptr, config_prof);
if (alignment != 0 && ((uintptr_t)ptr & ((uintptr_t)alignment-1))
!= 0) {
size_t usize, copysize;
/*
- * Existing object alignment is inadquate; allocate new space
+ * Existing object alignment is inadequate; allocate new space
* and copy.
*/
if (no_move)
@@ -711,7 +840,7 @@
*/
copysize = (size < oldsize) ? size : oldsize;
memcpy(ret, ptr, copysize);
- idalloc(ptr);
+ iqalloc(ptr);
return (ret);
}
diff --git a/include/jemalloc/internal/private_namespace.h b/include/jemalloc/internal/private_namespace.h
index de3042e..a962192 100644
--- a/include/jemalloc/internal/private_namespace.h
+++ b/include/jemalloc/internal/private_namespace.h
@@ -1,7 +1,9 @@
+#define arena_alloc_junk_small JEMALLOC_N(arena_alloc_junk_small)
#define arena_bin_index JEMALLOC_N(arena_bin_index)
#define arena_boot JEMALLOC_N(arena_boot)
#define arena_dalloc JEMALLOC_N(arena_dalloc)
#define arena_dalloc_bin JEMALLOC_N(arena_dalloc_bin)
+#define arena_dalloc_junk_small JEMALLOC_N(arena_dalloc_junk_small)
#define arena_dalloc_large JEMALLOC_N(arena_dalloc_large)
#define arena_malloc JEMALLOC_N(arena_malloc)
#define arena_malloc_large JEMALLOC_N(arena_malloc_large)
@@ -20,7 +22,6 @@
#define arena_ralloc_no_move JEMALLOC_N(arena_ralloc_no_move)
#define arena_run_regind JEMALLOC_N(arena_run_regind)
#define arena_salloc JEMALLOC_N(arena_salloc)
-#define arena_salloc_demote JEMALLOC_N(arena_salloc_demote)
#define arena_stats_merge JEMALLOC_N(arena_stats_merge)
#define arena_tcache_fill_small JEMALLOC_N(arena_tcache_fill_small)
#define arenas_bin_i_index JEMALLOC_N(arenas_bin_i_index)
@@ -136,6 +137,7 @@
#define idalloc JEMALLOC_N(idalloc)
#define imalloc JEMALLOC_N(imalloc)
#define ipalloc JEMALLOC_N(ipalloc)
+#define iqalloc JEMALLOC_N(iqalloc)
#define iralloc JEMALLOC_N(iralloc)
#define isalloc JEMALLOC_N(isalloc)
#define ivsalloc JEMALLOC_N(ivsalloc)
@@ -176,6 +178,7 @@
#define opt_utrace JEMALLOC_N(opt_utrace)
#define opt_xmalloc JEMALLOC_N(opt_xmalloc)
#define opt_zero JEMALLOC_N(opt_zero)
+#define p2rz JEMALLOC_N(p2rz)
#define pow2_ceil JEMALLOC_N(pow2_ceil)
#define prof_backtrace JEMALLOC_N(prof_backtrace)
#define prof_boot0 JEMALLOC_N(prof_boot0)
@@ -195,6 +198,8 @@
#define prof_tdata_init JEMALLOC_N(prof_tdata_init)
#define prof_tdata_tls JEMALLOC_N(prof_tdata_tls)
#define pthread_create JEMALLOC_N(pthread_create)
+#define quarantine JEMALLOC_N(quarantine)
+#define quarantine_boot JEMALLOC_N(quarantine_boot)
#define register_zone JEMALLOC_N(register_zone)
#define rtree_get JEMALLOC_N(rtree_get)
#define rtree_get_locked JEMALLOC_N(rtree_get_locked)
@@ -229,3 +234,4 @@
#define thread_allocated_get JEMALLOC_N(thread_allocated_get)
#define thread_allocated_get_hard JEMALLOC_N(thread_allocated_get_hard)
#define thread_allocated_tls JEMALLOC_N(thread_allocated_tls)
+#define u2rz JEMALLOC_N(u2rz)
diff --git a/include/jemalloc/internal/prof.h b/include/jemalloc/internal/prof.h
index 34929e7..a37bb44 100644
--- a/include/jemalloc/internal/prof.h
+++ b/include/jemalloc/internal/prof.h
@@ -378,7 +378,7 @@
cassert(config_prof);
assert(ptr != NULL);
- assert(size == isalloc(ptr));
+ assert(size == isalloc(ptr, true));
if (opt_lg_prof_sample != 0) {
if (prof_sample_accum_update(size)) {
@@ -427,7 +427,7 @@
assert(ptr != NULL || (uintptr_t)cnt <= (uintptr_t)1U);
if (ptr != NULL) {
- assert(size == isalloc(ptr));
+ assert(size == isalloc(ptr, true));
if (opt_lg_prof_sample != 0) {
if (prof_sample_accum_update(size)) {
/*
@@ -500,7 +500,7 @@
cassert(config_prof);
if ((uintptr_t)ctx > (uintptr_t)1) {
- assert(size == isalloc(ptr));
+ assert(size == isalloc(ptr, true));
prof_thr_cnt_t *tcnt = prof_lookup(ctx->bt);
if (tcnt != NULL) {
diff --git a/include/jemalloc/internal/quarantine.h b/include/jemalloc/internal/quarantine.h
new file mode 100644
index 0000000..38f3d69
--- /dev/null
+++ b/include/jemalloc/internal/quarantine.h
@@ -0,0 +1,24 @@
+/******************************************************************************/
+#ifdef JEMALLOC_H_TYPES
+
+/* Default per thread quarantine size if valgrind is enabled. */
+#define JEMALLOC_VALGRIND_QUARANTINE_DEFAULT (ZU(1) << 24)
+
+#endif /* JEMALLOC_H_TYPES */
+/******************************************************************************/
+#ifdef JEMALLOC_H_STRUCTS
+
+#endif /* JEMALLOC_H_STRUCTS */
+/******************************************************************************/
+#ifdef JEMALLOC_H_EXTERNS
+
+void quarantine(void *ptr);
+bool quarantine_boot(void);
+
+#endif /* JEMALLOC_H_EXTERNS */
+/******************************************************************************/
+#ifdef JEMALLOC_H_INLINES
+
+#endif /* JEMALLOC_H_INLINES */
+/******************************************************************************/
+
diff --git a/include/jemalloc/internal/tcache.h b/include/jemalloc/internal/tcache.h
index 93e721d..9d8c992 100644
--- a/include/jemalloc/internal/tcache.h
+++ b/include/jemalloc/internal/tcache.h
@@ -340,17 +340,24 @@
if (ret == NULL)
return (NULL);
}
- assert(arena_salloc(ret) == arena_bin_info[binind].reg_size);
+ assert(arena_salloc(ret, false) == arena_bin_info[binind].reg_size);
if (zero == false) {
if (config_fill) {
- if (opt_junk)
- memset(ret, 0xa5, size);
- else if (opt_zero)
+ if (opt_junk) {
+ arena_alloc_junk_small(ret,
+ &arena_bin_info[binind], false);
+ } else if (opt_zero)
memset(ret, 0, size);
}
- } else
+ } else {
+ if (config_fill && opt_junk) {
+ arena_alloc_junk_small(ret, &arena_bin_info[binind],
+ true);
+ }
+ VALGRIND_MAKE_MEM_UNDEFINED(ret, size);
memset(ret, 0, size);
+ }
if (config_stats)
tbin->tstats.nrequests++;
@@ -397,8 +404,10 @@
else if (opt_zero)
memset(ret, 0, size);
}
- } else
+ } else {
+ VALGRIND_MAKE_MEM_UNDEFINED(ret, size);
memset(ret, 0, size);
+ }
if (config_stats)
tbin->tstats.nrequests++;
@@ -422,7 +431,7 @@
size_t pageind, binind;
arena_chunk_map_t *mapelm;
- assert(arena_salloc(ptr) <= SMALL_MAXCLASS);
+ assert(arena_salloc(ptr, false) <= SMALL_MAXCLASS);
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
arena = chunk->arena;
@@ -436,7 +445,7 @@
assert(binind < NBINS);
if (config_fill && opt_junk)
- memset(ptr, 0x5a, arena_bin_info[binind].reg_size);
+ arena_dalloc_junk_small(ptr, &arena_bin_info[binind]);
tbin = &tcache->tbins[binind];
tbin_info = &tcache_bin_info[binind];
@@ -459,8 +468,8 @@
tcache_bin_info_t *tbin_info;
assert((size & PAGE_MASK) == 0);
- assert(arena_salloc(ptr) > SMALL_MAXCLASS);
- assert(arena_salloc(ptr) <= tcache_maxclass);
+ assert(arena_salloc(ptr, false) > SMALL_MAXCLASS);
+ assert(arena_salloc(ptr, false) <= tcache_maxclass);
binind = NBINS + (size >> LG_PAGE) - 1;
diff --git a/include/jemalloc/internal/util.h b/include/jemalloc/internal/util.h
index 3d3ea3a..d360ae3 100644
--- a/include/jemalloc/internal/util.h
+++ b/include/jemalloc/internal/util.h
@@ -140,7 +140,6 @@
je_malloc_message(NULL, s);
}
-
#endif
#endif /* JEMALLOC_H_INLINES */
diff --git a/include/jemalloc/jemalloc_defs.h.in b/include/jemalloc/jemalloc_defs.h.in
index 8e7442d..7770a7a 100644
--- a/include/jemalloc/jemalloc_defs.h.in
+++ b/include/jemalloc/jemalloc_defs.h.in
@@ -148,7 +148,7 @@
*/
#undef JEMALLOC_DSS
-/* Support memory filling (junk/zero). */
+/* Support memory filling (junk/zero/quarantine/redzone). */
#undef JEMALLOC_FILL
/* Support the experimental API. */
@@ -157,6 +157,9 @@
/* Support utrace(2)-based tracing. */
#undef JEMALLOC_UTRACE
+/* Support Valgrind. */
+#undef JEMALLOC_VALGRIND
+
/* Support optional abort() on OOM. */
#undef JEMALLOC_XMALLOC
diff --git a/src/arena.c b/src/arena.c
index c84aaf4..1d4f61e 100644
--- a/src/arena.c
+++ b/src/arena.c
@@ -140,7 +140,7 @@
regind = bitmap_sfu(bitmap, &bin_info->bitmap_info);
ret = (void *)((uintptr_t)run + (uintptr_t)bin_info->reg0_offset +
- (uintptr_t)(bin_info->reg_size * regind));
+ (uintptr_t)(bin_info->reg_interval * regind));
run->nfree--;
if (regind == run->nextind)
run->nextind++;
@@ -161,8 +161,8 @@
assert(run->nfree < bin_info->nregs);
/* Freeing an interior pointer can cause assertion failure. */
assert(((uintptr_t)ptr - ((uintptr_t)run +
- (uintptr_t)bin_info->reg0_offset)) % (uintptr_t)bin_info->reg_size
- == 0);
+ (uintptr_t)bin_info->reg0_offset)) %
+ (uintptr_t)bin_info->reg_interval == 0);
assert((uintptr_t)ptr >= (uintptr_t)run +
(uintptr_t)bin_info->reg0_offset);
/* Freeing an unallocated pointer can cause assertion failure. */
@@ -260,10 +260,18 @@
for (i = 0; i < need_pages; i++) {
if ((chunk->map[run_ind+i-map_bias].bits
& CHUNK_MAP_UNZEROED) != 0) {
+ VALGRIND_MAKE_MEM_UNDEFINED(
+ (void *)((uintptr_t)
+ chunk + ((run_ind+i) <<
+ LG_PAGE)), PAGE);
memset((void *)((uintptr_t)
chunk + ((run_ind+i) <<
LG_PAGE)), 0, PAGE);
} else if (config_debug) {
+ VALGRIND_MAKE_MEM_DEFINED(
+ (void *)((uintptr_t)
+ chunk + ((run_ind+i) <<
+ LG_PAGE)), PAGE);
arena_chunk_validate_zeroed(
chunk, run_ind+i);
}
@@ -273,6 +281,9 @@
* The run is dirty, so all pages must be
* zeroed.
*/
+ VALGRIND_MAKE_MEM_UNDEFINED((void
+ *)((uintptr_t)chunk + (run_ind <<
+ LG_PAGE)), (need_pages << LG_PAGE));
memset((void *)((uintptr_t)chunk + (run_ind <<
LG_PAGE)), 0, (need_pages << LG_PAGE));
}
@@ -1245,6 +1256,10 @@
ptr = arena_bin_malloc_hard(arena, bin);
if (ptr == NULL)
break;
+ if (config_fill && opt_junk) {
+ arena_alloc_junk_small(ptr, &arena_bin_info[binind],
+ true);
+ }
/* Insert such that low regions get used first. */
tbin->avail[nfill - 1 - i] = ptr;
}
@@ -1259,6 +1274,55 @@
tbin->ncached = i;
}
+void
+arena_alloc_junk_small(void *ptr, arena_bin_info_t *bin_info, bool zero)
+{
+
+ if (zero) {
+ size_t redzone_size = bin_info->redzone_size;
+ memset((void *)((uintptr_t)ptr - redzone_size), 0xa5,
+ redzone_size);
+ memset((void *)((uintptr_t)ptr + bin_info->reg_size), 0xa5,
+ redzone_size);
+ } else {
+ memset((void *)((uintptr_t)ptr - bin_info->redzone_size), 0xa5,
+ bin_info->reg_interval);
+ }
+}
+
+void
+arena_dalloc_junk_small(void *ptr, arena_bin_info_t *bin_info)
+{
+ size_t size = bin_info->reg_size;
+ size_t redzone_size = bin_info->redzone_size;
+ size_t i;
+ bool error = false;
+
+ for (i = 1; i <= redzone_size; i++) {
+ unsigned byte;
+ if ((byte = *(uint8_t *)((uintptr_t)ptr - i)) != 0xa5) {
+ error = true;
+ malloc_printf("<jemalloc>: Corrupt redzone "
+ "%zu byte%s before %p (size %zu), byte=%#x\n", i,
+ (i == 1) ? "" : "s", ptr, size, byte);
+ }
+ }
+ for (i = 0; i < redzone_size; i++) {
+ unsigned byte;
+ if ((byte = *(uint8_t *)((uintptr_t)ptr + size + i)) != 0xa5) {
+ error = true;
+ malloc_printf("<jemalloc>: Corrupt redzone "
+ "%zu byte%s after end of %p (size %zu), byte=%#x\n",
+ i, (i == 1) ? "" : "s", ptr, size, byte);
+ }
+ }
+ if (opt_abort && error)
+ abort();
+
+ memset((void *)((uintptr_t)ptr - redzone_size), 0x5a,
+ bin_info->reg_interval);
+}
+
void *
arena_malloc_small(arena_t *arena, size_t size, bool zero)
{
@@ -1297,13 +1361,20 @@
if (zero == false) {
if (config_fill) {
- if (opt_junk)
- memset(ret, 0xa5, size);
- else if (opt_zero)
+ if (opt_junk) {
+ arena_alloc_junk_small(ret,
+ &arena_bin_info[binind], false);
+ } else if (opt_zero)
memset(ret, 0, size);
}
- } else
+ } else {
+ if (config_fill && opt_junk) {
+ arena_alloc_junk_small(ret, &arena_bin_info[binind],
+ true);
+ }
+ VALGRIND_MAKE_MEM_UNDEFINED(ret, size);
memset(ret, 0, size);
+ }
return (ret);
}
@@ -1412,7 +1483,7 @@
/* Return the size of the allocation pointed to by ptr. */
size_t
-arena_salloc(const void *ptr)
+arena_salloc(const void *ptr, bool demote)
{
size_t ret;
arena_chunk_t *chunk;
@@ -1431,12 +1502,19 @@
size_t binind = arena_bin_index(chunk->arena, run->bin);
arena_bin_info_t *bin_info = &arena_bin_info[binind];
assert(((uintptr_t)ptr - ((uintptr_t)run +
- (uintptr_t)bin_info->reg0_offset)) % bin_info->reg_size ==
- 0);
+ (uintptr_t)bin_info->reg0_offset)) % bin_info->reg_interval
+ == 0);
ret = bin_info->reg_size;
} else {
assert(((uintptr_t)ptr & PAGE_MASK) == 0);
ret = mapbits & ~PAGE_MASK;
+ if (demote && prof_promote && ret == PAGE && (mapbits &
+ CHUNK_MAP_CLASS_MASK) != 0) {
+ size_t binind = ((mapbits & CHUNK_MAP_CLASS_MASK) >>
+ CHUNK_MAP_CLASS_SHIFT) - 1;
+ assert(binind < NBINS);
+ ret = arena_bin_info[binind].reg_size;
+ }
assert(ret != 0);
}
@@ -1449,9 +1527,11 @@
arena_chunk_t *chunk;
size_t pageind, binind;
+ assert(config_prof);
assert(ptr != NULL);
assert(CHUNK_ADDR2BASE(ptr) != ptr);
- assert(isalloc(ptr) == PAGE);
+ assert(isalloc(ptr, false) == PAGE);
+ assert(isalloc(ptr, true) == PAGE);
assert(size <= SMALL_MAXCLASS);
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
@@ -1460,45 +1540,9 @@
assert(binind < NBINS);
chunk->map[pageind-map_bias].bits = (chunk->map[pageind-map_bias].bits &
~CHUNK_MAP_CLASS_MASK) | ((binind+1) << CHUNK_MAP_CLASS_SHIFT);
-}
-size_t
-arena_salloc_demote(const void *ptr)
-{
- size_t ret;
- arena_chunk_t *chunk;
- size_t pageind, mapbits;
-
- assert(ptr != NULL);
- assert(CHUNK_ADDR2BASE(ptr) != ptr);
-
- chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
- pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
- mapbits = chunk->map[pageind-map_bias].bits;
- assert((mapbits & CHUNK_MAP_ALLOCATED) != 0);
- if ((mapbits & CHUNK_MAP_LARGE) == 0) {
- arena_run_t *run = (arena_run_t *)((uintptr_t)chunk +
- (uintptr_t)((pageind - (mapbits >> LG_PAGE)) << LG_PAGE));
- size_t binind = arena_bin_index(chunk->arena, run->bin);
- arena_bin_info_t *bin_info = &arena_bin_info[binind];
- assert(((uintptr_t)ptr - ((uintptr_t)run +
- (uintptr_t)bin_info->reg0_offset)) % bin_info->reg_size ==
- 0);
- ret = bin_info->reg_size;
- } else {
- assert(((uintptr_t)ptr & PAGE_MASK) == 0);
- ret = mapbits & ~PAGE_MASK;
- if (prof_promote && ret == PAGE && (mapbits &
- CHUNK_MAP_CLASS_MASK) != 0) {
- size_t binind = ((mapbits & CHUNK_MAP_CLASS_MASK) >>
- CHUNK_MAP_CLASS_SHIFT) - 1;
- assert(binind < NBINS);
- ret = arena_bin_info[binind].reg_size;
- }
- assert(ret != 0);
- }
-
- return (ret);
+ assert(isalloc(ptr, false) == PAGE);
+ assert(isalloc(ptr, true) == size);
}
static void
@@ -1545,7 +1589,8 @@
run_ind = (size_t)(((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE);
past = (size_t)(PAGE_CEILING((uintptr_t)run +
(uintptr_t)bin_info->reg0_offset + (uintptr_t)(run->nextind *
- bin_info->reg_size) - (uintptr_t)chunk) >> LG_PAGE);
+ bin_info->reg_interval - bin_info->redzone_size) -
+ (uintptr_t)chunk) >> LG_PAGE);
malloc_mutex_lock(&arena->lock);
/*
@@ -1617,7 +1662,7 @@
size = bin_info->reg_size;
if (config_fill && opt_junk)
- memset(ptr, 0x5a, size);
+ arena_dalloc_junk_small(ptr, bin_info);
arena_run_reg_dalloc(run, ptr);
if (run->nfree == bin_info->nregs) {
@@ -1936,7 +1981,7 @@
*/
copysize = (size < oldsize) ? size : oldsize;
memcpy(ret, ptr, copysize);
- idalloc(ptr);
+ iqalloc(ptr);
return (ret);
}
@@ -2007,17 +2052,41 @@
static size_t
bin_info_run_size_calc(arena_bin_info_t *bin_info, size_t min_run_size)
{
+ size_t pad_size;
size_t try_run_size, good_run_size;
uint32_t try_nregs, good_nregs;
uint32_t try_hdr_size, good_hdr_size;
uint32_t try_bitmap_offset, good_bitmap_offset;
uint32_t try_ctx0_offset, good_ctx0_offset;
- uint32_t try_reg0_offset, good_reg0_offset;
+ uint32_t try_redzone0_offset, good_redzone0_offset;
assert(min_run_size >= PAGE);
assert(min_run_size <= arena_maxclass);
/*
+ * Determine redzone size based on minimum alignment and minimum
+ * redzone size. Add padding to the end of the run if it is needed to
+ * align the regions. The padding allows each redzone to be half the
+ * minimum alignment; without the padding, each redzone would have to
+ * be twice as large in order to maintain alignment.
+ */
+ if (config_fill && opt_redzone) {
+ size_t align_min = ZU(1) << (ffs(bin_info->reg_size) - 1);
+ if (align_min <= REDZONE_MINSIZE) {
+ bin_info->redzone_size = REDZONE_MINSIZE;
+ pad_size = 0;
+ } else {
+ bin_info->redzone_size = align_min >> 1;
+ pad_size = bin_info->redzone_size;
+ }
+ } else {
+ bin_info->redzone_size = 0;
+ pad_size = 0;
+ }
+ bin_info->reg_interval = bin_info->reg_size +
+ (bin_info->redzone_size << 1);
+
+ /*
* Calculate known-valid settings before entering the run_size
* expansion loop, so that the first part of the loop always copies
* valid settings.
@@ -2028,7 +2097,8 @@
* header's mask length and the number of regions.
*/
try_run_size = min_run_size;
- try_nregs = ((try_run_size - sizeof(arena_run_t)) / bin_info->reg_size)
+ try_nregs = ((try_run_size - sizeof(arena_run_t)) /
+ bin_info->reg_interval)
+ 1; /* Counter-act try_nregs-- in loop. */
if (try_nregs > RUN_MAXREGS) {
try_nregs = RUN_MAXREGS
@@ -2050,9 +2120,9 @@
try_hdr_size += try_nregs * sizeof(prof_ctx_t *);
} else
try_ctx0_offset = 0;
- try_reg0_offset = try_run_size - (try_nregs *
- bin_info->reg_size);
- } while (try_hdr_size > try_reg0_offset);
+ try_redzone0_offset = try_run_size - (try_nregs *
+ bin_info->reg_interval) - pad_size;
+ } while (try_hdr_size > try_redzone0_offset);
/* run_size expansion loop. */
do {
@@ -2064,12 +2134,12 @@
good_hdr_size = try_hdr_size;
good_bitmap_offset = try_bitmap_offset;
good_ctx0_offset = try_ctx0_offset;
- good_reg0_offset = try_reg0_offset;
+ good_redzone0_offset = try_redzone0_offset;
/* Try more aggressive settings. */
try_run_size += PAGE;
- try_nregs = ((try_run_size - sizeof(arena_run_t)) /
- bin_info->reg_size)
+ try_nregs = ((try_run_size - sizeof(arena_run_t) - pad_size) /
+ bin_info->reg_interval)
+ 1; /* Counter-act try_nregs-- in loop. */
if (try_nregs > RUN_MAXREGS) {
try_nregs = RUN_MAXREGS
@@ -2093,23 +2163,27 @@
try_hdr_size += try_nregs *
sizeof(prof_ctx_t *);
}
- try_reg0_offset = try_run_size - (try_nregs *
- bin_info->reg_size);
- } while (try_hdr_size > try_reg0_offset);
+ try_redzone0_offset = try_run_size - (try_nregs *
+ bin_info->reg_interval) - pad_size;
+ } while (try_hdr_size > try_redzone0_offset);
} while (try_run_size <= arena_maxclass
&& try_run_size <= arena_maxclass
- && RUN_MAX_OVRHD * (bin_info->reg_size << 3) > RUN_MAX_OVRHD_RELAX
- && (try_reg0_offset << RUN_BFP) > RUN_MAX_OVRHD * try_run_size
+ && RUN_MAX_OVRHD * (bin_info->reg_interval << 3) >
+ RUN_MAX_OVRHD_RELAX
+ && (try_redzone0_offset << RUN_BFP) > RUN_MAX_OVRHD * try_run_size
&& try_nregs < RUN_MAXREGS);
- assert(good_hdr_size <= good_reg0_offset);
+ assert(good_hdr_size <= good_redzone0_offset);
/* Copy final settings. */
bin_info->run_size = good_run_size;
bin_info->nregs = good_nregs;
bin_info->bitmap_offset = good_bitmap_offset;
bin_info->ctx0_offset = good_ctx0_offset;
- bin_info->reg0_offset = good_reg0_offset;
+ bin_info->reg0_offset = good_redzone0_offset + bin_info->redzone_size;
+
+ assert(bin_info->reg0_offset - bin_info->redzone_size + (bin_info->nregs
+ * bin_info->reg_interval) + pad_size == bin_info->run_size);
return (good_run_size);
}
diff --git a/src/ctl.c b/src/ctl.c
index a75ffef..6be4056 100644
--- a/src/ctl.c
+++ b/src/ctl.c
@@ -57,6 +57,7 @@
CTL_PROTO(config_tcache)
CTL_PROTO(config_tls)
CTL_PROTO(config_utrace)
+CTL_PROTO(config_valgrind)
CTL_PROTO(config_xmalloc)
CTL_PROTO(opt_abort)
CTL_PROTO(opt_lg_chunk)
@@ -65,7 +66,10 @@
CTL_PROTO(opt_stats_print)
CTL_PROTO(opt_junk)
CTL_PROTO(opt_zero)
+CTL_PROTO(opt_quarantine)
+CTL_PROTO(opt_redzone)
CTL_PROTO(opt_utrace)
+CTL_PROTO(opt_valgrind)
CTL_PROTO(opt_xmalloc)
CTL_PROTO(opt_tcache)
CTL_PROTO(opt_lg_tcache_max)
@@ -179,6 +183,7 @@
{NAME("tcache"), CTL(config_tcache)},
{NAME("tls"), CTL(config_tls)},
{NAME("utrace"), CTL(config_utrace)},
+ {NAME("valgrind"), CTL(config_valgrind)},
{NAME("xmalloc"), CTL(config_xmalloc)}
};
@@ -190,7 +195,10 @@
{NAME("stats_print"), CTL(opt_stats_print)},
{NAME("junk"), CTL(opt_junk)},
{NAME("zero"), CTL(opt_zero)},
+ {NAME("quarantine"), CTL(opt_quarantine)},
+ {NAME("redzone"), CTL(opt_redzone)},
{NAME("utrace"), CTL(opt_utrace)},
+ {NAME("valgrind"), CTL(opt_valgrind)},
{NAME("xmalloc"), CTL(opt_xmalloc)},
{NAME("tcache"), CTL(opt_tcache)},
{NAME("lg_tcache_max"), CTL(opt_lg_tcache_max)},
@@ -1050,7 +1058,8 @@
/* Set new arena association. */
if (config_tcache) {
tcache_t *tcache;
- if ((tcache = *tcache_tsd_get()) != NULL) {
+ if ((uintptr_t)(tcache = *tcache_tsd_get()) >
+ (uintptr_t)TCACHE_STATE_MAX) {
tcache_arena_dissociate(tcache);
tcache_arena_associate(tcache, arena);
}
@@ -1085,6 +1094,7 @@
CTL_RO_BOOL_CONFIG_GEN(config_tcache)
CTL_RO_BOOL_CONFIG_GEN(config_tls)
CTL_RO_BOOL_CONFIG_GEN(config_utrace)
+CTL_RO_BOOL_CONFIG_GEN(config_valgrind)
CTL_RO_BOOL_CONFIG_GEN(config_xmalloc)
/******************************************************************************/
@@ -1096,7 +1106,10 @@
CTL_RO_NL_GEN(opt_stats_print, opt_stats_print, bool)
CTL_RO_NL_CGEN(config_fill, opt_junk, opt_junk, bool)
CTL_RO_NL_CGEN(config_fill, opt_zero, opt_zero, bool)
+CTL_RO_NL_CGEN(config_fill, opt_quarantine, opt_quarantine, size_t)
+CTL_RO_NL_CGEN(config_fill, opt_redzone, opt_redzone, bool)
CTL_RO_NL_CGEN(config_utrace, opt_utrace, opt_utrace, bool)
+CTL_RO_NL_CGEN(config_valgrind, opt_valgrind, opt_valgrind, bool)
CTL_RO_NL_CGEN(config_xmalloc, opt_xmalloc, opt_xmalloc, bool)
CTL_RO_NL_CGEN(config_tcache, opt_tcache, opt_tcache, bool)
CTL_RO_NL_CGEN(config_tcache, opt_lg_tcache_max, opt_lg_tcache_max, ssize_t)
diff --git a/src/huge.c b/src/huge.c
index 43c8f3b..daf0c62 100644
--- a/src/huge.c
+++ b/src/huge.c
@@ -174,7 +174,7 @@
#endif
{
memcpy(ret, ptr, copysize);
- idalloc(ptr);
+ iqalloc(ptr);
}
return (ret);
}
diff --git a/src/jemalloc.c b/src/jemalloc.c
index cde998c..237dd58 100644
--- a/src/jemalloc.c
+++ b/src/jemalloc.c
@@ -14,14 +14,19 @@
bool opt_abort = true;
# ifdef JEMALLOC_FILL
bool opt_junk = true;
+bool opt_redzone = true;
# else
bool opt_junk = false;
+bool opt_redzone = false;
# endif
#else
bool opt_abort = false;
bool opt_junk = false;
+bool opt_redzone = false;
#endif
+size_t opt_quarantine = ZU(0);
bool opt_utrace = false;
+bool opt_valgrind = false;
bool opt_xmalloc = false;
bool opt_zero = false;
size_t opt_narenas = 0;
@@ -419,7 +424,7 @@
while (*opts != '\0' && malloc_conf_next(&opts, &k, &klen, &v,
&vlen) == false) {
-#define CONF_HANDLE_BOOL(o, n) \
+#define CONF_HANDLE_BOOL_HIT(o, n, hit) \
if (sizeof(#n)-1 == klen && strncmp(#n, k, \
klen) == 0) { \
if (strncmp("true", v, vlen) == 0 && \
@@ -433,12 +438,19 @@
"Invalid conf value", \
k, klen, v, vlen); \
} \
+ hit = true; \
+ } else \
+ hit = false;
+#define CONF_HANDLE_BOOL(o, n) { \
+ bool hit; \
+ CONF_HANDLE_BOOL_HIT(o, n, hit); \
+ if (hit) \
continue; \
- }
+}
#define CONF_HANDLE_SIZE_T(o, n, min, max) \
if (sizeof(#n)-1 == klen && strncmp(#n, k, \
klen) == 0) { \
- uintmax_t um; \
+ uintmax_t um; \
char *end; \
\
errno = 0; \
@@ -502,11 +514,30 @@
CONF_HANDLE_BOOL(opt_stats_print, stats_print)
if (config_fill) {
CONF_HANDLE_BOOL(opt_junk, junk)
+ CONF_HANDLE_SIZE_T(opt_quarantine, quarantine,
+ 0, SIZE_T_MAX)
+ CONF_HANDLE_BOOL(opt_redzone, redzone)
CONF_HANDLE_BOOL(opt_zero, zero)
}
if (config_utrace) {
CONF_HANDLE_BOOL(opt_utrace, utrace)
}
+ if (config_valgrind) {
+ bool hit;
+ CONF_HANDLE_BOOL_HIT(opt_valgrind,
+ valgrind, hit)
+ if (config_fill && opt_valgrind && hit) {
+ opt_junk = false;
+ opt_zero = false;
+ if (opt_quarantine == 0) {
+ opt_quarantine =
+ JEMALLOC_VALGRIND_QUARANTINE_DEFAULT;
+ }
+ opt_redzone = true;
+ }
+ if (hit)
+ continue;
+ }
if (config_xmalloc) {
CONF_HANDLE_BOOL(opt_xmalloc, xmalloc)
}
@@ -662,6 +693,11 @@
return (true);
}
+ if (config_fill && quarantine_boot()) {
+ malloc_mutex_unlock(&init_lock);
+ return (true);
+ }
+
if (config_prof && prof_boot2()) {
malloc_mutex_unlock(&init_lock);
return (true);
@@ -763,7 +799,7 @@
} else
ret = imalloc(size);
} else {
- if (config_stats)
+ if (config_stats || (config_valgrind && opt_valgrind))
usize = s2u(size);
ret = imalloc(size);
}
@@ -780,10 +816,11 @@
if (config_prof && opt_prof && ret != NULL)
prof_malloc(ret, usize, cnt);
if (config_stats && ret != NULL) {
- assert(usize == isalloc(ret));
+ assert(usize == isalloc(ret, config_prof));
thread_allocated_tsd_get()->allocated += usize;
}
UTRACE(0, size, ret);
+ JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, usize, false);
return (ret);
}
@@ -872,7 +909,7 @@
label_return:
if (config_stats && result != NULL) {
- assert(usize == isalloc(result));
+ assert(usize == isalloc(result, config_prof));
thread_allocated_tsd_get()->allocated += usize;
}
if (config_prof && opt_prof && result != NULL)
@@ -886,8 +923,10 @@
int
je_posix_memalign(void **memptr, size_t alignment, size_t size)
{
-
- return imemalign(memptr, alignment, size, sizeof(void *));
+ int ret = imemalign(memptr, alignment, size, sizeof(void *));
+ JEMALLOC_VALGRIND_MALLOC(ret == 0, *memptr, isalloc(*memptr,
+ config_prof), false);
+ return (ret);
}
JEMALLOC_ATTR(malloc)
@@ -902,6 +941,8 @@
ret = NULL;
errno = err;
}
+ JEMALLOC_VALGRIND_MALLOC(err == 0, ret, isalloc(ret, config_prof),
+ false);
return (ret);
}
@@ -956,7 +997,7 @@
} else
ret = icalloc(num_size);
} else {
- if (config_stats)
+ if (config_stats || (config_valgrind && opt_valgrind))
usize = s2u(num_size);
ret = icalloc(num_size);
}
@@ -974,10 +1015,11 @@
if (config_prof && opt_prof && ret != NULL)
prof_malloc(ret, usize, cnt);
if (config_stats && ret != NULL) {
- assert(usize == isalloc(ret));
+ assert(usize == isalloc(ret, config_prof));
thread_allocated_tsd_get()->allocated += usize;
}
UTRACE(0, num_size, ret);
+ JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, usize, true);
return (ret);
}
@@ -988,19 +1030,30 @@
void *ret;
size_t usize;
size_t old_size = 0;
+ size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0);
prof_thr_cnt_t *cnt JEMALLOC_CC_SILENCE_INIT(NULL);
prof_ctx_t *old_ctx JEMALLOC_CC_SILENCE_INIT(NULL);
if (size == 0) {
if (ptr != NULL) {
/* realloc(ptr, 0) is equivalent to free(p). */
- if (config_prof || config_stats)
- old_size = isalloc(ptr);
+ if (config_prof) {
+ old_size = isalloc(ptr, true);
+ if (config_valgrind && opt_valgrind)
+ old_rzsize = p2rz(ptr);
+ } else if (config_stats) {
+ old_size = isalloc(ptr, false);
+ if (config_valgrind && opt_valgrind)
+ old_rzsize = u2rz(old_size);
+ } else if (config_valgrind && opt_valgrind) {
+ old_size = isalloc(ptr, false);
+ old_rzsize = u2rz(old_size);
+ }
if (config_prof && opt_prof) {
old_ctx = prof_ctx_get(ptr);
cnt = NULL;
}
- idalloc(ptr);
+ iqalloc(ptr);
ret = NULL;
goto label_return;
} else
@@ -1010,8 +1063,18 @@
if (ptr != NULL) {
assert(malloc_initialized || IS_INITIALIZER);
- if (config_prof || config_stats)
- old_size = isalloc(ptr);
+ if (config_prof) {
+ old_size = isalloc(ptr, true);
+ if (config_valgrind && opt_valgrind)
+ old_rzsize = p2rz(ptr);
+ } else if (config_stats) {
+ old_size = isalloc(ptr, false);
+ if (config_valgrind && opt_valgrind)
+ old_rzsize = u2rz(old_size);
+ } else if (config_valgrind && opt_valgrind) {
+ old_size = isalloc(ptr, false);
+ old_rzsize = u2rz(old_size);
+ }
if (config_prof && opt_prof) {
usize = s2u(size);
old_ctx = prof_ctx_get(ptr);
@@ -1035,7 +1098,7 @@
old_ctx = NULL;
}
} else {
- if (config_stats)
+ if (config_stats || (config_valgrind && opt_valgrind))
usize = s2u(size);
ret = iralloc(ptr, size, 0, 0, false, false);
}
@@ -1076,7 +1139,8 @@
ret = imalloc(size);
}
} else {
- if (config_stats)
+ if (config_stats || (config_valgrind &&
+ opt_valgrind))
usize = s2u(size);
ret = imalloc(size);
}
@@ -1097,12 +1161,13 @@
prof_realloc(ret, usize, cnt, old_size, old_ctx);
if (config_stats && ret != NULL) {
thread_allocated_t *ta;
- assert(usize == isalloc(ret));
+ assert(usize == isalloc(ret, config_prof));
ta = thread_allocated_tsd_get();
ta->allocated += usize;
ta->deallocated += old_size;
}
UTRACE(ptr, size, ret);
+ JEMALLOC_VALGRIND_REALLOC(ret, usize, ptr, old_size, old_rzsize, false);
return (ret);
}
@@ -1114,18 +1179,21 @@
UTRACE(ptr, 0, 0);
if (ptr != NULL) {
size_t usize;
+ size_t rzsize JEMALLOC_CC_SILENCE_INIT(0);
assert(malloc_initialized || IS_INITIALIZER);
if (config_prof && opt_prof) {
- usize = isalloc(ptr);
+ usize = isalloc(ptr, config_prof);
prof_free(ptr, usize);
- } else if (config_stats) {
- usize = isalloc(ptr);
- }
+ } else if (config_stats || config_valgrind)
+ usize = isalloc(ptr, config_prof);
if (config_stats)
thread_allocated_tsd_get()->deallocated += usize;
- idalloc(ptr);
+ if (config_valgrind && opt_valgrind)
+ rzsize = p2rz(ptr);
+ iqalloc(ptr);
+ JEMALLOC_VALGRIND_FREE(ptr, rzsize);
}
}
@@ -1145,6 +1213,7 @@
{
void *ret JEMALLOC_CC_SILENCE_INIT(NULL);
imemalign(&ret, alignment, size, 1);
+ JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, size, false);
return (ret);
}
#endif
@@ -1157,6 +1226,7 @@
{
void *ret JEMALLOC_CC_SILENCE_INIT(NULL);
imemalign(&ret, PAGE, size, 1);
+ JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, size, false);
return (ret);
}
#endif
@@ -1209,9 +1279,9 @@
assert(malloc_initialized || IS_INITIALIZER);
if (config_ivsalloc)
- ret = ivsalloc(ptr);
+ ret = ivsalloc(ptr, config_prof);
else
- ret = (ptr != NULL) ? isalloc(ptr) : 0;
+ ret = (ptr != NULL) ? isalloc(ptr, config_prof) : 0;
return (ret);
}
@@ -1336,10 +1406,11 @@
*ptr = p;
if (config_stats) {
- assert(usize == isalloc(p));
+ assert(usize == isalloc(p, config_prof));
thread_allocated_tsd_get()->allocated += usize;
}
UTRACE(0, size, p);
+ JEMALLOC_VALGRIND_MALLOC(true, p, usize, zero);
return (ALLOCM_SUCCESS);
label_oom:
if (config_xmalloc && opt_xmalloc) {
@@ -1360,6 +1431,7 @@
void *p, *q;
size_t usize;
size_t old_size;
+ size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0);
size_t alignment = (ZU(1) << (flags & ALLOCM_LG_ALIGN_MASK)
& (SIZE_T_MAX-1));
bool zero = flags & ALLOCM_ZERO;
@@ -1384,7 +1456,9 @@
size_t max_usize = (alignment == 0) ? s2u(size+extra) :
sa2u(size+extra, alignment, NULL);
prof_ctx_t *old_ctx = prof_ctx_get(p);
- old_size = isalloc(p);
+ old_size = isalloc(p, true);
+ if (config_valgrind && opt_valgrind)
+ old_rzsize = p2rz(p);
PROF_ALLOC_PREP(1, max_usize, cnt);
if (cnt == NULL)
goto label_oom;
@@ -1403,27 +1477,33 @@
usize = max_usize;
arena_prof_promoted(q, usize);
} else
- usize = isalloc(q);
+ usize = isalloc(q, config_prof);
} else {
q = iralloc(p, size, extra, alignment, zero, no_move);
if (q == NULL)
goto label_err;
- usize = isalloc(q);
+ usize = isalloc(q, config_prof);
}
prof_realloc(q, usize, cnt, old_size, old_ctx);
if (rsize != NULL)
*rsize = usize;
} else {
- if (config_stats)
- old_size = isalloc(p);
+ if (config_stats) {
+ old_size = isalloc(p, false);
+ if (config_valgrind && opt_valgrind)
+ old_rzsize = u2rz(old_size);
+ } else if (config_valgrind && opt_valgrind) {
+ old_size = isalloc(p, false);
+ old_rzsize = u2rz(old_size);
+ }
q = iralloc(p, size, extra, alignment, zero, no_move);
if (q == NULL)
goto label_err;
if (config_stats)
- usize = isalloc(q);
+ usize = isalloc(q, config_prof);
if (rsize != NULL) {
if (config_stats == false)
- usize = isalloc(q);
+ usize = isalloc(q, config_prof);
*rsize = usize;
}
}
@@ -1436,6 +1516,7 @@
ta->deallocated += old_size;
}
UTRACE(p, size, q);
+ JEMALLOC_VALGRIND_REALLOC(q, usize, p, old_size, old_rzsize, zero);
return (ALLOCM_SUCCESS);
label_err:
if (no_move) {
@@ -1462,10 +1543,10 @@
assert(malloc_initialized || IS_INITIALIZER);
if (config_ivsalloc)
- sz = ivsalloc(ptr);
+ sz = ivsalloc(ptr, config_prof);
else {
assert(ptr != NULL);
- sz = isalloc(ptr);
+ sz = isalloc(ptr, config_prof);
}
assert(rsize != NULL);
*rsize = sz;
@@ -1479,21 +1560,25 @@
je_dallocm(void *ptr, int flags)
{
size_t usize;
+ size_t rzsize JEMALLOC_CC_SILENCE_INIT(0);
assert(ptr != NULL);
assert(malloc_initialized || IS_INITIALIZER);
UTRACE(ptr, 0, 0);
- if (config_stats)
- usize = isalloc(ptr);
+ if (config_stats || config_valgrind)
+ usize = isalloc(ptr, config_prof);
if (config_prof && opt_prof) {
- if (config_stats == false)
- usize = isalloc(ptr);
+ if (config_stats == false && config_valgrind == false)
+ usize = isalloc(ptr, config_prof);
prof_free(ptr, usize);
}
if (config_stats)
thread_allocated_tsd_get()->deallocated += usize;
- idalloc(ptr);
+ if (config_valgrind && opt_valgrind)
+ rzsize = p2rz(ptr);
+ iqalloc(ptr);
+ JEMALLOC_VALGRIND_FREE(ptr, rzsize);
return (ALLOCM_SUCCESS);
}
diff --git a/src/quarantine.c b/src/quarantine.c
new file mode 100644
index 0000000..89a25c6
--- /dev/null
+++ b/src/quarantine.c
@@ -0,0 +1,163 @@
+#include "jemalloc/internal/jemalloc_internal.h"
+
+/******************************************************************************/
+/* Data. */
+
+typedef struct quarantine_s quarantine_t;
+
+struct quarantine_s {
+ size_t curbytes;
+ size_t curobjs;
+ size_t first;
+#define LG_MAXOBJS_INIT 10
+ size_t lg_maxobjs;
+ void *objs[1]; /* Dynamically sized ring buffer. */
+};
+
+static void quarantine_cleanup(void *arg);
+
+malloc_tsd_data(static, quarantine, quarantine_t *, NULL)
+malloc_tsd_funcs(JEMALLOC_INLINE, quarantine, quarantine_t *, NULL,
+ quarantine_cleanup)
+
+/******************************************************************************/
+/* Function prototypes for non-inline static functions. */
+
+static quarantine_t *quarantine_init(size_t lg_maxobjs);
+static quarantine_t *quarantine_grow(quarantine_t *quarantine);
+static void quarantine_drain(quarantine_t *quarantine, size_t upper_bound);
+
+/******************************************************************************/
+
+static quarantine_t *
+quarantine_init(size_t lg_maxobjs)
+{
+ quarantine_t *quarantine;
+
+ quarantine = (quarantine_t *)imalloc(offsetof(quarantine_t, objs) +
+ ((ZU(1) << lg_maxobjs) * sizeof(void *)));
+ if (quarantine == NULL)
+ return (NULL);
+ quarantine->curbytes = 0;
+ quarantine->curobjs = 0;
+ quarantine->first = 0;
+ quarantine->lg_maxobjs = lg_maxobjs;
+
+ quarantine_tsd_set(&quarantine);
+
+ return (quarantine);
+}
+
+static quarantine_t *
+quarantine_grow(quarantine_t *quarantine)
+{
+ quarantine_t *ret;
+
+ ret = quarantine_init(quarantine->lg_maxobjs + 1);
+ if (ret == NULL)
+ return (quarantine);
+
+ ret->curbytes = quarantine->curbytes;
+ if (quarantine->first + quarantine->curobjs < (ZU(1) <<
+ quarantine->lg_maxobjs)) {
+ /* objs ring buffer data are contiguous. */
+ memcpy(ret->objs, &quarantine->objs[quarantine->first],
+ quarantine->curobjs * sizeof(void *));
+ ret->curobjs = quarantine->curobjs;
+ } else {
+ /* objs ring buffer data wrap around. */
+ size_t ncopy = (ZU(1) << quarantine->lg_maxobjs) -
+ quarantine->first;
+ memcpy(ret->objs, &quarantine->objs[quarantine->first], ncopy *
+ sizeof(void *));
+ ret->curobjs = ncopy;
+ if (quarantine->curobjs != 0) {
+ memcpy(&ret->objs[ret->curobjs], quarantine->objs,
+ quarantine->curobjs - ncopy);
+ }
+ }
+
+ return (ret);
+}
+
+static void
+quarantine_drain(quarantine_t *quarantine, size_t upper_bound)
+{
+
+ while (quarantine->curbytes > upper_bound && quarantine->curobjs > 0) {
+ void *ptr = quarantine->objs[quarantine->first];
+ size_t usize = isalloc(ptr, config_prof);
+ idalloc(ptr);
+ quarantine->curbytes -= usize;
+ quarantine->curobjs--;
+ quarantine->first = (quarantine->first + 1) & ((ZU(1) <<
+ quarantine->lg_maxobjs) - 1);
+ }
+}
+
+void
+quarantine(void *ptr)
+{
+ quarantine_t *quarantine;
+ size_t usize = isalloc(ptr, config_prof);
+
+ assert(config_fill);
+ assert(opt_quarantine);
+
+ quarantine = *quarantine_tsd_get();
+ if (quarantine == NULL && (quarantine =
+ quarantine_init(LG_MAXOBJS_INIT)) == NULL) {
+ idalloc(ptr);
+ return;
+ }
+ /*
+ * Drain one or more objects if the quarantine size limit would be
+ * exceeded by appending ptr.
+ */
+ if (quarantine->curbytes + usize > opt_quarantine) {
+ size_t upper_bound = (opt_quarantine >= usize) ? opt_quarantine
+ - usize : 0;
+ quarantine_drain(quarantine, upper_bound);
+ }
+ /* Grow the quarantine ring buffer if it's full. */
+ if (quarantine->curobjs == (ZU(1) << quarantine->lg_maxobjs))
+ quarantine = quarantine_grow(quarantine);
+ /* quarantine_grow() must free a slot if it fails to grow. */
+ assert(quarantine->curobjs < (ZU(1) << quarantine->lg_maxobjs));
+ /* Append ptr if its size doesn't exceed the quarantine size. */
+ if (quarantine->curbytes + usize <= opt_quarantine) {
+ size_t offset = (quarantine->first + quarantine->curobjs) &
+ ((ZU(1) << quarantine->lg_maxobjs) - 1);
+ quarantine->objs[offset] = ptr;
+ quarantine->curbytes += usize;
+ quarantine->curobjs++;
+ if (opt_junk)
+ memset(ptr, 0x5a, usize);
+ } else {
+ assert(quarantine->curbytes == 0);
+ idalloc(ptr);
+ }
+}
+
+static void
+quarantine_cleanup(void *arg)
+{
+ quarantine_t *quarantine = *(quarantine_t **)arg;
+
+ if (quarantine != NULL) {
+ quarantine_drain(quarantine, 0);
+ idalloc(quarantine);
+ }
+}
+
+bool
+quarantine_boot(void)
+{
+
+ assert(config_fill);
+
+ if (quarantine_tsd_boot())
+ return (true);
+
+ return (false);
+}
diff --git a/src/stats.c b/src/stats.c
index 0cd70b0..4cad214 100644
--- a/src/stats.c
+++ b/src/stats.c
@@ -382,8 +382,11 @@
OPT_WRITE_SSIZE_T(lg_dirty_mult)
OPT_WRITE_BOOL(stats_print)
OPT_WRITE_BOOL(junk)
+ OPT_WRITE_SIZE_T(quarantine)
+ OPT_WRITE_BOOL(redzone)
OPT_WRITE_BOOL(zero)
OPT_WRITE_BOOL(utrace)
+ OPT_WRITE_BOOL(valgrind)
OPT_WRITE_BOOL(xmalloc)
OPT_WRITE_BOOL(tcache)
OPT_WRITE_SSIZE_T(lg_tcache_max)
diff --git a/src/tcache.c b/src/tcache.c
index 99a657b..be26b59 100644
--- a/src/tcache.c
+++ b/src/tcache.c
@@ -75,6 +75,10 @@
(uintptr_t)chunk) >> LG_PAGE;
arena_chunk_map_t *mapelm =
&chunk->map[pageind-map_bias];
+ if (config_fill && opt_junk) {
+ arena_alloc_junk_small(ptr,
+ &arena_bin_info[binind], true);
+ }
arena_dalloc_bin(arena, chunk, ptr, mapelm);
} else {
/*
@@ -298,7 +302,7 @@
malloc_mutex_unlock(&tcache->arena->lock);
}
- tcache_size = arena_salloc(tcache);
+ tcache_size = arena_salloc(tcache, false);
if (tcache_size <= SMALL_MAXCLASS) {
arena_chunk_t *chunk = CHUNK_ADDR2BASE(tcache);
arena_t *arena = chunk->arena;
diff --git a/src/zone.c b/src/zone.c
index a50c129..cde5d49 100644
--- a/src/zone.c
+++ b/src/zone.c
@@ -56,7 +56,7 @@
* not work in practice, we must check all pointers to assure that they
* reside within a mapped chunk before determining size.
*/
- return (ivsalloc(ptr));
+ return (ivsalloc(ptr, config_prof));
}
static void *
@@ -87,7 +87,7 @@
zone_free(malloc_zone_t *zone, void *ptr)
{
- if (ivsalloc(ptr) != 0) {
+ if (ivsalloc(ptr, config_prof) != 0) {
je_free(ptr);
return;
}
@@ -99,7 +99,7 @@
zone_realloc(malloc_zone_t *zone, void *ptr, size_t size)
{
- if (ivsalloc(ptr) != 0)
+ if (ivsalloc(ptr, config_prof) != 0)
return (je_realloc(ptr, size));
return (realloc(ptr, size));
@@ -122,8 +122,8 @@
zone_free_definite_size(malloc_zone_t *zone, void *ptr, size_t size)
{
- if (ivsalloc(ptr) != 0) {
- assert(ivsalloc(ptr) == size);
+ if (ivsalloc(ptr, config_prof) != 0) {
+ assert(ivsalloc(ptr, config_prof) == size);
je_free(ptr);
return;
}