Modify pages_map() to support mapping uncommitted virtual memory.
If the OS overcommits:
- Commit all mappings in pages_map() regardless of whether the caller
requested committed memory.
- Linux-specific: Specify MAP_NORESERVE to avoid
unfortunate interactions with heuristic overcommit mode during
fork(2).
This resolves #193.
diff --git a/configure.ac b/configure.ac
index 1a1c970..7f19715 100644
--- a/configure.ac
+++ b/configure.ac
@@ -305,6 +305,7 @@
*-*-freebsd*)
CFLAGS="$CFLAGS"
abi="elf"
+ AC_DEFINE([JEMALLOC_SYSCTL_VM_OVERCOMMIT], [ ])
AC_DEFINE([JEMALLOC_PURGE_MADVISE_FREE], [ ])
force_lazy_lock="1"
;;
@@ -329,6 +330,7 @@
CPPFLAGS="$CPPFLAGS -D_GNU_SOURCE"
abi="elf"
AC_DEFINE([JEMALLOC_HAS_ALLOCA_H])
+ AC_DEFINE([JEMALLOC_PROC_SYS_VM_OVERCOMMIT_MEMORY], [ ])
AC_DEFINE([JEMALLOC_PURGE_MADVISE_DONTNEED], [ ])
AC_DEFINE([JEMALLOC_THREADED_INIT], [ ])
AC_DEFINE([JEMALLOC_USE_CXX_THROW], [ ])
diff --git a/include/jemalloc/internal/jemalloc_internal_defs.h.in b/include/jemalloc/internal/jemalloc_internal_defs.h.in
index 2c75371..7de0cf7 100644
--- a/include/jemalloc/internal/jemalloc_internal_defs.h.in
+++ b/include/jemalloc/internal/jemalloc_internal_defs.h.in
@@ -215,6 +215,15 @@
#undef JEMALLOC_ZONE_VERSION
/*
+ * Methods for determining whether the OS overcommits.
+ * JEMALLOC_PROC_SYS_VM_OVERCOMMIT_MEMORY: Linux's
+ * /proc/sys/vm.overcommit_memory file.
+ * JEMALLOC_SYSCTL_VM_OVERCOMMIT: FreeBSD's vm.overcommit sysctl.
+ */
+#undef JEMALLOC_SYSCTL_VM_OVERCOMMIT
+#undef JEMALLOC_PROC_SYS_VM_OVERCOMMIT_MEMORY
+
+/*
* Methods for purging unused pages differ between operating systems.
*
* madvise(..., MADV_DONTNEED) : On Linux, this immediately discards pages,
diff --git a/include/jemalloc/internal/pages.h b/include/jemalloc/internal/pages.h
index da7eb96..e21effd 100644
--- a/include/jemalloc/internal/pages.h
+++ b/include/jemalloc/internal/pages.h
@@ -9,13 +9,14 @@
/******************************************************************************/
#ifdef JEMALLOC_H_EXTERNS
-void *pages_map(void *addr, size_t size);
+void *pages_map(void *addr, size_t size, bool *commit);
void pages_unmap(void *addr, size_t size);
void *pages_trim(void *addr, size_t alloc_size, size_t leadsize,
- size_t size);
+ size_t size, bool *commit);
bool pages_commit(void *addr, size_t size);
bool pages_decommit(void *addr, size_t size);
bool pages_purge(void *addr, size_t size);
+void pages_boot(void);
#endif /* JEMALLOC_H_EXTERNS */
/******************************************************************************/
diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt
index 7958a4f..0f9b99e 100644
--- a/include/jemalloc/internal/private_symbols.txt
+++ b/include/jemalloc/internal/private_symbols.txt
@@ -398,6 +398,7 @@
opt_xmalloc
opt_zero
p2rz
+pages_boot
pages_commit
pages_decommit
pages_map
diff --git a/src/chunk_mmap.c b/src/chunk_mmap.c
index e2e66bc..f95ae75 100644
--- a/src/chunk_mmap.c
+++ b/src/chunk_mmap.c
@@ -16,18 +16,16 @@
do {
void *pages;
size_t leadsize;
- pages = pages_map(NULL, alloc_size);
+ pages = pages_map(NULL, alloc_size, commit);
if (pages == NULL)
return (NULL);
leadsize = ALIGNMENT_CEILING((uintptr_t)pages, alignment) -
(uintptr_t)pages;
- ret = pages_trim(pages, alloc_size, leadsize, size);
+ ret = pages_trim(pages, alloc_size, leadsize, size, commit);
} while (ret == NULL);
assert(ret != NULL);
*zero = true;
- if (!*commit)
- *commit = pages_decommit(ret, size);
return (ret);
}
@@ -54,7 +52,7 @@
assert(alignment != 0);
assert((alignment & chunksize_mask) == 0);
- ret = pages_map(new_addr, size);
+ ret = pages_map(new_addr, size, commit);
if (ret == NULL || ret == new_addr)
return (ret);
assert(new_addr == NULL);
@@ -66,8 +64,6 @@
assert(ret != NULL);
*zero = true;
- if (!*commit)
- *commit = pages_decommit(ret, size);
return (ret);
}
diff --git a/src/jemalloc.c b/src/jemalloc.c
index cd97ea1..1a26a44 100644
--- a/src/jemalloc.c
+++ b/src/jemalloc.c
@@ -1272,6 +1272,7 @@
abort();
}
}
+ pages_boot();
if (base_boot())
return (true);
if (chunk_boot())
diff --git a/src/pages.c b/src/pages.c
index 83a167f..2a9b7e3 100644
--- a/src/pages.c
+++ b/src/pages.c
@@ -1,29 +1,49 @@
#define JEMALLOC_PAGES_C_
#include "jemalloc/internal/jemalloc_internal.h"
+#ifdef JEMALLOC_SYSCTL_VM_OVERCOMMIT
+#include <sys/sysctl.h>
+#endif
+
+/******************************************************************************/
+/* Data. */
+
+#ifndef _WIN32
+# define PAGES_PROT_COMMIT (PROT_READ | PROT_WRITE)
+# define PAGES_PROT_DECOMMIT (PROT_NONE)
+static int mmap_flags;
+#endif
+static bool os_overcommits;
+
/******************************************************************************/
void *
-pages_map(void *addr, size_t size)
+pages_map(void *addr, size_t size, bool *commit)
{
void *ret;
assert(size != 0);
+ if (os_overcommits)
+ *commit = true;
+
#ifdef _WIN32
/*
* If VirtualAlloc can't allocate at the given address when one is
* given, it fails and returns NULL.
*/
- ret = VirtualAlloc(addr, size, MEM_COMMIT | MEM_RESERVE,
+ ret = VirtualAlloc(addr, size, MEM_RESERVE | (*commit ? MEM_COMMIT : 0),
PAGE_READWRITE);
#else
/*
* We don't use MAP_FIXED here, because it can cause the *replacement*
* of existing mappings, and we only want to create new mappings.
*/
- ret = mmap(addr, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON,
- -1, 0);
+ {
+ int prot = *commit ? PAGES_PROT_COMMIT : PAGES_PROT_DECOMMIT;
+
+ ret = mmap(addr, size, prot, mmap_flags, -1, 0);
+ }
assert(ret != NULL);
if (ret == MAP_FAILED)
@@ -67,7 +87,8 @@
}
void *
-pages_trim(void *addr, size_t alloc_size, size_t leadsize, size_t size)
+pages_trim(void *addr, size_t alloc_size, size_t leadsize, size_t size,
+ bool *commit)
{
void *ret = (void *)((uintptr_t)addr + leadsize);
@@ -77,7 +98,7 @@
void *new_addr;
pages_unmap(addr, alloc_size);
- new_addr = pages_map(ret, size);
+ new_addr = pages_map(ret, size, commit);
if (new_addr == ret)
return (ret);
if (new_addr)
@@ -101,17 +122,17 @@
pages_commit_impl(void *addr, size_t size, bool commit)
{
-#ifndef _WIN32
- /*
- * The following decommit/commit implementation is functional, but
- * always disabled because it doesn't add value beyong improved
- * debugging (at the cost of extra system calls) on systems that
- * overcommit.
- */
- if (false) {
- int prot = commit ? (PROT_READ | PROT_WRITE) : PROT_NONE;
- void *result = mmap(addr, size, prot, MAP_PRIVATE | MAP_ANON |
- MAP_FIXED, -1, 0);
+ if (os_overcommits)
+ return (true);
+
+#ifdef _WIN32
+ return (commit ? (addr != VirtualAlloc(addr, size, MEM_COMMIT,
+ PAGE_READWRITE)) : (!VirtualFree(addr, size, MEM_DECOMMIT)));
+#else
+ {
+ int prot = commit ? PAGES_PROT_COMMIT : PAGES_PROT_DECOMMIT;
+ void *result = mmap(addr, size, prot, mmap_flags | MAP_FIXED,
+ -1, 0);
if (result == MAP_FAILED)
return (true);
if (result != addr) {
@@ -125,7 +146,6 @@
return (false);
}
#endif
- return (true);
}
bool
@@ -171,3 +191,63 @@
return (unzeroed);
}
+#ifdef JEMALLOC_SYSCTL_VM_OVERCOMMIT
+static bool
+os_overcommits_sysctl(void)
+{
+ int vm_overcommit;
+ size_t sz;
+
+ sz = sizeof(vm_overcommit);
+ if (sysctlbyname("vm.overcommit", &vm_overcommit, &sz, NULL, 0) != 0)
+ return (false); /* Error. */
+
+ return ((vm_overcommit & 0x3) == 0);
+}
+#endif
+
+#ifdef JEMALLOC_PROC_SYS_VM_OVERCOMMIT_MEMORY
+static bool
+os_overcommits_proc(void)
+{
+ int fd;
+ char buf[1];
+ ssize_t nread;
+
+ fd = open("/proc/sys/vm/overcommit_memory", O_RDONLY);
+ if (fd == -1)
+ return (false); /* Error. */
+
+ nread = read(fd, &buf, sizeof(buf));
+ if (nread < 1)
+ return (false); /* Error. */
+ /*
+ * /proc/sys/vm/overcommit_memory meanings:
+ * 0: Heuristic overcommit.
+ * 1: Always overcommit.
+ * 2: Never overcommit.
+ */
+ return (buf[0] == '0' || buf[0] == '1');
+}
+#endif
+
+void
+pages_boot(void)
+{
+
+#ifndef _WIN32
+ mmap_flags = MAP_PRIVATE | MAP_ANON;
+#endif
+
+#ifdef JEMALLOC_SYSCTL_VM_OVERCOMMIT
+ os_overcommits = os_overcommits_sysctl();
+#elif defined(JEMALLOC_PROC_SYS_VM_OVERCOMMIT_MEMORY)
+ os_overcommits = os_overcommits_proc();
+# ifdef MAP_NORESERVE
+ if (os_overcommits)
+ mmap_flags |= MAP_NORESERVE;
+# endif
+#else
+ os_overcommits = false;
+#endif
+}