Jason Evans | e476f8a | 2010-01-16 09:53:50 -0800 | [diff] [blame] | 1 | #define JEMALLOC_C_ |
Jason Evans | 376b152 | 2010-02-11 14:45:59 -0800 | [diff] [blame] | 2 | #include "jemalloc/internal/jemalloc_internal.h" |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 3 | |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 4 | /******************************************************************************/ |
Jason Evans | e476f8a | 2010-01-16 09:53:50 -0800 | [diff] [blame] | 5 | /* Data. */ |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 6 | |
Jason Evans | cd9a134 | 2012-03-21 18:33:03 -0700 | [diff] [blame] | 7 | malloc_tsd_data(, arenas, arena_t *, NULL) |
| 8 | malloc_tsd_data(, thread_allocated, thread_allocated_t, |
| 9 | THREAD_ALLOCATED_INITIALIZER) |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 10 | |
Jason Evans | e476f8a | 2010-01-16 09:53:50 -0800 | [diff] [blame] | 11 | /* Runtime configuration options. */ |
Mike Hommey | da99e31 | 2012-04-30 12:38:29 +0200 | [diff] [blame] | 12 | const char *je_malloc_conf; |
Jason Evans | b7924f5 | 2009-06-23 19:01:18 -0700 | [diff] [blame] | 13 | #ifdef JEMALLOC_DEBUG |
Jason Evans | e476f8a | 2010-01-16 09:53:50 -0800 | [diff] [blame] | 14 | bool opt_abort = true; |
Jason Evans | b7924f5 | 2009-06-23 19:01:18 -0700 | [diff] [blame] | 15 | # ifdef JEMALLOC_FILL |
Jason Evans | e476f8a | 2010-01-16 09:53:50 -0800 | [diff] [blame] | 16 | bool opt_junk = true; |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 17 | # else |
| 18 | bool opt_junk = false; |
Jason Evans | b7924f5 | 2009-06-23 19:01:18 -0700 | [diff] [blame] | 19 | # endif |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 20 | #else |
Jason Evans | e476f8a | 2010-01-16 09:53:50 -0800 | [diff] [blame] | 21 | bool opt_abort = false; |
Jason Evans | e476f8a | 2010-01-16 09:53:50 -0800 | [diff] [blame] | 22 | bool opt_junk = false; |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 23 | #endif |
Jason Evans | 122449b | 2012-04-06 00:35:09 -0700 | [diff] [blame] | 24 | size_t opt_quarantine = ZU(0); |
Jason Evans | d6abcbb | 2012-04-12 17:09:54 -0700 | [diff] [blame] | 25 | bool opt_redzone = false; |
Jason Evans | b147611 | 2012-04-05 13:36:17 -0700 | [diff] [blame] | 26 | bool opt_utrace = false; |
Jason Evans | 122449b | 2012-04-06 00:35:09 -0700 | [diff] [blame] | 27 | bool opt_valgrind = false; |
Jason Evans | e476f8a | 2010-01-16 09:53:50 -0800 | [diff] [blame] | 28 | bool opt_xmalloc = false; |
Jason Evans | e476f8a | 2010-01-16 09:53:50 -0800 | [diff] [blame] | 29 | bool opt_zero = false; |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 30 | size_t opt_narenas = 0; |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 31 | |
Jason Evans | cd9a134 | 2012-03-21 18:33:03 -0700 | [diff] [blame] | 32 | unsigned ncpus; |
| 33 | |
| 34 | malloc_mutex_t arenas_lock; |
| 35 | arena_t **arenas; |
| 36 | unsigned narenas; |
| 37 | |
| 38 | /* Set to true once the allocator has been initialized. */ |
Jason Evans | 4eeb52f | 2012-04-02 01:46:25 -0700 | [diff] [blame] | 39 | static bool malloc_initialized = false; |
Jason Evans | cd9a134 | 2012-03-21 18:33:03 -0700 | [diff] [blame] | 40 | |
Jason Evans | 41b6afb | 2012-02-02 22:04:57 -0800 | [diff] [blame] | 41 | #ifdef JEMALLOC_THREADED_INIT |
Jason Evans | cd9a134 | 2012-03-21 18:33:03 -0700 | [diff] [blame] | 42 | /* Used to let the initializing thread recursively allocate. */ |
Jason Evans | 02b2312 | 2012-04-05 11:06:23 -0700 | [diff] [blame] | 43 | # define NO_INITIALIZER ((unsigned long)0) |
Jason Evans | 41b6afb | 2012-02-02 22:04:57 -0800 | [diff] [blame] | 44 | # define INITIALIZER pthread_self() |
| 45 | # define IS_INITIALIZER (malloc_initializer == pthread_self()) |
Jason Evans | 02b2312 | 2012-04-05 11:06:23 -0700 | [diff] [blame] | 46 | static pthread_t malloc_initializer = NO_INITIALIZER; |
Jason Evans | 41b6afb | 2012-02-02 22:04:57 -0800 | [diff] [blame] | 47 | #else |
Jason Evans | 02b2312 | 2012-04-05 11:06:23 -0700 | [diff] [blame] | 48 | # define NO_INITIALIZER false |
Jason Evans | 41b6afb | 2012-02-02 22:04:57 -0800 | [diff] [blame] | 49 | # define INITIALIZER true |
| 50 | # define IS_INITIALIZER malloc_initializer |
Jason Evans | 02b2312 | 2012-04-05 11:06:23 -0700 | [diff] [blame] | 51 | static bool malloc_initializer = NO_INITIALIZER; |
Jason Evans | 41b6afb | 2012-02-02 22:04:57 -0800 | [diff] [blame] | 52 | #endif |
Jason Evans | cd9a134 | 2012-03-21 18:33:03 -0700 | [diff] [blame] | 53 | |
| 54 | /* Used to avoid initialization races. */ |
Mike Hommey | a19e87f | 2012-04-21 21:27:46 -0700 | [diff] [blame] | 55 | #ifdef _WIN32 |
| 56 | static malloc_mutex_t init_lock; |
| 57 | |
| 58 | JEMALLOC_ATTR(constructor) |
Mike Hommey | fd97b1d | 2012-04-30 12:38:31 +0200 | [diff] [blame^] | 59 | static void WINAPI |
| 60 | _init_init_lock(void) |
Mike Hommey | a19e87f | 2012-04-21 21:27:46 -0700 | [diff] [blame] | 61 | { |
| 62 | |
| 63 | malloc_mutex_init(&init_lock); |
| 64 | } |
Mike Hommey | fd97b1d | 2012-04-30 12:38:31 +0200 | [diff] [blame^] | 65 | |
| 66 | #ifdef _MSC_VER |
| 67 | # pragma section(".CRT$XCU", read) |
| 68 | JEMALLOC_SECTION(".CRT$XCU") JEMALLOC_ATTR(used) |
| 69 | static const void (WINAPI *init_init_lock)(void) = _init_init_lock; |
| 70 | #endif |
| 71 | |
Mike Hommey | a19e87f | 2012-04-21 21:27:46 -0700 | [diff] [blame] | 72 | #else |
Jason Evans | cd9a134 | 2012-03-21 18:33:03 -0700 | [diff] [blame] | 73 | static malloc_mutex_t init_lock = MALLOC_MUTEX_INITIALIZER; |
Mike Hommey | a19e87f | 2012-04-21 21:27:46 -0700 | [diff] [blame] | 74 | #endif |
Jason Evans | cd9a134 | 2012-03-21 18:33:03 -0700 | [diff] [blame] | 75 | |
Jason Evans | b147611 | 2012-04-05 13:36:17 -0700 | [diff] [blame] | 76 | typedef struct { |
| 77 | void *p; /* Input pointer (as in realloc(p, s)). */ |
| 78 | size_t s; /* Request size. */ |
| 79 | void *r; /* Result pointer. */ |
| 80 | } malloc_utrace_t; |
| 81 | |
| 82 | #ifdef JEMALLOC_UTRACE |
| 83 | # define UTRACE(a, b, c) do { \ |
| 84 | if (opt_utrace) { \ |
| 85 | malloc_utrace_t ut; \ |
| 86 | ut.p = (a); \ |
| 87 | ut.s = (b); \ |
| 88 | ut.r = (c); \ |
| 89 | utrace(&ut, sizeof(ut)); \ |
| 90 | } \ |
| 91 | } while (0) |
| 92 | #else |
| 93 | # define UTRACE(a, b, c) |
| 94 | #endif |
| 95 | |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 96 | /******************************************************************************/ |
Jason Evans | e476f8a | 2010-01-16 09:53:50 -0800 | [diff] [blame] | 97 | /* Function prototypes for non-inline static functions. */ |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 98 | |
Jason Evans | 03c2237 | 2010-01-03 12:10:42 -0800 | [diff] [blame] | 99 | static void stats_print_atexit(void); |
Jason Evans | c9658dd | 2009-06-22 14:44:08 -0700 | [diff] [blame] | 100 | static unsigned malloc_ncpus(void); |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 101 | static bool malloc_conf_next(char const **opts_p, char const **k_p, |
| 102 | size_t *klen_p, char const **v_p, size_t *vlen_p); |
| 103 | static void malloc_conf_error(const char *msg, const char *k, size_t klen, |
| 104 | const char *v, size_t vlen); |
| 105 | static void malloc_conf_init(void); |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 106 | static bool malloc_init_hard(void); |
Jason Evans | 5965631 | 2012-02-28 21:37:38 -0800 | [diff] [blame] | 107 | static int imemalign(void **memptr, size_t alignment, size_t size, |
Jason Evans | 0a0bbf6 | 2012-03-13 12:55:21 -0700 | [diff] [blame] | 108 | size_t min_alignment); |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 109 | |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 110 | /******************************************************************************/ |
Jason Evans | c9658dd | 2009-06-22 14:44:08 -0700 | [diff] [blame] | 111 | /* |
Jason Evans | e476f8a | 2010-01-16 09:53:50 -0800 | [diff] [blame] | 112 | * Begin miscellaneous support functions. |
Jason Evans | b7924f5 | 2009-06-23 19:01:18 -0700 | [diff] [blame] | 113 | */ |
| 114 | |
Jason Evans | e476f8a | 2010-01-16 09:53:50 -0800 | [diff] [blame] | 115 | /* Create a new arena and insert it into the arenas array at index ind. */ |
| 116 | arena_t * |
| 117 | arenas_extend(unsigned ind) |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 118 | { |
| 119 | arena_t *ret; |
| 120 | |
Jason Evans | b172610 | 2012-02-28 16:50:47 -0800 | [diff] [blame] | 121 | ret = (arena_t *)base_alloc(sizeof(arena_t)); |
Jason Evans | e476f8a | 2010-01-16 09:53:50 -0800 | [diff] [blame] | 122 | if (ret != NULL && arena_new(ret, ind) == false) { |
| 123 | arenas[ind] = ret; |
| 124 | return (ret); |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 125 | } |
Jason Evans | e476f8a | 2010-01-16 09:53:50 -0800 | [diff] [blame] | 126 | /* Only reached if there is an OOM error. */ |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 127 | |
Jason Evans | e476f8a | 2010-01-16 09:53:50 -0800 | [diff] [blame] | 128 | /* |
| 129 | * OOM here is quite inconvenient to propagate, since dealing with it |
| 130 | * would require a check for failure in the fast path. Instead, punt |
| 131 | * by using arenas[0]. In practice, this is an extremely unlikely |
| 132 | * failure. |
| 133 | */ |
Jason Evans | 698805c | 2010-03-03 17:45:38 -0800 | [diff] [blame] | 134 | malloc_write("<jemalloc>: Error initializing arena\n"); |
Jason Evans | e476f8a | 2010-01-16 09:53:50 -0800 | [diff] [blame] | 135 | if (opt_abort) |
| 136 | abort(); |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 137 | |
Jason Evans | e476f8a | 2010-01-16 09:53:50 -0800 | [diff] [blame] | 138 | return (arenas[0]); |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 139 | } |
| 140 | |
Jason Evans | 4c2faa8 | 2012-03-13 11:09:23 -0700 | [diff] [blame] | 141 | /* Slow path, called only by choose_arena(). */ |
Jason Evans | e476f8a | 2010-01-16 09:53:50 -0800 | [diff] [blame] | 142 | arena_t * |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 143 | choose_arena_hard(void) |
| 144 | { |
| 145 | arena_t *ret; |
| 146 | |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 147 | if (narenas > 1) { |
Jason Evans | 597632b | 2011-03-18 13:41:33 -0700 | [diff] [blame] | 148 | unsigned i, choose, first_null; |
| 149 | |
| 150 | choose = 0; |
| 151 | first_null = narenas; |
Jason Evans | 3ee7a5c | 2009-12-29 00:09:15 -0800 | [diff] [blame] | 152 | malloc_mutex_lock(&arenas_lock); |
Jason Evans | 0657f12 | 2011-03-18 17:56:14 -0700 | [diff] [blame] | 153 | assert(arenas[0] != NULL); |
Jason Evans | 597632b | 2011-03-18 13:41:33 -0700 | [diff] [blame] | 154 | for (i = 1; i < narenas; i++) { |
| 155 | if (arenas[i] != NULL) { |
| 156 | /* |
| 157 | * Choose the first arena that has the lowest |
| 158 | * number of threads assigned to it. |
| 159 | */ |
| 160 | if (arenas[i]->nthreads < |
| 161 | arenas[choose]->nthreads) |
| 162 | choose = i; |
| 163 | } else if (first_null == narenas) { |
| 164 | /* |
| 165 | * Record the index of the first uninitialized |
| 166 | * arena, in case all extant arenas are in use. |
| 167 | * |
| 168 | * NB: It is possible for there to be |
| 169 | * discontinuities in terms of initialized |
| 170 | * versus uninitialized arenas, due to the |
| 171 | * "thread.arena" mallctl. |
| 172 | */ |
| 173 | first_null = i; |
| 174 | } |
| 175 | } |
| 176 | |
Jason Evans | 41b6afb | 2012-02-02 22:04:57 -0800 | [diff] [blame] | 177 | if (arenas[choose]->nthreads == 0 || first_null == narenas) { |
Jason Evans | 597632b | 2011-03-18 13:41:33 -0700 | [diff] [blame] | 178 | /* |
| 179 | * Use an unloaded arena, or the least loaded arena if |
| 180 | * all arenas are already initialized. |
| 181 | */ |
| 182 | ret = arenas[choose]; |
| 183 | } else { |
| 184 | /* Initialize a new arena. */ |
| 185 | ret = arenas_extend(first_null); |
| 186 | } |
| 187 | ret->nthreads++; |
Jason Evans | 3ee7a5c | 2009-12-29 00:09:15 -0800 | [diff] [blame] | 188 | malloc_mutex_unlock(&arenas_lock); |
Jason Evans | 597632b | 2011-03-18 13:41:33 -0700 | [diff] [blame] | 189 | } else { |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 190 | ret = arenas[0]; |
Jason Evans | 597632b | 2011-03-18 13:41:33 -0700 | [diff] [blame] | 191 | malloc_mutex_lock(&arenas_lock); |
| 192 | ret->nthreads++; |
| 193 | malloc_mutex_unlock(&arenas_lock); |
| 194 | } |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 195 | |
Jason Evans | cd9a134 | 2012-03-21 18:33:03 -0700 | [diff] [blame] | 196 | arenas_tsd_set(&ret); |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 197 | |
| 198 | return (ret); |
| 199 | } |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 200 | |
Jason Evans | 03c2237 | 2010-01-03 12:10:42 -0800 | [diff] [blame] | 201 | static void |
| 202 | stats_print_atexit(void) |
| 203 | { |
| 204 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 205 | if (config_tcache && config_stats) { |
| 206 | unsigned i; |
Jason Evans | 03c2237 | 2010-01-03 12:10:42 -0800 | [diff] [blame] | 207 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 208 | /* |
| 209 | * Merge stats from extant threads. This is racy, since |
| 210 | * individual threads do not lock when recording tcache stats |
| 211 | * events. As a consequence, the final stats may be slightly |
| 212 | * out of date by the time they are reported, if other threads |
| 213 | * continue to allocate. |
| 214 | */ |
| 215 | for (i = 0; i < narenas; i++) { |
| 216 | arena_t *arena = arenas[i]; |
| 217 | if (arena != NULL) { |
| 218 | tcache_t *tcache; |
Jason Evans | 03c2237 | 2010-01-03 12:10:42 -0800 | [diff] [blame] | 219 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 220 | /* |
| 221 | * tcache_stats_merge() locks bins, so if any |
| 222 | * code is introduced that acquires both arena |
| 223 | * and bin locks in the opposite order, |
| 224 | * deadlocks may result. |
| 225 | */ |
| 226 | malloc_mutex_lock(&arena->lock); |
| 227 | ql_foreach(tcache, &arena->tcache_ql, link) { |
| 228 | tcache_stats_merge(tcache, arena); |
| 229 | } |
| 230 | malloc_mutex_unlock(&arena->lock); |
Jason Evans | 03c2237 | 2010-01-03 12:10:42 -0800 | [diff] [blame] | 231 | } |
Jason Evans | 03c2237 | 2010-01-03 12:10:42 -0800 | [diff] [blame] | 232 | } |
| 233 | } |
Jason Evans | 0a5489e | 2012-03-01 17:19:20 -0800 | [diff] [blame] | 234 | je_malloc_stats_print(NULL, NULL, NULL); |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 235 | } |
| 236 | |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 237 | /* |
Jason Evans | e476f8a | 2010-01-16 09:53:50 -0800 | [diff] [blame] | 238 | * End miscellaneous support functions. |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 239 | */ |
| 240 | /******************************************************************************/ |
| 241 | /* |
Jason Evans | e476f8a | 2010-01-16 09:53:50 -0800 | [diff] [blame] | 242 | * Begin initialization functions. |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 243 | */ |
| 244 | |
Jason Evans | c9658dd | 2009-06-22 14:44:08 -0700 | [diff] [blame] | 245 | static unsigned |
| 246 | malloc_ncpus(void) |
| 247 | { |
| 248 | unsigned ret; |
Jason Evans | b7924f5 | 2009-06-23 19:01:18 -0700 | [diff] [blame] | 249 | long result; |
Jason Evans | c9658dd | 2009-06-22 14:44:08 -0700 | [diff] [blame] | 250 | |
Mike Hommey | a19e87f | 2012-04-21 21:27:46 -0700 | [diff] [blame] | 251 | #ifdef _WIN32 |
| 252 | SYSTEM_INFO si; |
| 253 | GetSystemInfo(&si); |
| 254 | result = si.dwNumberOfProcessors; |
| 255 | #else |
Jason Evans | b7924f5 | 2009-06-23 19:01:18 -0700 | [diff] [blame] | 256 | result = sysconf(_SC_NPROCESSORS_ONLN); |
| 257 | if (result == -1) { |
| 258 | /* Error. */ |
| 259 | ret = 1; |
Jason Evans | c9658dd | 2009-06-22 14:44:08 -0700 | [diff] [blame] | 260 | } |
Mike Hommey | a19e87f | 2012-04-21 21:27:46 -0700 | [diff] [blame] | 261 | #endif |
Jason Evans | b7924f5 | 2009-06-23 19:01:18 -0700 | [diff] [blame] | 262 | ret = (unsigned)result; |
Jason Evans | c9658dd | 2009-06-22 14:44:08 -0700 | [diff] [blame] | 263 | |
| 264 | return (ret); |
| 265 | } |
Jason Evans | b7924f5 | 2009-06-23 19:01:18 -0700 | [diff] [blame] | 266 | |
Jason Evans | cd9a134 | 2012-03-21 18:33:03 -0700 | [diff] [blame] | 267 | void |
Jason Evans | 597632b | 2011-03-18 13:41:33 -0700 | [diff] [blame] | 268 | arenas_cleanup(void *arg) |
| 269 | { |
Jason Evans | cd9a134 | 2012-03-21 18:33:03 -0700 | [diff] [blame] | 270 | arena_t *arena = *(arena_t **)arg; |
Jason Evans | 597632b | 2011-03-18 13:41:33 -0700 | [diff] [blame] | 271 | |
| 272 | malloc_mutex_lock(&arenas_lock); |
| 273 | arena->nthreads--; |
| 274 | malloc_mutex_unlock(&arenas_lock); |
| 275 | } |
| 276 | |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 277 | static inline bool |
| 278 | malloc_init(void) |
| 279 | { |
| 280 | |
| 281 | if (malloc_initialized == false) |
| 282 | return (malloc_init_hard()); |
| 283 | |
| 284 | return (false); |
| 285 | } |
| 286 | |
| 287 | static bool |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 288 | malloc_conf_next(char const **opts_p, char const **k_p, size_t *klen_p, |
| 289 | char const **v_p, size_t *vlen_p) |
| 290 | { |
| 291 | bool accept; |
| 292 | const char *opts = *opts_p; |
| 293 | |
| 294 | *k_p = opts; |
| 295 | |
| 296 | for (accept = false; accept == false;) { |
| 297 | switch (*opts) { |
Jason Evans | d81e4bd | 2012-03-06 14:57:45 -0800 | [diff] [blame] | 298 | case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': |
| 299 | case 'G': case 'H': case 'I': case 'J': case 'K': case 'L': |
| 300 | case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R': |
| 301 | case 'S': case 'T': case 'U': case 'V': case 'W': case 'X': |
| 302 | case 'Y': case 'Z': |
| 303 | case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': |
| 304 | case 'g': case 'h': case 'i': case 'j': case 'k': case 'l': |
| 305 | case 'm': case 'n': case 'o': case 'p': case 'q': case 'r': |
| 306 | case 's': case 't': case 'u': case 'v': case 'w': case 'x': |
| 307 | case 'y': case 'z': |
| 308 | case '0': case '1': case '2': case '3': case '4': case '5': |
| 309 | case '6': case '7': case '8': case '9': |
| 310 | case '_': |
| 311 | opts++; |
| 312 | break; |
| 313 | case ':': |
| 314 | opts++; |
| 315 | *klen_p = (uintptr_t)opts - 1 - (uintptr_t)*k_p; |
| 316 | *v_p = opts; |
| 317 | accept = true; |
| 318 | break; |
| 319 | case '\0': |
| 320 | if (opts != *opts_p) { |
| 321 | malloc_write("<jemalloc>: Conf string ends " |
| 322 | "with key\n"); |
| 323 | } |
| 324 | return (true); |
| 325 | default: |
| 326 | malloc_write("<jemalloc>: Malformed conf string\n"); |
| 327 | return (true); |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 328 | } |
| 329 | } |
| 330 | |
| 331 | for (accept = false; accept == false;) { |
| 332 | switch (*opts) { |
Jason Evans | d81e4bd | 2012-03-06 14:57:45 -0800 | [diff] [blame] | 333 | case ',': |
| 334 | opts++; |
| 335 | /* |
| 336 | * Look ahead one character here, because the next time |
| 337 | * this function is called, it will assume that end of |
| 338 | * input has been cleanly reached if no input remains, |
| 339 | * but we have optimistically already consumed the |
| 340 | * comma if one exists. |
| 341 | */ |
| 342 | if (*opts == '\0') { |
| 343 | malloc_write("<jemalloc>: Conf string ends " |
| 344 | "with comma\n"); |
| 345 | } |
| 346 | *vlen_p = (uintptr_t)opts - 1 - (uintptr_t)*v_p; |
| 347 | accept = true; |
| 348 | break; |
| 349 | case '\0': |
| 350 | *vlen_p = (uintptr_t)opts - (uintptr_t)*v_p; |
| 351 | accept = true; |
| 352 | break; |
| 353 | default: |
| 354 | opts++; |
| 355 | break; |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 356 | } |
| 357 | } |
| 358 | |
| 359 | *opts_p = opts; |
| 360 | return (false); |
| 361 | } |
| 362 | |
| 363 | static void |
| 364 | malloc_conf_error(const char *msg, const char *k, size_t klen, const char *v, |
| 365 | size_t vlen) |
| 366 | { |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 367 | |
Jason Evans | d81e4bd | 2012-03-06 14:57:45 -0800 | [diff] [blame] | 368 | malloc_printf("<jemalloc>: %s: %.*s:%.*s\n", msg, (int)klen, k, |
| 369 | (int)vlen, v); |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 370 | } |
| 371 | |
| 372 | static void |
| 373 | malloc_conf_init(void) |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 374 | { |
| 375 | unsigned i; |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 376 | char buf[PATH_MAX + 1]; |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 377 | const char *opts, *k, *v; |
| 378 | size_t klen, vlen; |
| 379 | |
| 380 | for (i = 0; i < 3; i++) { |
| 381 | /* Get runtime configuration. */ |
| 382 | switch (i) { |
| 383 | case 0: |
Jason Evans | 0a5489e | 2012-03-01 17:19:20 -0800 | [diff] [blame] | 384 | if (je_malloc_conf != NULL) { |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 385 | /* |
| 386 | * Use options that were compiled into the |
| 387 | * program. |
| 388 | */ |
Jason Evans | 0a5489e | 2012-03-01 17:19:20 -0800 | [diff] [blame] | 389 | opts = je_malloc_conf; |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 390 | } else { |
| 391 | /* No configuration specified. */ |
| 392 | buf[0] = '\0'; |
| 393 | opts = buf; |
| 394 | } |
| 395 | break; |
| 396 | case 1: { |
Mike Hommey | a19e87f | 2012-04-21 21:27:46 -0700 | [diff] [blame] | 397 | #ifndef _WIN32 |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 398 | int linklen; |
| 399 | const char *linkname = |
Mike Hommey | a19e87f | 2012-04-21 21:27:46 -0700 | [diff] [blame] | 400 | # ifdef JEMALLOC_PREFIX |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 401 | "/etc/"JEMALLOC_PREFIX"malloc.conf" |
Mike Hommey | a19e87f | 2012-04-21 21:27:46 -0700 | [diff] [blame] | 402 | # else |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 403 | "/etc/malloc.conf" |
Mike Hommey | a19e87f | 2012-04-21 21:27:46 -0700 | [diff] [blame] | 404 | # endif |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 405 | ; |
| 406 | |
| 407 | if ((linklen = readlink(linkname, buf, |
| 408 | sizeof(buf) - 1)) != -1) { |
| 409 | /* |
| 410 | * Use the contents of the "/etc/malloc.conf" |
| 411 | * symbolic link's name. |
| 412 | */ |
| 413 | buf[linklen] = '\0'; |
| 414 | opts = buf; |
Mike Hommey | a19e87f | 2012-04-21 21:27:46 -0700 | [diff] [blame] | 415 | } else |
| 416 | #endif |
| 417 | { |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 418 | /* No configuration specified. */ |
| 419 | buf[0] = '\0'; |
| 420 | opts = buf; |
| 421 | } |
| 422 | break; |
Jason Evans | d81e4bd | 2012-03-06 14:57:45 -0800 | [diff] [blame] | 423 | } case 2: { |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 424 | const char *envname = |
| 425 | #ifdef JEMALLOC_PREFIX |
| 426 | JEMALLOC_CPREFIX"MALLOC_CONF" |
| 427 | #else |
| 428 | "MALLOC_CONF" |
| 429 | #endif |
| 430 | ; |
| 431 | |
| 432 | if ((opts = getenv(envname)) != NULL) { |
| 433 | /* |
| 434 | * Do nothing; opts is already initialized to |
Jason Evans | 8ad0eac | 2010-12-17 18:07:53 -0800 | [diff] [blame] | 435 | * the value of the MALLOC_CONF environment |
| 436 | * variable. |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 437 | */ |
| 438 | } else { |
| 439 | /* No configuration specified. */ |
| 440 | buf[0] = '\0'; |
| 441 | opts = buf; |
| 442 | } |
| 443 | break; |
Jason Evans | d81e4bd | 2012-03-06 14:57:45 -0800 | [diff] [blame] | 444 | } default: |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 445 | /* NOTREACHED */ |
| 446 | assert(false); |
| 447 | buf[0] = '\0'; |
| 448 | opts = buf; |
| 449 | } |
| 450 | |
| 451 | while (*opts != '\0' && malloc_conf_next(&opts, &k, &klen, &v, |
| 452 | &vlen) == false) { |
Jason Evans | 122449b | 2012-04-06 00:35:09 -0700 | [diff] [blame] | 453 | #define CONF_HANDLE_BOOL_HIT(o, n, hit) \ |
Jason Evans | 606f1fd | 2012-04-20 21:39:14 -0700 | [diff] [blame] | 454 | if (sizeof(n)-1 == klen && strncmp(n, k, \ |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 455 | klen) == 0) { \ |
| 456 | if (strncmp("true", v, vlen) == 0 && \ |
| 457 | vlen == sizeof("true")-1) \ |
Jason Evans | d81e4bd | 2012-03-06 14:57:45 -0800 | [diff] [blame] | 458 | o = true; \ |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 459 | else if (strncmp("false", v, vlen) == \ |
| 460 | 0 && vlen == sizeof("false")-1) \ |
Jason Evans | d81e4bd | 2012-03-06 14:57:45 -0800 | [diff] [blame] | 461 | o = false; \ |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 462 | else { \ |
| 463 | malloc_conf_error( \ |
| 464 | "Invalid conf value", \ |
| 465 | k, klen, v, vlen); \ |
| 466 | } \ |
Jason Evans | 122449b | 2012-04-06 00:35:09 -0700 | [diff] [blame] | 467 | hit = true; \ |
| 468 | } else \ |
| 469 | hit = false; |
| 470 | #define CONF_HANDLE_BOOL(o, n) { \ |
| 471 | bool hit; \ |
| 472 | CONF_HANDLE_BOOL_HIT(o, n, hit); \ |
| 473 | if (hit) \ |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 474 | continue; \ |
Jason Evans | 122449b | 2012-04-06 00:35:09 -0700 | [diff] [blame] | 475 | } |
Jason Evans | d81e4bd | 2012-03-06 14:57:45 -0800 | [diff] [blame] | 476 | #define CONF_HANDLE_SIZE_T(o, n, min, max) \ |
Jason Evans | 606f1fd | 2012-04-20 21:39:14 -0700 | [diff] [blame] | 477 | if (sizeof(n)-1 == klen && strncmp(n, k, \ |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 478 | klen) == 0) { \ |
Jason Evans | 122449b | 2012-04-06 00:35:09 -0700 | [diff] [blame] | 479 | uintmax_t um; \ |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 480 | char *end; \ |
| 481 | \ |
Mike Hommey | a14bce8 | 2012-04-30 12:38:26 +0200 | [diff] [blame] | 482 | set_errno(0); \ |
Jason Evans | 41b6afb | 2012-02-02 22:04:57 -0800 | [diff] [blame] | 483 | um = malloc_strtoumax(v, &end, 0); \ |
Mike Hommey | a14bce8 | 2012-04-30 12:38:26 +0200 | [diff] [blame] | 484 | if (get_errno() != 0 || (uintptr_t)end -\ |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 485 | (uintptr_t)v != vlen) { \ |
| 486 | malloc_conf_error( \ |
| 487 | "Invalid conf value", \ |
| 488 | k, klen, v, vlen); \ |
Jason Evans | 41b6afb | 2012-02-02 22:04:57 -0800 | [diff] [blame] | 489 | } else if (um < min || um > max) { \ |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 490 | malloc_conf_error( \ |
| 491 | "Out-of-range conf value", \ |
| 492 | k, klen, v, vlen); \ |
| 493 | } else \ |
Jason Evans | 41b6afb | 2012-02-02 22:04:57 -0800 | [diff] [blame] | 494 | o = um; \ |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 495 | continue; \ |
| 496 | } |
Jason Evans | d81e4bd | 2012-03-06 14:57:45 -0800 | [diff] [blame] | 497 | #define CONF_HANDLE_SSIZE_T(o, n, min, max) \ |
Jason Evans | 606f1fd | 2012-04-20 21:39:14 -0700 | [diff] [blame] | 498 | if (sizeof(n)-1 == klen && strncmp(n, k, \ |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 499 | klen) == 0) { \ |
| 500 | long l; \ |
| 501 | char *end; \ |
| 502 | \ |
Mike Hommey | a14bce8 | 2012-04-30 12:38:26 +0200 | [diff] [blame] | 503 | set_errno(0); \ |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 504 | l = strtol(v, &end, 0); \ |
Mike Hommey | a14bce8 | 2012-04-30 12:38:26 +0200 | [diff] [blame] | 505 | if (get_errno() != 0 || (uintptr_t)end -\ |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 506 | (uintptr_t)v != vlen) { \ |
| 507 | malloc_conf_error( \ |
| 508 | "Invalid conf value", \ |
| 509 | k, klen, v, vlen); \ |
| 510 | } else if (l < (ssize_t)min || l > \ |
| 511 | (ssize_t)max) { \ |
| 512 | malloc_conf_error( \ |
| 513 | "Out-of-range conf value", \ |
| 514 | k, klen, v, vlen); \ |
| 515 | } else \ |
Jason Evans | d81e4bd | 2012-03-06 14:57:45 -0800 | [diff] [blame] | 516 | o = l; \ |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 517 | continue; \ |
| 518 | } |
Jason Evans | d81e4bd | 2012-03-06 14:57:45 -0800 | [diff] [blame] | 519 | #define CONF_HANDLE_CHAR_P(o, n, d) \ |
Jason Evans | 606f1fd | 2012-04-20 21:39:14 -0700 | [diff] [blame] | 520 | if (sizeof(n)-1 == klen && strncmp(n, k, \ |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 521 | klen) == 0) { \ |
| 522 | size_t cpylen = (vlen <= \ |
Jason Evans | d81e4bd | 2012-03-06 14:57:45 -0800 | [diff] [blame] | 523 | sizeof(o)-1) ? vlen : \ |
| 524 | sizeof(o)-1; \ |
| 525 | strncpy(o, v, cpylen); \ |
| 526 | o[cpylen] = '\0'; \ |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 527 | continue; \ |
| 528 | } |
| 529 | |
Jason Evans | 606f1fd | 2012-04-20 21:39:14 -0700 | [diff] [blame] | 530 | CONF_HANDLE_BOOL(opt_abort, "abort") |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 531 | /* |
Jason Evans | 7ca0fdf | 2012-04-12 20:20:58 -0700 | [diff] [blame] | 532 | * Chunks always require at least one header page, plus |
| 533 | * one data page in the absence of redzones, or three |
| 534 | * pages in the presence of redzones. In order to |
| 535 | * simplify options processing, fix the limit based on |
| 536 | * config_fill. |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 537 | */ |
Jason Evans | 606f1fd | 2012-04-20 21:39:14 -0700 | [diff] [blame] | 538 | CONF_HANDLE_SIZE_T(opt_lg_chunk, "lg_chunk", LG_PAGE + |
Jason Evans | 7ca0fdf | 2012-04-12 20:20:58 -0700 | [diff] [blame] | 539 | (config_fill ? 2 : 1), (sizeof(size_t) << 3) - 1) |
Jason Evans | 606f1fd | 2012-04-20 21:39:14 -0700 | [diff] [blame] | 540 | CONF_HANDLE_SIZE_T(opt_narenas, "narenas", 1, |
| 541 | SIZE_T_MAX) |
| 542 | CONF_HANDLE_SSIZE_T(opt_lg_dirty_mult, "lg_dirty_mult", |
Jason Evans | d81e4bd | 2012-03-06 14:57:45 -0800 | [diff] [blame] | 543 | -1, (sizeof(size_t) << 3) - 1) |
Jason Evans | 606f1fd | 2012-04-20 21:39:14 -0700 | [diff] [blame] | 544 | CONF_HANDLE_BOOL(opt_stats_print, "stats_print") |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 545 | if (config_fill) { |
Jason Evans | 606f1fd | 2012-04-20 21:39:14 -0700 | [diff] [blame] | 546 | CONF_HANDLE_BOOL(opt_junk, "junk") |
| 547 | CONF_HANDLE_SIZE_T(opt_quarantine, "quarantine", |
Jason Evans | 122449b | 2012-04-06 00:35:09 -0700 | [diff] [blame] | 548 | 0, SIZE_T_MAX) |
Jason Evans | 606f1fd | 2012-04-20 21:39:14 -0700 | [diff] [blame] | 549 | CONF_HANDLE_BOOL(opt_redzone, "redzone") |
| 550 | CONF_HANDLE_BOOL(opt_zero, "zero") |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 551 | } |
Jason Evans | b147611 | 2012-04-05 13:36:17 -0700 | [diff] [blame] | 552 | if (config_utrace) { |
Jason Evans | 606f1fd | 2012-04-20 21:39:14 -0700 | [diff] [blame] | 553 | CONF_HANDLE_BOOL(opt_utrace, "utrace") |
Jason Evans | b147611 | 2012-04-05 13:36:17 -0700 | [diff] [blame] | 554 | } |
Jason Evans | 122449b | 2012-04-06 00:35:09 -0700 | [diff] [blame] | 555 | if (config_valgrind) { |
| 556 | bool hit; |
| 557 | CONF_HANDLE_BOOL_HIT(opt_valgrind, |
Jason Evans | 606f1fd | 2012-04-20 21:39:14 -0700 | [diff] [blame] | 558 | "valgrind", hit) |
Jason Evans | 122449b | 2012-04-06 00:35:09 -0700 | [diff] [blame] | 559 | if (config_fill && opt_valgrind && hit) { |
| 560 | opt_junk = false; |
| 561 | opt_zero = false; |
| 562 | if (opt_quarantine == 0) { |
| 563 | opt_quarantine = |
| 564 | JEMALLOC_VALGRIND_QUARANTINE_DEFAULT; |
| 565 | } |
| 566 | opt_redzone = true; |
| 567 | } |
| 568 | if (hit) |
| 569 | continue; |
| 570 | } |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 571 | if (config_xmalloc) { |
Jason Evans | 606f1fd | 2012-04-20 21:39:14 -0700 | [diff] [blame] | 572 | CONF_HANDLE_BOOL(opt_xmalloc, "xmalloc") |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 573 | } |
| 574 | if (config_tcache) { |
Jason Evans | 606f1fd | 2012-04-20 21:39:14 -0700 | [diff] [blame] | 575 | CONF_HANDLE_BOOL(opt_tcache, "tcache") |
Jason Evans | d81e4bd | 2012-03-06 14:57:45 -0800 | [diff] [blame] | 576 | CONF_HANDLE_SSIZE_T(opt_lg_tcache_max, |
Jason Evans | 606f1fd | 2012-04-20 21:39:14 -0700 | [diff] [blame] | 577 | "lg_tcache_max", -1, |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 578 | (sizeof(size_t) << 3) - 1) |
| 579 | } |
| 580 | if (config_prof) { |
Jason Evans | 606f1fd | 2012-04-20 21:39:14 -0700 | [diff] [blame] | 581 | CONF_HANDLE_BOOL(opt_prof, "prof") |
| 582 | CONF_HANDLE_CHAR_P(opt_prof_prefix, |
| 583 | "prof_prefix", "jeprof") |
| 584 | CONF_HANDLE_BOOL(opt_prof_active, "prof_active") |
Jason Evans | d81e4bd | 2012-03-06 14:57:45 -0800 | [diff] [blame] | 585 | CONF_HANDLE_SSIZE_T(opt_lg_prof_sample, |
Jason Evans | 606f1fd | 2012-04-20 21:39:14 -0700 | [diff] [blame] | 586 | "lg_prof_sample", 0, |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 587 | (sizeof(uint64_t) << 3) - 1) |
Jason Evans | 606f1fd | 2012-04-20 21:39:14 -0700 | [diff] [blame] | 588 | CONF_HANDLE_BOOL(opt_prof_accum, "prof_accum") |
Jason Evans | d81e4bd | 2012-03-06 14:57:45 -0800 | [diff] [blame] | 589 | CONF_HANDLE_SSIZE_T(opt_lg_prof_interval, |
Jason Evans | 606f1fd | 2012-04-20 21:39:14 -0700 | [diff] [blame] | 590 | "lg_prof_interval", -1, |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 591 | (sizeof(uint64_t) << 3) - 1) |
Jason Evans | 606f1fd | 2012-04-20 21:39:14 -0700 | [diff] [blame] | 592 | CONF_HANDLE_BOOL(opt_prof_gdump, "prof_gdump") |
| 593 | CONF_HANDLE_BOOL(opt_prof_final, "prof_final") |
| 594 | CONF_HANDLE_BOOL(opt_prof_leak, "prof_leak") |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 595 | } |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 596 | malloc_conf_error("Invalid conf pair", k, klen, v, |
| 597 | vlen); |
| 598 | #undef CONF_HANDLE_BOOL |
| 599 | #undef CONF_HANDLE_SIZE_T |
| 600 | #undef CONF_HANDLE_SSIZE_T |
| 601 | #undef CONF_HANDLE_CHAR_P |
| 602 | } |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 603 | } |
| 604 | } |
| 605 | |
| 606 | static bool |
| 607 | malloc_init_hard(void) |
| 608 | { |
Jason Evans | b7924f5 | 2009-06-23 19:01:18 -0700 | [diff] [blame] | 609 | arena_t *init_arenas[1]; |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 610 | |
| 611 | malloc_mutex_lock(&init_lock); |
Jason Evans | 41b6afb | 2012-02-02 22:04:57 -0800 | [diff] [blame] | 612 | if (malloc_initialized || IS_INITIALIZER) { |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 613 | /* |
| 614 | * Another thread initialized the allocator before this one |
Jason Evans | a25d0a8 | 2009-11-09 14:57:38 -0800 | [diff] [blame] | 615 | * acquired init_lock, or this thread is the initializing |
| 616 | * thread, and it is recursively allocating. |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 617 | */ |
| 618 | malloc_mutex_unlock(&init_lock); |
| 619 | return (false); |
| 620 | } |
Jason Evans | 41b6afb | 2012-02-02 22:04:57 -0800 | [diff] [blame] | 621 | #ifdef JEMALLOC_THREADED_INIT |
Jason Evans | 02b2312 | 2012-04-05 11:06:23 -0700 | [diff] [blame] | 622 | if (malloc_initializer != NO_INITIALIZER && IS_INITIALIZER == false) { |
Jason Evans | b7924f5 | 2009-06-23 19:01:18 -0700 | [diff] [blame] | 623 | /* Busy-wait until the initializing thread completes. */ |
| 624 | do { |
| 625 | malloc_mutex_unlock(&init_lock); |
| 626 | CPU_SPINWAIT; |
| 627 | malloc_mutex_lock(&init_lock); |
| 628 | } while (malloc_initialized == false); |
Jason Evans | 2541e1b | 2010-07-22 11:35:59 -0700 | [diff] [blame] | 629 | malloc_mutex_unlock(&init_lock); |
Jason Evans | b7924f5 | 2009-06-23 19:01:18 -0700 | [diff] [blame] | 630 | return (false); |
| 631 | } |
Jason Evans | 41b6afb | 2012-02-02 22:04:57 -0800 | [diff] [blame] | 632 | #endif |
| 633 | malloc_initializer = INITIALIZER; |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 634 | |
Jason Evans | cd9a134 | 2012-03-21 18:33:03 -0700 | [diff] [blame] | 635 | malloc_tsd_boot(); |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 636 | if (config_prof) |
| 637 | prof_boot0(); |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 638 | |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 639 | malloc_conf_init(); |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 640 | |
Mike Hommey | a19e87f | 2012-04-21 21:27:46 -0700 | [diff] [blame] | 641 | #if (!defined(JEMALLOC_MUTEX_INIT_CB) && !defined(JEMALLOC_ZONE) \ |
| 642 | && !defined(_WIN32)) |
Jason Evans | a0bf242 | 2010-01-29 14:30:41 -0800 | [diff] [blame] | 643 | /* Register fork handlers. */ |
Jason Evans | 4e2e3dd | 2012-03-13 16:31:41 -0700 | [diff] [blame] | 644 | if (pthread_atfork(jemalloc_prefork, jemalloc_postfork_parent, |
| 645 | jemalloc_postfork_child) != 0) { |
Jason Evans | 698805c | 2010-03-03 17:45:38 -0800 | [diff] [blame] | 646 | malloc_write("<jemalloc>: Error in pthread_atfork()\n"); |
Jason Evans | a0bf242 | 2010-01-29 14:30:41 -0800 | [diff] [blame] | 647 | if (opt_abort) |
| 648 | abort(); |
| 649 | } |
Jason Evans | 41b6afb | 2012-02-02 22:04:57 -0800 | [diff] [blame] | 650 | #endif |
Jason Evans | 3c23435 | 2010-01-27 13:10:55 -0800 | [diff] [blame] | 651 | |
Jason Evans | 03c2237 | 2010-01-03 12:10:42 -0800 | [diff] [blame] | 652 | if (opt_stats_print) { |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 653 | /* Print statistics at exit. */ |
Jason Evans | a0bf242 | 2010-01-29 14:30:41 -0800 | [diff] [blame] | 654 | if (atexit(stats_print_atexit) != 0) { |
Jason Evans | 698805c | 2010-03-03 17:45:38 -0800 | [diff] [blame] | 655 | malloc_write("<jemalloc>: Error in atexit()\n"); |
Jason Evans | a0bf242 | 2010-01-29 14:30:41 -0800 | [diff] [blame] | 656 | if (opt_abort) |
| 657 | abort(); |
| 658 | } |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 659 | } |
| 660 | |
Mike Hommey | b8325f9 | 2012-04-12 15:15:35 +0200 | [diff] [blame] | 661 | if (base_boot()) { |
Jason Evans | a0bf242 | 2010-01-29 14:30:41 -0800 | [diff] [blame] | 662 | malloc_mutex_unlock(&init_lock); |
| 663 | return (true); |
| 664 | } |
Jason Evans | c9658dd | 2009-06-22 14:44:08 -0700 | [diff] [blame] | 665 | |
Jason Evans | a8f8d75 | 2012-04-21 19:17:21 -0700 | [diff] [blame] | 666 | if (chunk_boot()) { |
Jason Evans | 3c23435 | 2010-01-27 13:10:55 -0800 | [diff] [blame] | 667 | malloc_mutex_unlock(&init_lock); |
| 668 | return (true); |
| 669 | } |
| 670 | |
Jason Evans | 41b6afb | 2012-02-02 22:04:57 -0800 | [diff] [blame] | 671 | if (ctl_boot()) { |
| 672 | malloc_mutex_unlock(&init_lock); |
| 673 | return (true); |
| 674 | } |
| 675 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 676 | if (config_prof) |
| 677 | prof_boot1(); |
Jason Evans | 3383af6 | 2010-02-11 08:59:06 -0800 | [diff] [blame] | 678 | |
Jason Evans | b172610 | 2012-02-28 16:50:47 -0800 | [diff] [blame] | 679 | arena_boot(); |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 680 | |
Jason Evans | cd9a134 | 2012-03-21 18:33:03 -0700 | [diff] [blame] | 681 | if (config_tcache && tcache_boot0()) { |
Jason Evans | 84c8eef | 2011-03-16 10:30:13 -0700 | [diff] [blame] | 682 | malloc_mutex_unlock(&init_lock); |
| 683 | return (true); |
| 684 | } |
Jason Evans | 84cbbcb | 2009-12-29 00:09:15 -0800 | [diff] [blame] | 685 | |
Jason Evans | e476f8a | 2010-01-16 09:53:50 -0800 | [diff] [blame] | 686 | if (huge_boot()) { |
Jason Evans | c9658dd | 2009-06-22 14:44:08 -0700 | [diff] [blame] | 687 | malloc_mutex_unlock(&init_lock); |
| 688 | return (true); |
| 689 | } |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 690 | |
Jason Evans | 8e6f8b4 | 2011-11-03 18:40:03 -0700 | [diff] [blame] | 691 | if (malloc_mutex_init(&arenas_lock)) |
| 692 | return (true); |
| 693 | |
Jason Evans | b7924f5 | 2009-06-23 19:01:18 -0700 | [diff] [blame] | 694 | /* |
| 695 | * Create enough scaffolding to allow recursive allocation in |
| 696 | * malloc_ncpus(). |
| 697 | */ |
| 698 | narenas = 1; |
| 699 | arenas = init_arenas; |
| 700 | memset(arenas, 0, sizeof(arena_t *) * narenas); |
| 701 | |
| 702 | /* |
| 703 | * Initialize one arena here. The rest are lazily created in |
| 704 | * choose_arena_hard(). |
| 705 | */ |
| 706 | arenas_extend(0); |
| 707 | if (arenas[0] == NULL) { |
| 708 | malloc_mutex_unlock(&init_lock); |
| 709 | return (true); |
| 710 | } |
| 711 | |
Jason Evans | cd9a134 | 2012-03-21 18:33:03 -0700 | [diff] [blame] | 712 | /* Initialize allocation counters before any allocations can occur. */ |
| 713 | if (config_stats && thread_allocated_tsd_boot()) { |
| 714 | malloc_mutex_unlock(&init_lock); |
| 715 | return (true); |
| 716 | } |
Jason Evans | b7924f5 | 2009-06-23 19:01:18 -0700 | [diff] [blame] | 717 | |
Jason Evans | cd9a134 | 2012-03-21 18:33:03 -0700 | [diff] [blame] | 718 | if (arenas_tsd_boot()) { |
| 719 | malloc_mutex_unlock(&init_lock); |
| 720 | return (true); |
| 721 | } |
| 722 | |
| 723 | if (config_tcache && tcache_boot1()) { |
| 724 | malloc_mutex_unlock(&init_lock); |
| 725 | return (true); |
| 726 | } |
| 727 | |
Jason Evans | 122449b | 2012-04-06 00:35:09 -0700 | [diff] [blame] | 728 | if (config_fill && quarantine_boot()) { |
| 729 | malloc_mutex_unlock(&init_lock); |
| 730 | return (true); |
| 731 | } |
| 732 | |
Jason Evans | 6da5418 | 2012-03-23 18:05:51 -0700 | [diff] [blame] | 733 | if (config_prof && prof_boot2()) { |
| 734 | malloc_mutex_unlock(&init_lock); |
| 735 | return (true); |
| 736 | } |
| 737 | |
Jason Evans | b7924f5 | 2009-06-23 19:01:18 -0700 | [diff] [blame] | 738 | /* Get number of CPUs. */ |
Jason Evans | b7924f5 | 2009-06-23 19:01:18 -0700 | [diff] [blame] | 739 | malloc_mutex_unlock(&init_lock); |
| 740 | ncpus = malloc_ncpus(); |
| 741 | malloc_mutex_lock(&init_lock); |
| 742 | |
Jason Evans | 633aaff | 2012-04-03 08:47:07 -0700 | [diff] [blame] | 743 | if (mutex_boot()) { |
| 744 | malloc_mutex_unlock(&init_lock); |
| 745 | return (true); |
| 746 | } |
| 747 | |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 748 | if (opt_narenas == 0) { |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 749 | /* |
Jason Evans | 5463a52 | 2009-12-29 00:09:15 -0800 | [diff] [blame] | 750 | * For SMP systems, create more than one arena per CPU by |
| 751 | * default. |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 752 | */ |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 753 | if (ncpus > 1) |
| 754 | opt_narenas = ncpus << 2; |
| 755 | else |
| 756 | opt_narenas = 1; |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 757 | } |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 758 | narenas = opt_narenas; |
| 759 | /* |
| 760 | * Make sure that the arenas array can be allocated. In practice, this |
| 761 | * limit is enough to allow the allocator to function, but the ctl |
| 762 | * machinery will fail to allocate memory at far lower limits. |
| 763 | */ |
| 764 | if (narenas > chunksize / sizeof(arena_t *)) { |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 765 | narenas = chunksize / sizeof(arena_t *); |
Jason Evans | d81e4bd | 2012-03-06 14:57:45 -0800 | [diff] [blame] | 766 | malloc_printf("<jemalloc>: Reducing narenas to limit (%d)\n", |
| 767 | narenas); |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 768 | } |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 769 | |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 770 | /* Allocate and initialize arenas. */ |
| 771 | arenas = (arena_t **)base_alloc(sizeof(arena_t *) * narenas); |
| 772 | if (arenas == NULL) { |
| 773 | malloc_mutex_unlock(&init_lock); |
| 774 | return (true); |
| 775 | } |
| 776 | /* |
| 777 | * Zero the array. In practice, this should always be pre-zeroed, |
| 778 | * since it was just mmap()ed, but let's be sure. |
| 779 | */ |
| 780 | memset(arenas, 0, sizeof(arena_t *) * narenas); |
Jason Evans | b7924f5 | 2009-06-23 19:01:18 -0700 | [diff] [blame] | 781 | /* Copy the pointer to the one arena that was already initialized. */ |
| 782 | arenas[0] = init_arenas[0]; |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 783 | |
| 784 | malloc_initialized = true; |
| 785 | malloc_mutex_unlock(&init_lock); |
| 786 | return (false); |
| 787 | } |
| 788 | |
| 789 | /* |
Jason Evans | e476f8a | 2010-01-16 09:53:50 -0800 | [diff] [blame] | 790 | * End initialization functions. |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 791 | */ |
| 792 | /******************************************************************************/ |
| 793 | /* |
| 794 | * Begin malloc(3)-compatible functions. |
| 795 | */ |
| 796 | |
| 797 | void * |
Jason Evans | 0a5489e | 2012-03-01 17:19:20 -0800 | [diff] [blame] | 798 | je_malloc(size_t size) |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 799 | { |
| 800 | void *ret; |
Jason Evans | 8694e2e | 2012-04-23 13:05:32 -0700 | [diff] [blame] | 801 | size_t usize JEMALLOC_CC_SILENCE_INIT(0); |
Jason Evans | 9225a19 | 2012-03-23 15:39:07 -0700 | [diff] [blame] | 802 | prof_thr_cnt_t *cnt JEMALLOC_CC_SILENCE_INIT(NULL); |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 803 | |
| 804 | if (malloc_init()) { |
| 805 | ret = NULL; |
Jason Evans | a1ee783 | 2012-04-10 15:07:44 -0700 | [diff] [blame] | 806 | goto label_oom; |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 807 | } |
| 808 | |
Jason Evans | c90ad71 | 2012-02-28 20:31:37 -0800 | [diff] [blame] | 809 | if (size == 0) |
| 810 | size = 1; |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 811 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 812 | if (config_prof && opt_prof) { |
Jason Evans | 9344368 | 2010-10-20 17:39:18 -0700 | [diff] [blame] | 813 | usize = s2u(size); |
Jason Evans | a507004 | 2011-08-12 13:48:27 -0700 | [diff] [blame] | 814 | PROF_ALLOC_PREP(1, usize, cnt); |
| 815 | if (cnt == NULL) { |
Jason Evans | 0b270a9 | 2010-03-31 16:45:04 -0700 | [diff] [blame] | 816 | ret = NULL; |
Jason Evans | a1ee783 | 2012-04-10 15:07:44 -0700 | [diff] [blame] | 817 | goto label_oom; |
Jason Evans | 0b270a9 | 2010-03-31 16:45:04 -0700 | [diff] [blame] | 818 | } |
Jason Evans | 9344368 | 2010-10-20 17:39:18 -0700 | [diff] [blame] | 819 | if (prof_promote && (uintptr_t)cnt != (uintptr_t)1U && usize <= |
Jason Evans | b172610 | 2012-02-28 16:50:47 -0800 | [diff] [blame] | 820 | SMALL_MAXCLASS) { |
| 821 | ret = imalloc(SMALL_MAXCLASS+1); |
Jason Evans | 0b270a9 | 2010-03-31 16:45:04 -0700 | [diff] [blame] | 822 | if (ret != NULL) |
Jason Evans | 9344368 | 2010-10-20 17:39:18 -0700 | [diff] [blame] | 823 | arena_prof_promoted(ret, usize); |
Jason Evans | 0b270a9 | 2010-03-31 16:45:04 -0700 | [diff] [blame] | 824 | } else |
| 825 | ret = imalloc(size); |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 826 | } else { |
Jason Evans | 122449b | 2012-04-06 00:35:09 -0700 | [diff] [blame] | 827 | if (config_stats || (config_valgrind && opt_valgrind)) |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 828 | usize = s2u(size); |
Jason Evans | 0b270a9 | 2010-03-31 16:45:04 -0700 | [diff] [blame] | 829 | ret = imalloc(size); |
Jason Evans | 9344368 | 2010-10-20 17:39:18 -0700 | [diff] [blame] | 830 | } |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 831 | |
Jason Evans | a1ee783 | 2012-04-10 15:07:44 -0700 | [diff] [blame] | 832 | label_oom: |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 833 | if (ret == NULL) { |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 834 | if (config_xmalloc && opt_xmalloc) { |
Jason Evans | 698805c | 2010-03-03 17:45:38 -0800 | [diff] [blame] | 835 | malloc_write("<jemalloc>: Error in malloc(): " |
| 836 | "out of memory\n"); |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 837 | abort(); |
| 838 | } |
Mike Hommey | a14bce8 | 2012-04-30 12:38:26 +0200 | [diff] [blame] | 839 | set_errno(ENOMEM); |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 840 | } |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 841 | if (config_prof && opt_prof && ret != NULL) |
Jason Evans | 9344368 | 2010-10-20 17:39:18 -0700 | [diff] [blame] | 842 | prof_malloc(ret, usize, cnt); |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 843 | if (config_stats && ret != NULL) { |
Jason Evans | 122449b | 2012-04-06 00:35:09 -0700 | [diff] [blame] | 844 | assert(usize == isalloc(ret, config_prof)); |
Jason Evans | cd9a134 | 2012-03-21 18:33:03 -0700 | [diff] [blame] | 845 | thread_allocated_tsd_get()->allocated += usize; |
Jason Evans | 9344368 | 2010-10-20 17:39:18 -0700 | [diff] [blame] | 846 | } |
Jason Evans | b147611 | 2012-04-05 13:36:17 -0700 | [diff] [blame] | 847 | UTRACE(0, size, ret); |
Jason Evans | 122449b | 2012-04-06 00:35:09 -0700 | [diff] [blame] | 848 | JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, usize, false); |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 849 | return (ret); |
| 850 | } |
| 851 | |
Jason Evans | 9ad4823 | 2010-01-03 11:59:20 -0800 | [diff] [blame] | 852 | JEMALLOC_ATTR(nonnull(1)) |
Jason Evans | a507004 | 2011-08-12 13:48:27 -0700 | [diff] [blame] | 853 | #ifdef JEMALLOC_PROF |
| 854 | /* |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 855 | * Avoid any uncertainty as to how many backtrace frames to ignore in |
Jason Evans | a507004 | 2011-08-12 13:48:27 -0700 | [diff] [blame] | 856 | * PROF_ALLOC_PREP(). |
| 857 | */ |
| 858 | JEMALLOC_ATTR(noinline) |
| 859 | #endif |
| 860 | static int |
Jason Evans | 5965631 | 2012-02-28 21:37:38 -0800 | [diff] [blame] | 861 | imemalign(void **memptr, size_t alignment, size_t size, |
Jason Evans | 0a0bbf6 | 2012-03-13 12:55:21 -0700 | [diff] [blame] | 862 | size_t min_alignment) |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 863 | { |
| 864 | int ret; |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 865 | size_t usize; |
Jason Evans | 38d9210 | 2011-03-23 00:37:29 -0700 | [diff] [blame] | 866 | void *result; |
Jason Evans | 9225a19 | 2012-03-23 15:39:07 -0700 | [diff] [blame] | 867 | prof_thr_cnt_t *cnt JEMALLOC_CC_SILENCE_INIT(NULL); |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 868 | |
Jason Evans | 0a0bbf6 | 2012-03-13 12:55:21 -0700 | [diff] [blame] | 869 | assert(min_alignment != 0); |
| 870 | |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 871 | if (malloc_init()) |
| 872 | result = NULL; |
| 873 | else { |
Jason Evans | c90ad71 | 2012-02-28 20:31:37 -0800 | [diff] [blame] | 874 | if (size == 0) |
| 875 | size = 1; |
Jason Evans | f251814 | 2009-12-29 00:09:15 -0800 | [diff] [blame] | 876 | |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 877 | /* Make sure that alignment is a large enough power of 2. */ |
| 878 | if (((alignment - 1) & alignment) != 0 |
Jason Evans | 0a0bbf6 | 2012-03-13 12:55:21 -0700 | [diff] [blame] | 879 | || (alignment < min_alignment)) { |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 880 | if (config_xmalloc && opt_xmalloc) { |
Jason Evans | 0a0bbf6 | 2012-03-13 12:55:21 -0700 | [diff] [blame] | 881 | malloc_write("<jemalloc>: Error allocating " |
| 882 | "aligned memory: invalid alignment\n"); |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 883 | abort(); |
| 884 | } |
| 885 | result = NULL; |
| 886 | ret = EINVAL; |
Jason Evans | a1ee783 | 2012-04-10 15:07:44 -0700 | [diff] [blame] | 887 | goto label_return; |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 888 | } |
| 889 | |
Jason Evans | 5ff709c | 2012-04-11 18:13:45 -0700 | [diff] [blame] | 890 | usize = sa2u(size, alignment); |
Jason Evans | 38d9210 | 2011-03-23 00:37:29 -0700 | [diff] [blame] | 891 | if (usize == 0) { |
| 892 | result = NULL; |
| 893 | ret = ENOMEM; |
Jason Evans | a1ee783 | 2012-04-10 15:07:44 -0700 | [diff] [blame] | 894 | goto label_return; |
Jason Evans | 38d9210 | 2011-03-23 00:37:29 -0700 | [diff] [blame] | 895 | } |
| 896 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 897 | if (config_prof && opt_prof) { |
Jason Evans | a507004 | 2011-08-12 13:48:27 -0700 | [diff] [blame] | 898 | PROF_ALLOC_PREP(2, usize, cnt); |
| 899 | if (cnt == NULL) { |
Jason Evans | 0b270a9 | 2010-03-31 16:45:04 -0700 | [diff] [blame] | 900 | result = NULL; |
| 901 | ret = EINVAL; |
| 902 | } else { |
| 903 | if (prof_promote && (uintptr_t)cnt != |
Jason Evans | b172610 | 2012-02-28 16:50:47 -0800 | [diff] [blame] | 904 | (uintptr_t)1U && usize <= SMALL_MAXCLASS) { |
| 905 | assert(sa2u(SMALL_MAXCLASS+1, |
Jason Evans | 5ff709c | 2012-04-11 18:13:45 -0700 | [diff] [blame] | 906 | alignment) != 0); |
Jason Evans | b172610 | 2012-02-28 16:50:47 -0800 | [diff] [blame] | 907 | result = ipalloc(sa2u(SMALL_MAXCLASS+1, |
Jason Evans | 5ff709c | 2012-04-11 18:13:45 -0700 | [diff] [blame] | 908 | alignment), alignment, false); |
Jason Evans | 0b270a9 | 2010-03-31 16:45:04 -0700 | [diff] [blame] | 909 | if (result != NULL) { |
| 910 | arena_prof_promoted(result, |
Jason Evans | 9344368 | 2010-10-20 17:39:18 -0700 | [diff] [blame] | 911 | usize); |
Jason Evans | 0b270a9 | 2010-03-31 16:45:04 -0700 | [diff] [blame] | 912 | } |
Jason Evans | 8e3c3c6 | 2010-09-17 15:46:18 -0700 | [diff] [blame] | 913 | } else { |
Jason Evans | 38d9210 | 2011-03-23 00:37:29 -0700 | [diff] [blame] | 914 | result = ipalloc(usize, alignment, |
Jason Evans | 8e3c3c6 | 2010-09-17 15:46:18 -0700 | [diff] [blame] | 915 | false); |
| 916 | } |
Jason Evans | 0b270a9 | 2010-03-31 16:45:04 -0700 | [diff] [blame] | 917 | } |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 918 | } else |
Jason Evans | 38d9210 | 2011-03-23 00:37:29 -0700 | [diff] [blame] | 919 | result = ipalloc(usize, alignment, false); |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 920 | } |
| 921 | |
| 922 | if (result == NULL) { |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 923 | if (config_xmalloc && opt_xmalloc) { |
Jason Evans | 0a0bbf6 | 2012-03-13 12:55:21 -0700 | [diff] [blame] | 924 | malloc_write("<jemalloc>: Error allocating aligned " |
| 925 | "memory: out of memory\n"); |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 926 | abort(); |
| 927 | } |
| 928 | ret = ENOMEM; |
Jason Evans | a1ee783 | 2012-04-10 15:07:44 -0700 | [diff] [blame] | 929 | goto label_return; |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 930 | } |
| 931 | |
| 932 | *memptr = result; |
| 933 | ret = 0; |
| 934 | |
Jason Evans | a1ee783 | 2012-04-10 15:07:44 -0700 | [diff] [blame] | 935 | label_return: |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 936 | if (config_stats && result != NULL) { |
Jason Evans | 122449b | 2012-04-06 00:35:09 -0700 | [diff] [blame] | 937 | assert(usize == isalloc(result, config_prof)); |
Jason Evans | cd9a134 | 2012-03-21 18:33:03 -0700 | [diff] [blame] | 938 | thread_allocated_tsd_get()->allocated += usize; |
Jason Evans | 9344368 | 2010-10-20 17:39:18 -0700 | [diff] [blame] | 939 | } |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 940 | if (config_prof && opt_prof && result != NULL) |
Jason Evans | 9344368 | 2010-10-20 17:39:18 -0700 | [diff] [blame] | 941 | prof_malloc(result, usize, cnt); |
Jason Evans | b147611 | 2012-04-05 13:36:17 -0700 | [diff] [blame] | 942 | UTRACE(0, size, result); |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 943 | return (ret); |
| 944 | } |
| 945 | |
Jason Evans | a507004 | 2011-08-12 13:48:27 -0700 | [diff] [blame] | 946 | int |
Jason Evans | 0a5489e | 2012-03-01 17:19:20 -0800 | [diff] [blame] | 947 | je_posix_memalign(void **memptr, size_t alignment, size_t size) |
Jason Evans | a507004 | 2011-08-12 13:48:27 -0700 | [diff] [blame] | 948 | { |
Jason Evans | 122449b | 2012-04-06 00:35:09 -0700 | [diff] [blame] | 949 | int ret = imemalign(memptr, alignment, size, sizeof(void *)); |
| 950 | JEMALLOC_VALGRIND_MALLOC(ret == 0, *memptr, isalloc(*memptr, |
| 951 | config_prof), false); |
| 952 | return (ret); |
Jason Evans | 0a0bbf6 | 2012-03-13 12:55:21 -0700 | [diff] [blame] | 953 | } |
| 954 | |
Jason Evans | 0a0bbf6 | 2012-03-13 12:55:21 -0700 | [diff] [blame] | 955 | void * |
| 956 | je_aligned_alloc(size_t alignment, size_t size) |
| 957 | { |
| 958 | void *ret; |
| 959 | int err; |
| 960 | |
| 961 | if ((err = imemalign(&ret, alignment, size, 1)) != 0) { |
| 962 | ret = NULL; |
Mike Hommey | a14bce8 | 2012-04-30 12:38:26 +0200 | [diff] [blame] | 963 | set_errno(err); |
Jason Evans | 0a0bbf6 | 2012-03-13 12:55:21 -0700 | [diff] [blame] | 964 | } |
Jason Evans | 122449b | 2012-04-06 00:35:09 -0700 | [diff] [blame] | 965 | JEMALLOC_VALGRIND_MALLOC(err == 0, ret, isalloc(ret, config_prof), |
| 966 | false); |
Jason Evans | 0a0bbf6 | 2012-03-13 12:55:21 -0700 | [diff] [blame] | 967 | return (ret); |
Jason Evans | a507004 | 2011-08-12 13:48:27 -0700 | [diff] [blame] | 968 | } |
| 969 | |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 970 | void * |
Jason Evans | 0a5489e | 2012-03-01 17:19:20 -0800 | [diff] [blame] | 971 | je_calloc(size_t num, size_t size) |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 972 | { |
| 973 | void *ret; |
| 974 | size_t num_size; |
Jason Evans | 8694e2e | 2012-04-23 13:05:32 -0700 | [diff] [blame] | 975 | size_t usize JEMALLOC_CC_SILENCE_INIT(0); |
Jason Evans | 9225a19 | 2012-03-23 15:39:07 -0700 | [diff] [blame] | 976 | prof_thr_cnt_t *cnt JEMALLOC_CC_SILENCE_INIT(NULL); |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 977 | |
| 978 | if (malloc_init()) { |
| 979 | num_size = 0; |
| 980 | ret = NULL; |
Jason Evans | a1ee783 | 2012-04-10 15:07:44 -0700 | [diff] [blame] | 981 | goto label_return; |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 982 | } |
| 983 | |
| 984 | num_size = num * size; |
| 985 | if (num_size == 0) { |
Jason Evans | c90ad71 | 2012-02-28 20:31:37 -0800 | [diff] [blame] | 986 | if (num == 0 || size == 0) |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 987 | num_size = 1; |
| 988 | else { |
| 989 | ret = NULL; |
Jason Evans | a1ee783 | 2012-04-10 15:07:44 -0700 | [diff] [blame] | 990 | goto label_return; |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 991 | } |
| 992 | /* |
| 993 | * Try to avoid division here. We know that it isn't possible to |
| 994 | * overflow during multiplication if neither operand uses any of the |
| 995 | * most significant half of the bits in a size_t. |
| 996 | */ |
| 997 | } else if (((num | size) & (SIZE_T_MAX << (sizeof(size_t) << 2))) |
| 998 | && (num_size / size != num)) { |
| 999 | /* size_t overflow. */ |
| 1000 | ret = NULL; |
Jason Evans | a1ee783 | 2012-04-10 15:07:44 -0700 | [diff] [blame] | 1001 | goto label_return; |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 1002 | } |
| 1003 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 1004 | if (config_prof && opt_prof) { |
Jason Evans | 9344368 | 2010-10-20 17:39:18 -0700 | [diff] [blame] | 1005 | usize = s2u(num_size); |
Jason Evans | a507004 | 2011-08-12 13:48:27 -0700 | [diff] [blame] | 1006 | PROF_ALLOC_PREP(1, usize, cnt); |
| 1007 | if (cnt == NULL) { |
Jason Evans | 0b270a9 | 2010-03-31 16:45:04 -0700 | [diff] [blame] | 1008 | ret = NULL; |
Jason Evans | a1ee783 | 2012-04-10 15:07:44 -0700 | [diff] [blame] | 1009 | goto label_return; |
Jason Evans | 0b270a9 | 2010-03-31 16:45:04 -0700 | [diff] [blame] | 1010 | } |
Jason Evans | 9344368 | 2010-10-20 17:39:18 -0700 | [diff] [blame] | 1011 | if (prof_promote && (uintptr_t)cnt != (uintptr_t)1U && usize |
Jason Evans | b172610 | 2012-02-28 16:50:47 -0800 | [diff] [blame] | 1012 | <= SMALL_MAXCLASS) { |
| 1013 | ret = icalloc(SMALL_MAXCLASS+1); |
Jason Evans | 0b270a9 | 2010-03-31 16:45:04 -0700 | [diff] [blame] | 1014 | if (ret != NULL) |
Jason Evans | 9344368 | 2010-10-20 17:39:18 -0700 | [diff] [blame] | 1015 | arena_prof_promoted(ret, usize); |
Jason Evans | 0b270a9 | 2010-03-31 16:45:04 -0700 | [diff] [blame] | 1016 | } else |
| 1017 | ret = icalloc(num_size); |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 1018 | } else { |
Jason Evans | 122449b | 2012-04-06 00:35:09 -0700 | [diff] [blame] | 1019 | if (config_stats || (config_valgrind && opt_valgrind)) |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 1020 | usize = s2u(num_size); |
Jason Evans | 0b270a9 | 2010-03-31 16:45:04 -0700 | [diff] [blame] | 1021 | ret = icalloc(num_size); |
Jason Evans | 9344368 | 2010-10-20 17:39:18 -0700 | [diff] [blame] | 1022 | } |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 1023 | |
Jason Evans | a1ee783 | 2012-04-10 15:07:44 -0700 | [diff] [blame] | 1024 | label_return: |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 1025 | if (ret == NULL) { |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 1026 | if (config_xmalloc && opt_xmalloc) { |
Jason Evans | 698805c | 2010-03-03 17:45:38 -0800 | [diff] [blame] | 1027 | malloc_write("<jemalloc>: Error in calloc(): out of " |
| 1028 | "memory\n"); |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 1029 | abort(); |
| 1030 | } |
Mike Hommey | a14bce8 | 2012-04-30 12:38:26 +0200 | [diff] [blame] | 1031 | set_errno(ENOMEM); |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 1032 | } |
| 1033 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 1034 | if (config_prof && opt_prof && ret != NULL) |
Jason Evans | 9344368 | 2010-10-20 17:39:18 -0700 | [diff] [blame] | 1035 | prof_malloc(ret, usize, cnt); |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 1036 | if (config_stats && ret != NULL) { |
Jason Evans | 122449b | 2012-04-06 00:35:09 -0700 | [diff] [blame] | 1037 | assert(usize == isalloc(ret, config_prof)); |
Jason Evans | cd9a134 | 2012-03-21 18:33:03 -0700 | [diff] [blame] | 1038 | thread_allocated_tsd_get()->allocated += usize; |
Jason Evans | 9344368 | 2010-10-20 17:39:18 -0700 | [diff] [blame] | 1039 | } |
Jason Evans | b147611 | 2012-04-05 13:36:17 -0700 | [diff] [blame] | 1040 | UTRACE(0, num_size, ret); |
Jason Evans | 122449b | 2012-04-06 00:35:09 -0700 | [diff] [blame] | 1041 | JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, usize, true); |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 1042 | return (ret); |
| 1043 | } |
| 1044 | |
| 1045 | void * |
Jason Evans | 0a5489e | 2012-03-01 17:19:20 -0800 | [diff] [blame] | 1046 | je_realloc(void *ptr, size_t size) |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 1047 | { |
| 1048 | void *ret; |
Jason Evans | 8694e2e | 2012-04-23 13:05:32 -0700 | [diff] [blame] | 1049 | size_t usize JEMALLOC_CC_SILENCE_INIT(0); |
Jason Evans | 9344368 | 2010-10-20 17:39:18 -0700 | [diff] [blame] | 1050 | size_t old_size = 0; |
Jason Evans | 122449b | 2012-04-06 00:35:09 -0700 | [diff] [blame] | 1051 | size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0); |
Jason Evans | 9225a19 | 2012-03-23 15:39:07 -0700 | [diff] [blame] | 1052 | prof_thr_cnt_t *cnt JEMALLOC_CC_SILENCE_INIT(NULL); |
| 1053 | prof_ctx_t *old_ctx JEMALLOC_CC_SILENCE_INIT(NULL); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1054 | |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 1055 | if (size == 0) { |
Jason Evans | f081b88 | 2012-02-28 20:24:05 -0800 | [diff] [blame] | 1056 | if (ptr != NULL) { |
| 1057 | /* realloc(ptr, 0) is equivalent to free(p). */ |
Jason Evans | 122449b | 2012-04-06 00:35:09 -0700 | [diff] [blame] | 1058 | if (config_prof) { |
| 1059 | old_size = isalloc(ptr, true); |
| 1060 | if (config_valgrind && opt_valgrind) |
| 1061 | old_rzsize = p2rz(ptr); |
| 1062 | } else if (config_stats) { |
| 1063 | old_size = isalloc(ptr, false); |
| 1064 | if (config_valgrind && opt_valgrind) |
| 1065 | old_rzsize = u2rz(old_size); |
| 1066 | } else if (config_valgrind && opt_valgrind) { |
| 1067 | old_size = isalloc(ptr, false); |
| 1068 | old_rzsize = u2rz(old_size); |
| 1069 | } |
Jason Evans | f081b88 | 2012-02-28 20:24:05 -0800 | [diff] [blame] | 1070 | if (config_prof && opt_prof) { |
| 1071 | old_ctx = prof_ctx_get(ptr); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1072 | cnt = NULL; |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1073 | } |
Jason Evans | 122449b | 2012-04-06 00:35:09 -0700 | [diff] [blame] | 1074 | iqalloc(ptr); |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 1075 | ret = NULL; |
Jason Evans | a1ee783 | 2012-04-10 15:07:44 -0700 | [diff] [blame] | 1076 | goto label_return; |
Jason Evans | c90ad71 | 2012-02-28 20:31:37 -0800 | [diff] [blame] | 1077 | } else |
| 1078 | size = 1; |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 1079 | } |
| 1080 | |
| 1081 | if (ptr != NULL) { |
Jason Evans | 41b6afb | 2012-02-02 22:04:57 -0800 | [diff] [blame] | 1082 | assert(malloc_initialized || IS_INITIALIZER); |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 1083 | |
Jason Evans | 122449b | 2012-04-06 00:35:09 -0700 | [diff] [blame] | 1084 | if (config_prof) { |
| 1085 | old_size = isalloc(ptr, true); |
| 1086 | if (config_valgrind && opt_valgrind) |
| 1087 | old_rzsize = p2rz(ptr); |
| 1088 | } else if (config_stats) { |
| 1089 | old_size = isalloc(ptr, false); |
| 1090 | if (config_valgrind && opt_valgrind) |
| 1091 | old_rzsize = u2rz(old_size); |
| 1092 | } else if (config_valgrind && opt_valgrind) { |
| 1093 | old_size = isalloc(ptr, false); |
| 1094 | old_rzsize = u2rz(old_size); |
| 1095 | } |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 1096 | if (config_prof && opt_prof) { |
Jason Evans | 9344368 | 2010-10-20 17:39:18 -0700 | [diff] [blame] | 1097 | usize = s2u(size); |
Jason Evans | 5065156 | 2010-04-13 16:13:54 -0700 | [diff] [blame] | 1098 | old_ctx = prof_ctx_get(ptr); |
Jason Evans | a507004 | 2011-08-12 13:48:27 -0700 | [diff] [blame] | 1099 | PROF_ALLOC_PREP(1, usize, cnt); |
| 1100 | if (cnt == NULL) { |
Jason Evans | 46405e6 | 2011-08-30 23:37:29 -0700 | [diff] [blame] | 1101 | old_ctx = NULL; |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1102 | ret = NULL; |
Jason Evans | a1ee783 | 2012-04-10 15:07:44 -0700 | [diff] [blame] | 1103 | goto label_oom; |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1104 | } |
Jason Evans | 0b270a9 | 2010-03-31 16:45:04 -0700 | [diff] [blame] | 1105 | if (prof_promote && (uintptr_t)cnt != (uintptr_t)1U && |
Jason Evans | b172610 | 2012-02-28 16:50:47 -0800 | [diff] [blame] | 1106 | usize <= SMALL_MAXCLASS) { |
| 1107 | ret = iralloc(ptr, SMALL_MAXCLASS+1, 0, 0, |
Jason Evans | 8e3c3c6 | 2010-09-17 15:46:18 -0700 | [diff] [blame] | 1108 | false, false); |
Jason Evans | 0b270a9 | 2010-03-31 16:45:04 -0700 | [diff] [blame] | 1109 | if (ret != NULL) |
Jason Evans | 9344368 | 2010-10-20 17:39:18 -0700 | [diff] [blame] | 1110 | arena_prof_promoted(ret, usize); |
Jason Evans | 46405e6 | 2011-08-30 23:37:29 -0700 | [diff] [blame] | 1111 | else |
| 1112 | old_ctx = NULL; |
| 1113 | } else { |
Jason Evans | 8e3c3c6 | 2010-09-17 15:46:18 -0700 | [diff] [blame] | 1114 | ret = iralloc(ptr, size, 0, 0, false, false); |
Jason Evans | 46405e6 | 2011-08-30 23:37:29 -0700 | [diff] [blame] | 1115 | if (ret == NULL) |
| 1116 | old_ctx = NULL; |
| 1117 | } |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 1118 | } else { |
Jason Evans | 122449b | 2012-04-06 00:35:09 -0700 | [diff] [blame] | 1119 | if (config_stats || (config_valgrind && opt_valgrind)) |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 1120 | usize = s2u(size); |
Jason Evans | 8e3c3c6 | 2010-09-17 15:46:18 -0700 | [diff] [blame] | 1121 | ret = iralloc(ptr, size, 0, 0, false, false); |
Jason Evans | 9344368 | 2010-10-20 17:39:18 -0700 | [diff] [blame] | 1122 | } |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 1123 | |
Jason Evans | a1ee783 | 2012-04-10 15:07:44 -0700 | [diff] [blame] | 1124 | label_oom: |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 1125 | if (ret == NULL) { |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 1126 | if (config_xmalloc && opt_xmalloc) { |
Jason Evans | 698805c | 2010-03-03 17:45:38 -0800 | [diff] [blame] | 1127 | malloc_write("<jemalloc>: Error in realloc(): " |
| 1128 | "out of memory\n"); |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 1129 | abort(); |
| 1130 | } |
Mike Hommey | a14bce8 | 2012-04-30 12:38:26 +0200 | [diff] [blame] | 1131 | set_errno(ENOMEM); |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 1132 | } |
| 1133 | } else { |
Jason Evans | f081b88 | 2012-02-28 20:24:05 -0800 | [diff] [blame] | 1134 | /* realloc(NULL, size) is equivalent to malloc(size). */ |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 1135 | if (config_prof && opt_prof) |
Jason Evans | 5065156 | 2010-04-13 16:13:54 -0700 | [diff] [blame] | 1136 | old_ctx = NULL; |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1137 | if (malloc_init()) { |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 1138 | if (config_prof && opt_prof) |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1139 | cnt = NULL; |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1140 | ret = NULL; |
| 1141 | } else { |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 1142 | if (config_prof && opt_prof) { |
Jason Evans | 9344368 | 2010-10-20 17:39:18 -0700 | [diff] [blame] | 1143 | usize = s2u(size); |
Jason Evans | a507004 | 2011-08-12 13:48:27 -0700 | [diff] [blame] | 1144 | PROF_ALLOC_PREP(1, usize, cnt); |
| 1145 | if (cnt == NULL) |
Jason Evans | 0b270a9 | 2010-03-31 16:45:04 -0700 | [diff] [blame] | 1146 | ret = NULL; |
| 1147 | else { |
| 1148 | if (prof_promote && (uintptr_t)cnt != |
Jason Evans | 9344368 | 2010-10-20 17:39:18 -0700 | [diff] [blame] | 1149 | (uintptr_t)1U && usize <= |
Jason Evans | b172610 | 2012-02-28 16:50:47 -0800 | [diff] [blame] | 1150 | SMALL_MAXCLASS) { |
| 1151 | ret = imalloc(SMALL_MAXCLASS+1); |
Jason Evans | 0b270a9 | 2010-03-31 16:45:04 -0700 | [diff] [blame] | 1152 | if (ret != NULL) { |
| 1153 | arena_prof_promoted(ret, |
Jason Evans | 9344368 | 2010-10-20 17:39:18 -0700 | [diff] [blame] | 1154 | usize); |
Jason Evans | 0b270a9 | 2010-03-31 16:45:04 -0700 | [diff] [blame] | 1155 | } |
| 1156 | } else |
| 1157 | ret = imalloc(size); |
| 1158 | } |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 1159 | } else { |
Jason Evans | 122449b | 2012-04-06 00:35:09 -0700 | [diff] [blame] | 1160 | if (config_stats || (config_valgrind && |
| 1161 | opt_valgrind)) |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 1162 | usize = s2u(size); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1163 | ret = imalloc(size); |
Jason Evans | 9344368 | 2010-10-20 17:39:18 -0700 | [diff] [blame] | 1164 | } |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1165 | } |
Jason Evans | 569432c | 2009-12-29 00:09:15 -0800 | [diff] [blame] | 1166 | |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 1167 | if (ret == NULL) { |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 1168 | if (config_xmalloc && opt_xmalloc) { |
Jason Evans | 698805c | 2010-03-03 17:45:38 -0800 | [diff] [blame] | 1169 | malloc_write("<jemalloc>: Error in realloc(): " |
| 1170 | "out of memory\n"); |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 1171 | abort(); |
| 1172 | } |
Mike Hommey | a14bce8 | 2012-04-30 12:38:26 +0200 | [diff] [blame] | 1173 | set_errno(ENOMEM); |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 1174 | } |
| 1175 | } |
| 1176 | |
Jason Evans | a1ee783 | 2012-04-10 15:07:44 -0700 | [diff] [blame] | 1177 | label_return: |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 1178 | if (config_prof && opt_prof) |
Jason Evans | e4f7846 | 2010-10-22 10:45:59 -0700 | [diff] [blame] | 1179 | prof_realloc(ret, usize, cnt, old_size, old_ctx); |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 1180 | if (config_stats && ret != NULL) { |
Jason Evans | cd9a134 | 2012-03-21 18:33:03 -0700 | [diff] [blame] | 1181 | thread_allocated_t *ta; |
Jason Evans | 122449b | 2012-04-06 00:35:09 -0700 | [diff] [blame] | 1182 | assert(usize == isalloc(ret, config_prof)); |
Jason Evans | cd9a134 | 2012-03-21 18:33:03 -0700 | [diff] [blame] | 1183 | ta = thread_allocated_tsd_get(); |
| 1184 | ta->allocated += usize; |
| 1185 | ta->deallocated += old_size; |
Jason Evans | 9344368 | 2010-10-20 17:39:18 -0700 | [diff] [blame] | 1186 | } |
Jason Evans | b147611 | 2012-04-05 13:36:17 -0700 | [diff] [blame] | 1187 | UTRACE(ptr, size, ret); |
Jason Evans | 122449b | 2012-04-06 00:35:09 -0700 | [diff] [blame] | 1188 | JEMALLOC_VALGRIND_REALLOC(ret, usize, ptr, old_size, old_rzsize, false); |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 1189 | return (ret); |
| 1190 | } |
| 1191 | |
| 1192 | void |
Jason Evans | 0a5489e | 2012-03-01 17:19:20 -0800 | [diff] [blame] | 1193 | je_free(void *ptr) |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 1194 | { |
| 1195 | |
Jason Evans | b147611 | 2012-04-05 13:36:17 -0700 | [diff] [blame] | 1196 | UTRACE(ptr, 0, 0); |
Jason Evans | f004737 | 2012-04-02 15:18:24 -0700 | [diff] [blame] | 1197 | if (ptr != NULL) { |
| 1198 | size_t usize; |
Jason Evans | 122449b | 2012-04-06 00:35:09 -0700 | [diff] [blame] | 1199 | size_t rzsize JEMALLOC_CC_SILENCE_INIT(0); |
Jason Evans | e4f7846 | 2010-10-22 10:45:59 -0700 | [diff] [blame] | 1200 | |
Jason Evans | f004737 | 2012-04-02 15:18:24 -0700 | [diff] [blame] | 1201 | assert(malloc_initialized || IS_INITIALIZER); |
| 1202 | |
| 1203 | if (config_prof && opt_prof) { |
Jason Evans | 122449b | 2012-04-06 00:35:09 -0700 | [diff] [blame] | 1204 | usize = isalloc(ptr, config_prof); |
Jason Evans | f004737 | 2012-04-02 15:18:24 -0700 | [diff] [blame] | 1205 | prof_free(ptr, usize); |
Jason Evans | 122449b | 2012-04-06 00:35:09 -0700 | [diff] [blame] | 1206 | } else if (config_stats || config_valgrind) |
| 1207 | usize = isalloc(ptr, config_prof); |
Jason Evans | f004737 | 2012-04-02 15:18:24 -0700 | [diff] [blame] | 1208 | if (config_stats) |
| 1209 | thread_allocated_tsd_get()->deallocated += usize; |
Jason Evans | 122449b | 2012-04-06 00:35:09 -0700 | [diff] [blame] | 1210 | if (config_valgrind && opt_valgrind) |
| 1211 | rzsize = p2rz(ptr); |
| 1212 | iqalloc(ptr); |
| 1213 | JEMALLOC_VALGRIND_FREE(ptr, rzsize); |
Jason Evans | f004737 | 2012-04-02 15:18:24 -0700 | [diff] [blame] | 1214 | } |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 1215 | } |
| 1216 | |
| 1217 | /* |
| 1218 | * End malloc(3)-compatible functions. |
| 1219 | */ |
| 1220 | /******************************************************************************/ |
| 1221 | /* |
Jason Evans | 6a0d291 | 2010-09-20 16:44:23 -0700 | [diff] [blame] | 1222 | * Begin non-standard override functions. |
Jason Evans | 6a0d291 | 2010-09-20 16:44:23 -0700 | [diff] [blame] | 1223 | */ |
Jason Evans | 6a0d291 | 2010-09-20 16:44:23 -0700 | [diff] [blame] | 1224 | |
| 1225 | #ifdef JEMALLOC_OVERRIDE_MEMALIGN |
Jason Evans | 6a0d291 | 2010-09-20 16:44:23 -0700 | [diff] [blame] | 1226 | void * |
Jason Evans | 0a5489e | 2012-03-01 17:19:20 -0800 | [diff] [blame] | 1227 | je_memalign(size_t alignment, size_t size) |
Jason Evans | 6a0d291 | 2010-09-20 16:44:23 -0700 | [diff] [blame] | 1228 | { |
Jason Evans | 9225a19 | 2012-03-23 15:39:07 -0700 | [diff] [blame] | 1229 | void *ret JEMALLOC_CC_SILENCE_INIT(NULL); |
Jason Evans | 0a0bbf6 | 2012-03-13 12:55:21 -0700 | [diff] [blame] | 1230 | imemalign(&ret, alignment, size, 1); |
Jason Evans | 122449b | 2012-04-06 00:35:09 -0700 | [diff] [blame] | 1231 | JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, size, false); |
Jason Evans | 6a0d291 | 2010-09-20 16:44:23 -0700 | [diff] [blame] | 1232 | return (ret); |
| 1233 | } |
| 1234 | #endif |
| 1235 | |
| 1236 | #ifdef JEMALLOC_OVERRIDE_VALLOC |
Jason Evans | 6a0d291 | 2010-09-20 16:44:23 -0700 | [diff] [blame] | 1237 | void * |
Jason Evans | 0a5489e | 2012-03-01 17:19:20 -0800 | [diff] [blame] | 1238 | je_valloc(size_t size) |
Jason Evans | 6a0d291 | 2010-09-20 16:44:23 -0700 | [diff] [blame] | 1239 | { |
Jason Evans | 9225a19 | 2012-03-23 15:39:07 -0700 | [diff] [blame] | 1240 | void *ret JEMALLOC_CC_SILENCE_INIT(NULL); |
Jason Evans | ae4c7b4 | 2012-04-02 07:04:34 -0700 | [diff] [blame] | 1241 | imemalign(&ret, PAGE, size, 1); |
Jason Evans | 122449b | 2012-04-06 00:35:09 -0700 | [diff] [blame] | 1242 | JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, size, false); |
Jason Evans | 6a0d291 | 2010-09-20 16:44:23 -0700 | [diff] [blame] | 1243 | return (ret); |
| 1244 | } |
| 1245 | #endif |
| 1246 | |
Mike Hommey | 5c89c50 | 2012-03-26 17:46:57 +0200 | [diff] [blame] | 1247 | /* |
| 1248 | * is_malloc(je_malloc) is some macro magic to detect if jemalloc_defs.h has |
| 1249 | * #define je_malloc malloc |
| 1250 | */ |
| 1251 | #define malloc_is_malloc 1 |
| 1252 | #define is_malloc_(a) malloc_is_ ## a |
| 1253 | #define is_malloc(a) is_malloc_(a) |
| 1254 | |
| 1255 | #if ((is_malloc(je_malloc) == 1) && defined(__GLIBC__) && !defined(__UCLIBC__)) |
Jason Evans | 4bb0983 | 2012-02-29 10:37:27 -0800 | [diff] [blame] | 1256 | /* |
| 1257 | * glibc provides the RTLD_DEEPBIND flag for dlopen which can make it possible |
| 1258 | * to inconsistently reference libc's malloc(3)-compatible functions |
| 1259 | * (https://bugzilla.mozilla.org/show_bug.cgi?id=493541). |
| 1260 | * |
Mike Hommey | 3c2ba0d | 2012-03-27 14:20:13 +0200 | [diff] [blame] | 1261 | * These definitions interpose hooks in glibc. The functions are actually |
Jason Evans | 4bb0983 | 2012-02-29 10:37:27 -0800 | [diff] [blame] | 1262 | * passed an extra argument for the caller return address, which will be |
| 1263 | * ignored. |
| 1264 | */ |
Mike Hommey | da99e31 | 2012-04-30 12:38:29 +0200 | [diff] [blame] | 1265 | JEMALLOC_EXPORT void (* const __free_hook)(void *ptr) = je_free; |
| 1266 | JEMALLOC_EXPORT void *(* const __malloc_hook)(size_t size) = je_malloc; |
| 1267 | JEMALLOC_EXPORT void *(* const __realloc_hook)(void *ptr, size_t size) = |
| 1268 | je_realloc; |
| 1269 | JEMALLOC_EXPORT void *(* const __memalign_hook)(size_t alignment, size_t size) = |
| 1270 | je_memalign; |
Jason Evans | 4bb0983 | 2012-02-29 10:37:27 -0800 | [diff] [blame] | 1271 | #endif |
| 1272 | |
Jason Evans | 6a0d291 | 2010-09-20 16:44:23 -0700 | [diff] [blame] | 1273 | /* |
| 1274 | * End non-standard override functions. |
| 1275 | */ |
| 1276 | /******************************************************************************/ |
| 1277 | /* |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 1278 | * Begin non-standard functions. |
| 1279 | */ |
| 1280 | |
| 1281 | size_t |
Jason Evans | 0a5489e | 2012-03-01 17:19:20 -0800 | [diff] [blame] | 1282 | je_malloc_usable_size(const void *ptr) |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 1283 | { |
Jason Evans | 569432c | 2009-12-29 00:09:15 -0800 | [diff] [blame] | 1284 | size_t ret; |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 1285 | |
Jason Evans | 41b6afb | 2012-02-02 22:04:57 -0800 | [diff] [blame] | 1286 | assert(malloc_initialized || IS_INITIALIZER); |
Jason Evans | 8e3c3c6 | 2010-09-17 15:46:18 -0700 | [diff] [blame] | 1287 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 1288 | if (config_ivsalloc) |
Jason Evans | 122449b | 2012-04-06 00:35:09 -0700 | [diff] [blame] | 1289 | ret = ivsalloc(ptr, config_prof); |
Jason Evans | 2465bdf | 2012-03-26 13:13:55 -0700 | [diff] [blame] | 1290 | else |
Jason Evans | 122449b | 2012-04-06 00:35:09 -0700 | [diff] [blame] | 1291 | ret = (ptr != NULL) ? isalloc(ptr, config_prof) : 0; |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 1292 | |
Jason Evans | 569432c | 2009-12-29 00:09:15 -0800 | [diff] [blame] | 1293 | return (ret); |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 1294 | } |
| 1295 | |
Jason Evans | 4201af0 | 2010-01-24 02:53:40 -0800 | [diff] [blame] | 1296 | void |
Jason Evans | 0a5489e | 2012-03-01 17:19:20 -0800 | [diff] [blame] | 1297 | je_malloc_stats_print(void (*write_cb)(void *, const char *), void *cbopaque, |
| 1298 | const char *opts) |
Jason Evans | 4201af0 | 2010-01-24 02:53:40 -0800 | [diff] [blame] | 1299 | { |
| 1300 | |
Jason Evans | 698805c | 2010-03-03 17:45:38 -0800 | [diff] [blame] | 1301 | stats_print(write_cb, cbopaque, opts); |
Jason Evans | 4201af0 | 2010-01-24 02:53:40 -0800 | [diff] [blame] | 1302 | } |
| 1303 | |
Jason Evans | 3c23435 | 2010-01-27 13:10:55 -0800 | [diff] [blame] | 1304 | int |
Jason Evans | 0a5489e | 2012-03-01 17:19:20 -0800 | [diff] [blame] | 1305 | je_mallctl(const char *name, void *oldp, size_t *oldlenp, void *newp, |
Jason Evans | 3c23435 | 2010-01-27 13:10:55 -0800 | [diff] [blame] | 1306 | size_t newlen) |
| 1307 | { |
| 1308 | |
Jason Evans | 9583331 | 2010-01-27 13:45:21 -0800 | [diff] [blame] | 1309 | if (malloc_init()) |
| 1310 | return (EAGAIN); |
| 1311 | |
Jason Evans | 3c23435 | 2010-01-27 13:10:55 -0800 | [diff] [blame] | 1312 | return (ctl_byname(name, oldp, oldlenp, newp, newlen)); |
| 1313 | } |
| 1314 | |
Jason Evans | 3c23435 | 2010-01-27 13:10:55 -0800 | [diff] [blame] | 1315 | int |
Jason Evans | 0a5489e | 2012-03-01 17:19:20 -0800 | [diff] [blame] | 1316 | je_mallctlnametomib(const char *name, size_t *mibp, size_t *miblenp) |
Jason Evans | 3c23435 | 2010-01-27 13:10:55 -0800 | [diff] [blame] | 1317 | { |
| 1318 | |
Jason Evans | 9583331 | 2010-01-27 13:45:21 -0800 | [diff] [blame] | 1319 | if (malloc_init()) |
| 1320 | return (EAGAIN); |
| 1321 | |
Jason Evans | 3c23435 | 2010-01-27 13:10:55 -0800 | [diff] [blame] | 1322 | return (ctl_nametomib(name, mibp, miblenp)); |
| 1323 | } |
| 1324 | |
Jason Evans | 3c23435 | 2010-01-27 13:10:55 -0800 | [diff] [blame] | 1325 | int |
Jason Evans | 0a5489e | 2012-03-01 17:19:20 -0800 | [diff] [blame] | 1326 | je_mallctlbymib(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, |
| 1327 | void *newp, size_t newlen) |
Jason Evans | 3c23435 | 2010-01-27 13:10:55 -0800 | [diff] [blame] | 1328 | { |
| 1329 | |
Jason Evans | 9583331 | 2010-01-27 13:45:21 -0800 | [diff] [blame] | 1330 | if (malloc_init()) |
| 1331 | return (EAGAIN); |
| 1332 | |
Jason Evans | 3c23435 | 2010-01-27 13:10:55 -0800 | [diff] [blame] | 1333 | return (ctl_bymib(mib, miblen, oldp, oldlenp, newp, newlen)); |
| 1334 | } |
| 1335 | |
Jason Evans | 7e77eaf | 2012-03-02 17:47:37 -0800 | [diff] [blame] | 1336 | /* |
| 1337 | * End non-standard functions. |
| 1338 | */ |
| 1339 | /******************************************************************************/ |
| 1340 | /* |
| 1341 | * Begin experimental functions. |
| 1342 | */ |
| 1343 | #ifdef JEMALLOC_EXPERIMENTAL |
| 1344 | |
Jason Evans | 8e3c3c6 | 2010-09-17 15:46:18 -0700 | [diff] [blame] | 1345 | JEMALLOC_INLINE void * |
Jason Evans | 38d9210 | 2011-03-23 00:37:29 -0700 | [diff] [blame] | 1346 | iallocm(size_t usize, size_t alignment, bool zero) |
Jason Evans | 8e3c3c6 | 2010-09-17 15:46:18 -0700 | [diff] [blame] | 1347 | { |
| 1348 | |
Jason Evans | 5ff709c | 2012-04-11 18:13:45 -0700 | [diff] [blame] | 1349 | assert(usize == ((alignment == 0) ? s2u(usize) : sa2u(usize, |
| 1350 | alignment))); |
Jason Evans | 38d9210 | 2011-03-23 00:37:29 -0700 | [diff] [blame] | 1351 | |
Jason Evans | 8e3c3c6 | 2010-09-17 15:46:18 -0700 | [diff] [blame] | 1352 | if (alignment != 0) |
Jason Evans | 38d9210 | 2011-03-23 00:37:29 -0700 | [diff] [blame] | 1353 | return (ipalloc(usize, alignment, zero)); |
Jason Evans | 8e3c3c6 | 2010-09-17 15:46:18 -0700 | [diff] [blame] | 1354 | else if (zero) |
Jason Evans | 38d9210 | 2011-03-23 00:37:29 -0700 | [diff] [blame] | 1355 | return (icalloc(usize)); |
Jason Evans | 8e3c3c6 | 2010-09-17 15:46:18 -0700 | [diff] [blame] | 1356 | else |
Jason Evans | 38d9210 | 2011-03-23 00:37:29 -0700 | [diff] [blame] | 1357 | return (imalloc(usize)); |
Jason Evans | 8e3c3c6 | 2010-09-17 15:46:18 -0700 | [diff] [blame] | 1358 | } |
| 1359 | |
Jason Evans | 8e3c3c6 | 2010-09-17 15:46:18 -0700 | [diff] [blame] | 1360 | int |
Jason Evans | 0a5489e | 2012-03-01 17:19:20 -0800 | [diff] [blame] | 1361 | je_allocm(void **ptr, size_t *rsize, size_t size, int flags) |
Jason Evans | 8e3c3c6 | 2010-09-17 15:46:18 -0700 | [diff] [blame] | 1362 | { |
| 1363 | void *p; |
Jason Evans | 9344368 | 2010-10-20 17:39:18 -0700 | [diff] [blame] | 1364 | size_t usize; |
Jason Evans | 8e3c3c6 | 2010-09-17 15:46:18 -0700 | [diff] [blame] | 1365 | size_t alignment = (ZU(1) << (flags & ALLOCM_LG_ALIGN_MASK) |
| 1366 | & (SIZE_T_MAX-1)); |
| 1367 | bool zero = flags & ALLOCM_ZERO; |
Jason Evans | 8e3c3c6 | 2010-09-17 15:46:18 -0700 | [diff] [blame] | 1368 | |
| 1369 | assert(ptr != NULL); |
| 1370 | assert(size != 0); |
| 1371 | |
| 1372 | if (malloc_init()) |
Jason Evans | a1ee783 | 2012-04-10 15:07:44 -0700 | [diff] [blame] | 1373 | goto label_oom; |
Jason Evans | 8e3c3c6 | 2010-09-17 15:46:18 -0700 | [diff] [blame] | 1374 | |
Jason Evans | 5ff709c | 2012-04-11 18:13:45 -0700 | [diff] [blame] | 1375 | usize = (alignment == 0) ? s2u(size) : sa2u(size, alignment); |
Jason Evans | 38d9210 | 2011-03-23 00:37:29 -0700 | [diff] [blame] | 1376 | if (usize == 0) |
Jason Evans | a1ee783 | 2012-04-10 15:07:44 -0700 | [diff] [blame] | 1377 | goto label_oom; |
Jason Evans | 38d9210 | 2011-03-23 00:37:29 -0700 | [diff] [blame] | 1378 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 1379 | if (config_prof && opt_prof) { |
Jason Evans | 3fb50b0 | 2012-04-25 13:13:44 -0700 | [diff] [blame] | 1380 | prof_thr_cnt_t *cnt; |
| 1381 | |
Jason Evans | a507004 | 2011-08-12 13:48:27 -0700 | [diff] [blame] | 1382 | PROF_ALLOC_PREP(1, usize, cnt); |
| 1383 | if (cnt == NULL) |
Jason Evans | a1ee783 | 2012-04-10 15:07:44 -0700 | [diff] [blame] | 1384 | goto label_oom; |
Jason Evans | 9344368 | 2010-10-20 17:39:18 -0700 | [diff] [blame] | 1385 | if (prof_promote && (uintptr_t)cnt != (uintptr_t)1U && usize <= |
Jason Evans | b172610 | 2012-02-28 16:50:47 -0800 | [diff] [blame] | 1386 | SMALL_MAXCLASS) { |
Jason Evans | 38d9210 | 2011-03-23 00:37:29 -0700 | [diff] [blame] | 1387 | size_t usize_promoted = (alignment == 0) ? |
Jason Evans | b172610 | 2012-02-28 16:50:47 -0800 | [diff] [blame] | 1388 | s2u(SMALL_MAXCLASS+1) : sa2u(SMALL_MAXCLASS+1, |
Jason Evans | 5ff709c | 2012-04-11 18:13:45 -0700 | [diff] [blame] | 1389 | alignment); |
Jason Evans | 38d9210 | 2011-03-23 00:37:29 -0700 | [diff] [blame] | 1390 | assert(usize_promoted != 0); |
| 1391 | p = iallocm(usize_promoted, alignment, zero); |
Jason Evans | 8e3c3c6 | 2010-09-17 15:46:18 -0700 | [diff] [blame] | 1392 | if (p == NULL) |
Jason Evans | a1ee783 | 2012-04-10 15:07:44 -0700 | [diff] [blame] | 1393 | goto label_oom; |
Jason Evans | 9344368 | 2010-10-20 17:39:18 -0700 | [diff] [blame] | 1394 | arena_prof_promoted(p, usize); |
Jason Evans | 8e3c3c6 | 2010-09-17 15:46:18 -0700 | [diff] [blame] | 1395 | } else { |
Jason Evans | 38d9210 | 2011-03-23 00:37:29 -0700 | [diff] [blame] | 1396 | p = iallocm(usize, alignment, zero); |
Jason Evans | 8e3c3c6 | 2010-09-17 15:46:18 -0700 | [diff] [blame] | 1397 | if (p == NULL) |
Jason Evans | a1ee783 | 2012-04-10 15:07:44 -0700 | [diff] [blame] | 1398 | goto label_oom; |
Jason Evans | 8e3c3c6 | 2010-09-17 15:46:18 -0700 | [diff] [blame] | 1399 | } |
Jason Evans | 749c2a0 | 2011-08-12 18:37:54 -0700 | [diff] [blame] | 1400 | prof_malloc(p, usize, cnt); |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 1401 | } else { |
Jason Evans | 38d9210 | 2011-03-23 00:37:29 -0700 | [diff] [blame] | 1402 | p = iallocm(usize, alignment, zero); |
Jason Evans | 8e3c3c6 | 2010-09-17 15:46:18 -0700 | [diff] [blame] | 1403 | if (p == NULL) |
Jason Evans | a1ee783 | 2012-04-10 15:07:44 -0700 | [diff] [blame] | 1404 | goto label_oom; |
Jason Evans | 8e3c3c6 | 2010-09-17 15:46:18 -0700 | [diff] [blame] | 1405 | } |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 1406 | if (rsize != NULL) |
| 1407 | *rsize = usize; |
Jason Evans | 8e3c3c6 | 2010-09-17 15:46:18 -0700 | [diff] [blame] | 1408 | |
| 1409 | *ptr = p; |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 1410 | if (config_stats) { |
Jason Evans | 122449b | 2012-04-06 00:35:09 -0700 | [diff] [blame] | 1411 | assert(usize == isalloc(p, config_prof)); |
Jason Evans | cd9a134 | 2012-03-21 18:33:03 -0700 | [diff] [blame] | 1412 | thread_allocated_tsd_get()->allocated += usize; |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 1413 | } |
Jason Evans | b147611 | 2012-04-05 13:36:17 -0700 | [diff] [blame] | 1414 | UTRACE(0, size, p); |
Jason Evans | 122449b | 2012-04-06 00:35:09 -0700 | [diff] [blame] | 1415 | JEMALLOC_VALGRIND_MALLOC(true, p, usize, zero); |
Jason Evans | 8e3c3c6 | 2010-09-17 15:46:18 -0700 | [diff] [blame] | 1416 | return (ALLOCM_SUCCESS); |
Jason Evans | a1ee783 | 2012-04-10 15:07:44 -0700 | [diff] [blame] | 1417 | label_oom: |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 1418 | if (config_xmalloc && opt_xmalloc) { |
Jason Evans | 8e3c3c6 | 2010-09-17 15:46:18 -0700 | [diff] [blame] | 1419 | malloc_write("<jemalloc>: Error in allocm(): " |
| 1420 | "out of memory\n"); |
| 1421 | abort(); |
| 1422 | } |
Jason Evans | 8e3c3c6 | 2010-09-17 15:46:18 -0700 | [diff] [blame] | 1423 | *ptr = NULL; |
Jason Evans | b147611 | 2012-04-05 13:36:17 -0700 | [diff] [blame] | 1424 | UTRACE(0, size, 0); |
Jason Evans | 8e3c3c6 | 2010-09-17 15:46:18 -0700 | [diff] [blame] | 1425 | return (ALLOCM_ERR_OOM); |
| 1426 | } |
| 1427 | |
Jason Evans | 8e3c3c6 | 2010-09-17 15:46:18 -0700 | [diff] [blame] | 1428 | int |
Jason Evans | 0a5489e | 2012-03-01 17:19:20 -0800 | [diff] [blame] | 1429 | je_rallocm(void **ptr, size_t *rsize, size_t size, size_t extra, int flags) |
Jason Evans | 8e3c3c6 | 2010-09-17 15:46:18 -0700 | [diff] [blame] | 1430 | { |
| 1431 | void *p, *q; |
Jason Evans | 9344368 | 2010-10-20 17:39:18 -0700 | [diff] [blame] | 1432 | size_t usize; |
Jason Evans | 9344368 | 2010-10-20 17:39:18 -0700 | [diff] [blame] | 1433 | size_t old_size; |
Jason Evans | 122449b | 2012-04-06 00:35:09 -0700 | [diff] [blame] | 1434 | size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0); |
Jason Evans | 8e3c3c6 | 2010-09-17 15:46:18 -0700 | [diff] [blame] | 1435 | size_t alignment = (ZU(1) << (flags & ALLOCM_LG_ALIGN_MASK) |
| 1436 | & (SIZE_T_MAX-1)); |
| 1437 | bool zero = flags & ALLOCM_ZERO; |
| 1438 | bool no_move = flags & ALLOCM_NO_MOVE; |
Jason Evans | 8e3c3c6 | 2010-09-17 15:46:18 -0700 | [diff] [blame] | 1439 | |
| 1440 | assert(ptr != NULL); |
| 1441 | assert(*ptr != NULL); |
| 1442 | assert(size != 0); |
| 1443 | assert(SIZE_T_MAX - size >= extra); |
Jason Evans | 41b6afb | 2012-02-02 22:04:57 -0800 | [diff] [blame] | 1444 | assert(malloc_initialized || IS_INITIALIZER); |
Jason Evans | 8e3c3c6 | 2010-09-17 15:46:18 -0700 | [diff] [blame] | 1445 | |
| 1446 | p = *ptr; |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 1447 | if (config_prof && opt_prof) { |
Jason Evans | 3fb50b0 | 2012-04-25 13:13:44 -0700 | [diff] [blame] | 1448 | prof_thr_cnt_t *cnt; |
| 1449 | |
Jason Evans | 9344368 | 2010-10-20 17:39:18 -0700 | [diff] [blame] | 1450 | /* |
| 1451 | * usize isn't knowable before iralloc() returns when extra is |
| 1452 | * non-zero. Therefore, compute its maximum possible value and |
Jason Evans | a507004 | 2011-08-12 13:48:27 -0700 | [diff] [blame] | 1453 | * use that in PROF_ALLOC_PREP() to decide whether to capture a |
Jason Evans | 9344368 | 2010-10-20 17:39:18 -0700 | [diff] [blame] | 1454 | * backtrace. prof_realloc() will use the actual usize to |
| 1455 | * decide whether to sample. |
| 1456 | */ |
| 1457 | size_t max_usize = (alignment == 0) ? s2u(size+extra) : |
Jason Evans | 5ff709c | 2012-04-11 18:13:45 -0700 | [diff] [blame] | 1458 | sa2u(size+extra, alignment); |
Jason Evans | 46405e6 | 2011-08-30 23:37:29 -0700 | [diff] [blame] | 1459 | prof_ctx_t *old_ctx = prof_ctx_get(p); |
Jason Evans | 122449b | 2012-04-06 00:35:09 -0700 | [diff] [blame] | 1460 | old_size = isalloc(p, true); |
| 1461 | if (config_valgrind && opt_valgrind) |
| 1462 | old_rzsize = p2rz(p); |
Jason Evans | a507004 | 2011-08-12 13:48:27 -0700 | [diff] [blame] | 1463 | PROF_ALLOC_PREP(1, max_usize, cnt); |
| 1464 | if (cnt == NULL) |
Jason Evans | a1ee783 | 2012-04-10 15:07:44 -0700 | [diff] [blame] | 1465 | goto label_oom; |
Jason Evans | 183ba50 | 2011-08-11 22:51:00 -0700 | [diff] [blame] | 1466 | /* |
| 1467 | * Use minimum usize to determine whether promotion may happen. |
| 1468 | */ |
| 1469 | if (prof_promote && (uintptr_t)cnt != (uintptr_t)1U |
Jason Evans | 5ff709c | 2012-04-11 18:13:45 -0700 | [diff] [blame] | 1470 | && ((alignment == 0) ? s2u(size) : sa2u(size, alignment)) |
| 1471 | <= SMALL_MAXCLASS) { |
Jason Evans | b172610 | 2012-02-28 16:50:47 -0800 | [diff] [blame] | 1472 | q = iralloc(p, SMALL_MAXCLASS+1, (SMALL_MAXCLASS+1 >= |
| 1473 | size+extra) ? 0 : size+extra - (SMALL_MAXCLASS+1), |
Jason Evans | 8e3c3c6 | 2010-09-17 15:46:18 -0700 | [diff] [blame] | 1474 | alignment, zero, no_move); |
| 1475 | if (q == NULL) |
Jason Evans | a1ee783 | 2012-04-10 15:07:44 -0700 | [diff] [blame] | 1476 | goto label_err; |
Jason Evans | ae4c7b4 | 2012-04-02 07:04:34 -0700 | [diff] [blame] | 1477 | if (max_usize < PAGE) { |
Jason Evans | 183ba50 | 2011-08-11 22:51:00 -0700 | [diff] [blame] | 1478 | usize = max_usize; |
| 1479 | arena_prof_promoted(q, usize); |
Jason Evans | b493ce2 | 2011-08-12 11:28:47 -0700 | [diff] [blame] | 1480 | } else |
Jason Evans | 122449b | 2012-04-06 00:35:09 -0700 | [diff] [blame] | 1481 | usize = isalloc(q, config_prof); |
Jason Evans | 8e3c3c6 | 2010-09-17 15:46:18 -0700 | [diff] [blame] | 1482 | } else { |
| 1483 | q = iralloc(p, size, extra, alignment, zero, no_move); |
| 1484 | if (q == NULL) |
Jason Evans | a1ee783 | 2012-04-10 15:07:44 -0700 | [diff] [blame] | 1485 | goto label_err; |
Jason Evans | 122449b | 2012-04-06 00:35:09 -0700 | [diff] [blame] | 1486 | usize = isalloc(q, config_prof); |
Jason Evans | 8e3c3c6 | 2010-09-17 15:46:18 -0700 | [diff] [blame] | 1487 | } |
Jason Evans | e4f7846 | 2010-10-22 10:45:59 -0700 | [diff] [blame] | 1488 | prof_realloc(q, usize, cnt, old_size, old_ctx); |
Jason Evans | eacb896 | 2011-03-23 00:30:30 -0700 | [diff] [blame] | 1489 | if (rsize != NULL) |
| 1490 | *rsize = usize; |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 1491 | } else { |
Jason Evans | 122449b | 2012-04-06 00:35:09 -0700 | [diff] [blame] | 1492 | if (config_stats) { |
| 1493 | old_size = isalloc(p, false); |
| 1494 | if (config_valgrind && opt_valgrind) |
| 1495 | old_rzsize = u2rz(old_size); |
| 1496 | } else if (config_valgrind && opt_valgrind) { |
| 1497 | old_size = isalloc(p, false); |
| 1498 | old_rzsize = u2rz(old_size); |
| 1499 | } |
Jason Evans | 8e3c3c6 | 2010-09-17 15:46:18 -0700 | [diff] [blame] | 1500 | q = iralloc(p, size, extra, alignment, zero, no_move); |
| 1501 | if (q == NULL) |
Jason Evans | a1ee783 | 2012-04-10 15:07:44 -0700 | [diff] [blame] | 1502 | goto label_err; |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 1503 | if (config_stats) |
Jason Evans | 122449b | 2012-04-06 00:35:09 -0700 | [diff] [blame] | 1504 | usize = isalloc(q, config_prof); |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 1505 | if (rsize != NULL) { |
| 1506 | if (config_stats == false) |
Jason Evans | 122449b | 2012-04-06 00:35:09 -0700 | [diff] [blame] | 1507 | usize = isalloc(q, config_prof); |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 1508 | *rsize = usize; |
Jason Evans | 9344368 | 2010-10-20 17:39:18 -0700 | [diff] [blame] | 1509 | } |
Jason Evans | 8e3c3c6 | 2010-09-17 15:46:18 -0700 | [diff] [blame] | 1510 | } |
| 1511 | |
| 1512 | *ptr = q; |
Jason Evans | cd9a134 | 2012-03-21 18:33:03 -0700 | [diff] [blame] | 1513 | if (config_stats) { |
| 1514 | thread_allocated_t *ta; |
| 1515 | ta = thread_allocated_tsd_get(); |
| 1516 | ta->allocated += usize; |
| 1517 | ta->deallocated += old_size; |
| 1518 | } |
Jason Evans | b147611 | 2012-04-05 13:36:17 -0700 | [diff] [blame] | 1519 | UTRACE(p, size, q); |
Jason Evans | 122449b | 2012-04-06 00:35:09 -0700 | [diff] [blame] | 1520 | JEMALLOC_VALGRIND_REALLOC(q, usize, p, old_size, old_rzsize, zero); |
Jason Evans | 8e3c3c6 | 2010-09-17 15:46:18 -0700 | [diff] [blame] | 1521 | return (ALLOCM_SUCCESS); |
Jason Evans | a1ee783 | 2012-04-10 15:07:44 -0700 | [diff] [blame] | 1522 | label_err: |
Jason Evans | b147611 | 2012-04-05 13:36:17 -0700 | [diff] [blame] | 1523 | if (no_move) { |
| 1524 | UTRACE(p, size, q); |
Jason Evans | 8e3c3c6 | 2010-09-17 15:46:18 -0700 | [diff] [blame] | 1525 | return (ALLOCM_ERR_NOT_MOVED); |
Jason Evans | b147611 | 2012-04-05 13:36:17 -0700 | [diff] [blame] | 1526 | } |
Jason Evans | a1ee783 | 2012-04-10 15:07:44 -0700 | [diff] [blame] | 1527 | label_oom: |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 1528 | if (config_xmalloc && opt_xmalloc) { |
Jason Evans | 8e3c3c6 | 2010-09-17 15:46:18 -0700 | [diff] [blame] | 1529 | malloc_write("<jemalloc>: Error in rallocm(): " |
| 1530 | "out of memory\n"); |
| 1531 | abort(); |
| 1532 | } |
Jason Evans | b147611 | 2012-04-05 13:36:17 -0700 | [diff] [blame] | 1533 | UTRACE(p, size, 0); |
Jason Evans | 8e3c3c6 | 2010-09-17 15:46:18 -0700 | [diff] [blame] | 1534 | return (ALLOCM_ERR_OOM); |
| 1535 | } |
| 1536 | |
Jason Evans | 8e3c3c6 | 2010-09-17 15:46:18 -0700 | [diff] [blame] | 1537 | int |
Jason Evans | 0a5489e | 2012-03-01 17:19:20 -0800 | [diff] [blame] | 1538 | je_sallocm(const void *ptr, size_t *rsize, int flags) |
Jason Evans | 8e3c3c6 | 2010-09-17 15:46:18 -0700 | [diff] [blame] | 1539 | { |
| 1540 | size_t sz; |
| 1541 | |
Jason Evans | 41b6afb | 2012-02-02 22:04:57 -0800 | [diff] [blame] | 1542 | assert(malloc_initialized || IS_INITIALIZER); |
Jason Evans | 8e3c3c6 | 2010-09-17 15:46:18 -0700 | [diff] [blame] | 1543 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 1544 | if (config_ivsalloc) |
Jason Evans | 122449b | 2012-04-06 00:35:09 -0700 | [diff] [blame] | 1545 | sz = ivsalloc(ptr, config_prof); |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 1546 | else { |
| 1547 | assert(ptr != NULL); |
Jason Evans | 122449b | 2012-04-06 00:35:09 -0700 | [diff] [blame] | 1548 | sz = isalloc(ptr, config_prof); |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 1549 | } |
Jason Evans | 8e3c3c6 | 2010-09-17 15:46:18 -0700 | [diff] [blame] | 1550 | assert(rsize != NULL); |
| 1551 | *rsize = sz; |
| 1552 | |
| 1553 | return (ALLOCM_SUCCESS); |
| 1554 | } |
| 1555 | |
Jason Evans | 8e3c3c6 | 2010-09-17 15:46:18 -0700 | [diff] [blame] | 1556 | int |
Jason Evans | 0a5489e | 2012-03-01 17:19:20 -0800 | [diff] [blame] | 1557 | je_dallocm(void *ptr, int flags) |
Jason Evans | 8e3c3c6 | 2010-09-17 15:46:18 -0700 | [diff] [blame] | 1558 | { |
Jason Evans | e4f7846 | 2010-10-22 10:45:59 -0700 | [diff] [blame] | 1559 | size_t usize; |
Jason Evans | 122449b | 2012-04-06 00:35:09 -0700 | [diff] [blame] | 1560 | size_t rzsize JEMALLOC_CC_SILENCE_INIT(0); |
Jason Evans | 8e3c3c6 | 2010-09-17 15:46:18 -0700 | [diff] [blame] | 1561 | |
| 1562 | assert(ptr != NULL); |
Jason Evans | 41b6afb | 2012-02-02 22:04:57 -0800 | [diff] [blame] | 1563 | assert(malloc_initialized || IS_INITIALIZER); |
Jason Evans | 8e3c3c6 | 2010-09-17 15:46:18 -0700 | [diff] [blame] | 1564 | |
Jason Evans | b147611 | 2012-04-05 13:36:17 -0700 | [diff] [blame] | 1565 | UTRACE(ptr, 0, 0); |
Jason Evans | 122449b | 2012-04-06 00:35:09 -0700 | [diff] [blame] | 1566 | if (config_stats || config_valgrind) |
| 1567 | usize = isalloc(ptr, config_prof); |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 1568 | if (config_prof && opt_prof) { |
Jason Evans | 122449b | 2012-04-06 00:35:09 -0700 | [diff] [blame] | 1569 | if (config_stats == false && config_valgrind == false) |
| 1570 | usize = isalloc(ptr, config_prof); |
Jason Evans | e4f7846 | 2010-10-22 10:45:59 -0700 | [diff] [blame] | 1571 | prof_free(ptr, usize); |
| 1572 | } |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 1573 | if (config_stats) |
Jason Evans | cd9a134 | 2012-03-21 18:33:03 -0700 | [diff] [blame] | 1574 | thread_allocated_tsd_get()->deallocated += usize; |
Jason Evans | 122449b | 2012-04-06 00:35:09 -0700 | [diff] [blame] | 1575 | if (config_valgrind && opt_valgrind) |
| 1576 | rzsize = p2rz(ptr); |
| 1577 | iqalloc(ptr); |
| 1578 | JEMALLOC_VALGRIND_FREE(ptr, rzsize); |
Jason Evans | 8e3c3c6 | 2010-09-17 15:46:18 -0700 | [diff] [blame] | 1579 | |
| 1580 | return (ALLOCM_SUCCESS); |
| 1581 | } |
| 1582 | |
Jason Evans | 7e15dab | 2012-02-29 12:56:37 -0800 | [diff] [blame] | 1583 | int |
Jason Evans | 0a5489e | 2012-03-01 17:19:20 -0800 | [diff] [blame] | 1584 | je_nallocm(size_t *rsize, size_t size, int flags) |
Jason Evans | 7e15dab | 2012-02-29 12:56:37 -0800 | [diff] [blame] | 1585 | { |
| 1586 | size_t usize; |
| 1587 | size_t alignment = (ZU(1) << (flags & ALLOCM_LG_ALIGN_MASK) |
| 1588 | & (SIZE_T_MAX-1)); |
| 1589 | |
| 1590 | assert(size != 0); |
| 1591 | |
| 1592 | if (malloc_init()) |
| 1593 | return (ALLOCM_ERR_OOM); |
| 1594 | |
Jason Evans | 5ff709c | 2012-04-11 18:13:45 -0700 | [diff] [blame] | 1595 | usize = (alignment == 0) ? s2u(size) : sa2u(size, alignment); |
Jason Evans | 7e15dab | 2012-02-29 12:56:37 -0800 | [diff] [blame] | 1596 | if (usize == 0) |
| 1597 | return (ALLOCM_ERR_OOM); |
| 1598 | |
| 1599 | if (rsize != NULL) |
| 1600 | *rsize = usize; |
| 1601 | return (ALLOCM_SUCCESS); |
| 1602 | } |
| 1603 | |
Jason Evans | 7e77eaf | 2012-03-02 17:47:37 -0800 | [diff] [blame] | 1604 | #endif |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 1605 | /* |
Jason Evans | 7e77eaf | 2012-03-02 17:47:37 -0800 | [diff] [blame] | 1606 | * End experimental functions. |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 1607 | */ |
| 1608 | /******************************************************************************/ |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 1609 | /* |
| 1610 | * The following functions are used by threading libraries for protection of |
Jason Evans | 28177d4 | 2010-09-20 11:24:24 -0700 | [diff] [blame] | 1611 | * malloc during fork(). |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 1612 | */ |
| 1613 | |
Jason Evans | 41b6afb | 2012-02-02 22:04:57 -0800 | [diff] [blame] | 1614 | #ifndef JEMALLOC_MUTEX_INIT_CB |
Jason Evans | 2dbecf1 | 2010-09-05 10:35:13 -0700 | [diff] [blame] | 1615 | void |
Jason Evans | 804c9ec | 2009-06-22 17:44:33 -0700 | [diff] [blame] | 1616 | jemalloc_prefork(void) |
Jason Evans | 41b6afb | 2012-02-02 22:04:57 -0800 | [diff] [blame] | 1617 | #else |
Mike Hommey | da99e31 | 2012-04-30 12:38:29 +0200 | [diff] [blame] | 1618 | JEMALLOC_EXPORT void |
Jason Evans | 41b6afb | 2012-02-02 22:04:57 -0800 | [diff] [blame] | 1619 | _malloc_prefork(void) |
| 1620 | #endif |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 1621 | { |
Jason Evans | fbbb624 | 2010-01-24 17:56:48 -0800 | [diff] [blame] | 1622 | unsigned i; |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 1623 | |
| 1624 | /* Acquire all mutexes in a safe order. */ |
Jason Evans | 4e2e3dd | 2012-03-13 16:31:41 -0700 | [diff] [blame] | 1625 | malloc_mutex_prefork(&arenas_lock); |
Jason Evans | fbbb624 | 2010-01-24 17:56:48 -0800 | [diff] [blame] | 1626 | for (i = 0; i < narenas; i++) { |
| 1627 | if (arenas[i] != NULL) |
Jason Evans | 4e2e3dd | 2012-03-13 16:31:41 -0700 | [diff] [blame] | 1628 | arena_prefork(arenas[i]); |
Jason Evans | fbbb624 | 2010-01-24 17:56:48 -0800 | [diff] [blame] | 1629 | } |
Jason Evans | 4e2e3dd | 2012-03-13 16:31:41 -0700 | [diff] [blame] | 1630 | base_prefork(); |
| 1631 | huge_prefork(); |
| 1632 | chunk_dss_prefork(); |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 1633 | } |
| 1634 | |
Jason Evans | 41b6afb | 2012-02-02 22:04:57 -0800 | [diff] [blame] | 1635 | #ifndef JEMALLOC_MUTEX_INIT_CB |
Jason Evans | 2dbecf1 | 2010-09-05 10:35:13 -0700 | [diff] [blame] | 1636 | void |
Jason Evans | 4e2e3dd | 2012-03-13 16:31:41 -0700 | [diff] [blame] | 1637 | jemalloc_postfork_parent(void) |
Jason Evans | 41b6afb | 2012-02-02 22:04:57 -0800 | [diff] [blame] | 1638 | #else |
Mike Hommey | da99e31 | 2012-04-30 12:38:29 +0200 | [diff] [blame] | 1639 | JEMALLOC_EXPORT void |
Jason Evans | 41b6afb | 2012-02-02 22:04:57 -0800 | [diff] [blame] | 1640 | _malloc_postfork(void) |
| 1641 | #endif |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 1642 | { |
| 1643 | unsigned i; |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 1644 | |
| 1645 | /* Release all mutexes, now that fork() has completed. */ |
Jason Evans | 4e2e3dd | 2012-03-13 16:31:41 -0700 | [diff] [blame] | 1646 | chunk_dss_postfork_parent(); |
| 1647 | huge_postfork_parent(); |
| 1648 | base_postfork_parent(); |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 1649 | for (i = 0; i < narenas; i++) { |
Jason Evans | fbbb624 | 2010-01-24 17:56:48 -0800 | [diff] [blame] | 1650 | if (arenas[i] != NULL) |
Jason Evans | 4e2e3dd | 2012-03-13 16:31:41 -0700 | [diff] [blame] | 1651 | arena_postfork_parent(arenas[i]); |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 1652 | } |
Jason Evans | 4e2e3dd | 2012-03-13 16:31:41 -0700 | [diff] [blame] | 1653 | malloc_mutex_postfork_parent(&arenas_lock); |
| 1654 | } |
| 1655 | |
| 1656 | void |
| 1657 | jemalloc_postfork_child(void) |
| 1658 | { |
| 1659 | unsigned i; |
| 1660 | |
| 1661 | /* Release all mutexes, now that fork() has completed. */ |
| 1662 | chunk_dss_postfork_child(); |
| 1663 | huge_postfork_child(); |
| 1664 | base_postfork_child(); |
| 1665 | for (i = 0; i < narenas; i++) { |
| 1666 | if (arenas[i] != NULL) |
| 1667 | arena_postfork_child(arenas[i]); |
| 1668 | } |
| 1669 | malloc_mutex_postfork_child(&arenas_lock); |
Jason Evans | 289053c | 2009-06-22 12:08:42 -0700 | [diff] [blame] | 1670 | } |
Jason Evans | 2dbecf1 | 2010-09-05 10:35:13 -0700 | [diff] [blame] | 1671 | |
| 1672 | /******************************************************************************/ |
Jason Evans | 01b3fe5 | 2012-04-03 09:28:00 -0700 | [diff] [blame] | 1673 | /* |
| 1674 | * The following functions are used for TLS allocation/deallocation in static |
| 1675 | * binaries on FreeBSD. The primary difference between these and i[mcd]alloc() |
| 1676 | * is that these avoid accessing TLS variables. |
| 1677 | */ |
| 1678 | |
| 1679 | static void * |
| 1680 | a0alloc(size_t size, bool zero) |
| 1681 | { |
| 1682 | |
| 1683 | if (malloc_init()) |
| 1684 | return (NULL); |
| 1685 | |
| 1686 | if (size == 0) |
| 1687 | size = 1; |
| 1688 | |
| 1689 | if (size <= arena_maxclass) |
| 1690 | return (arena_malloc(arenas[0], size, zero, false)); |
| 1691 | else |
| 1692 | return (huge_malloc(size, zero)); |
| 1693 | } |
| 1694 | |
| 1695 | void * |
| 1696 | a0malloc(size_t size) |
| 1697 | { |
| 1698 | |
| 1699 | return (a0alloc(size, false)); |
| 1700 | } |
| 1701 | |
| 1702 | void * |
| 1703 | a0calloc(size_t num, size_t size) |
| 1704 | { |
| 1705 | |
| 1706 | return (a0alloc(num * size, true)); |
| 1707 | } |
| 1708 | |
| 1709 | void |
| 1710 | a0free(void *ptr) |
| 1711 | { |
| 1712 | arena_chunk_t *chunk; |
| 1713 | |
| 1714 | if (ptr == NULL) |
| 1715 | return; |
| 1716 | |
| 1717 | chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); |
| 1718 | if (chunk != ptr) |
| 1719 | arena_dalloc(chunk->arena, chunk, ptr, false); |
| 1720 | else |
| 1721 | huge_dalloc(ptr, true); |
| 1722 | } |
| 1723 | |
| 1724 | /******************************************************************************/ |