Jason Evans | 2dbecf1 | 2010-09-05 10:35:13 -0700 | [diff] [blame] | 1 | #include "jemalloc/internal/jemalloc_internal.h" |
| 2 | #ifndef JEMALLOC_ZONE |
| 3 | # error "This source file is for zones on Darwin (OS X)." |
| 4 | #endif |
| 5 | |
Mike Hommey | 3c2ba0d | 2012-03-27 14:20:13 +0200 | [diff] [blame] | 6 | /* |
| 7 | * The malloc_default_purgeable_zone function is only available on >= 10.6. |
| 8 | * We need to check whether it is present at runtime, thus the weak_import. |
| 9 | */ |
| 10 | extern malloc_zone_t *malloc_default_purgeable_zone(void) |
| 11 | JEMALLOC_ATTR(weak_import); |
| 12 | |
Jason Evans | 2dbecf1 | 2010-09-05 10:35:13 -0700 | [diff] [blame] | 13 | /******************************************************************************/ |
| 14 | /* Data. */ |
| 15 | |
Mike Hommey | 154829d | 2012-03-20 18:01:38 +0100 | [diff] [blame] | 16 | static malloc_zone_t zone; |
| 17 | static struct malloc_introspection_t zone_introspect; |
Jason Evans | 2dbecf1 | 2010-09-05 10:35:13 -0700 | [diff] [blame] | 18 | |
| 19 | /******************************************************************************/ |
| 20 | /* Function prototypes for non-inline static functions. */ |
| 21 | |
| 22 | static size_t zone_size(malloc_zone_t *zone, void *ptr); |
| 23 | static void *zone_malloc(malloc_zone_t *zone, size_t size); |
| 24 | static void *zone_calloc(malloc_zone_t *zone, size_t num, size_t size); |
| 25 | static void *zone_valloc(malloc_zone_t *zone, size_t size); |
| 26 | static void zone_free(malloc_zone_t *zone, void *ptr); |
| 27 | static void *zone_realloc(malloc_zone_t *zone, void *ptr, size_t size); |
Mike Hommey | 154829d | 2012-03-20 18:01:38 +0100 | [diff] [blame] | 28 | #if (JEMALLOC_ZONE_VERSION >= 5) |
Jason Evans | 2dbecf1 | 2010-09-05 10:35:13 -0700 | [diff] [blame] | 29 | static void *zone_memalign(malloc_zone_t *zone, size_t alignment, |
Mike Hommey | 154829d | 2012-03-20 18:01:38 +0100 | [diff] [blame] | 30 | #endif |
| 31 | #if (JEMALLOC_ZONE_VERSION >= 6) |
Jason Evans | 2dbecf1 | 2010-09-05 10:35:13 -0700 | [diff] [blame] | 32 | size_t size); |
| 33 | static void zone_free_definite_size(malloc_zone_t *zone, void *ptr, |
| 34 | size_t size); |
| 35 | #endif |
| 36 | static void *zone_destroy(malloc_zone_t *zone); |
| 37 | static size_t zone_good_size(malloc_zone_t *zone, size_t size); |
| 38 | static void zone_force_lock(malloc_zone_t *zone); |
| 39 | static void zone_force_unlock(malloc_zone_t *zone); |
Jason Evans | 2dbecf1 | 2010-09-05 10:35:13 -0700 | [diff] [blame] | 40 | |
| 41 | /******************************************************************************/ |
| 42 | /* |
| 43 | * Functions. |
| 44 | */ |
| 45 | |
| 46 | static size_t |
| 47 | zone_size(malloc_zone_t *zone, void *ptr) |
| 48 | { |
| 49 | |
| 50 | /* |
| 51 | * There appear to be places within Darwin (such as setenv(3)) that |
| 52 | * cause calls to this function with pointers that *no* zone owns. If |
| 53 | * we knew that all pointers were owned by *some* zone, we could split |
| 54 | * our zone into two parts, and use one as the default allocator and |
| 55 | * the other as the default deallocator/reallocator. Since that will |
| 56 | * not work in practice, we must check all pointers to assure that they |
| 57 | * reside within a mapped chunk before determining size. |
| 58 | */ |
Jason Evans | 122449b | 2012-04-06 00:35:09 -0700 | [diff] [blame] | 59 | return (ivsalloc(ptr, config_prof)); |
Jason Evans | 2dbecf1 | 2010-09-05 10:35:13 -0700 | [diff] [blame] | 60 | } |
| 61 | |
| 62 | static void * |
| 63 | zone_malloc(malloc_zone_t *zone, size_t size) |
| 64 | { |
| 65 | |
Jason Evans | 0a5489e | 2012-03-01 17:19:20 -0800 | [diff] [blame] | 66 | return (je_malloc(size)); |
Jason Evans | 2dbecf1 | 2010-09-05 10:35:13 -0700 | [diff] [blame] | 67 | } |
| 68 | |
| 69 | static void * |
| 70 | zone_calloc(malloc_zone_t *zone, size_t num, size_t size) |
| 71 | { |
| 72 | |
Jason Evans | 0a5489e | 2012-03-01 17:19:20 -0800 | [diff] [blame] | 73 | return (je_calloc(num, size)); |
Jason Evans | 2dbecf1 | 2010-09-05 10:35:13 -0700 | [diff] [blame] | 74 | } |
| 75 | |
| 76 | static void * |
| 77 | zone_valloc(malloc_zone_t *zone, size_t size) |
| 78 | { |
| 79 | void *ret = NULL; /* Assignment avoids useless compiler warning. */ |
| 80 | |
Jason Evans | ae4c7b4 | 2012-04-02 07:04:34 -0700 | [diff] [blame] | 81 | je_posix_memalign(&ret, PAGE, size); |
Jason Evans | 2dbecf1 | 2010-09-05 10:35:13 -0700 | [diff] [blame] | 82 | |
| 83 | return (ret); |
| 84 | } |
| 85 | |
| 86 | static void |
| 87 | zone_free(malloc_zone_t *zone, void *ptr) |
| 88 | { |
| 89 | |
Jason Evans | 122449b | 2012-04-06 00:35:09 -0700 | [diff] [blame] | 90 | if (ivsalloc(ptr, config_prof) != 0) { |
Mike Hommey | 5b3db09 | 2012-03-26 18:39:35 +0200 | [diff] [blame] | 91 | je_free(ptr); |
| 92 | return; |
| 93 | } |
| 94 | |
| 95 | free(ptr); |
Jason Evans | 2dbecf1 | 2010-09-05 10:35:13 -0700 | [diff] [blame] | 96 | } |
| 97 | |
| 98 | static void * |
| 99 | zone_realloc(malloc_zone_t *zone, void *ptr, size_t size) |
| 100 | { |
| 101 | |
Jason Evans | 122449b | 2012-04-06 00:35:09 -0700 | [diff] [blame] | 102 | if (ivsalloc(ptr, config_prof) != 0) |
Mike Hommey | 5b3db09 | 2012-03-26 18:39:35 +0200 | [diff] [blame] | 103 | return (je_realloc(ptr, size)); |
| 104 | |
| 105 | return (realloc(ptr, size)); |
Jason Evans | 2dbecf1 | 2010-09-05 10:35:13 -0700 | [diff] [blame] | 106 | } |
| 107 | |
Mike Hommey | 154829d | 2012-03-20 18:01:38 +0100 | [diff] [blame] | 108 | #if (JEMALLOC_ZONE_VERSION >= 5) |
Jason Evans | 2dbecf1 | 2010-09-05 10:35:13 -0700 | [diff] [blame] | 109 | static void * |
| 110 | zone_memalign(malloc_zone_t *zone, size_t alignment, size_t size) |
| 111 | { |
| 112 | void *ret = NULL; /* Assignment avoids useless compiler warning. */ |
| 113 | |
Jason Evans | 0a5489e | 2012-03-01 17:19:20 -0800 | [diff] [blame] | 114 | je_posix_memalign(&ret, alignment, size); |
Jason Evans | 2dbecf1 | 2010-09-05 10:35:13 -0700 | [diff] [blame] | 115 | |
| 116 | return (ret); |
| 117 | } |
Mike Hommey | 154829d | 2012-03-20 18:01:38 +0100 | [diff] [blame] | 118 | #endif |
Jason Evans | 2dbecf1 | 2010-09-05 10:35:13 -0700 | [diff] [blame] | 119 | |
Mike Hommey | 154829d | 2012-03-20 18:01:38 +0100 | [diff] [blame] | 120 | #if (JEMALLOC_ZONE_VERSION >= 6) |
Jason Evans | 2dbecf1 | 2010-09-05 10:35:13 -0700 | [diff] [blame] | 121 | static void |
| 122 | zone_free_definite_size(malloc_zone_t *zone, void *ptr, size_t size) |
| 123 | { |
| 124 | |
Jason Evans | 122449b | 2012-04-06 00:35:09 -0700 | [diff] [blame] | 125 | if (ivsalloc(ptr, config_prof) != 0) { |
| 126 | assert(ivsalloc(ptr, config_prof) == size); |
Mike Hommey | 5b3db09 | 2012-03-26 18:39:35 +0200 | [diff] [blame] | 127 | je_free(ptr); |
| 128 | return; |
| 129 | } |
| 130 | |
| 131 | free(ptr); |
Jason Evans | 2dbecf1 | 2010-09-05 10:35:13 -0700 | [diff] [blame] | 132 | } |
| 133 | #endif |
| 134 | |
| 135 | static void * |
| 136 | zone_destroy(malloc_zone_t *zone) |
| 137 | { |
| 138 | |
| 139 | /* This function should never be called. */ |
Jason Evans | 6556e28 | 2013-10-21 14:56:27 -0700 | [diff] [blame] | 140 | not_reached(); |
Jason Evans | 2dbecf1 | 2010-09-05 10:35:13 -0700 | [diff] [blame] | 141 | return (NULL); |
| 142 | } |
| 143 | |
| 144 | static size_t |
| 145 | zone_good_size(malloc_zone_t *zone, size_t size) |
| 146 | { |
Jason Evans | 2dbecf1 | 2010-09-05 10:35:13 -0700 | [diff] [blame] | 147 | |
Jason Evans | 166a745 | 2012-02-29 12:58:39 -0800 | [diff] [blame] | 148 | if (size == 0) |
| 149 | size = 1; |
| 150 | return (s2u(size)); |
Jason Evans | 2dbecf1 | 2010-09-05 10:35:13 -0700 | [diff] [blame] | 151 | } |
| 152 | |
| 153 | static void |
| 154 | zone_force_lock(malloc_zone_t *zone) |
| 155 | { |
| 156 | |
| 157 | if (isthreaded) |
| 158 | jemalloc_prefork(); |
| 159 | } |
| 160 | |
| 161 | static void |
| 162 | zone_force_unlock(malloc_zone_t *zone) |
| 163 | { |
| 164 | |
| 165 | if (isthreaded) |
Mike Hommey | f4d0fc3 | 2012-03-20 18:03:09 +0100 | [diff] [blame] | 166 | jemalloc_postfork_parent(); |
Jason Evans | 2dbecf1 | 2010-09-05 10:35:13 -0700 | [diff] [blame] | 167 | } |
| 168 | |
Mike Hommey | 80b2593 | 2012-04-02 09:04:54 +0200 | [diff] [blame] | 169 | JEMALLOC_ATTR(constructor) |
Mike Hommey | 71a93b8 | 2012-03-27 14:20:12 +0200 | [diff] [blame] | 170 | void |
| 171 | register_zone(void) |
Jason Evans | 2dbecf1 | 2010-09-05 10:35:13 -0700 | [diff] [blame] | 172 | { |
| 173 | |
Mike Hommey | 847ff22 | 2012-10-23 08:42:48 +0200 | [diff] [blame] | 174 | /* |
| 175 | * If something else replaced the system default zone allocator, don't |
| 176 | * register jemalloc's. |
| 177 | */ |
| 178 | malloc_zone_t *default_zone = malloc_default_zone(); |
Mike Hommey | 6f533c1 | 2014-06-10 18:18:22 +0900 | [diff] [blame] | 179 | malloc_zone_t *purgeable_zone = NULL; |
Mike Hommey | 847ff22 | 2012-10-23 08:42:48 +0200 | [diff] [blame] | 180 | if (!default_zone->zone_name || |
| 181 | strcmp(default_zone->zone_name, "DefaultMallocZone") != 0) { |
| 182 | return; |
| 183 | } |
| 184 | |
Jason Evans | 2dbecf1 | 2010-09-05 10:35:13 -0700 | [diff] [blame] | 185 | zone.size = (void *)zone_size; |
| 186 | zone.malloc = (void *)zone_malloc; |
| 187 | zone.calloc = (void *)zone_calloc; |
| 188 | zone.valloc = (void *)zone_valloc; |
| 189 | zone.free = (void *)zone_free; |
| 190 | zone.realloc = (void *)zone_realloc; |
| 191 | zone.destroy = (void *)zone_destroy; |
| 192 | zone.zone_name = "jemalloc_zone"; |
| 193 | zone.batch_malloc = NULL; |
| 194 | zone.batch_free = NULL; |
| 195 | zone.introspect = &zone_introspect; |
| 196 | zone.version = JEMALLOC_ZONE_VERSION; |
Mike Hommey | 154829d | 2012-03-20 18:01:38 +0100 | [diff] [blame] | 197 | #if (JEMALLOC_ZONE_VERSION >= 5) |
Jason Evans | 2dbecf1 | 2010-09-05 10:35:13 -0700 | [diff] [blame] | 198 | zone.memalign = zone_memalign; |
Mike Hommey | 154829d | 2012-03-20 18:01:38 +0100 | [diff] [blame] | 199 | #endif |
| 200 | #if (JEMALLOC_ZONE_VERSION >= 6) |
Jason Evans | 2dbecf1 | 2010-09-05 10:35:13 -0700 | [diff] [blame] | 201 | zone.free_definite_size = zone_free_definite_size; |
| 202 | #endif |
Mike Hommey | 154829d | 2012-03-20 18:01:38 +0100 | [diff] [blame] | 203 | #if (JEMALLOC_ZONE_VERSION >= 8) |
| 204 | zone.pressure_relief = NULL; |
| 205 | #endif |
Jason Evans | 2dbecf1 | 2010-09-05 10:35:13 -0700 | [diff] [blame] | 206 | |
| 207 | zone_introspect.enumerator = NULL; |
| 208 | zone_introspect.good_size = (void *)zone_good_size; |
| 209 | zone_introspect.check = NULL; |
| 210 | zone_introspect.print = NULL; |
| 211 | zone_introspect.log = NULL; |
| 212 | zone_introspect.force_lock = (void *)zone_force_lock; |
| 213 | zone_introspect.force_unlock = (void *)zone_force_unlock; |
| 214 | zone_introspect.statistics = NULL; |
| 215 | #if (JEMALLOC_ZONE_VERSION >= 6) |
| 216 | zone_introspect.zone_locked = NULL; |
| 217 | #endif |
Mike Hommey | 154829d | 2012-03-20 18:01:38 +0100 | [diff] [blame] | 218 | #if (JEMALLOC_ZONE_VERSION >= 7) |
| 219 | zone_introspect.enable_discharge_checking = NULL; |
| 220 | zone_introspect.disable_discharge_checking = NULL; |
| 221 | zone_introspect.discharge = NULL; |
| 222 | #ifdef __BLOCKS__ |
| 223 | zone_introspect.enumerate_discharged_pointers = NULL; |
| 224 | #else |
| 225 | zone_introspect.enumerate_unavailable_without_blocks = NULL; |
| 226 | #endif |
| 227 | #endif |
Mike Hommey | 71a93b8 | 2012-03-27 14:20:12 +0200 | [diff] [blame] | 228 | |
Mike Hommey | 3c2ba0d | 2012-03-27 14:20:13 +0200 | [diff] [blame] | 229 | /* |
| 230 | * The default purgeable zone is created lazily by OSX's libc. It uses |
| 231 | * the default zone when it is created for "small" allocations |
| 232 | * (< 15 KiB), but assumes the default zone is a scalable_zone. This |
| 233 | * obviously fails when the default zone is the jemalloc zone, so |
| 234 | * malloc_default_purgeable_zone is called beforehand so that the |
| 235 | * default purgeable zone is created when the default zone is still |
| 236 | * a scalable_zone. As purgeable zones only exist on >= 10.6, we need |
| 237 | * to check for the existence of malloc_default_purgeable_zone() at |
| 238 | * run time. |
| 239 | */ |
| 240 | if (malloc_default_purgeable_zone != NULL) |
Mike Hommey | 6f533c1 | 2014-06-10 18:18:22 +0900 | [diff] [blame] | 241 | purgeable_zone = malloc_default_purgeable_zone(); |
Mike Hommey | 3c2ba0d | 2012-03-27 14:20:13 +0200 | [diff] [blame] | 242 | |
| 243 | /* Register the custom zone. At this point it won't be the default. */ |
Mike Hommey | 71a93b8 | 2012-03-27 14:20:12 +0200 | [diff] [blame] | 244 | malloc_zone_register(&zone); |
| 245 | |
Mike Hommey | 71a93b8 | 2012-03-27 14:20:12 +0200 | [diff] [blame] | 246 | do { |
Mike Hommey | 847ff22 | 2012-10-23 08:42:48 +0200 | [diff] [blame] | 247 | default_zone = malloc_default_zone(); |
Mike Hommey | 6f533c1 | 2014-06-10 18:18:22 +0900 | [diff] [blame] | 248 | /* |
| 249 | * Unregister and reregister the default zone. On OSX >= 10.6, |
| 250 | * unregistering takes the last registered zone and places it |
| 251 | * at the location of the specified zone. Unregistering the |
| 252 | * default zone thus makes the last registered one the default. |
| 253 | * On OSX < 10.6, unregistering shifts all registered zones. |
| 254 | * The first registered zone then becomes the default. |
| 255 | */ |
Mike Hommey | 71a93b8 | 2012-03-27 14:20:12 +0200 | [diff] [blame] | 256 | malloc_zone_unregister(default_zone); |
| 257 | malloc_zone_register(default_zone); |
Mike Hommey | 6f533c1 | 2014-06-10 18:18:22 +0900 | [diff] [blame] | 258 | /* |
| 259 | * On OSX 10.6, having the default purgeable zone appear before |
| 260 | * the default zone makes some things crash because it thinks it |
Jason Evans | c21b05e | 2014-09-04 22:27:26 -0700 | [diff] [blame] | 261 | * owns the default zone allocated pointers. We thus |
| 262 | * unregister/re-register it in order to ensure it's always |
| 263 | * after the default zone. On OSX < 10.6, there is no purgeable |
| 264 | * zone, so this does nothing. On OSX >= 10.6, unregistering |
| 265 | * replaces the purgeable zone with the last registered zone |
Jason Evans | e12eaf9 | 2014-12-08 14:40:14 -0800 | [diff] [blame^] | 266 | * above, i.e. the default zone. Registering it again then puts |
Jason Evans | c21b05e | 2014-09-04 22:27:26 -0700 | [diff] [blame] | 267 | * it at the end, obviously after the default zone. |
Mike Hommey | 6f533c1 | 2014-06-10 18:18:22 +0900 | [diff] [blame] | 268 | */ |
| 269 | if (purgeable_zone) { |
| 270 | malloc_zone_unregister(purgeable_zone); |
| 271 | malloc_zone_register(purgeable_zone); |
| 272 | } |
Mike Hommey | 71a93b8 | 2012-03-27 14:20:12 +0200 | [diff] [blame] | 273 | } while (malloc_default_zone() != &zone); |
Jason Evans | 2dbecf1 | 2010-09-05 10:35:13 -0700 | [diff] [blame] | 274 | } |