Jason Evans | e476f8a | 2010-01-16 09:53:50 -0800 | [diff] [blame] | 1 | #define JEMALLOC_BASE_C_ |
Jason Evans | 376b152 | 2010-02-11 14:45:59 -0800 | [diff] [blame] | 2 | #include "jemalloc/internal/jemalloc_internal.h" |
Jason Evans | e476f8a | 2010-01-16 09:53:50 -0800 | [diff] [blame] | 3 | |
Jason Evans | 4201af0 | 2010-01-24 02:53:40 -0800 | [diff] [blame] | 4 | /******************************************************************************/ |
| 5 | /* Data. */ |
Jason Evans | e476f8a | 2010-01-16 09:53:50 -0800 | [diff] [blame] | 6 | |
Jason Evans | 4e2e3dd | 2012-03-13 16:31:41 -0700 | [diff] [blame] | 7 | static malloc_mutex_t base_mtx; |
Jason Evans | f500a10 | 2015-01-30 21:49:19 -0800 | [diff] [blame] | 8 | static extent_tree_t base_avail_szad; |
Jason Evans | e476f8a | 2010-01-16 09:53:50 -0800 | [diff] [blame] | 9 | static extent_node_t *base_nodes; |
Jason Evans | 4581b97 | 2014-11-27 17:22:36 -0200 | [diff] [blame] | 10 | static size_t base_allocated; |
Jason Evans | 4acd75a | 2015-03-23 17:25:57 -0700 | [diff] [blame] | 11 | static size_t base_resident; |
| 12 | static size_t base_mapped; |
Jason Evans | 4581b97 | 2014-11-27 17:22:36 -0200 | [diff] [blame] | 13 | |
Jason Evans | 4201af0 | 2010-01-24 02:53:40 -0800 | [diff] [blame] | 14 | /******************************************************************************/ |
Jason Evans | e476f8a | 2010-01-16 09:53:50 -0800 | [diff] [blame] | 15 | |
Jason Evans | f500a10 | 2015-01-30 21:49:19 -0800 | [diff] [blame] | 16 | static extent_node_t * |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 17 | base_node_try_alloc(tsdn_t *tsdn) |
Jason Evans | e476f8a | 2010-01-16 09:53:50 -0800 | [diff] [blame] | 18 | { |
Jason Evans | f500a10 | 2015-01-30 21:49:19 -0800 | [diff] [blame] | 19 | extent_node_t *node; |
Jason Evans | e476f8a | 2010-01-16 09:53:50 -0800 | [diff] [blame] | 20 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 21 | malloc_mutex_assert_owner(tsdn, &base_mtx); |
Jason Evans | d9394d0 | 2016-04-17 12:33:39 -0700 | [diff] [blame] | 22 | |
Jason Evans | f500a10 | 2015-01-30 21:49:19 -0800 | [diff] [blame] | 23 | if (base_nodes == NULL) |
| 24 | return (NULL); |
| 25 | node = base_nodes; |
| 26 | base_nodes = *(extent_node_t **)node; |
| 27 | JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(node, sizeof(extent_node_t)); |
| 28 | return (node); |
Jason Evans | e476f8a | 2010-01-16 09:53:50 -0800 | [diff] [blame] | 29 | } |
| 30 | |
Jason Evans | f500a10 | 2015-01-30 21:49:19 -0800 | [diff] [blame] | 31 | static void |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 32 | base_node_dalloc(tsdn_t *tsdn, extent_node_t *node) |
Jason Evans | f500a10 | 2015-01-30 21:49:19 -0800 | [diff] [blame] | 33 | { |
| 34 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 35 | malloc_mutex_assert_owner(tsdn, &base_mtx); |
Jason Evans | d9394d0 | 2016-04-17 12:33:39 -0700 | [diff] [blame] | 36 | |
Jason Evans | f500a10 | 2015-01-30 21:49:19 -0800 | [diff] [blame] | 37 | JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(node, sizeof(extent_node_t)); |
| 38 | *(extent_node_t **)node = base_nodes; |
| 39 | base_nodes = node; |
| 40 | } |
| 41 | |
Jason Evans | f500a10 | 2015-01-30 21:49:19 -0800 | [diff] [blame] | 42 | static extent_node_t * |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 43 | base_chunk_alloc(tsdn_t *tsdn, size_t minsize) |
Jason Evans | f500a10 | 2015-01-30 21:49:19 -0800 | [diff] [blame] | 44 | { |
| 45 | extent_node_t *node; |
| 46 | size_t csize, nsize; |
| 47 | void *addr; |
| 48 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 49 | malloc_mutex_assert_owner(tsdn, &base_mtx); |
Jason Evans | f500a10 | 2015-01-30 21:49:19 -0800 | [diff] [blame] | 50 | assert(minsize != 0); |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 51 | node = base_node_try_alloc(tsdn); |
Jason Evans | f500a10 | 2015-01-30 21:49:19 -0800 | [diff] [blame] | 52 | /* Allocate enough space to also carve a node out if necessary. */ |
| 53 | nsize = (node == NULL) ? CACHELINE_CEILING(sizeof(extent_node_t)) : 0; |
| 54 | csize = CHUNK_CEILING(minsize + nsize); |
| 55 | addr = chunk_alloc_base(csize); |
| 56 | if (addr == NULL) { |
| 57 | if (node != NULL) |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 58 | base_node_dalloc(tsdn, node); |
Jason Evans | f500a10 | 2015-01-30 21:49:19 -0800 | [diff] [blame] | 59 | return (NULL); |
| 60 | } |
Jason Evans | 4acd75a | 2015-03-23 17:25:57 -0700 | [diff] [blame] | 61 | base_mapped += csize; |
Jason Evans | f500a10 | 2015-01-30 21:49:19 -0800 | [diff] [blame] | 62 | if (node == NULL) { |
Jason Evans | 4acd75a | 2015-03-23 17:25:57 -0700 | [diff] [blame] | 63 | node = (extent_node_t *)addr; |
| 64 | addr = (void *)((uintptr_t)addr + nsize); |
Jason Evans | f500a10 | 2015-01-30 21:49:19 -0800 | [diff] [blame] | 65 | csize -= nsize; |
Jason Evans | 4acd75a | 2015-03-23 17:25:57 -0700 | [diff] [blame] | 66 | if (config_stats) { |
Jason Evans | f500a10 | 2015-01-30 21:49:19 -0800 | [diff] [blame] | 67 | base_allocated += nsize; |
Jason Evans | 4acd75a | 2015-03-23 17:25:57 -0700 | [diff] [blame] | 68 | base_resident += PAGE_CEILING(nsize); |
| 69 | } |
Jason Evans | f500a10 | 2015-01-30 21:49:19 -0800 | [diff] [blame] | 70 | } |
Jason Evans | b49a334 | 2015-07-28 11:28:19 -0400 | [diff] [blame] | 71 | extent_node_init(node, NULL, addr, csize, true, true); |
Jason Evans | f500a10 | 2015-01-30 21:49:19 -0800 | [diff] [blame] | 72 | return (node); |
| 73 | } |
| 74 | |
Jason Evans | cbf3a6d | 2015-02-11 12:24:27 -0800 | [diff] [blame] | 75 | /* |
| 76 | * base_alloc() guarantees demand-zeroed memory, in order to make multi-page |
| 77 | * sparse data structures such as radix tree nodes efficient with respect to |
| 78 | * physical memory usage. |
| 79 | */ |
| 80 | void * |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 81 | base_alloc(tsdn_t *tsdn, size_t size) |
Jason Evans | f500a10 | 2015-01-30 21:49:19 -0800 | [diff] [blame] | 82 | { |
| 83 | void *ret; |
Jason Evans | 5707d6f | 2015-03-06 17:14:05 -0800 | [diff] [blame] | 84 | size_t csize, usize; |
Jason Evans | f500a10 | 2015-01-30 21:49:19 -0800 | [diff] [blame] | 85 | extent_node_t *node; |
| 86 | extent_node_t key; |
| 87 | |
| 88 | /* |
| 89 | * Round size up to nearest multiple of the cacheline size, so that |
| 90 | * there is no chance of false cache line sharing. |
| 91 | */ |
| 92 | csize = CACHELINE_CEILING(size); |
| 93 | |
Jason Evans | 5707d6f | 2015-03-06 17:14:05 -0800 | [diff] [blame] | 94 | usize = s2u(csize); |
Jason Evans | 8fadb1a | 2015-08-04 10:49:46 -0700 | [diff] [blame] | 95 | extent_node_init(&key, NULL, NULL, usize, false, false); |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 96 | malloc_mutex_lock(tsdn, &base_mtx); |
Jason Evans | f500a10 | 2015-01-30 21:49:19 -0800 | [diff] [blame] | 97 | node = extent_tree_szad_nsearch(&base_avail_szad, &key); |
| 98 | if (node != NULL) { |
| 99 | /* Use existing space. */ |
| 100 | extent_tree_szad_remove(&base_avail_szad, node); |
| 101 | } else { |
| 102 | /* Try to allocate more space. */ |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 103 | node = base_chunk_alloc(tsdn, csize); |
Jason Evans | f500a10 | 2015-01-30 21:49:19 -0800 | [diff] [blame] | 104 | } |
Jason Evans | cbf3a6d | 2015-02-11 12:24:27 -0800 | [diff] [blame] | 105 | if (node == NULL) { |
| 106 | ret = NULL; |
| 107 | goto label_return; |
| 108 | } |
Jason Evans | f500a10 | 2015-01-30 21:49:19 -0800 | [diff] [blame] | 109 | |
Jason Evans | ee41ad4 | 2015-02-15 18:04:46 -0800 | [diff] [blame] | 110 | ret = extent_node_addr_get(node); |
| 111 | if (extent_node_size_get(node) > csize) { |
| 112 | extent_node_addr_set(node, (void *)((uintptr_t)ret + csize)); |
| 113 | extent_node_size_set(node, extent_node_size_get(node) - csize); |
Jason Evans | f500a10 | 2015-01-30 21:49:19 -0800 | [diff] [blame] | 114 | extent_tree_szad_insert(&base_avail_szad, node); |
| 115 | } else |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 116 | base_node_dalloc(tsdn, node); |
Jason Evans | 4acd75a | 2015-03-23 17:25:57 -0700 | [diff] [blame] | 117 | if (config_stats) { |
Jason Evans | f500a10 | 2015-01-30 21:49:19 -0800 | [diff] [blame] | 118 | base_allocated += csize; |
Jason Evans | 4acd75a | 2015-03-23 17:25:57 -0700 | [diff] [blame] | 119 | /* |
| 120 | * Add one PAGE to base_resident for every page boundary that is |
| 121 | * crossed by the new allocation. |
| 122 | */ |
| 123 | base_resident += PAGE_CEILING((uintptr_t)ret + csize) - |
| 124 | PAGE_CEILING((uintptr_t)ret); |
| 125 | } |
Jason Evans | 4f6f2b1 | 2015-06-22 14:38:06 -0700 | [diff] [blame] | 126 | JEMALLOC_VALGRIND_MAKE_MEM_DEFINED(ret, csize); |
Jason Evans | cbf3a6d | 2015-02-11 12:24:27 -0800 | [diff] [blame] | 127 | label_return: |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 128 | malloc_mutex_unlock(tsdn, &base_mtx); |
Jason Evans | 41b6afb | 2012-02-02 22:04:57 -0800 | [diff] [blame] | 129 | return (ret); |
| 130 | } |
| 131 | |
Jason Evans | 4acd75a | 2015-03-23 17:25:57 -0700 | [diff] [blame] | 132 | void |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 133 | base_stats_get(tsdn_t *tsdn, size_t *allocated, size_t *resident, |
| 134 | size_t *mapped) |
Jason Evans | 4581b97 | 2014-11-27 17:22:36 -0200 | [diff] [blame] | 135 | { |
Jason Evans | 4581b97 | 2014-11-27 17:22:36 -0200 | [diff] [blame] | 136 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 137 | malloc_mutex_lock(tsdn, &base_mtx); |
Jason Evans | 56048ba | 2015-05-28 15:03:58 -0700 | [diff] [blame] | 138 | assert(base_allocated <= base_resident); |
| 139 | assert(base_resident <= base_mapped); |
Jason Evans | 4acd75a | 2015-03-23 17:25:57 -0700 | [diff] [blame] | 140 | *allocated = base_allocated; |
| 141 | *resident = base_resident; |
| 142 | *mapped = base_mapped; |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 143 | malloc_mutex_unlock(tsdn, &base_mtx); |
Jason Evans | 4581b97 | 2014-11-27 17:22:36 -0200 | [diff] [blame] | 144 | } |
| 145 | |
Jason Evans | e476f8a | 2010-01-16 09:53:50 -0800 | [diff] [blame] | 146 | bool |
| 147 | base_boot(void) |
| 148 | { |
| 149 | |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 150 | if (malloc_mutex_init(&base_mtx, "base", WITNESS_RANK_BASE)) |
Jason Evans | e476f8a | 2010-01-16 09:53:50 -0800 | [diff] [blame] | 151 | return (true); |
Jason Evans | f500a10 | 2015-01-30 21:49:19 -0800 | [diff] [blame] | 152 | extent_tree_szad_new(&base_avail_szad); |
| 153 | base_nodes = NULL; |
Jason Evans | e476f8a | 2010-01-16 09:53:50 -0800 | [diff] [blame] | 154 | |
| 155 | return (false); |
| 156 | } |
Jason Evans | 4e2e3dd | 2012-03-13 16:31:41 -0700 | [diff] [blame] | 157 | |
| 158 | void |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 159 | base_prefork(tsdn_t *tsdn) |
Jason Evans | 4e2e3dd | 2012-03-13 16:31:41 -0700 | [diff] [blame] | 160 | { |
| 161 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 162 | malloc_mutex_prefork(tsdn, &base_mtx); |
Jason Evans | 4e2e3dd | 2012-03-13 16:31:41 -0700 | [diff] [blame] | 163 | } |
| 164 | |
| 165 | void |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 166 | base_postfork_parent(tsdn_t *tsdn) |
Jason Evans | 4e2e3dd | 2012-03-13 16:31:41 -0700 | [diff] [blame] | 167 | { |
| 168 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 169 | malloc_mutex_postfork_parent(tsdn, &base_mtx); |
Jason Evans | 4e2e3dd | 2012-03-13 16:31:41 -0700 | [diff] [blame] | 170 | } |
| 171 | |
| 172 | void |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 173 | base_postfork_child(tsdn_t *tsdn) |
Jason Evans | 4e2e3dd | 2012-03-13 16:31:41 -0700 | [diff] [blame] | 174 | { |
| 175 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 176 | malloc_mutex_postfork_child(tsdn, &base_mtx); |
Jason Evans | 4e2e3dd | 2012-03-13 16:31:41 -0700 | [diff] [blame] | 177 | } |