Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1 | #define JEMALLOC_PROF_C_ |
Jason Evans | 376b152 | 2010-02-11 14:45:59 -0800 | [diff] [blame] | 2 | #include "jemalloc/internal/jemalloc_internal.h" |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 3 | /******************************************************************************/ |
| 4 | |
| 5 | #ifdef JEMALLOC_PROF_LIBUNWIND |
| 6 | #define UNW_LOCAL_ONLY |
| 7 | #include <libunwind.h> |
| 8 | #endif |
| 9 | |
Jason Evans | 77f350b | 2011-03-15 22:23:12 -0700 | [diff] [blame] | 10 | #ifdef JEMALLOC_PROF_LIBGCC |
| 11 | #include <unwind.h> |
| 12 | #endif |
| 13 | |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 14 | /******************************************************************************/ |
| 15 | /* Data. */ |
| 16 | |
| 17 | bool opt_prof = false; |
Jason Evans | f18c982 | 2010-03-31 18:43:24 -0700 | [diff] [blame] | 18 | bool opt_prof_active = true; |
Jason Evans | fc12c0b | 2014-10-03 23:25:30 -0700 | [diff] [blame] | 19 | bool opt_prof_thread_active_init = true; |
Jason Evans | b9477e7 | 2010-03-01 20:15:26 -0800 | [diff] [blame] | 20 | size_t opt_lg_prof_sample = LG_PROF_SAMPLE_DEFAULT; |
Jason Evans | a02fc08 | 2010-03-31 17:35:51 -0700 | [diff] [blame] | 21 | ssize_t opt_lg_prof_interval = LG_PROF_INTERVAL_DEFAULT; |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 22 | bool opt_prof_gdump = false; |
Jason Evans | 57efa7b | 2014-10-08 17:57:19 -0700 | [diff] [blame] | 23 | bool opt_prof_final = false; |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 24 | bool opt_prof_leak = false; |
Jason Evans | 0b25fe7 | 2012-04-17 16:39:33 -0700 | [diff] [blame] | 25 | bool opt_prof_accum = false; |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 26 | char opt_prof_prefix[ |
| 27 | /* Minimize memory bloat for non-prof builds. */ |
| 28 | #ifdef JEMALLOC_PROF |
| 29 | PATH_MAX + |
| 30 | #endif |
Jason Evans | eefdd02 | 2014-01-16 18:04:30 -0800 | [diff] [blame] | 31 | 1]; |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 32 | |
Jason Evans | fc12c0b | 2014-10-03 23:25:30 -0700 | [diff] [blame] | 33 | /* |
| 34 | * Initialized as opt_prof_active, and accessed via |
| 35 | * prof_active_[gs]et{_unlocked,}(). |
| 36 | */ |
| 37 | bool prof_active; |
| 38 | static malloc_mutex_t prof_active_mtx; |
| 39 | |
| 40 | /* |
| 41 | * Initialized as opt_prof_thread_active_init, and accessed via |
| 42 | * prof_thread_active_init_[gs]et(). |
| 43 | */ |
| 44 | static bool prof_thread_active_init; |
| 45 | static malloc_mutex_t prof_thread_active_init_mtx; |
| 46 | |
Jason Evans | 5b8ed5b | 2015-01-25 21:16:57 -0800 | [diff] [blame] | 47 | /* |
| 48 | * Initialized as opt_prof_gdump, and accessed via |
| 49 | * prof_gdump_[gs]et{_unlocked,}(). |
| 50 | */ |
| 51 | bool prof_gdump_val; |
| 52 | static malloc_mutex_t prof_gdump_mtx; |
| 53 | |
Jason Evans | a3b3386 | 2012-11-13 12:56:27 -0800 | [diff] [blame] | 54 | uint64_t prof_interval = 0; |
Jason Evans | d34f9e7 | 2010-02-11 13:19:21 -0800 | [diff] [blame] | 55 | |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 56 | size_t lg_prof_sample; |
| 57 | |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 58 | /* |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 59 | * Table of mutexes that are shared among gctx's. These are leaf locks, so |
| 60 | * there is no problem with using them for more than one gctx at the same time. |
| 61 | * The primary motivation for this sharing though is that gctx's are ephemeral, |
Jason Evans | 6da5418 | 2012-03-23 18:05:51 -0700 | [diff] [blame] | 62 | * and destroying mutexes causes complications for systems that allocate when |
| 63 | * creating/destroying mutexes. |
| 64 | */ |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 65 | static malloc_mutex_t *gctx_locks; |
| 66 | static unsigned cum_gctxs; /* Atomic counter. */ |
Jason Evans | 6da5418 | 2012-03-23 18:05:51 -0700 | [diff] [blame] | 67 | |
| 68 | /* |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 69 | * Table of mutexes that are shared among tdata's. No operations require |
| 70 | * holding multiple tdata locks, so there is no problem with using them for more |
| 71 | * than one tdata at the same time, even though a gctx lock may be acquired |
| 72 | * while holding a tdata lock. |
| 73 | */ |
| 74 | static malloc_mutex_t *tdata_locks; |
| 75 | |
| 76 | /* |
| 77 | * Global hash of (prof_bt_t *)-->(prof_gctx_t *). This is the master data |
Jason Evans | a881cd2 | 2010-10-02 15:18:50 -0700 | [diff] [blame] | 78 | * structure that knows about all backtraces currently captured. |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 79 | */ |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 80 | static ckh_t bt2gctx; |
| 81 | static malloc_mutex_t bt2gctx_mtx; |
| 82 | |
| 83 | /* |
| 84 | * Tree of all extant prof_tdata_t structures, regardless of state, |
| 85 | * {attached,detached,expired}. |
| 86 | */ |
| 87 | static prof_tdata_tree_t tdatas; |
| 88 | static malloc_mutex_t tdatas_mtx; |
| 89 | |
| 90 | static uint64_t next_thr_uid; |
Jason Evans | 9d8f3d2 | 2014-09-11 18:06:30 -0700 | [diff] [blame] | 91 | static malloc_mutex_t next_thr_uid_mtx; |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 92 | |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 93 | static malloc_mutex_t prof_dump_seq_mtx; |
| 94 | static uint64_t prof_dump_seq; |
| 95 | static uint64_t prof_dump_iseq; |
| 96 | static uint64_t prof_dump_mseq; |
| 97 | static uint64_t prof_dump_useq; |
| 98 | |
| 99 | /* |
| 100 | * This buffer is rather large for stack allocation, so use a single buffer for |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 101 | * all profile dumps. |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 102 | */ |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 103 | static malloc_mutex_t prof_dump_mtx; |
| 104 | static char prof_dump_buf[ |
| 105 | /* Minimize memory bloat for non-prof builds. */ |
| 106 | #ifdef JEMALLOC_PROF |
| 107 | PROF_DUMP_BUFSIZE |
| 108 | #else |
| 109 | 1 |
| 110 | #endif |
| 111 | ]; |
Jason Evans | 42ce80e | 2016-02-25 20:51:00 -0800 | [diff] [blame] | 112 | static size_t prof_dump_buf_end; |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 113 | static int prof_dump_fd; |
| 114 | |
| 115 | /* Do not dump any profiles until bootstrapping is complete. */ |
| 116 | static bool prof_booted = false; |
| 117 | |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 118 | /******************************************************************************/ |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 119 | /* |
| 120 | * Function prototypes for static functions that are referenced prior to |
| 121 | * definition. |
| 122 | */ |
| 123 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 124 | static bool prof_tctx_should_destroy(tsdn_t *tsdn, prof_tctx_t *tctx); |
Jason Evans | 5460aa6 | 2014-09-22 21:09:23 -0700 | [diff] [blame] | 125 | static void prof_tctx_destroy(tsd_t *tsd, prof_tctx_t *tctx); |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 126 | static bool prof_tdata_should_destroy(tsdn_t *tsdn, prof_tdata_t *tdata, |
Jason Evans | f04a0be | 2014-10-04 15:03:49 -0700 | [diff] [blame] | 127 | bool even_if_attached); |
Jason Evans | b54d160 | 2016-10-20 23:59:12 -0700 | [diff] [blame^] | 128 | static void prof_tdata_destroy(tsd_t *tsd, prof_tdata_t *tdata, |
Jason Evans | f04a0be | 2014-10-04 15:03:49 -0700 | [diff] [blame] | 129 | bool even_if_attached); |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 130 | static char *prof_thread_name_alloc(tsdn_t *tsdn, const char *thread_name); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 131 | |
| 132 | /******************************************************************************/ |
| 133 | /* Red-black trees. */ |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 134 | |
Jason Evans | 3a81cbd | 2014-08-16 12:58:55 -0700 | [diff] [blame] | 135 | JEMALLOC_INLINE_C int |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 136 | prof_tctx_comp(const prof_tctx_t *a, const prof_tctx_t *b) |
Jason Evans | 3a81cbd | 2014-08-16 12:58:55 -0700 | [diff] [blame] | 137 | { |
Jason Evans | 04211e2 | 2015-03-16 15:11:06 -0700 | [diff] [blame] | 138 | uint64_t a_thr_uid = a->thr_uid; |
| 139 | uint64_t b_thr_uid = b->thr_uid; |
| 140 | int ret = (a_thr_uid > b_thr_uid) - (a_thr_uid < b_thr_uid); |
Jason Evans | d69964b | 2015-03-12 16:25:18 -0700 | [diff] [blame] | 141 | if (ret == 0) { |
Jason Evans | a00b107 | 2015-09-09 23:16:10 -0700 | [diff] [blame] | 142 | uint64_t a_thr_discrim = a->thr_discrim; |
| 143 | uint64_t b_thr_discrim = b->thr_discrim; |
| 144 | ret = (a_thr_discrim > b_thr_discrim) - (a_thr_discrim < |
| 145 | b_thr_discrim); |
| 146 | if (ret == 0) { |
| 147 | uint64_t a_tctx_uid = a->tctx_uid; |
| 148 | uint64_t b_tctx_uid = b->tctx_uid; |
| 149 | ret = (a_tctx_uid > b_tctx_uid) - (a_tctx_uid < |
| 150 | b_tctx_uid); |
| 151 | } |
Jason Evans | d69964b | 2015-03-12 16:25:18 -0700 | [diff] [blame] | 152 | } |
| 153 | return (ret); |
Jason Evans | 3a81cbd | 2014-08-16 12:58:55 -0700 | [diff] [blame] | 154 | } |
| 155 | |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 156 | rb_gen(static UNUSED, tctx_tree_, prof_tctx_tree_t, prof_tctx_t, |
| 157 | tctx_link, prof_tctx_comp) |
Jason Evans | 3a81cbd | 2014-08-16 12:58:55 -0700 | [diff] [blame] | 158 | |
| 159 | JEMALLOC_INLINE_C int |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 160 | prof_gctx_comp(const prof_gctx_t *a, const prof_gctx_t *b) |
Jason Evans | 3a81cbd | 2014-08-16 12:58:55 -0700 | [diff] [blame] | 161 | { |
| 162 | unsigned a_len = a->bt.len; |
| 163 | unsigned b_len = b->bt.len; |
| 164 | unsigned comp_len = (a_len < b_len) ? a_len : b_len; |
| 165 | int ret = memcmp(a->bt.vec, b->bt.vec, comp_len * sizeof(void *)); |
| 166 | if (ret == 0) |
| 167 | ret = (a_len > b_len) - (a_len < b_len); |
| 168 | return (ret); |
| 169 | } |
| 170 | |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 171 | rb_gen(static UNUSED, gctx_tree_, prof_gctx_tree_t, prof_gctx_t, dump_link, |
| 172 | prof_gctx_comp) |
| 173 | |
| 174 | JEMALLOC_INLINE_C int |
| 175 | prof_tdata_comp(const prof_tdata_t *a, const prof_tdata_t *b) |
| 176 | { |
Jason Evans | 20c31de | 2014-10-02 23:01:10 -0700 | [diff] [blame] | 177 | int ret; |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 178 | uint64_t a_uid = a->thr_uid; |
| 179 | uint64_t b_uid = b->thr_uid; |
| 180 | |
Jason Evans | 20c31de | 2014-10-02 23:01:10 -0700 | [diff] [blame] | 181 | ret = ((a_uid > b_uid) - (a_uid < b_uid)); |
| 182 | if (ret == 0) { |
| 183 | uint64_t a_discrim = a->thr_discrim; |
| 184 | uint64_t b_discrim = b->thr_discrim; |
| 185 | |
| 186 | ret = ((a_discrim > b_discrim) - (a_discrim < b_discrim)); |
| 187 | } |
| 188 | return (ret); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 189 | } |
| 190 | |
| 191 | rb_gen(static UNUSED, tdata_tree_, prof_tdata_tree_t, prof_tdata_t, tdata_link, |
| 192 | prof_tdata_comp) |
| 193 | |
| 194 | /******************************************************************************/ |
| 195 | |
| 196 | void |
Jason Evans | 5460aa6 | 2014-09-22 21:09:23 -0700 | [diff] [blame] | 197 | prof_alloc_rollback(tsd_t *tsd, prof_tctx_t *tctx, bool updated) |
Jason Evans | 6e73dc1 | 2014-09-09 19:37:26 -0700 | [diff] [blame] | 198 | { |
| 199 | prof_tdata_t *tdata; |
| 200 | |
| 201 | cassert(config_prof); |
| 202 | |
| 203 | if (updated) { |
| 204 | /* |
| 205 | * Compute a new sample threshold. This isn't very important in |
| 206 | * practice, because this function is rarely executed, so the |
| 207 | * potential for sample bias is minimal except in contrived |
| 208 | * programs. |
| 209 | */ |
Jason Evans | 5460aa6 | 2014-09-22 21:09:23 -0700 | [diff] [blame] | 210 | tdata = prof_tdata_get(tsd, true); |
| 211 | if (tdata != NULL) |
Jason Evans | 3ca0cf6 | 2015-09-17 14:47:39 -0700 | [diff] [blame] | 212 | prof_sample_threshold_update(tdata); |
Jason Evans | 6e73dc1 | 2014-09-09 19:37:26 -0700 | [diff] [blame] | 213 | } |
| 214 | |
| 215 | if ((uintptr_t)tctx > (uintptr_t)1U) { |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 216 | malloc_mutex_lock(tsd_tsdn(tsd), tctx->tdata->lock); |
Jason Evans | 6e73dc1 | 2014-09-09 19:37:26 -0700 | [diff] [blame] | 217 | tctx->prepared = false; |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 218 | if (prof_tctx_should_destroy(tsd_tsdn(tsd), tctx)) |
Jason Evans | 5460aa6 | 2014-09-22 21:09:23 -0700 | [diff] [blame] | 219 | prof_tctx_destroy(tsd, tctx); |
Jason Evans | 6e73dc1 | 2014-09-09 19:37:26 -0700 | [diff] [blame] | 220 | else |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 221 | malloc_mutex_unlock(tsd_tsdn(tsd), tctx->tdata->lock); |
Jason Evans | 6e73dc1 | 2014-09-09 19:37:26 -0700 | [diff] [blame] | 222 | } |
| 223 | } |
| 224 | |
| 225 | void |
Jason Evans | db72272 | 2016-03-23 20:29:33 -0700 | [diff] [blame] | 226 | prof_malloc_sample_object(tsdn_t *tsdn, extent_t *extent, const void *ptr, |
| 227 | size_t usize, prof_tctx_t *tctx) |
Jason Evans | cfc5706 | 2014-10-30 23:18:45 -0700 | [diff] [blame] | 228 | { |
| 229 | |
Jason Evans | db72272 | 2016-03-23 20:29:33 -0700 | [diff] [blame] | 230 | prof_tctx_set(tsdn, extent, ptr, usize, tctx); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 231 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 232 | malloc_mutex_lock(tsdn, tctx->tdata->lock); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 233 | tctx->cnts.curobjs++; |
| 234 | tctx->cnts.curbytes += usize; |
| 235 | if (opt_prof_accum) { |
| 236 | tctx->cnts.accumobjs++; |
| 237 | tctx->cnts.accumbytes += usize; |
| 238 | } |
Jason Evans | 6e73dc1 | 2014-09-09 19:37:26 -0700 | [diff] [blame] | 239 | tctx->prepared = false; |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 240 | malloc_mutex_unlock(tsdn, tctx->tdata->lock); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 241 | } |
| 242 | |
| 243 | void |
Jason Evans | 5460aa6 | 2014-09-22 21:09:23 -0700 | [diff] [blame] | 244 | prof_free_sampled_object(tsd_t *tsd, size_t usize, prof_tctx_t *tctx) |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 245 | { |
| 246 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 247 | malloc_mutex_lock(tsd_tsdn(tsd), tctx->tdata->lock); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 248 | assert(tctx->cnts.curobjs > 0); |
| 249 | assert(tctx->cnts.curbytes >= usize); |
| 250 | tctx->cnts.curobjs--; |
| 251 | tctx->cnts.curbytes -= usize; |
| 252 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 253 | if (prof_tctx_should_destroy(tsd_tsdn(tsd), tctx)) |
Jason Evans | 5460aa6 | 2014-09-22 21:09:23 -0700 | [diff] [blame] | 254 | prof_tctx_destroy(tsd, tctx); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 255 | else |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 256 | malloc_mutex_unlock(tsd_tsdn(tsd), tctx->tdata->lock); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 257 | } |
Jason Evans | 3a81cbd | 2014-08-16 12:58:55 -0700 | [diff] [blame] | 258 | |
Jason Evans | 4d6a134 | 2010-10-20 19:05:59 -0700 | [diff] [blame] | 259 | void |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 260 | bt_init(prof_bt_t *bt, void **vec) |
| 261 | { |
| 262 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 263 | cassert(config_prof); |
| 264 | |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 265 | bt->vec = vec; |
| 266 | bt->len = 0; |
| 267 | } |
| 268 | |
Jason Evans | af1f592 | 2014-10-30 16:38:08 -0700 | [diff] [blame] | 269 | JEMALLOC_INLINE_C void |
Jason Evans | c93ed81 | 2014-10-30 16:50:33 -0700 | [diff] [blame] | 270 | prof_enter(tsd_t *tsd, prof_tdata_t *tdata) |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 271 | { |
| 272 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 273 | cassert(config_prof); |
Jason Evans | c93ed81 | 2014-10-30 16:50:33 -0700 | [diff] [blame] | 274 | assert(tdata == prof_tdata_get(tsd, false)); |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 275 | |
Jason Evans | 82cb603 | 2014-11-01 00:20:28 -0700 | [diff] [blame] | 276 | if (tdata != NULL) { |
| 277 | assert(!tdata->enq); |
| 278 | tdata->enq = true; |
| 279 | } |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 280 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 281 | malloc_mutex_lock(tsd_tsdn(tsd), &bt2gctx_mtx); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 282 | } |
| 283 | |
Jason Evans | af1f592 | 2014-10-30 16:38:08 -0700 | [diff] [blame] | 284 | JEMALLOC_INLINE_C void |
Jason Evans | c93ed81 | 2014-10-30 16:50:33 -0700 | [diff] [blame] | 285 | prof_leave(tsd_t *tsd, prof_tdata_t *tdata) |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 286 | { |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 287 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 288 | cassert(config_prof); |
Jason Evans | c93ed81 | 2014-10-30 16:50:33 -0700 | [diff] [blame] | 289 | assert(tdata == prof_tdata_get(tsd, false)); |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 290 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 291 | malloc_mutex_unlock(tsd_tsdn(tsd), &bt2gctx_mtx); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 292 | |
Jason Evans | 82cb603 | 2014-11-01 00:20:28 -0700 | [diff] [blame] | 293 | if (tdata != NULL) { |
| 294 | bool idump, gdump; |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 295 | |
Jason Evans | 82cb603 | 2014-11-01 00:20:28 -0700 | [diff] [blame] | 296 | assert(tdata->enq); |
| 297 | tdata->enq = false; |
| 298 | idump = tdata->enq_idump; |
| 299 | tdata->enq_idump = false; |
| 300 | gdump = tdata->enq_gdump; |
| 301 | tdata->enq_gdump = false; |
| 302 | |
| 303 | if (idump) |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 304 | prof_idump(tsd_tsdn(tsd)); |
Jason Evans | 82cb603 | 2014-11-01 00:20:28 -0700 | [diff] [blame] | 305 | if (gdump) |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 306 | prof_gdump(tsd_tsdn(tsd)); |
Jason Evans | 82cb603 | 2014-11-01 00:20:28 -0700 | [diff] [blame] | 307 | } |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 308 | } |
| 309 | |
Jason Evans | 77f350b | 2011-03-15 22:23:12 -0700 | [diff] [blame] | 310 | #ifdef JEMALLOC_PROF_LIBUNWIND |
Jason Evans | 4d6a134 | 2010-10-20 19:05:59 -0700 | [diff] [blame] | 311 | void |
Jason Evans | 6f00105 | 2014-04-22 18:41:15 -0700 | [diff] [blame] | 312 | prof_backtrace(prof_bt_t *bt) |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 313 | { |
Jason Evans | 6f00105 | 2014-04-22 18:41:15 -0700 | [diff] [blame] | 314 | int nframes; |
| 315 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 316 | cassert(config_prof); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 317 | assert(bt->len == 0); |
| 318 | assert(bt->vec != NULL); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 319 | |
Jason Evans | 6f00105 | 2014-04-22 18:41:15 -0700 | [diff] [blame] | 320 | nframes = unw_backtrace(bt->vec, PROF_BT_MAX); |
| 321 | if (nframes <= 0) |
Lucian Adrian Grijincu | 9d4e13f | 2014-04-21 20:52:35 -0700 | [diff] [blame] | 322 | return; |
Jason Evans | 6f00105 | 2014-04-22 18:41:15 -0700 | [diff] [blame] | 323 | bt->len = nframes; |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 324 | } |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 325 | #elif (defined(JEMALLOC_PROF_LIBGCC)) |
Jason Evans | 77f350b | 2011-03-15 22:23:12 -0700 | [diff] [blame] | 326 | static _Unwind_Reason_Code |
| 327 | prof_unwind_init_callback(struct _Unwind_Context *context, void *arg) |
| 328 | { |
| 329 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 330 | cassert(config_prof); |
| 331 | |
Jason Evans | 77f350b | 2011-03-15 22:23:12 -0700 | [diff] [blame] | 332 | return (_URC_NO_REASON); |
| 333 | } |
| 334 | |
| 335 | static _Unwind_Reason_Code |
| 336 | prof_unwind_callback(struct _Unwind_Context *context, void *arg) |
| 337 | { |
| 338 | prof_unwind_data_t *data = (prof_unwind_data_t *)arg; |
Jason Evans | 6f00105 | 2014-04-22 18:41:15 -0700 | [diff] [blame] | 339 | void *ip; |
Jason Evans | 77f350b | 2011-03-15 22:23:12 -0700 | [diff] [blame] | 340 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 341 | cassert(config_prof); |
| 342 | |
Jason Evans | 6f00105 | 2014-04-22 18:41:15 -0700 | [diff] [blame] | 343 | ip = (void *)_Unwind_GetIP(context); |
| 344 | if (ip == NULL) |
| 345 | return (_URC_END_OF_STACK); |
| 346 | data->bt->vec[data->bt->len] = ip; |
| 347 | data->bt->len++; |
| 348 | if (data->bt->len == data->max) |
| 349 | return (_URC_END_OF_STACK); |
Jason Evans | 77f350b | 2011-03-15 22:23:12 -0700 | [diff] [blame] | 350 | |
| 351 | return (_URC_NO_REASON); |
| 352 | } |
| 353 | |
| 354 | void |
Jason Evans | 6f00105 | 2014-04-22 18:41:15 -0700 | [diff] [blame] | 355 | prof_backtrace(prof_bt_t *bt) |
Jason Evans | 77f350b | 2011-03-15 22:23:12 -0700 | [diff] [blame] | 356 | { |
Jason Evans | 6f00105 | 2014-04-22 18:41:15 -0700 | [diff] [blame] | 357 | prof_unwind_data_t data = {bt, PROF_BT_MAX}; |
Jason Evans | 77f350b | 2011-03-15 22:23:12 -0700 | [diff] [blame] | 358 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 359 | cassert(config_prof); |
| 360 | |
Jason Evans | 77f350b | 2011-03-15 22:23:12 -0700 | [diff] [blame] | 361 | _Unwind_Backtrace(prof_unwind_callback, &data); |
| 362 | } |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 363 | #elif (defined(JEMALLOC_PROF_GCC)) |
Jason Evans | 4d6a134 | 2010-10-20 19:05:59 -0700 | [diff] [blame] | 364 | void |
Jason Evans | 6f00105 | 2014-04-22 18:41:15 -0700 | [diff] [blame] | 365 | prof_backtrace(prof_bt_t *bt) |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 366 | { |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 367 | #define BT_FRAME(i) \ |
Jason Evans | 6f00105 | 2014-04-22 18:41:15 -0700 | [diff] [blame] | 368 | if ((i) < PROF_BT_MAX) { \ |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 369 | void *p; \ |
| 370 | if (__builtin_frame_address(i) == 0) \ |
Jason Evans | b27805b | 2010-02-10 18:15:53 -0800 | [diff] [blame] | 371 | return; \ |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 372 | p = __builtin_return_address(i); \ |
| 373 | if (p == NULL) \ |
Jason Evans | b27805b | 2010-02-10 18:15:53 -0800 | [diff] [blame] | 374 | return; \ |
Jason Evans | 6f00105 | 2014-04-22 18:41:15 -0700 | [diff] [blame] | 375 | bt->vec[(i)] = p; \ |
| 376 | bt->len = (i) + 1; \ |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 377 | } else \ |
Jason Evans | b27805b | 2010-02-10 18:15:53 -0800 | [diff] [blame] | 378 | return; |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 379 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 380 | cassert(config_prof); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 381 | |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 382 | BT_FRAME(0) |
| 383 | BT_FRAME(1) |
| 384 | BT_FRAME(2) |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 385 | BT_FRAME(3) |
| 386 | BT_FRAME(4) |
| 387 | BT_FRAME(5) |
| 388 | BT_FRAME(6) |
| 389 | BT_FRAME(7) |
| 390 | BT_FRAME(8) |
| 391 | BT_FRAME(9) |
| 392 | |
| 393 | BT_FRAME(10) |
| 394 | BT_FRAME(11) |
| 395 | BT_FRAME(12) |
| 396 | BT_FRAME(13) |
| 397 | BT_FRAME(14) |
| 398 | BT_FRAME(15) |
| 399 | BT_FRAME(16) |
| 400 | BT_FRAME(17) |
| 401 | BT_FRAME(18) |
| 402 | BT_FRAME(19) |
| 403 | |
| 404 | BT_FRAME(20) |
| 405 | BT_FRAME(21) |
| 406 | BT_FRAME(22) |
| 407 | BT_FRAME(23) |
| 408 | BT_FRAME(24) |
| 409 | BT_FRAME(25) |
| 410 | BT_FRAME(26) |
| 411 | BT_FRAME(27) |
| 412 | BT_FRAME(28) |
| 413 | BT_FRAME(29) |
| 414 | |
| 415 | BT_FRAME(30) |
| 416 | BT_FRAME(31) |
| 417 | BT_FRAME(32) |
| 418 | BT_FRAME(33) |
| 419 | BT_FRAME(34) |
| 420 | BT_FRAME(35) |
| 421 | BT_FRAME(36) |
| 422 | BT_FRAME(37) |
| 423 | BT_FRAME(38) |
| 424 | BT_FRAME(39) |
| 425 | |
| 426 | BT_FRAME(40) |
| 427 | BT_FRAME(41) |
| 428 | BT_FRAME(42) |
| 429 | BT_FRAME(43) |
| 430 | BT_FRAME(44) |
| 431 | BT_FRAME(45) |
| 432 | BT_FRAME(46) |
| 433 | BT_FRAME(47) |
| 434 | BT_FRAME(48) |
| 435 | BT_FRAME(49) |
| 436 | |
| 437 | BT_FRAME(50) |
| 438 | BT_FRAME(51) |
| 439 | BT_FRAME(52) |
| 440 | BT_FRAME(53) |
| 441 | BT_FRAME(54) |
| 442 | BT_FRAME(55) |
| 443 | BT_FRAME(56) |
| 444 | BT_FRAME(57) |
| 445 | BT_FRAME(58) |
| 446 | BT_FRAME(59) |
| 447 | |
| 448 | BT_FRAME(60) |
| 449 | BT_FRAME(61) |
| 450 | BT_FRAME(62) |
| 451 | BT_FRAME(63) |
| 452 | BT_FRAME(64) |
| 453 | BT_FRAME(65) |
| 454 | BT_FRAME(66) |
| 455 | BT_FRAME(67) |
| 456 | BT_FRAME(68) |
| 457 | BT_FRAME(69) |
| 458 | |
| 459 | BT_FRAME(70) |
| 460 | BT_FRAME(71) |
| 461 | BT_FRAME(72) |
| 462 | BT_FRAME(73) |
| 463 | BT_FRAME(74) |
| 464 | BT_FRAME(75) |
| 465 | BT_FRAME(76) |
| 466 | BT_FRAME(77) |
| 467 | BT_FRAME(78) |
| 468 | BT_FRAME(79) |
| 469 | |
| 470 | BT_FRAME(80) |
| 471 | BT_FRAME(81) |
| 472 | BT_FRAME(82) |
| 473 | BT_FRAME(83) |
| 474 | BT_FRAME(84) |
| 475 | BT_FRAME(85) |
| 476 | BT_FRAME(86) |
| 477 | BT_FRAME(87) |
| 478 | BT_FRAME(88) |
| 479 | BT_FRAME(89) |
| 480 | |
| 481 | BT_FRAME(90) |
| 482 | BT_FRAME(91) |
| 483 | BT_FRAME(92) |
| 484 | BT_FRAME(93) |
| 485 | BT_FRAME(94) |
| 486 | BT_FRAME(95) |
| 487 | BT_FRAME(96) |
| 488 | BT_FRAME(97) |
| 489 | BT_FRAME(98) |
| 490 | BT_FRAME(99) |
| 491 | |
| 492 | BT_FRAME(100) |
| 493 | BT_FRAME(101) |
| 494 | BT_FRAME(102) |
| 495 | BT_FRAME(103) |
| 496 | BT_FRAME(104) |
| 497 | BT_FRAME(105) |
| 498 | BT_FRAME(106) |
| 499 | BT_FRAME(107) |
| 500 | BT_FRAME(108) |
| 501 | BT_FRAME(109) |
| 502 | |
| 503 | BT_FRAME(110) |
| 504 | BT_FRAME(111) |
| 505 | BT_FRAME(112) |
| 506 | BT_FRAME(113) |
| 507 | BT_FRAME(114) |
| 508 | BT_FRAME(115) |
| 509 | BT_FRAME(116) |
| 510 | BT_FRAME(117) |
| 511 | BT_FRAME(118) |
| 512 | BT_FRAME(119) |
| 513 | |
| 514 | BT_FRAME(120) |
| 515 | BT_FRAME(121) |
| 516 | BT_FRAME(122) |
| 517 | BT_FRAME(123) |
| 518 | BT_FRAME(124) |
| 519 | BT_FRAME(125) |
| 520 | BT_FRAME(126) |
| 521 | BT_FRAME(127) |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 522 | #undef BT_FRAME |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 523 | } |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 524 | #else |
| 525 | void |
Jason Evans | 6f00105 | 2014-04-22 18:41:15 -0700 | [diff] [blame] | 526 | prof_backtrace(prof_bt_t *bt) |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 527 | { |
| 528 | |
| 529 | cassert(config_prof); |
Jason Evans | 6556e28 | 2013-10-21 14:56:27 -0700 | [diff] [blame] | 530 | not_reached(); |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 531 | } |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 532 | #endif |
| 533 | |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 534 | static malloc_mutex_t * |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 535 | prof_gctx_mutex_choose(void) |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 536 | { |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 537 | unsigned ngctxs = atomic_add_u(&cum_gctxs, 1); |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 538 | |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 539 | return (&gctx_locks[(ngctxs - 1) % PROF_NCTX_LOCKS]); |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 540 | } |
| 541 | |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 542 | static malloc_mutex_t * |
| 543 | prof_tdata_mutex_choose(uint64_t thr_uid) |
| 544 | { |
| 545 | |
| 546 | return (&tdata_locks[thr_uid % PROF_NTDATA_LOCKS]); |
| 547 | } |
| 548 | |
| 549 | static prof_gctx_t * |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 550 | prof_gctx_create(tsdn_t *tsdn, prof_bt_t *bt) |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 551 | { |
Jason Evans | ab532e9 | 2014-08-15 15:05:12 -0700 | [diff] [blame] | 552 | /* |
| 553 | * Create a single allocation that has space for vec of length bt->len. |
| 554 | */ |
Qi Wang | f4a0f32 | 2015-10-27 15:12:10 -0700 | [diff] [blame] | 555 | size_t size = offsetof(prof_gctx_t, vec) + (bt->len * sizeof(void *)); |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 556 | prof_gctx_t *gctx = (prof_gctx_t *)iallocztm(tsdn, size, |
| 557 | size2index(size), false, NULL, true, arena_get(TSDN_NULL, 0, true), |
Jason Evans | 66cd953 | 2016-04-22 14:34:14 -0700 | [diff] [blame] | 558 | true); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 559 | if (gctx == NULL) |
Jason Evans | ab532e9 | 2014-08-15 15:05:12 -0700 | [diff] [blame] | 560 | return (NULL); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 561 | gctx->lock = prof_gctx_mutex_choose(); |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 562 | /* |
| 563 | * Set nlimbo to 1, in order to avoid a race condition with |
Jason Evans | 20c31de | 2014-10-02 23:01:10 -0700 | [diff] [blame] | 564 | * prof_tctx_destroy()/prof_gctx_try_destroy(). |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 565 | */ |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 566 | gctx->nlimbo = 1; |
| 567 | tctx_tree_new(&gctx->tctxs); |
Jason Evans | ab532e9 | 2014-08-15 15:05:12 -0700 | [diff] [blame] | 568 | /* Duplicate bt. */ |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 569 | memcpy(gctx->vec, bt->vec, bt->len * sizeof(void *)); |
| 570 | gctx->bt.vec = gctx->vec; |
| 571 | gctx->bt.len = bt->len; |
| 572 | return (gctx); |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 573 | } |
| 574 | |
| 575 | static void |
Jason Evans | c93ed81 | 2014-10-30 16:50:33 -0700 | [diff] [blame] | 576 | prof_gctx_try_destroy(tsd_t *tsd, prof_tdata_t *tdata_self, prof_gctx_t *gctx, |
| 577 | prof_tdata_t *tdata) |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 578 | { |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 579 | |
| 580 | cassert(config_prof); |
| 581 | |
| 582 | /* |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 583 | * Check that gctx is still unused by any thread cache before destroying |
| 584 | * it. prof_lookup() increments gctx->nlimbo in order to avoid a race |
| 585 | * condition with this function, as does prof_tctx_destroy() in order to |
| 586 | * avoid a race between the main body of prof_tctx_destroy() and entry |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 587 | * into this function. |
| 588 | */ |
Jason Evans | c93ed81 | 2014-10-30 16:50:33 -0700 | [diff] [blame] | 589 | prof_enter(tsd, tdata_self); |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 590 | malloc_mutex_lock(tsd_tsdn(tsd), gctx->lock); |
Jason Evans | 20c31de | 2014-10-02 23:01:10 -0700 | [diff] [blame] | 591 | assert(gctx->nlimbo != 0); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 592 | if (tctx_tree_empty(&gctx->tctxs) && gctx->nlimbo == 1) { |
| 593 | /* Remove gctx from bt2gctx. */ |
Jason Evans | b54d160 | 2016-10-20 23:59:12 -0700 | [diff] [blame^] | 594 | if (ckh_remove(tsd, &bt2gctx, &gctx->bt, NULL, NULL)) |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 595 | not_reached(); |
Jason Evans | c93ed81 | 2014-10-30 16:50:33 -0700 | [diff] [blame] | 596 | prof_leave(tsd, tdata_self); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 597 | /* Destroy gctx. */ |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 598 | malloc_mutex_unlock(tsd_tsdn(tsd), gctx->lock); |
Jason Evans | 8c9be3e | 2016-04-16 00:36:11 -0700 | [diff] [blame] | 599 | idalloctm(tsd_tsdn(tsd), iealloc(tsd_tsdn(tsd), gctx), gctx, |
| 600 | NULL, true, true); |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 601 | } else { |
| 602 | /* |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 603 | * Compensate for increment in prof_tctx_destroy() or |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 604 | * prof_lookup(). |
| 605 | */ |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 606 | gctx->nlimbo--; |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 607 | malloc_mutex_unlock(tsd_tsdn(tsd), gctx->lock); |
Jason Evans | c93ed81 | 2014-10-30 16:50:33 -0700 | [diff] [blame] | 608 | prof_leave(tsd, tdata_self); |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 609 | } |
| 610 | } |
| 611 | |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 612 | static bool |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 613 | prof_tctx_should_destroy(tsdn_t *tsdn, prof_tctx_t *tctx) |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 614 | { |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 615 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 616 | malloc_mutex_assert_owner(tsdn, tctx->tdata->lock); |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 617 | |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 618 | if (opt_prof_accum) |
| 619 | return (false); |
| 620 | if (tctx->cnts.curobjs != 0) |
| 621 | return (false); |
Jason Evans | 6e73dc1 | 2014-09-09 19:37:26 -0700 | [diff] [blame] | 622 | if (tctx->prepared) |
| 623 | return (false); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 624 | return (true); |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 625 | } |
| 626 | |
Jason Evans | fb1775e | 2014-01-14 17:04:34 -0800 | [diff] [blame] | 627 | static bool |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 628 | prof_gctx_should_destroy(prof_gctx_t *gctx) |
| 629 | { |
| 630 | |
| 631 | if (opt_prof_accum) |
| 632 | return (false); |
Jason Evans | 551ebc4 | 2014-10-03 10:16:09 -0700 | [diff] [blame] | 633 | if (!tctx_tree_empty(&gctx->tctxs)) |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 634 | return (false); |
| 635 | if (gctx->nlimbo != 0) |
| 636 | return (false); |
| 637 | return (true); |
| 638 | } |
| 639 | |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 640 | static void |
Jason Evans | 5460aa6 | 2014-09-22 21:09:23 -0700 | [diff] [blame] | 641 | prof_tctx_destroy(tsd_t *tsd, prof_tctx_t *tctx) |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 642 | { |
Jason Evans | 6fd53da | 2014-09-09 12:45:53 -0700 | [diff] [blame] | 643 | prof_tdata_t *tdata = tctx->tdata; |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 644 | prof_gctx_t *gctx = tctx->gctx; |
Jason Evans | bf40641 | 2014-10-06 16:35:11 -0700 | [diff] [blame] | 645 | bool destroy_tdata, destroy_tctx, destroy_gctx; |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 646 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 647 | malloc_mutex_assert_owner(tsd_tsdn(tsd), tctx->tdata->lock); |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 648 | |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 649 | assert(tctx->cnts.curobjs == 0); |
| 650 | assert(tctx->cnts.curbytes == 0); |
Jason Evans | 551ebc4 | 2014-10-03 10:16:09 -0700 | [diff] [blame] | 651 | assert(!opt_prof_accum); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 652 | assert(tctx->cnts.accumobjs == 0); |
| 653 | assert(tctx->cnts.accumbytes == 0); |
| 654 | |
Jason Evans | b54d160 | 2016-10-20 23:59:12 -0700 | [diff] [blame^] | 655 | ckh_remove(tsd, &tdata->bt2tctx, &gctx->bt, NULL, NULL); |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 656 | destroy_tdata = prof_tdata_should_destroy(tsd_tsdn(tsd), tdata, false); |
| 657 | malloc_mutex_unlock(tsd_tsdn(tsd), tdata->lock); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 658 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 659 | malloc_mutex_lock(tsd_tsdn(tsd), gctx->lock); |
Jason Evans | 764b000 | 2015-03-14 14:01:35 -0700 | [diff] [blame] | 660 | switch (tctx->state) { |
Jason Evans | 04211e2 | 2015-03-16 15:11:06 -0700 | [diff] [blame] | 661 | case prof_tctx_state_nominal: |
Jason Evans | bf40641 | 2014-10-06 16:35:11 -0700 | [diff] [blame] | 662 | tctx_tree_remove(&gctx->tctxs, tctx); |
| 663 | destroy_tctx = true; |
| 664 | if (prof_gctx_should_destroy(gctx)) { |
| 665 | /* |
| 666 | * Increment gctx->nlimbo in order to keep another |
| 667 | * thread from winning the race to destroy gctx while |
| 668 | * this one has gctx->lock dropped. Without this, it |
| 669 | * would be possible for another thread to: |
| 670 | * |
| 671 | * 1) Sample an allocation associated with gctx. |
| 672 | * 2) Deallocate the sampled object. |
| 673 | * 3) Successfully prof_gctx_try_destroy(gctx). |
| 674 | * |
| 675 | * The result would be that gctx no longer exists by the |
| 676 | * time this thread accesses it in |
| 677 | * prof_gctx_try_destroy(). |
| 678 | */ |
| 679 | gctx->nlimbo++; |
| 680 | destroy_gctx = true; |
| 681 | } else |
| 682 | destroy_gctx = false; |
Jason Evans | 764b000 | 2015-03-14 14:01:35 -0700 | [diff] [blame] | 683 | break; |
| 684 | case prof_tctx_state_dumping: |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 685 | /* |
Jason Evans | bf40641 | 2014-10-06 16:35:11 -0700 | [diff] [blame] | 686 | * A dumping thread needs tctx to remain valid until dumping |
| 687 | * has finished. Change state such that the dumping thread will |
| 688 | * complete destruction during a late dump iteration phase. |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 689 | */ |
Jason Evans | bf40641 | 2014-10-06 16:35:11 -0700 | [diff] [blame] | 690 | tctx->state = prof_tctx_state_purgatory; |
| 691 | destroy_tctx = false; |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 692 | destroy_gctx = false; |
Jason Evans | 764b000 | 2015-03-14 14:01:35 -0700 | [diff] [blame] | 693 | break; |
| 694 | default: |
| 695 | not_reached(); |
Jason Evans | 262146d | 2015-03-14 14:34:16 -0700 | [diff] [blame] | 696 | destroy_tctx = false; |
| 697 | destroy_gctx = false; |
Jason Evans | bf40641 | 2014-10-06 16:35:11 -0700 | [diff] [blame] | 698 | } |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 699 | malloc_mutex_unlock(tsd_tsdn(tsd), gctx->lock); |
Jason Evans | c93ed81 | 2014-10-30 16:50:33 -0700 | [diff] [blame] | 700 | if (destroy_gctx) { |
| 701 | prof_gctx_try_destroy(tsd, prof_tdata_get(tsd, false), gctx, |
| 702 | tdata); |
| 703 | } |
Jason Evans | 6fd53da | 2014-09-09 12:45:53 -0700 | [diff] [blame] | 704 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 705 | malloc_mutex_assert_not_owner(tsd_tsdn(tsd), tctx->tdata->lock); |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 706 | |
Jason Evans | 6fd53da | 2014-09-09 12:45:53 -0700 | [diff] [blame] | 707 | if (destroy_tdata) |
Jason Evans | b54d160 | 2016-10-20 23:59:12 -0700 | [diff] [blame^] | 708 | prof_tdata_destroy(tsd, tdata, false); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 709 | |
Jason Evans | bf40641 | 2014-10-06 16:35:11 -0700 | [diff] [blame] | 710 | if (destroy_tctx) |
Jason Evans | 8c9be3e | 2016-04-16 00:36:11 -0700 | [diff] [blame] | 711 | idalloctm(tsd_tsdn(tsd), iealloc(tsd_tsdn(tsd), tctx), tctx, |
| 712 | NULL, true, true); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 713 | } |
| 714 | |
| 715 | static bool |
Jason Evans | 5460aa6 | 2014-09-22 21:09:23 -0700 | [diff] [blame] | 716 | prof_lookup_global(tsd_t *tsd, prof_bt_t *bt, prof_tdata_t *tdata, |
| 717 | void **p_btkey, prof_gctx_t **p_gctx, bool *p_new_gctx) |
Jason Evans | fb1775e | 2014-01-14 17:04:34 -0800 | [diff] [blame] | 718 | { |
| 719 | union { |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 720 | prof_gctx_t *p; |
Jason Evans | fb1775e | 2014-01-14 17:04:34 -0800 | [diff] [blame] | 721 | void *v; |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 722 | } gctx; |
Jason Evans | fb1775e | 2014-01-14 17:04:34 -0800 | [diff] [blame] | 723 | union { |
| 724 | prof_bt_t *p; |
| 725 | void *v; |
| 726 | } btkey; |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 727 | bool new_gctx; |
Jason Evans | fb1775e | 2014-01-14 17:04:34 -0800 | [diff] [blame] | 728 | |
Jason Evans | c93ed81 | 2014-10-30 16:50:33 -0700 | [diff] [blame] | 729 | prof_enter(tsd, tdata); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 730 | if (ckh_search(&bt2gctx, bt, &btkey.v, &gctx.v)) { |
Jason Evans | fb1775e | 2014-01-14 17:04:34 -0800 | [diff] [blame] | 731 | /* bt has never been seen before. Insert it. */ |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 732 | gctx.p = prof_gctx_create(tsd_tsdn(tsd), bt); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 733 | if (gctx.v == NULL) { |
Jason Evans | c93ed81 | 2014-10-30 16:50:33 -0700 | [diff] [blame] | 734 | prof_leave(tsd, tdata); |
Jason Evans | fb1775e | 2014-01-14 17:04:34 -0800 | [diff] [blame] | 735 | return (true); |
| 736 | } |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 737 | btkey.p = &gctx.p->bt; |
Jason Evans | b54d160 | 2016-10-20 23:59:12 -0700 | [diff] [blame^] | 738 | if (ckh_insert(tsd, &bt2gctx, btkey.v, gctx.v)) { |
Jason Evans | fb1775e | 2014-01-14 17:04:34 -0800 | [diff] [blame] | 739 | /* OOM. */ |
Jason Evans | c93ed81 | 2014-10-30 16:50:33 -0700 | [diff] [blame] | 740 | prof_leave(tsd, tdata); |
Jason Evans | 8c9be3e | 2016-04-16 00:36:11 -0700 | [diff] [blame] | 741 | idalloctm(tsd_tsdn(tsd), iealloc(tsd_tsdn(tsd), gctx.v), |
| 742 | gctx.v, NULL, true, true); |
Jason Evans | fb1775e | 2014-01-14 17:04:34 -0800 | [diff] [blame] | 743 | return (true); |
| 744 | } |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 745 | new_gctx = true; |
Jason Evans | fb1775e | 2014-01-14 17:04:34 -0800 | [diff] [blame] | 746 | } else { |
| 747 | /* |
| 748 | * Increment nlimbo, in order to avoid a race condition with |
Jason Evans | 20c31de | 2014-10-02 23:01:10 -0700 | [diff] [blame] | 749 | * prof_tctx_destroy()/prof_gctx_try_destroy(). |
Jason Evans | fb1775e | 2014-01-14 17:04:34 -0800 | [diff] [blame] | 750 | */ |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 751 | malloc_mutex_lock(tsd_tsdn(tsd), gctx.p->lock); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 752 | gctx.p->nlimbo++; |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 753 | malloc_mutex_unlock(tsd_tsdn(tsd), gctx.p->lock); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 754 | new_gctx = false; |
Jason Evans | fb1775e | 2014-01-14 17:04:34 -0800 | [diff] [blame] | 755 | } |
Jason Evans | c93ed81 | 2014-10-30 16:50:33 -0700 | [diff] [blame] | 756 | prof_leave(tsd, tdata); |
Jason Evans | fb1775e | 2014-01-14 17:04:34 -0800 | [diff] [blame] | 757 | |
| 758 | *p_btkey = btkey.v; |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 759 | *p_gctx = gctx.p; |
| 760 | *p_new_gctx = new_gctx; |
Jason Evans | fb1775e | 2014-01-14 17:04:34 -0800 | [diff] [blame] | 761 | return (false); |
| 762 | } |
| 763 | |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 764 | prof_tctx_t * |
Jason Evans | 5460aa6 | 2014-09-22 21:09:23 -0700 | [diff] [blame] | 765 | prof_lookup(tsd_t *tsd, prof_bt_t *bt) |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 766 | { |
Jason Evans | 075e77c | 2010-09-20 19:53:25 -0700 | [diff] [blame] | 767 | union { |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 768 | prof_tctx_t *p; |
Jason Evans | 075e77c | 2010-09-20 19:53:25 -0700 | [diff] [blame] | 769 | void *v; |
| 770 | } ret; |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 771 | prof_tdata_t *tdata; |
| 772 | bool not_found; |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 773 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 774 | cassert(config_prof); |
| 775 | |
Jason Evans | 5460aa6 | 2014-09-22 21:09:23 -0700 | [diff] [blame] | 776 | tdata = prof_tdata_get(tsd, false); |
| 777 | if (tdata == NULL) |
Jason Evans | 52386b2 | 2012-04-22 16:00:11 -0700 | [diff] [blame] | 778 | return (NULL); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 779 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 780 | malloc_mutex_lock(tsd_tsdn(tsd), tdata->lock); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 781 | not_found = ckh_search(&tdata->bt2tctx, bt, NULL, &ret.v); |
Jason Evans | 6e73dc1 | 2014-09-09 19:37:26 -0700 | [diff] [blame] | 782 | if (!not_found) /* Note double negative! */ |
| 783 | ret.p->prepared = true; |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 784 | malloc_mutex_unlock(tsd_tsdn(tsd), tdata->lock); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 785 | if (not_found) { |
Jason Evans | fb1775e | 2014-01-14 17:04:34 -0800 | [diff] [blame] | 786 | void *btkey; |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 787 | prof_gctx_t *gctx; |
| 788 | bool new_gctx, error; |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 789 | |
| 790 | /* |
| 791 | * This thread's cache lacks bt. Look for it in the global |
| 792 | * cache. |
| 793 | */ |
Jason Evans | 5460aa6 | 2014-09-22 21:09:23 -0700 | [diff] [blame] | 794 | if (prof_lookup_global(tsd, bt, tdata, &btkey, &gctx, |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 795 | &new_gctx)) |
Jason Evans | fb1775e | 2014-01-14 17:04:34 -0800 | [diff] [blame] | 796 | return (NULL); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 797 | |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 798 | /* Link a prof_tctx_t into gctx for this thread. */ |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 799 | ret.v = iallocztm(tsd_tsdn(tsd), sizeof(prof_tctx_t), |
Jason Evans | 66cd953 | 2016-04-22 14:34:14 -0700 | [diff] [blame] | 800 | size2index(sizeof(prof_tctx_t)), false, NULL, true, |
Jason Evans | b54d160 | 2016-10-20 23:59:12 -0700 | [diff] [blame^] | 801 | arena_ichoose(tsd, NULL), true); |
Jason Evans | b41ccdb | 2014-08-15 15:01:15 -0700 | [diff] [blame] | 802 | if (ret.p == NULL) { |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 803 | if (new_gctx) |
Jason Evans | c93ed81 | 2014-10-30 16:50:33 -0700 | [diff] [blame] | 804 | prof_gctx_try_destroy(tsd, tdata, gctx, tdata); |
Jason Evans | b41ccdb | 2014-08-15 15:01:15 -0700 | [diff] [blame] | 805 | return (NULL); |
Jason Evans | a881cd2 | 2010-10-02 15:18:50 -0700 | [diff] [blame] | 806 | } |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 807 | ret.p->tdata = tdata; |
Jason Evans | 44c97b7 | 2014-10-12 13:03:20 -0700 | [diff] [blame] | 808 | ret.p->thr_uid = tdata->thr_uid; |
Jason Evans | a00b107 | 2015-09-09 23:16:10 -0700 | [diff] [blame] | 809 | ret.p->thr_discrim = tdata->thr_discrim; |
Jason Evans | 075e77c | 2010-09-20 19:53:25 -0700 | [diff] [blame] | 810 | memset(&ret.p->cnts, 0, sizeof(prof_cnt_t)); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 811 | ret.p->gctx = gctx; |
Jason Evans | 04211e2 | 2015-03-16 15:11:06 -0700 | [diff] [blame] | 812 | ret.p->tctx_uid = tdata->tctx_uid_next++; |
Jason Evans | 6e73dc1 | 2014-09-09 19:37:26 -0700 | [diff] [blame] | 813 | ret.p->prepared = true; |
Jason Evans | 6ef80d6 | 2014-09-24 22:14:21 -0700 | [diff] [blame] | 814 | ret.p->state = prof_tctx_state_initializing; |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 815 | malloc_mutex_lock(tsd_tsdn(tsd), tdata->lock); |
Jason Evans | b54d160 | 2016-10-20 23:59:12 -0700 | [diff] [blame^] | 816 | error = ckh_insert(tsd, &tdata->bt2tctx, btkey, ret.v); |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 817 | malloc_mutex_unlock(tsd_tsdn(tsd), tdata->lock); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 818 | if (error) { |
| 819 | if (new_gctx) |
Jason Evans | c93ed81 | 2014-10-30 16:50:33 -0700 | [diff] [blame] | 820 | prof_gctx_try_destroy(tsd, tdata, gctx, tdata); |
Jason Evans | 8c9be3e | 2016-04-16 00:36:11 -0700 | [diff] [blame] | 821 | idalloctm(tsd_tsdn(tsd), iealloc(tsd_tsdn(tsd), ret.v), |
| 822 | ret.v, NULL, true, true); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 823 | return (NULL); |
| 824 | } |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 825 | malloc_mutex_lock(tsd_tsdn(tsd), gctx->lock); |
Jason Evans | 6ef80d6 | 2014-09-24 22:14:21 -0700 | [diff] [blame] | 826 | ret.p->state = prof_tctx_state_nominal; |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 827 | tctx_tree_insert(&gctx->tctxs, ret.p); |
| 828 | gctx->nlimbo--; |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 829 | malloc_mutex_unlock(tsd_tsdn(tsd), gctx->lock); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 830 | } |
| 831 | |
Jason Evans | 075e77c | 2010-09-20 19:53:25 -0700 | [diff] [blame] | 832 | return (ret.p); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 833 | } |
| 834 | |
Jason Evans | dc391ad | 2016-05-04 12:14:36 -0700 | [diff] [blame] | 835 | /* |
| 836 | * The bodies of this function and prof_leakcheck() are compiled out unless heap |
| 837 | * profiling is enabled, so that it is possible to compile jemalloc with |
| 838 | * floating point support completely disabled. Avoiding floating point code is |
| 839 | * important on memory-constrained systems, but it also enables a workaround for |
| 840 | * versions of glibc that don't properly save/restore floating point registers |
| 841 | * during dynamic lazy symbol loading (which internally calls into whatever |
| 842 | * malloc implementation happens to be integrated into the application). Note |
| 843 | * that some compilers (e.g. gcc 4.8) may use floating point registers for fast |
| 844 | * memory moves, so jemalloc must be compiled with such optimizations disabled |
| 845 | * (e.g. |
| 846 | * -mno-sse) in order for the workaround to be complete. |
| 847 | */ |
Ben Maurer | 6c39f9e | 2014-04-15 13:47:13 -0700 | [diff] [blame] | 848 | void |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 849 | prof_sample_threshold_update(prof_tdata_t *tdata) |
Ben Maurer | 6c39f9e | 2014-04-15 13:47:13 -0700 | [diff] [blame] | 850 | { |
Ben Maurer | 6c39f9e | 2014-04-15 13:47:13 -0700 | [diff] [blame] | 851 | #ifdef JEMALLOC_PROF |
| 852 | uint64_t r; |
| 853 | double u; |
| 854 | |
| 855 | if (!config_prof) |
| 856 | return; |
| 857 | |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 858 | if (lg_prof_sample == 0) { |
| 859 | tdata->bytes_until_sample = 0; |
Ben Maurer | 6c39f9e | 2014-04-15 13:47:13 -0700 | [diff] [blame] | 860 | return; |
| 861 | } |
| 862 | |
| 863 | /* |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 864 | * Compute sample interval as a geometrically distributed random |
| 865 | * variable with mean (2^lg_prof_sample). |
Ben Maurer | 6c39f9e | 2014-04-15 13:47:13 -0700 | [diff] [blame] | 866 | * |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 867 | * __ __ |
| 868 | * | log(u) | 1 |
| 869 | * tdata->bytes_until_sample = | -------- |, where p = --------------- |
| 870 | * | log(1-p) | lg_prof_sample |
| 871 | * 2 |
Ben Maurer | 6c39f9e | 2014-04-15 13:47:13 -0700 | [diff] [blame] | 872 | * |
| 873 | * For more information on the math, see: |
| 874 | * |
| 875 | * Non-Uniform Random Variate Generation |
| 876 | * Luc Devroye |
| 877 | * Springer-Verlag, New York, 1986 |
| 878 | * pp 500 |
| 879 | * (http://luc.devroye.org/rnbookindex.html) |
| 880 | */ |
Jason Evans | b46261d | 2016-05-27 18:57:15 -0700 | [diff] [blame] | 881 | r = prng_lg_range(&tdata->prng_state, 53, false); |
Ben Maurer | 6c39f9e | 2014-04-15 13:47:13 -0700 | [diff] [blame] | 882 | u = (double)r * (1.0/9007199254740992.0L); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 883 | tdata->bytes_until_sample = (uint64_t)(log(u) / |
| 884 | log(1.0 - (1.0 / (double)((uint64_t)1U << lg_prof_sample)))) |
Ben Maurer | 6c39f9e | 2014-04-15 13:47:13 -0700 | [diff] [blame] | 885 | + (uint64_t)1U; |
| 886 | #endif |
| 887 | } |
| 888 | |
Jason Evans | 772163b | 2014-01-17 15:40:52 -0800 | [diff] [blame] | 889 | #ifdef JEMALLOC_JET |
Jason Evans | 20c31de | 2014-10-02 23:01:10 -0700 | [diff] [blame] | 890 | static prof_tdata_t * |
| 891 | prof_tdata_count_iter(prof_tdata_tree_t *tdatas, prof_tdata_t *tdata, void *arg) |
| 892 | { |
| 893 | size_t *tdata_count = (size_t *)arg; |
| 894 | |
| 895 | (*tdata_count)++; |
| 896 | |
| 897 | return (NULL); |
| 898 | } |
| 899 | |
| 900 | size_t |
| 901 | prof_tdata_count(void) |
| 902 | { |
| 903 | size_t tdata_count = 0; |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 904 | tsdn_t *tsdn; |
Jason Evans | 20c31de | 2014-10-02 23:01:10 -0700 | [diff] [blame] | 905 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 906 | tsdn = tsdn_fetch(); |
| 907 | malloc_mutex_lock(tsdn, &tdatas_mtx); |
Jason Evans | 20c31de | 2014-10-02 23:01:10 -0700 | [diff] [blame] | 908 | tdata_tree_iter(&tdatas, NULL, prof_tdata_count_iter, |
| 909 | (void *)&tdata_count); |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 910 | malloc_mutex_unlock(tsdn, &tdatas_mtx); |
Jason Evans | 20c31de | 2014-10-02 23:01:10 -0700 | [diff] [blame] | 911 | |
| 912 | return (tdata_count); |
| 913 | } |
| 914 | #endif |
| 915 | |
| 916 | #ifdef JEMALLOC_JET |
Jason Evans | 772163b | 2014-01-17 15:40:52 -0800 | [diff] [blame] | 917 | size_t |
| 918 | prof_bt_count(void) |
| 919 | { |
| 920 | size_t bt_count; |
Jason Evans | 5460aa6 | 2014-09-22 21:09:23 -0700 | [diff] [blame] | 921 | tsd_t *tsd; |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 922 | prof_tdata_t *tdata; |
Jason Evans | 772163b | 2014-01-17 15:40:52 -0800 | [diff] [blame] | 923 | |
Jason Evans | 029d44c | 2014-10-04 11:12:53 -0700 | [diff] [blame] | 924 | tsd = tsd_fetch(); |
Jason Evans | 5460aa6 | 2014-09-22 21:09:23 -0700 | [diff] [blame] | 925 | tdata = prof_tdata_get(tsd, false); |
| 926 | if (tdata == NULL) |
Jason Evans | 772163b | 2014-01-17 15:40:52 -0800 | [diff] [blame] | 927 | return (0); |
| 928 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 929 | malloc_mutex_lock(tsd_tsdn(tsd), &bt2gctx_mtx); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 930 | bt_count = ckh_count(&bt2gctx); |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 931 | malloc_mutex_unlock(tsd_tsdn(tsd), &bt2gctx_mtx); |
Jason Evans | 772163b | 2014-01-17 15:40:52 -0800 | [diff] [blame] | 932 | |
| 933 | return (bt_count); |
| 934 | } |
| 935 | #endif |
| 936 | |
| 937 | #ifdef JEMALLOC_JET |
| 938 | #undef prof_dump_open |
| 939 | #define prof_dump_open JEMALLOC_N(prof_dump_open_impl) |
| 940 | #endif |
| 941 | static int |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 942 | prof_dump_open(bool propagate_err, const char *filename) |
| 943 | { |
Jason Evans | 772163b | 2014-01-17 15:40:52 -0800 | [diff] [blame] | 944 | int fd; |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 945 | |
Jason Evans | 772163b | 2014-01-17 15:40:52 -0800 | [diff] [blame] | 946 | fd = creat(filename, 0644); |
Jason Evans | 551ebc4 | 2014-10-03 10:16:09 -0700 | [diff] [blame] | 947 | if (fd == -1 && !propagate_err) { |
Jason Evans | 772163b | 2014-01-17 15:40:52 -0800 | [diff] [blame] | 948 | malloc_printf("<jemalloc>: creat(\"%s\"), 0644) failed\n", |
| 949 | filename); |
| 950 | if (opt_abort) |
| 951 | abort(); |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 952 | } |
| 953 | |
Jason Evans | 772163b | 2014-01-17 15:40:52 -0800 | [diff] [blame] | 954 | return (fd); |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 955 | } |
Jason Evans | 772163b | 2014-01-17 15:40:52 -0800 | [diff] [blame] | 956 | #ifdef JEMALLOC_JET |
| 957 | #undef prof_dump_open |
| 958 | #define prof_dump_open JEMALLOC_N(prof_dump_open) |
| 959 | prof_dump_open_t *prof_dump_open = JEMALLOC_N(prof_dump_open_impl); |
| 960 | #endif |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 961 | |
| 962 | static bool |
| 963 | prof_dump_flush(bool propagate_err) |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 964 | { |
Jason Evans | 22ca855 | 2010-03-02 11:57:30 -0800 | [diff] [blame] | 965 | bool ret = false; |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 966 | ssize_t err; |
| 967 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 968 | cassert(config_prof); |
| 969 | |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 970 | err = write(prof_dump_fd, prof_dump_buf, prof_dump_buf_end); |
| 971 | if (err == -1) { |
Jason Evans | 551ebc4 | 2014-10-03 10:16:09 -0700 | [diff] [blame] | 972 | if (!propagate_err) { |
Jason Evans | 698805c | 2010-03-03 17:45:38 -0800 | [diff] [blame] | 973 | malloc_write("<jemalloc>: write() failed during heap " |
| 974 | "profile flush\n"); |
Jason Evans | 22ca855 | 2010-03-02 11:57:30 -0800 | [diff] [blame] | 975 | if (opt_abort) |
| 976 | abort(); |
| 977 | } |
| 978 | ret = true; |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 979 | } |
| 980 | prof_dump_buf_end = 0; |
Jason Evans | 22ca855 | 2010-03-02 11:57:30 -0800 | [diff] [blame] | 981 | |
| 982 | return (ret); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 983 | } |
| 984 | |
Jason Evans | 22ca855 | 2010-03-02 11:57:30 -0800 | [diff] [blame] | 985 | static bool |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 986 | prof_dump_close(bool propagate_err) |
| 987 | { |
| 988 | bool ret; |
| 989 | |
| 990 | assert(prof_dump_fd != -1); |
| 991 | ret = prof_dump_flush(propagate_err); |
| 992 | close(prof_dump_fd); |
| 993 | prof_dump_fd = -1; |
| 994 | |
| 995 | return (ret); |
| 996 | } |
| 997 | |
| 998 | static bool |
| 999 | prof_dump_write(bool propagate_err, const char *s) |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1000 | { |
Jason Evans | ca8fffb | 2016-02-24 13:16:51 -0800 | [diff] [blame] | 1001 | size_t i, slen, n; |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1002 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 1003 | cassert(config_prof); |
| 1004 | |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1005 | i = 0; |
| 1006 | slen = strlen(s); |
| 1007 | while (i < slen) { |
| 1008 | /* Flush the buffer if it is full. */ |
Jason Evans | cd9a134 | 2012-03-21 18:33:03 -0700 | [diff] [blame] | 1009 | if (prof_dump_buf_end == PROF_DUMP_BUFSIZE) |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 1010 | if (prof_dump_flush(propagate_err) && propagate_err) |
Jason Evans | 22ca855 | 2010-03-02 11:57:30 -0800 | [diff] [blame] | 1011 | return (true); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1012 | |
Jason Evans | cd9a134 | 2012-03-21 18:33:03 -0700 | [diff] [blame] | 1013 | if (prof_dump_buf_end + slen <= PROF_DUMP_BUFSIZE) { |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1014 | /* Finish writing. */ |
| 1015 | n = slen - i; |
| 1016 | } else { |
| 1017 | /* Write as much of s as will fit. */ |
Jason Evans | cd9a134 | 2012-03-21 18:33:03 -0700 | [diff] [blame] | 1018 | n = PROF_DUMP_BUFSIZE - prof_dump_buf_end; |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1019 | } |
| 1020 | memcpy(&prof_dump_buf[prof_dump_buf_end], &s[i], n); |
| 1021 | prof_dump_buf_end += n; |
| 1022 | i += n; |
| 1023 | } |
Jason Evans | 22ca855 | 2010-03-02 11:57:30 -0800 | [diff] [blame] | 1024 | |
| 1025 | return (false); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1026 | } |
| 1027 | |
Jason Evans | e42c309 | 2015-07-22 15:44:47 -0700 | [diff] [blame] | 1028 | JEMALLOC_FORMAT_PRINTF(2, 3) |
Jason Evans | d81e4bd | 2012-03-06 14:57:45 -0800 | [diff] [blame] | 1029 | static bool |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 1030 | prof_dump_printf(bool propagate_err, const char *format, ...) |
Jason Evans | d81e4bd | 2012-03-06 14:57:45 -0800 | [diff] [blame] | 1031 | { |
| 1032 | bool ret; |
| 1033 | va_list ap; |
Jason Evans | cd9a134 | 2012-03-21 18:33:03 -0700 | [diff] [blame] | 1034 | char buf[PROF_PRINTF_BUFSIZE]; |
Jason Evans | d81e4bd | 2012-03-06 14:57:45 -0800 | [diff] [blame] | 1035 | |
| 1036 | va_start(ap, format); |
Jason Evans | 6da5418 | 2012-03-23 18:05:51 -0700 | [diff] [blame] | 1037 | malloc_vsnprintf(buf, sizeof(buf), format, ap); |
Jason Evans | d81e4bd | 2012-03-06 14:57:45 -0800 | [diff] [blame] | 1038 | va_end(ap); |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 1039 | ret = prof_dump_write(propagate_err, buf); |
Jason Evans | d81e4bd | 2012-03-06 14:57:45 -0800 | [diff] [blame] | 1040 | |
| 1041 | return (ret); |
| 1042 | } |
| 1043 | |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1044 | static void |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 1045 | prof_tctx_merge_tdata(tsdn_t *tsdn, prof_tctx_t *tctx, prof_tdata_t *tdata) |
Jason Evans | 3a81cbd | 2014-08-16 12:58:55 -0700 | [diff] [blame] | 1046 | { |
Jason Evans | 3a81cbd | 2014-08-16 12:58:55 -0700 | [diff] [blame] | 1047 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 1048 | malloc_mutex_assert_owner(tsdn, tctx->tdata->lock); |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 1049 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 1050 | malloc_mutex_lock(tsdn, tctx->gctx->lock); |
Jason Evans | 764b000 | 2015-03-14 14:01:35 -0700 | [diff] [blame] | 1051 | |
| 1052 | switch (tctx->state) { |
| 1053 | case prof_tctx_state_initializing: |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 1054 | malloc_mutex_unlock(tsdn, tctx->gctx->lock); |
Jason Evans | 6ef80d6 | 2014-09-24 22:14:21 -0700 | [diff] [blame] | 1055 | return; |
Jason Evans | 764b000 | 2015-03-14 14:01:35 -0700 | [diff] [blame] | 1056 | case prof_tctx_state_nominal: |
| 1057 | tctx->state = prof_tctx_state_dumping; |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 1058 | malloc_mutex_unlock(tsdn, tctx->gctx->lock); |
Jason Evans | 6ef80d6 | 2014-09-24 22:14:21 -0700 | [diff] [blame] | 1059 | |
Jason Evans | 764b000 | 2015-03-14 14:01:35 -0700 | [diff] [blame] | 1060 | memcpy(&tctx->dump_cnts, &tctx->cnts, sizeof(prof_cnt_t)); |
Jason Evans | 3a81cbd | 2014-08-16 12:58:55 -0700 | [diff] [blame] | 1061 | |
Jason Evans | 764b000 | 2015-03-14 14:01:35 -0700 | [diff] [blame] | 1062 | tdata->cnt_summed.curobjs += tctx->dump_cnts.curobjs; |
| 1063 | tdata->cnt_summed.curbytes += tctx->dump_cnts.curbytes; |
| 1064 | if (opt_prof_accum) { |
| 1065 | tdata->cnt_summed.accumobjs += |
| 1066 | tctx->dump_cnts.accumobjs; |
| 1067 | tdata->cnt_summed.accumbytes += |
| 1068 | tctx->dump_cnts.accumbytes; |
| 1069 | } |
| 1070 | break; |
| 1071 | case prof_tctx_state_dumping: |
| 1072 | case prof_tctx_state_purgatory: |
| 1073 | not_reached(); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1074 | } |
| 1075 | } |
| 1076 | |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1077 | static void |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 1078 | prof_tctx_merge_gctx(tsdn_t *tsdn, prof_tctx_t *tctx, prof_gctx_t *gctx) |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1079 | { |
| 1080 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 1081 | malloc_mutex_assert_owner(tsdn, gctx->lock); |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 1082 | |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1083 | gctx->cnt_summed.curobjs += tctx->dump_cnts.curobjs; |
| 1084 | gctx->cnt_summed.curbytes += tctx->dump_cnts.curbytes; |
| 1085 | if (opt_prof_accum) { |
| 1086 | gctx->cnt_summed.accumobjs += tctx->dump_cnts.accumobjs; |
| 1087 | gctx->cnt_summed.accumbytes += tctx->dump_cnts.accumbytes; |
| 1088 | } |
| 1089 | } |
| 1090 | |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1091 | static prof_tctx_t * |
| 1092 | prof_tctx_merge_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *arg) |
| 1093 | { |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 1094 | tsdn_t *tsdn = (tsdn_t *)arg; |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 1095 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 1096 | malloc_mutex_assert_owner(tsdn, tctx->gctx->lock); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1097 | |
| 1098 | switch (tctx->state) { |
| 1099 | case prof_tctx_state_nominal: |
| 1100 | /* New since dumping started; ignore. */ |
| 1101 | break; |
| 1102 | case prof_tctx_state_dumping: |
| 1103 | case prof_tctx_state_purgatory: |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 1104 | prof_tctx_merge_gctx(tsdn, tctx, tctx->gctx); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1105 | break; |
| 1106 | default: |
| 1107 | not_reached(); |
Jason Evans | 3a81cbd | 2014-08-16 12:58:55 -0700 | [diff] [blame] | 1108 | } |
| 1109 | |
| 1110 | return (NULL); |
| 1111 | } |
| 1112 | |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 1113 | struct prof_tctx_dump_iter_arg_s { |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 1114 | tsdn_t *tsdn; |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 1115 | bool propagate_err; |
| 1116 | }; |
| 1117 | |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1118 | static prof_tctx_t * |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 1119 | prof_tctx_dump_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *opaque) |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1120 | { |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 1121 | struct prof_tctx_dump_iter_arg_s *arg = |
| 1122 | (struct prof_tctx_dump_iter_arg_s *)opaque; |
| 1123 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 1124 | malloc_mutex_assert_owner(arg->tsdn, tctx->gctx->lock); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1125 | |
Jason Evans | fb64ec2 | 2015-09-21 18:37:18 -0700 | [diff] [blame] | 1126 | switch (tctx->state) { |
| 1127 | case prof_tctx_state_initializing: |
| 1128 | case prof_tctx_state_nominal: |
| 1129 | /* Not captured by this dump. */ |
| 1130 | break; |
| 1131 | case prof_tctx_state_dumping: |
| 1132 | case prof_tctx_state_purgatory: |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 1133 | if (prof_dump_printf(arg->propagate_err, |
Jason Evans | fb64ec2 | 2015-09-21 18:37:18 -0700 | [diff] [blame] | 1134 | " t%"FMTu64": %"FMTu64": %"FMTu64" [%"FMTu64": " |
| 1135 | "%"FMTu64"]\n", tctx->thr_uid, tctx->dump_cnts.curobjs, |
| 1136 | tctx->dump_cnts.curbytes, tctx->dump_cnts.accumobjs, |
| 1137 | tctx->dump_cnts.accumbytes)) |
| 1138 | return (tctx); |
| 1139 | break; |
| 1140 | default: |
| 1141 | not_reached(); |
| 1142 | } |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1143 | return (NULL); |
| 1144 | } |
| 1145 | |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1146 | static prof_tctx_t * |
| 1147 | prof_tctx_finish_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *arg) |
| 1148 | { |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 1149 | tsdn_t *tsdn = (tsdn_t *)arg; |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1150 | prof_tctx_t *ret; |
| 1151 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 1152 | malloc_mutex_assert_owner(tsdn, tctx->gctx->lock); |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 1153 | |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1154 | switch (tctx->state) { |
| 1155 | case prof_tctx_state_nominal: |
| 1156 | /* New since dumping started; ignore. */ |
| 1157 | break; |
| 1158 | case prof_tctx_state_dumping: |
| 1159 | tctx->state = prof_tctx_state_nominal; |
| 1160 | break; |
| 1161 | case prof_tctx_state_purgatory: |
Jason Evans | 20c31de | 2014-10-02 23:01:10 -0700 | [diff] [blame] | 1162 | ret = tctx; |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1163 | goto label_return; |
| 1164 | default: |
| 1165 | not_reached(); |
| 1166 | } |
| 1167 | |
| 1168 | ret = NULL; |
| 1169 | label_return: |
| 1170 | return (ret); |
| 1171 | } |
| 1172 | |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1173 | static void |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 1174 | prof_dump_gctx_prep(tsdn_t *tsdn, prof_gctx_t *gctx, prof_gctx_tree_t *gctxs) |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1175 | { |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1176 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 1177 | cassert(config_prof); |
| 1178 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 1179 | malloc_mutex_lock(tsdn, gctx->lock); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1180 | |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 1181 | /* |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1182 | * Increment nlimbo so that gctx won't go away before dump. |
| 1183 | * Additionally, link gctx into the dump list so that it is included in |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 1184 | * prof_dump()'s second pass. |
| 1185 | */ |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1186 | gctx->nlimbo++; |
| 1187 | gctx_tree_insert(gctxs, gctx); |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 1188 | |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1189 | memset(&gctx->cnt_summed, 0, sizeof(prof_cnt_t)); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1190 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 1191 | malloc_mutex_unlock(tsdn, gctx->lock); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1192 | } |
Jason Evans | 9ce3bfd | 2010-10-02 22:39:59 -0700 | [diff] [blame] | 1193 | |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 1194 | struct prof_gctx_merge_iter_arg_s { |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 1195 | tsdn_t *tsdn; |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 1196 | size_t leak_ngctx; |
| 1197 | }; |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1198 | |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 1199 | static prof_gctx_t * |
| 1200 | prof_gctx_merge_iter(prof_gctx_tree_t *gctxs, prof_gctx_t *gctx, void *opaque) |
| 1201 | { |
| 1202 | struct prof_gctx_merge_iter_arg_s *arg = |
| 1203 | (struct prof_gctx_merge_iter_arg_s *)opaque; |
| 1204 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 1205 | malloc_mutex_lock(arg->tsdn, gctx->lock); |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 1206 | tctx_tree_iter(&gctx->tctxs, NULL, prof_tctx_merge_iter, |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 1207 | (void *)arg->tsdn); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1208 | if (gctx->cnt_summed.curobjs != 0) |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 1209 | arg->leak_ngctx++; |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 1210 | malloc_mutex_unlock(arg->tsdn, gctx->lock); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1211 | |
| 1212 | return (NULL); |
| 1213 | } |
| 1214 | |
Jason Evans | 20c31de | 2014-10-02 23:01:10 -0700 | [diff] [blame] | 1215 | static void |
| 1216 | prof_gctx_finish(tsd_t *tsd, prof_gctx_tree_t *gctxs) |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1217 | { |
Jason Evans | 5460aa6 | 2014-09-22 21:09:23 -0700 | [diff] [blame] | 1218 | prof_tdata_t *tdata = prof_tdata_get(tsd, false); |
Jason Evans | 20c31de | 2014-10-02 23:01:10 -0700 | [diff] [blame] | 1219 | prof_gctx_t *gctx; |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1220 | |
Jason Evans | 20c31de | 2014-10-02 23:01:10 -0700 | [diff] [blame] | 1221 | /* |
| 1222 | * Standard tree iteration won't work here, because as soon as we |
| 1223 | * decrement gctx->nlimbo and unlock gctx, another thread can |
| 1224 | * concurrently destroy it, which will corrupt the tree. Therefore, |
| 1225 | * tear down the tree one node at a time during iteration. |
| 1226 | */ |
| 1227 | while ((gctx = gctx_tree_first(gctxs)) != NULL) { |
| 1228 | gctx_tree_remove(gctxs, gctx); |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 1229 | malloc_mutex_lock(tsd_tsdn(tsd), gctx->lock); |
Jason Evans | 20c31de | 2014-10-02 23:01:10 -0700 | [diff] [blame] | 1230 | { |
| 1231 | prof_tctx_t *next; |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1232 | |
Jason Evans | 20c31de | 2014-10-02 23:01:10 -0700 | [diff] [blame] | 1233 | next = NULL; |
| 1234 | do { |
| 1235 | prof_tctx_t *to_destroy = |
| 1236 | tctx_tree_iter(&gctx->tctxs, next, |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 1237 | prof_tctx_finish_iter, |
| 1238 | (void *)tsd_tsdn(tsd)); |
Jason Evans | 20c31de | 2014-10-02 23:01:10 -0700 | [diff] [blame] | 1239 | if (to_destroy != NULL) { |
| 1240 | next = tctx_tree_next(&gctx->tctxs, |
| 1241 | to_destroy); |
| 1242 | tctx_tree_remove(&gctx->tctxs, |
| 1243 | to_destroy); |
Jason Evans | db72272 | 2016-03-23 20:29:33 -0700 | [diff] [blame] | 1244 | idalloctm(tsd_tsdn(tsd), |
Jason Evans | 8c9be3e | 2016-04-16 00:36:11 -0700 | [diff] [blame] | 1245 | iealloc(tsd_tsdn(tsd), to_destroy), |
| 1246 | to_destroy, NULL, true, true); |
Jason Evans | 20c31de | 2014-10-02 23:01:10 -0700 | [diff] [blame] | 1247 | } else |
| 1248 | next = NULL; |
| 1249 | } while (next != NULL); |
| 1250 | } |
| 1251 | gctx->nlimbo--; |
| 1252 | if (prof_gctx_should_destroy(gctx)) { |
| 1253 | gctx->nlimbo++; |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 1254 | malloc_mutex_unlock(tsd_tsdn(tsd), gctx->lock); |
Jason Evans | c93ed81 | 2014-10-30 16:50:33 -0700 | [diff] [blame] | 1255 | prof_gctx_try_destroy(tsd, tdata, gctx, tdata); |
Jason Evans | 20c31de | 2014-10-02 23:01:10 -0700 | [diff] [blame] | 1256 | } else |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 1257 | malloc_mutex_unlock(tsd_tsdn(tsd), gctx->lock); |
Jason Evans | 20c31de | 2014-10-02 23:01:10 -0700 | [diff] [blame] | 1258 | } |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1259 | } |
| 1260 | |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 1261 | struct prof_tdata_merge_iter_arg_s { |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 1262 | tsdn_t *tsdn; |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 1263 | prof_cnt_t cnt_all; |
| 1264 | }; |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1265 | |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 1266 | static prof_tdata_t * |
| 1267 | prof_tdata_merge_iter(prof_tdata_tree_t *tdatas, prof_tdata_t *tdata, |
| 1268 | void *opaque) |
| 1269 | { |
| 1270 | struct prof_tdata_merge_iter_arg_s *arg = |
| 1271 | (struct prof_tdata_merge_iter_arg_s *)opaque; |
| 1272 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 1273 | malloc_mutex_lock(arg->tsdn, tdata->lock); |
Jason Evans | 20c31de | 2014-10-02 23:01:10 -0700 | [diff] [blame] | 1274 | if (!tdata->expired) { |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1275 | size_t tabind; |
| 1276 | union { |
| 1277 | prof_tctx_t *p; |
| 1278 | void *v; |
| 1279 | } tctx; |
| 1280 | |
| 1281 | tdata->dumping = true; |
| 1282 | memset(&tdata->cnt_summed, 0, sizeof(prof_cnt_t)); |
Jason Evans | 551ebc4 | 2014-10-03 10:16:09 -0700 | [diff] [blame] | 1283 | for (tabind = 0; !ckh_iter(&tdata->bt2tctx, &tabind, NULL, |
| 1284 | &tctx.v);) |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 1285 | prof_tctx_merge_tdata(arg->tsdn, tctx.p, tdata); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1286 | |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 1287 | arg->cnt_all.curobjs += tdata->cnt_summed.curobjs; |
| 1288 | arg->cnt_all.curbytes += tdata->cnt_summed.curbytes; |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1289 | if (opt_prof_accum) { |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 1290 | arg->cnt_all.accumobjs += tdata->cnt_summed.accumobjs; |
| 1291 | arg->cnt_all.accumbytes += tdata->cnt_summed.accumbytes; |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1292 | } |
| 1293 | } else |
| 1294 | tdata->dumping = false; |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 1295 | malloc_mutex_unlock(arg->tsdn, tdata->lock); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1296 | |
| 1297 | return (NULL); |
| 1298 | } |
| 1299 | |
| 1300 | static prof_tdata_t * |
| 1301 | prof_tdata_dump_iter(prof_tdata_tree_t *tdatas, prof_tdata_t *tdata, void *arg) |
| 1302 | { |
| 1303 | bool propagate_err = *(bool *)arg; |
| 1304 | |
Jason Evans | 551ebc4 | 2014-10-03 10:16:09 -0700 | [diff] [blame] | 1305 | if (!tdata->dumping) |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1306 | return (NULL); |
| 1307 | |
| 1308 | if (prof_dump_printf(propagate_err, |
Jason Evans | 5fae7dc | 2015-07-23 13:56:25 -0700 | [diff] [blame] | 1309 | " t%"FMTu64": %"FMTu64": %"FMTu64" [%"FMTu64": %"FMTu64"]%s%s\n", |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1310 | tdata->thr_uid, tdata->cnt_summed.curobjs, |
| 1311 | tdata->cnt_summed.curbytes, tdata->cnt_summed.accumobjs, |
| 1312 | tdata->cnt_summed.accumbytes, |
| 1313 | (tdata->thread_name != NULL) ? " " : "", |
| 1314 | (tdata->thread_name != NULL) ? tdata->thread_name : "")) |
| 1315 | return (tdata); |
| 1316 | return (NULL); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1317 | } |
| 1318 | |
Jason Evans | 20c31de | 2014-10-02 23:01:10 -0700 | [diff] [blame] | 1319 | #ifdef JEMALLOC_JET |
| 1320 | #undef prof_dump_header |
| 1321 | #define prof_dump_header JEMALLOC_N(prof_dump_header_impl) |
| 1322 | #endif |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 1323 | static bool |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 1324 | prof_dump_header(tsdn_t *tsdn, bool propagate_err, const prof_cnt_t *cnt_all) |
Jason Evans | a881cd2 | 2010-10-02 15:18:50 -0700 | [diff] [blame] | 1325 | { |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1326 | bool ret; |
Jason Evans | a881cd2 | 2010-10-02 15:18:50 -0700 | [diff] [blame] | 1327 | |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1328 | if (prof_dump_printf(propagate_err, |
Jason Evans | 5fae7dc | 2015-07-23 13:56:25 -0700 | [diff] [blame] | 1329 | "heap_v2/%"FMTu64"\n" |
| 1330 | " t*: %"FMTu64": %"FMTu64" [%"FMTu64": %"FMTu64"]\n", |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1331 | ((uint64_t)1U << lg_prof_sample), cnt_all->curobjs, |
| 1332 | cnt_all->curbytes, cnt_all->accumobjs, cnt_all->accumbytes)) |
| 1333 | return (true); |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 1334 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 1335 | malloc_mutex_lock(tsdn, &tdatas_mtx); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1336 | ret = (tdata_tree_iter(&tdatas, NULL, prof_tdata_dump_iter, |
| 1337 | (void *)&propagate_err) != NULL); |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 1338 | malloc_mutex_unlock(tsdn, &tdatas_mtx); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1339 | return (ret); |
Jason Evans | a881cd2 | 2010-10-02 15:18:50 -0700 | [diff] [blame] | 1340 | } |
Jason Evans | 20c31de | 2014-10-02 23:01:10 -0700 | [diff] [blame] | 1341 | #ifdef JEMALLOC_JET |
| 1342 | #undef prof_dump_header |
| 1343 | #define prof_dump_header JEMALLOC_N(prof_dump_header) |
| 1344 | prof_dump_header_t *prof_dump_header = JEMALLOC_N(prof_dump_header_impl); |
| 1345 | #endif |
Jason Evans | a881cd2 | 2010-10-02 15:18:50 -0700 | [diff] [blame] | 1346 | |
Jason Evans | 22ca855 | 2010-03-02 11:57:30 -0800 | [diff] [blame] | 1347 | static bool |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 1348 | prof_dump_gctx(tsdn_t *tsdn, bool propagate_err, prof_gctx_t *gctx, |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 1349 | const prof_bt_t *bt, prof_gctx_tree_t *gctxs) |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1350 | { |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 1351 | bool ret; |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1352 | unsigned i; |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 1353 | struct prof_tctx_dump_iter_arg_s prof_tctx_dump_iter_arg; |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1354 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 1355 | cassert(config_prof); |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 1356 | malloc_mutex_assert_owner(tsdn, gctx->lock); |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 1357 | |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1358 | /* Avoid dumping such gctx's that have no useful data. */ |
Jason Evans | 551ebc4 | 2014-10-03 10:16:09 -0700 | [diff] [blame] | 1359 | if ((!opt_prof_accum && gctx->cnt_summed.curobjs == 0) || |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1360 | (opt_prof_accum && gctx->cnt_summed.accumobjs == 0)) { |
| 1361 | assert(gctx->cnt_summed.curobjs == 0); |
| 1362 | assert(gctx->cnt_summed.curbytes == 0); |
| 1363 | assert(gctx->cnt_summed.accumobjs == 0); |
| 1364 | assert(gctx->cnt_summed.accumbytes == 0); |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 1365 | ret = false; |
| 1366 | goto label_return; |
Jason Evans | a881cd2 | 2010-10-02 15:18:50 -0700 | [diff] [blame] | 1367 | } |
| 1368 | |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1369 | if (prof_dump_printf(propagate_err, "@")) { |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 1370 | ret = true; |
| 1371 | goto label_return; |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1372 | } |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 1373 | for (i = 0; i < bt->len; i++) { |
Jason Evans | 5fae7dc | 2015-07-23 13:56:25 -0700 | [diff] [blame] | 1374 | if (prof_dump_printf(propagate_err, " %#"FMTxPTR, |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 1375 | (uintptr_t)bt->vec[i])) { |
| 1376 | ret = true; |
| 1377 | goto label_return; |
| 1378 | } |
| 1379 | } |
Jason Evans | 22ca855 | 2010-03-02 11:57:30 -0800 | [diff] [blame] | 1380 | |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1381 | if (prof_dump_printf(propagate_err, |
| 1382 | "\n" |
Jason Evans | 5fae7dc | 2015-07-23 13:56:25 -0700 | [diff] [blame] | 1383 | " t*: %"FMTu64": %"FMTu64" [%"FMTu64": %"FMTu64"]\n", |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1384 | gctx->cnt_summed.curobjs, gctx->cnt_summed.curbytes, |
| 1385 | gctx->cnt_summed.accumobjs, gctx->cnt_summed.accumbytes)) { |
| 1386 | ret = true; |
| 1387 | goto label_return; |
| 1388 | } |
| 1389 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 1390 | prof_tctx_dump_iter_arg.tsdn = tsdn; |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 1391 | prof_tctx_dump_iter_arg.propagate_err = propagate_err; |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1392 | if (tctx_tree_iter(&gctx->tctxs, NULL, prof_tctx_dump_iter, |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 1393 | (void *)&prof_tctx_dump_iter_arg) != NULL) { |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 1394 | ret = true; |
| 1395 | goto label_return; |
| 1396 | } |
| 1397 | |
Jason Evans | 772163b | 2014-01-17 15:40:52 -0800 | [diff] [blame] | 1398 | ret = false; |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 1399 | label_return: |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 1400 | return (ret); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1401 | } |
| 1402 | |
Jason Evans | 788d29d | 2016-02-20 23:46:14 -0800 | [diff] [blame] | 1403 | #ifndef _WIN32 |
Jason Evans | e42c309 | 2015-07-22 15:44:47 -0700 | [diff] [blame] | 1404 | JEMALLOC_FORMAT_PRINTF(1, 2) |
Jason Evans | 8e33c21 | 2015-05-01 09:03:20 -0700 | [diff] [blame] | 1405 | static int |
| 1406 | prof_open_maps(const char *format, ...) |
| 1407 | { |
| 1408 | int mfd; |
| 1409 | va_list ap; |
| 1410 | char filename[PATH_MAX + 1]; |
| 1411 | |
| 1412 | va_start(ap, format); |
| 1413 | malloc_vsnprintf(filename, sizeof(filename), format, ap); |
| 1414 | va_end(ap); |
| 1415 | mfd = open(filename, O_RDONLY); |
| 1416 | |
| 1417 | return (mfd); |
| 1418 | } |
Jason Evans | 788d29d | 2016-02-20 23:46:14 -0800 | [diff] [blame] | 1419 | #endif |
| 1420 | |
| 1421 | static int |
| 1422 | prof_getpid(void) |
| 1423 | { |
| 1424 | |
| 1425 | #ifdef _WIN32 |
| 1426 | return (GetCurrentProcessId()); |
| 1427 | #else |
| 1428 | return (getpid()); |
| 1429 | #endif |
| 1430 | } |
Jason Evans | 8e33c21 | 2015-05-01 09:03:20 -0700 | [diff] [blame] | 1431 | |
Jason Evans | 22ca855 | 2010-03-02 11:57:30 -0800 | [diff] [blame] | 1432 | static bool |
| 1433 | prof_dump_maps(bool propagate_err) |
Jason Evans | c717718 | 2010-02-11 09:25:56 -0800 | [diff] [blame] | 1434 | { |
Jason Evans | 93f39f8 | 2013-10-21 15:07:40 -0700 | [diff] [blame] | 1435 | bool ret; |
Jason Evans | c717718 | 2010-02-11 09:25:56 -0800 | [diff] [blame] | 1436 | int mfd; |
Jason Evans | c717718 | 2010-02-11 09:25:56 -0800 | [diff] [blame] | 1437 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 1438 | cassert(config_prof); |
Harald Weppner | c2da259 | 2014-03-18 00:00:14 -0700 | [diff] [blame] | 1439 | #ifdef __FreeBSD__ |
Jason Evans | 8e33c21 | 2015-05-01 09:03:20 -0700 | [diff] [blame] | 1440 | mfd = prof_open_maps("/proc/curproc/map"); |
rustyx | 7f28398 | 2016-01-30 14:51:16 +0100 | [diff] [blame] | 1441 | #elif defined(_WIN32) |
| 1442 | mfd = -1; // Not implemented |
Harald Weppner | c2da259 | 2014-03-18 00:00:14 -0700 | [diff] [blame] | 1443 | #else |
Jason Evans | 8e33c21 | 2015-05-01 09:03:20 -0700 | [diff] [blame] | 1444 | { |
Jason Evans | 788d29d | 2016-02-20 23:46:14 -0800 | [diff] [blame] | 1445 | int pid = prof_getpid(); |
Jason Evans | 8e33c21 | 2015-05-01 09:03:20 -0700 | [diff] [blame] | 1446 | |
| 1447 | mfd = prof_open_maps("/proc/%d/task/%d/maps", pid, pid); |
| 1448 | if (mfd == -1) |
| 1449 | mfd = prof_open_maps("/proc/%d/maps", pid); |
| 1450 | } |
Harald Weppner | c2da259 | 2014-03-18 00:00:14 -0700 | [diff] [blame] | 1451 | #endif |
Jason Evans | c717718 | 2010-02-11 09:25:56 -0800 | [diff] [blame] | 1452 | if (mfd != -1) { |
| 1453 | ssize_t nread; |
| 1454 | |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 1455 | if (prof_dump_write(propagate_err, "\nMAPPED_LIBRARIES:\n") && |
Jason Evans | 93f39f8 | 2013-10-21 15:07:40 -0700 | [diff] [blame] | 1456 | propagate_err) { |
| 1457 | ret = true; |
| 1458 | goto label_return; |
| 1459 | } |
Jason Evans | c717718 | 2010-02-11 09:25:56 -0800 | [diff] [blame] | 1460 | nread = 0; |
| 1461 | do { |
| 1462 | prof_dump_buf_end += nread; |
Jason Evans | cd9a134 | 2012-03-21 18:33:03 -0700 | [diff] [blame] | 1463 | if (prof_dump_buf_end == PROF_DUMP_BUFSIZE) { |
Jason Evans | c717718 | 2010-02-11 09:25:56 -0800 | [diff] [blame] | 1464 | /* Make space in prof_dump_buf before read(). */ |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 1465 | if (prof_dump_flush(propagate_err) && |
Jason Evans | 93f39f8 | 2013-10-21 15:07:40 -0700 | [diff] [blame] | 1466 | propagate_err) { |
| 1467 | ret = true; |
| 1468 | goto label_return; |
| 1469 | } |
Jason Evans | c717718 | 2010-02-11 09:25:56 -0800 | [diff] [blame] | 1470 | } |
| 1471 | nread = read(mfd, &prof_dump_buf[prof_dump_buf_end], |
Jason Evans | cd9a134 | 2012-03-21 18:33:03 -0700 | [diff] [blame] | 1472 | PROF_DUMP_BUFSIZE - prof_dump_buf_end); |
Jason Evans | c717718 | 2010-02-11 09:25:56 -0800 | [diff] [blame] | 1473 | } while (nread > 0); |
Jason Evans | 93f39f8 | 2013-10-21 15:07:40 -0700 | [diff] [blame] | 1474 | } else { |
| 1475 | ret = true; |
| 1476 | goto label_return; |
| 1477 | } |
Jason Evans | 22ca855 | 2010-03-02 11:57:30 -0800 | [diff] [blame] | 1478 | |
Jason Evans | 93f39f8 | 2013-10-21 15:07:40 -0700 | [diff] [blame] | 1479 | ret = false; |
| 1480 | label_return: |
| 1481 | if (mfd != -1) |
| 1482 | close(mfd); |
| 1483 | return (ret); |
Jason Evans | c717718 | 2010-02-11 09:25:56 -0800 | [diff] [blame] | 1484 | } |
| 1485 | |
Jason Evans | dc391ad | 2016-05-04 12:14:36 -0700 | [diff] [blame] | 1486 | /* |
| 1487 | * See prof_sample_threshold_update() comment for why the body of this function |
| 1488 | * is conditionally compiled. |
| 1489 | */ |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 1490 | static void |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1491 | prof_leakcheck(const prof_cnt_t *cnt_all, size_t leak_ngctx, |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 1492 | const char *filename) |
| 1493 | { |
| 1494 | |
Jason Evans | dc391ad | 2016-05-04 12:14:36 -0700 | [diff] [blame] | 1495 | #ifdef JEMALLOC_PROF |
| 1496 | /* |
| 1497 | * Scaling is equivalent AdjustSamples() in jeprof, but the result may |
| 1498 | * differ slightly from what jeprof reports, because here we scale the |
| 1499 | * summary values, whereas jeprof scales each context individually and |
| 1500 | * reports the sums of the scaled values. |
| 1501 | */ |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 1502 | if (cnt_all->curbytes != 0) { |
Jason Evans | dc391ad | 2016-05-04 12:14:36 -0700 | [diff] [blame] | 1503 | double sample_period = (double)((uint64_t)1 << lg_prof_sample); |
| 1504 | double ratio = (((double)cnt_all->curbytes) / |
| 1505 | (double)cnt_all->curobjs) / sample_period; |
| 1506 | double scale_factor = 1.0 / (1.0 - exp(-ratio)); |
| 1507 | uint64_t curbytes = (uint64_t)round(((double)cnt_all->curbytes) |
| 1508 | * scale_factor); |
| 1509 | uint64_t curobjs = (uint64_t)round(((double)cnt_all->curobjs) * |
| 1510 | scale_factor); |
| 1511 | |
| 1512 | malloc_printf("<jemalloc>: Leak approximation summary: ~%"FMTu64 |
| 1513 | " byte%s, ~%"FMTu64" object%s, >= %zu context%s\n", |
| 1514 | curbytes, (curbytes != 1) ? "s" : "", curobjs, (curobjs != |
| 1515 | 1) ? "s" : "", leak_ngctx, (leak_ngctx != 1) ? "s" : ""); |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 1516 | malloc_printf( |
Jason Evans | 7041720 | 2015-05-01 12:31:12 -0700 | [diff] [blame] | 1517 | "<jemalloc>: Run jeprof on \"%s\" for leak detail\n", |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 1518 | filename); |
| 1519 | } |
Jason Evans | dc391ad | 2016-05-04 12:14:36 -0700 | [diff] [blame] | 1520 | #endif |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 1521 | } |
| 1522 | |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 1523 | struct prof_gctx_dump_iter_arg_s { |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 1524 | tsdn_t *tsdn; |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 1525 | bool propagate_err; |
| 1526 | }; |
| 1527 | |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1528 | static prof_gctx_t * |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 1529 | prof_gctx_dump_iter(prof_gctx_tree_t *gctxs, prof_gctx_t *gctx, void *opaque) |
Jason Evans | 3a81cbd | 2014-08-16 12:58:55 -0700 | [diff] [blame] | 1530 | { |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1531 | prof_gctx_t *ret; |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 1532 | struct prof_gctx_dump_iter_arg_s *arg = |
| 1533 | (struct prof_gctx_dump_iter_arg_s *)opaque; |
Jason Evans | 3a81cbd | 2014-08-16 12:58:55 -0700 | [diff] [blame] | 1534 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 1535 | malloc_mutex_lock(arg->tsdn, gctx->lock); |
Jason Evans | 3a81cbd | 2014-08-16 12:58:55 -0700 | [diff] [blame] | 1536 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 1537 | if (prof_dump_gctx(arg->tsdn, arg->propagate_err, gctx, &gctx->bt, |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 1538 | gctxs)) { |
Jason Evans | 20c31de | 2014-10-02 23:01:10 -0700 | [diff] [blame] | 1539 | ret = gctx; |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1540 | goto label_return; |
| 1541 | } |
Jason Evans | 3a81cbd | 2014-08-16 12:58:55 -0700 | [diff] [blame] | 1542 | |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1543 | ret = NULL; |
| 1544 | label_return: |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 1545 | malloc_mutex_unlock(arg->tsdn, gctx->lock); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1546 | return (ret); |
Jason Evans | 3a81cbd | 2014-08-16 12:58:55 -0700 | [diff] [blame] | 1547 | } |
| 1548 | |
Jason Evans | 22ca855 | 2010-03-02 11:57:30 -0800 | [diff] [blame] | 1549 | static bool |
Jason Evans | 5460aa6 | 2014-09-22 21:09:23 -0700 | [diff] [blame] | 1550 | prof_dump(tsd_t *tsd, bool propagate_err, const char *filename, bool leakcheck) |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1551 | { |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1552 | prof_tdata_t *tdata; |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 1553 | struct prof_tdata_merge_iter_arg_s prof_tdata_merge_iter_arg; |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1554 | size_t tabind; |
Jason Evans | 075e77c | 2010-09-20 19:53:25 -0700 | [diff] [blame] | 1555 | union { |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1556 | prof_gctx_t *p; |
Jason Evans | 075e77c | 2010-09-20 19:53:25 -0700 | [diff] [blame] | 1557 | void *v; |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1558 | } gctx; |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 1559 | struct prof_gctx_merge_iter_arg_s prof_gctx_merge_iter_arg; |
| 1560 | struct prof_gctx_dump_iter_arg_s prof_gctx_dump_iter_arg; |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1561 | prof_gctx_tree_t gctxs; |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1562 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 1563 | cassert(config_prof); |
| 1564 | |
Jason Evans | 20c31de | 2014-10-02 23:01:10 -0700 | [diff] [blame] | 1565 | tdata = prof_tdata_get(tsd, true); |
Jason Evans | 5460aa6 | 2014-09-22 21:09:23 -0700 | [diff] [blame] | 1566 | if (tdata == NULL) |
Jason Evans | 52386b2 | 2012-04-22 16:00:11 -0700 | [diff] [blame] | 1567 | return (true); |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 1568 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 1569 | malloc_mutex_lock(tsd_tsdn(tsd), &prof_dump_mtx); |
Jason Evans | c93ed81 | 2014-10-30 16:50:33 -0700 | [diff] [blame] | 1570 | prof_enter(tsd, tdata); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1571 | |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1572 | /* |
| 1573 | * Put gctx's in limbo and clear their counters in preparation for |
| 1574 | * summing. |
| 1575 | */ |
| 1576 | gctx_tree_new(&gctxs); |
Jason Evans | 551ebc4 | 2014-10-03 10:16:09 -0700 | [diff] [blame] | 1577 | for (tabind = 0; !ckh_iter(&bt2gctx, &tabind, NULL, &gctx.v);) |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 1578 | prof_dump_gctx_prep(tsd_tsdn(tsd), gctx.p, &gctxs); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1579 | |
| 1580 | /* |
| 1581 | * Iterate over tdatas, and for the non-expired ones snapshot their tctx |
| 1582 | * stats and merge them into the associated gctx's. |
| 1583 | */ |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 1584 | prof_tdata_merge_iter_arg.tsdn = tsd_tsdn(tsd); |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 1585 | memset(&prof_tdata_merge_iter_arg.cnt_all, 0, sizeof(prof_cnt_t)); |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 1586 | malloc_mutex_lock(tsd_tsdn(tsd), &tdatas_mtx); |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 1587 | tdata_tree_iter(&tdatas, NULL, prof_tdata_merge_iter, |
| 1588 | (void *)&prof_tdata_merge_iter_arg); |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 1589 | malloc_mutex_unlock(tsd_tsdn(tsd), &tdatas_mtx); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1590 | |
| 1591 | /* Merge tctx stats into gctx's. */ |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 1592 | prof_gctx_merge_iter_arg.tsdn = tsd_tsdn(tsd); |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 1593 | prof_gctx_merge_iter_arg.leak_ngctx = 0; |
| 1594 | gctx_tree_iter(&gctxs, NULL, prof_gctx_merge_iter, |
| 1595 | (void *)&prof_gctx_merge_iter_arg); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1596 | |
Jason Evans | c93ed81 | 2014-10-30 16:50:33 -0700 | [diff] [blame] | 1597 | prof_leave(tsd, tdata); |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 1598 | |
| 1599 | /* Create dump file. */ |
Jason Evans | 772163b | 2014-01-17 15:40:52 -0800 | [diff] [blame] | 1600 | if ((prof_dump_fd = prof_dump_open(propagate_err, filename)) == -1) |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 1601 | goto label_open_close_error; |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1602 | |
| 1603 | /* Dump profile header. */ |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 1604 | if (prof_dump_header(tsd_tsdn(tsd), propagate_err, |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 1605 | &prof_tdata_merge_iter_arg.cnt_all)) |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 1606 | goto label_write_error; |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1607 | |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1608 | /* Dump per gctx profile stats. */ |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 1609 | prof_gctx_dump_iter_arg.tsdn = tsd_tsdn(tsd); |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 1610 | prof_gctx_dump_iter_arg.propagate_err = propagate_err; |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1611 | if (gctx_tree_iter(&gctxs, NULL, prof_gctx_dump_iter, |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 1612 | (void *)&prof_gctx_dump_iter_arg) != NULL) |
Jason Evans | 3a81cbd | 2014-08-16 12:58:55 -0700 | [diff] [blame] | 1613 | goto label_write_error; |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1614 | |
Jason Evans | c717718 | 2010-02-11 09:25:56 -0800 | [diff] [blame] | 1615 | /* Dump /proc/<pid>/maps if possible. */ |
Jason Evans | 22ca855 | 2010-03-02 11:57:30 -0800 | [diff] [blame] | 1616 | if (prof_dump_maps(propagate_err)) |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 1617 | goto label_write_error; |
Jason Evans | c717718 | 2010-02-11 09:25:56 -0800 | [diff] [blame] | 1618 | |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 1619 | if (prof_dump_close(propagate_err)) |
| 1620 | goto label_open_close_error; |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1621 | |
Jason Evans | 20c31de | 2014-10-02 23:01:10 -0700 | [diff] [blame] | 1622 | prof_gctx_finish(tsd, &gctxs); |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 1623 | malloc_mutex_unlock(tsd_tsdn(tsd), &prof_dump_mtx); |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 1624 | |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 1625 | if (leakcheck) { |
| 1626 | prof_leakcheck(&prof_tdata_merge_iter_arg.cnt_all, |
| 1627 | prof_gctx_merge_iter_arg.leak_ngctx, filename); |
| 1628 | } |
Jason Evans | 22ca855 | 2010-03-02 11:57:30 -0800 | [diff] [blame] | 1629 | return (false); |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 1630 | label_write_error: |
| 1631 | prof_dump_close(propagate_err); |
| 1632 | label_open_close_error: |
Jason Evans | 20c31de | 2014-10-02 23:01:10 -0700 | [diff] [blame] | 1633 | prof_gctx_finish(tsd, &gctxs); |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 1634 | malloc_mutex_unlock(tsd_tsdn(tsd), &prof_dump_mtx); |
Jason Evans | 22ca855 | 2010-03-02 11:57:30 -0800 | [diff] [blame] | 1635 | return (true); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1636 | } |
| 1637 | |
Jason Evans | d81e4bd | 2012-03-06 14:57:45 -0800 | [diff] [blame] | 1638 | #define DUMP_FILENAME_BUFSIZE (PATH_MAX + 1) |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 1639 | #define VSEQ_INVALID UINT64_C(0xffffffffffffffff) |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1640 | static void |
Chris Peterson | 3e310b3 | 2014-05-28 19:04:06 -0700 | [diff] [blame] | 1641 | prof_dump_filename(char *filename, char v, uint64_t vseq) |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1642 | { |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1643 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 1644 | cassert(config_prof); |
| 1645 | |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 1646 | if (vseq != VSEQ_INVALID) { |
Jason Evans | d81e4bd | 2012-03-06 14:57:45 -0800 | [diff] [blame] | 1647 | /* "<prefix>.<pid>.<seq>.v<vseq>.heap" */ |
| 1648 | malloc_snprintf(filename, DUMP_FILENAME_BUFSIZE, |
Jason Evans | 5fae7dc | 2015-07-23 13:56:25 -0700 | [diff] [blame] | 1649 | "%s.%d.%"FMTu64".%c%"FMTu64".heap", |
Jason Evans | 788d29d | 2016-02-20 23:46:14 -0800 | [diff] [blame] | 1650 | opt_prof_prefix, prof_getpid(), prof_dump_seq, v, vseq); |
Jason Evans | d81e4bd | 2012-03-06 14:57:45 -0800 | [diff] [blame] | 1651 | } else { |
| 1652 | /* "<prefix>.<pid>.<seq>.<v>.heap" */ |
| 1653 | malloc_snprintf(filename, DUMP_FILENAME_BUFSIZE, |
Jason Evans | 5fae7dc | 2015-07-23 13:56:25 -0700 | [diff] [blame] | 1654 | "%s.%d.%"FMTu64".%c.heap", |
Jason Evans | 788d29d | 2016-02-20 23:46:14 -0800 | [diff] [blame] | 1655 | opt_prof_prefix, prof_getpid(), prof_dump_seq, v); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1656 | } |
Jason Evans | 52386b2 | 2012-04-22 16:00:11 -0700 | [diff] [blame] | 1657 | prof_dump_seq++; |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1658 | } |
| 1659 | |
| 1660 | static void |
| 1661 | prof_fdump(void) |
| 1662 | { |
Jason Evans | 5460aa6 | 2014-09-22 21:09:23 -0700 | [diff] [blame] | 1663 | tsd_t *tsd; |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1664 | char filename[DUMP_FILENAME_BUFSIZE]; |
| 1665 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 1666 | cassert(config_prof); |
Jason Evans | 57efa7b | 2014-10-08 17:57:19 -0700 | [diff] [blame] | 1667 | assert(opt_prof_final); |
| 1668 | assert(opt_prof_prefix[0] != '\0'); |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 1669 | |
Jason Evans | 551ebc4 | 2014-10-03 10:16:09 -0700 | [diff] [blame] | 1670 | if (!prof_booted) |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1671 | return; |
Jason Evans | 029d44c | 2014-10-04 11:12:53 -0700 | [diff] [blame] | 1672 | tsd = tsd_fetch(); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1673 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 1674 | malloc_mutex_lock(tsd_tsdn(tsd), &prof_dump_seq_mtx); |
Jason Evans | 57efa7b | 2014-10-08 17:57:19 -0700 | [diff] [blame] | 1675 | prof_dump_filename(filename, 'f', VSEQ_INVALID); |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 1676 | malloc_mutex_unlock(tsd_tsdn(tsd), &prof_dump_seq_mtx); |
Jason Evans | 57efa7b | 2014-10-08 17:57:19 -0700 | [diff] [blame] | 1677 | prof_dump(tsd, false, filename, opt_prof_leak); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1678 | } |
| 1679 | |
| 1680 | void |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 1681 | prof_idump(tsdn_t *tsdn) |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1682 | { |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 1683 | tsd_t *tsd; |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1684 | prof_tdata_t *tdata; |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1685 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 1686 | cassert(config_prof); |
| 1687 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 1688 | if (!prof_booted || tsdn_null(tsdn)) |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1689 | return; |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 1690 | tsd = tsdn_tsd(tsdn); |
Jason Evans | 5460aa6 | 2014-09-22 21:09:23 -0700 | [diff] [blame] | 1691 | tdata = prof_tdata_get(tsd, false); |
| 1692 | if (tdata == NULL) |
Jason Evans | 52386b2 | 2012-04-22 16:00:11 -0700 | [diff] [blame] | 1693 | return; |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1694 | if (tdata->enq) { |
| 1695 | tdata->enq_idump = true; |
Jason Evans | d34f9e7 | 2010-02-11 13:19:21 -0800 | [diff] [blame] | 1696 | return; |
| 1697 | } |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1698 | |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 1699 | if (opt_prof_prefix[0] != '\0') { |
Dmitry-Me | 78ae1ac | 2015-09-08 15:09:20 +0300 | [diff] [blame] | 1700 | char filename[PATH_MAX + 1]; |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 1701 | malloc_mutex_lock(tsd_tsdn(tsd), &prof_dump_seq_mtx); |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 1702 | prof_dump_filename(filename, 'i', prof_dump_iseq); |
| 1703 | prof_dump_iseq++; |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 1704 | malloc_mutex_unlock(tsd_tsdn(tsd), &prof_dump_seq_mtx); |
Jason Evans | 5460aa6 | 2014-09-22 21:09:23 -0700 | [diff] [blame] | 1705 | prof_dump(tsd, false, filename, false); |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 1706 | } |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1707 | } |
| 1708 | |
Jason Evans | 22ca855 | 2010-03-02 11:57:30 -0800 | [diff] [blame] | 1709 | bool |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 1710 | prof_mdump(tsd_t *tsd, const char *filename) |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1711 | { |
Jason Evans | 22ca855 | 2010-03-02 11:57:30 -0800 | [diff] [blame] | 1712 | char filename_buf[DUMP_FILENAME_BUFSIZE]; |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1713 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 1714 | cassert(config_prof); |
| 1715 | |
Jason Evans | 551ebc4 | 2014-10-03 10:16:09 -0700 | [diff] [blame] | 1716 | if (!opt_prof || !prof_booted) |
Jason Evans | 22ca855 | 2010-03-02 11:57:30 -0800 | [diff] [blame] | 1717 | return (true); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1718 | |
Jason Evans | 22ca855 | 2010-03-02 11:57:30 -0800 | [diff] [blame] | 1719 | if (filename == NULL) { |
| 1720 | /* No filename specified, so automatically generate one. */ |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 1721 | if (opt_prof_prefix[0] == '\0') |
| 1722 | return (true); |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 1723 | malloc_mutex_lock(tsd_tsdn(tsd), &prof_dump_seq_mtx); |
Jason Evans | 22ca855 | 2010-03-02 11:57:30 -0800 | [diff] [blame] | 1724 | prof_dump_filename(filename_buf, 'm', prof_dump_mseq); |
| 1725 | prof_dump_mseq++; |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 1726 | malloc_mutex_unlock(tsd_tsdn(tsd), &prof_dump_seq_mtx); |
Jason Evans | 22ca855 | 2010-03-02 11:57:30 -0800 | [diff] [blame] | 1727 | filename = filename_buf; |
| 1728 | } |
Jason Evans | 5460aa6 | 2014-09-22 21:09:23 -0700 | [diff] [blame] | 1729 | return (prof_dump(tsd, true, filename, false)); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1730 | } |
| 1731 | |
| 1732 | void |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 1733 | prof_gdump(tsdn_t *tsdn) |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1734 | { |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 1735 | tsd_t *tsd; |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1736 | prof_tdata_t *tdata; |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1737 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 1738 | cassert(config_prof); |
| 1739 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 1740 | if (!prof_booted || tsdn_null(tsdn)) |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1741 | return; |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 1742 | tsd = tsdn_tsd(tsdn); |
Jason Evans | 5460aa6 | 2014-09-22 21:09:23 -0700 | [diff] [blame] | 1743 | tdata = prof_tdata_get(tsd, false); |
| 1744 | if (tdata == NULL) |
Jason Evans | 52386b2 | 2012-04-22 16:00:11 -0700 | [diff] [blame] | 1745 | return; |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1746 | if (tdata->enq) { |
| 1747 | tdata->enq_gdump = true; |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1748 | return; |
| 1749 | } |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1750 | |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 1751 | if (opt_prof_prefix[0] != '\0') { |
Dmitry-Me | 78ae1ac | 2015-09-08 15:09:20 +0300 | [diff] [blame] | 1752 | char filename[DUMP_FILENAME_BUFSIZE]; |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 1753 | malloc_mutex_lock(tsdn, &prof_dump_seq_mtx); |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 1754 | prof_dump_filename(filename, 'u', prof_dump_useq); |
| 1755 | prof_dump_useq++; |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 1756 | malloc_mutex_unlock(tsdn, &prof_dump_seq_mtx); |
Jason Evans | 5460aa6 | 2014-09-22 21:09:23 -0700 | [diff] [blame] | 1757 | prof_dump(tsd, false, filename, false); |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 1758 | } |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1759 | } |
| 1760 | |
| 1761 | static void |
Jason Evans | ae03bf6 | 2013-01-22 12:02:08 -0800 | [diff] [blame] | 1762 | prof_bt_hash(const void *key, size_t r_hash[2]) |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1763 | { |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1764 | prof_bt_t *bt = (prof_bt_t *)key; |
| 1765 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 1766 | cassert(config_prof); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1767 | |
Jason Evans | ae03bf6 | 2013-01-22 12:02:08 -0800 | [diff] [blame] | 1768 | hash(bt->vec, bt->len * sizeof(void *), 0x94122f33U, r_hash); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1769 | } |
| 1770 | |
| 1771 | static bool |
| 1772 | prof_bt_keycomp(const void *k1, const void *k2) |
| 1773 | { |
| 1774 | const prof_bt_t *bt1 = (prof_bt_t *)k1; |
| 1775 | const prof_bt_t *bt2 = (prof_bt_t *)k2; |
| 1776 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 1777 | cassert(config_prof); |
| 1778 | |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1779 | if (bt1->len != bt2->len) |
| 1780 | return (false); |
| 1781 | return (memcmp(bt1->vec, bt2->vec, bt1->len * sizeof(void *)) == 0); |
| 1782 | } |
| 1783 | |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1784 | JEMALLOC_INLINE_C uint64_t |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 1785 | prof_thr_uid_alloc(tsdn_t *tsdn) |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1786 | { |
Jason Evans | 9d8f3d2 | 2014-09-11 18:06:30 -0700 | [diff] [blame] | 1787 | uint64_t thr_uid; |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1788 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 1789 | malloc_mutex_lock(tsdn, &next_thr_uid_mtx); |
Jason Evans | 9d8f3d2 | 2014-09-11 18:06:30 -0700 | [diff] [blame] | 1790 | thr_uid = next_thr_uid; |
| 1791 | next_thr_uid++; |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 1792 | malloc_mutex_unlock(tsdn, &next_thr_uid_mtx); |
Jason Evans | 9d8f3d2 | 2014-09-11 18:06:30 -0700 | [diff] [blame] | 1793 | |
| 1794 | return (thr_uid); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1795 | } |
| 1796 | |
| 1797 | static prof_tdata_t * |
Jason Evans | b54d160 | 2016-10-20 23:59:12 -0700 | [diff] [blame^] | 1798 | prof_tdata_init_impl(tsd_t *tsd, uint64_t thr_uid, uint64_t thr_discrim, |
Jason Evans | fc12c0b | 2014-10-03 23:25:30 -0700 | [diff] [blame] | 1799 | char *thread_name, bool active) |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1800 | { |
| 1801 | prof_tdata_t *tdata; |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1802 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 1803 | cassert(config_prof); |
| 1804 | |
Jason Evans | 4d6a134 | 2010-10-20 19:05:59 -0700 | [diff] [blame] | 1805 | /* Initialize an empty cache for this thread. */ |
Jason Evans | b54d160 | 2016-10-20 23:59:12 -0700 | [diff] [blame^] | 1806 | tdata = (prof_tdata_t *)iallocztm(tsd_tsdn(tsd), sizeof(prof_tdata_t), |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 1807 | size2index(sizeof(prof_tdata_t)), false, NULL, true, |
| 1808 | arena_get(TSDN_NULL, 0, true), true); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1809 | if (tdata == NULL) |
Jason Evans | 4d6a134 | 2010-10-20 19:05:59 -0700 | [diff] [blame] | 1810 | return (NULL); |
| 1811 | |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1812 | tdata->lock = prof_tdata_mutex_choose(thr_uid); |
| 1813 | tdata->thr_uid = thr_uid; |
Jason Evans | 20c31de | 2014-10-02 23:01:10 -0700 | [diff] [blame] | 1814 | tdata->thr_discrim = thr_discrim; |
Jason Evans | fc12c0b | 2014-10-03 23:25:30 -0700 | [diff] [blame] | 1815 | tdata->thread_name = thread_name; |
Jason Evans | 20c31de | 2014-10-02 23:01:10 -0700 | [diff] [blame] | 1816 | tdata->attached = true; |
| 1817 | tdata->expired = false; |
Jason Evans | 04211e2 | 2015-03-16 15:11:06 -0700 | [diff] [blame] | 1818 | tdata->tctx_uid_next = 0; |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1819 | |
Jason Evans | b54d160 | 2016-10-20 23:59:12 -0700 | [diff] [blame^] | 1820 | if (ckh_new(tsd, &tdata->bt2tctx, PROF_CKH_MINITEMS, prof_bt_hash, |
| 1821 | prof_bt_keycomp)) { |
| 1822 | idalloctm(tsd_tsdn(tsd), iealloc(tsd_tsdn(tsd), tdata), tdata, |
| 1823 | NULL, true, true); |
Jason Evans | 4d6a134 | 2010-10-20 19:05:59 -0700 | [diff] [blame] | 1824 | return (NULL); |
| 1825 | } |
Jason Evans | 4d6a134 | 2010-10-20 19:05:59 -0700 | [diff] [blame] | 1826 | |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1827 | tdata->prng_state = (uint64_t)(uintptr_t)tdata; |
| 1828 | prof_sample_threshold_update(tdata); |
Jason Evans | 4d6a134 | 2010-10-20 19:05:59 -0700 | [diff] [blame] | 1829 | |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1830 | tdata->enq = false; |
| 1831 | tdata->enq_idump = false; |
| 1832 | tdata->enq_gdump = false; |
Jason Evans | 52386b2 | 2012-04-22 16:00:11 -0700 | [diff] [blame] | 1833 | |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1834 | tdata->dumping = false; |
Jason Evans | fc12c0b | 2014-10-03 23:25:30 -0700 | [diff] [blame] | 1835 | tdata->active = active; |
Jason Evans | 4d6a134 | 2010-10-20 19:05:59 -0700 | [diff] [blame] | 1836 | |
Jason Evans | b54d160 | 2016-10-20 23:59:12 -0700 | [diff] [blame^] | 1837 | malloc_mutex_lock(tsd_tsdn(tsd), &tdatas_mtx); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1838 | tdata_tree_insert(&tdatas, tdata); |
Jason Evans | b54d160 | 2016-10-20 23:59:12 -0700 | [diff] [blame^] | 1839 | malloc_mutex_unlock(tsd_tsdn(tsd), &tdatas_mtx); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1840 | |
| 1841 | return (tdata); |
| 1842 | } |
| 1843 | |
| 1844 | prof_tdata_t * |
Jason Evans | b54d160 | 2016-10-20 23:59:12 -0700 | [diff] [blame^] | 1845 | prof_tdata_init(tsd_t *tsd) |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1846 | { |
| 1847 | |
Jason Evans | b54d160 | 2016-10-20 23:59:12 -0700 | [diff] [blame^] | 1848 | return (prof_tdata_init_impl(tsd, prof_thr_uid_alloc(tsd_tsdn(tsd)), 0, |
| 1849 | NULL, prof_thread_active_init_get(tsd_tsdn(tsd)))); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1850 | } |
| 1851 | |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1852 | static bool |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 1853 | prof_tdata_should_destroy_unlocked(prof_tdata_t *tdata, bool even_if_attached) |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1854 | { |
| 1855 | |
Jason Evans | f04a0be | 2014-10-04 15:03:49 -0700 | [diff] [blame] | 1856 | if (tdata->attached && !even_if_attached) |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1857 | return (false); |
| 1858 | if (ckh_count(&tdata->bt2tctx) != 0) |
| 1859 | return (false); |
| 1860 | return (true); |
| 1861 | } |
| 1862 | |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 1863 | static bool |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 1864 | prof_tdata_should_destroy(tsdn_t *tsdn, prof_tdata_t *tdata, |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 1865 | bool even_if_attached) |
| 1866 | { |
| 1867 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 1868 | malloc_mutex_assert_owner(tsdn, tdata->lock); |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 1869 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 1870 | return (prof_tdata_should_destroy_unlocked(tdata, even_if_attached)); |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 1871 | } |
| 1872 | |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1873 | static void |
Jason Evans | b54d160 | 2016-10-20 23:59:12 -0700 | [diff] [blame^] | 1874 | prof_tdata_destroy_locked(tsd_t *tsd, prof_tdata_t *tdata, |
Jason Evans | f04a0be | 2014-10-04 15:03:49 -0700 | [diff] [blame] | 1875 | bool even_if_attached) |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1876 | { |
| 1877 | |
Jason Evans | b54d160 | 2016-10-20 23:59:12 -0700 | [diff] [blame^] | 1878 | malloc_mutex_assert_owner(tsd_tsdn(tsd), &tdatas_mtx); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1879 | |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1880 | tdata_tree_remove(&tdatas, tdata); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1881 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 1882 | assert(prof_tdata_should_destroy_unlocked(tdata, even_if_attached)); |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 1883 | |
Jason Evans | db72272 | 2016-03-23 20:29:33 -0700 | [diff] [blame] | 1884 | if (tdata->thread_name != NULL) { |
Jason Evans | b54d160 | 2016-10-20 23:59:12 -0700 | [diff] [blame^] | 1885 | idalloctm(tsd_tsdn(tsd), iealloc(tsd_tsdn(tsd), |
| 1886 | tdata->thread_name), tdata->thread_name, NULL, true, true); |
Jason Evans | db72272 | 2016-03-23 20:29:33 -0700 | [diff] [blame] | 1887 | } |
Jason Evans | b54d160 | 2016-10-20 23:59:12 -0700 | [diff] [blame^] | 1888 | ckh_delete(tsd, &tdata->bt2tctx); |
| 1889 | idalloctm(tsd_tsdn(tsd), iealloc(tsd_tsdn(tsd), tdata), tdata, NULL, |
| 1890 | true, true); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1891 | } |
| 1892 | |
| 1893 | static void |
Jason Evans | b54d160 | 2016-10-20 23:59:12 -0700 | [diff] [blame^] | 1894 | prof_tdata_destroy(tsd_t *tsd, prof_tdata_t *tdata, bool even_if_attached) |
Jason Evans | 20c31de | 2014-10-02 23:01:10 -0700 | [diff] [blame] | 1895 | { |
| 1896 | |
Jason Evans | b54d160 | 2016-10-20 23:59:12 -0700 | [diff] [blame^] | 1897 | malloc_mutex_lock(tsd_tsdn(tsd), &tdatas_mtx); |
| 1898 | prof_tdata_destroy_locked(tsd, tdata, even_if_attached); |
| 1899 | malloc_mutex_unlock(tsd_tsdn(tsd), &tdatas_mtx); |
Jason Evans | 20c31de | 2014-10-02 23:01:10 -0700 | [diff] [blame] | 1900 | } |
| 1901 | |
| 1902 | static void |
| 1903 | prof_tdata_detach(tsd_t *tsd, prof_tdata_t *tdata) |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1904 | { |
| 1905 | bool destroy_tdata; |
| 1906 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 1907 | malloc_mutex_lock(tsd_tsdn(tsd), tdata->lock); |
Jason Evans | 20c31de | 2014-10-02 23:01:10 -0700 | [diff] [blame] | 1908 | if (tdata->attached) { |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 1909 | destroy_tdata = prof_tdata_should_destroy(tsd_tsdn(tsd), tdata, |
| 1910 | true); |
Jason Evans | f04a0be | 2014-10-04 15:03:49 -0700 | [diff] [blame] | 1911 | /* |
| 1912 | * Only detach if !destroy_tdata, because detaching would allow |
| 1913 | * another thread to win the race to destroy tdata. |
| 1914 | */ |
| 1915 | if (!destroy_tdata) |
| 1916 | tdata->attached = false; |
Jason Evans | 029d44c | 2014-10-04 11:12:53 -0700 | [diff] [blame] | 1917 | tsd_prof_tdata_set(tsd, NULL); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1918 | } else |
| 1919 | destroy_tdata = false; |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 1920 | malloc_mutex_unlock(tsd_tsdn(tsd), tdata->lock); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1921 | if (destroy_tdata) |
Jason Evans | b54d160 | 2016-10-20 23:59:12 -0700 | [diff] [blame^] | 1922 | prof_tdata_destroy(tsd, tdata, true); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1923 | } |
| 1924 | |
Jason Evans | 20c31de | 2014-10-02 23:01:10 -0700 | [diff] [blame] | 1925 | prof_tdata_t * |
| 1926 | prof_tdata_reinit(tsd_t *tsd, prof_tdata_t *tdata) |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1927 | { |
Jason Evans | 20c31de | 2014-10-02 23:01:10 -0700 | [diff] [blame] | 1928 | uint64_t thr_uid = tdata->thr_uid; |
| 1929 | uint64_t thr_discrim = tdata->thr_discrim + 1; |
Jason Evans | fc12c0b | 2014-10-03 23:25:30 -0700 | [diff] [blame] | 1930 | char *thread_name = (tdata->thread_name != NULL) ? |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 1931 | prof_thread_name_alloc(tsd_tsdn(tsd), tdata->thread_name) : NULL; |
Jason Evans | fc12c0b | 2014-10-03 23:25:30 -0700 | [diff] [blame] | 1932 | bool active = tdata->active; |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1933 | |
Jason Evans | 20c31de | 2014-10-02 23:01:10 -0700 | [diff] [blame] | 1934 | prof_tdata_detach(tsd, tdata); |
Jason Evans | b54d160 | 2016-10-20 23:59:12 -0700 | [diff] [blame^] | 1935 | return (prof_tdata_init_impl(tsd, thr_uid, thr_discrim, thread_name, |
| 1936 | active)); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1937 | } |
| 1938 | |
Jason Evans | 20c31de | 2014-10-02 23:01:10 -0700 | [diff] [blame] | 1939 | static bool |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 1940 | prof_tdata_expire(tsdn_t *tsdn, prof_tdata_t *tdata) |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1941 | { |
Jason Evans | 20c31de | 2014-10-02 23:01:10 -0700 | [diff] [blame] | 1942 | bool destroy_tdata; |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1943 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 1944 | malloc_mutex_lock(tsdn, tdata->lock); |
Jason Evans | 20c31de | 2014-10-02 23:01:10 -0700 | [diff] [blame] | 1945 | if (!tdata->expired) { |
| 1946 | tdata->expired = true; |
| 1947 | destroy_tdata = tdata->attached ? false : |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 1948 | prof_tdata_should_destroy(tsdn, tdata, false); |
Jason Evans | 20c31de | 2014-10-02 23:01:10 -0700 | [diff] [blame] | 1949 | } else |
| 1950 | destroy_tdata = false; |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 1951 | malloc_mutex_unlock(tsdn, tdata->lock); |
Jason Evans | 20c31de | 2014-10-02 23:01:10 -0700 | [diff] [blame] | 1952 | |
| 1953 | return (destroy_tdata); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1954 | } |
| 1955 | |
| 1956 | static prof_tdata_t * |
| 1957 | prof_tdata_reset_iter(prof_tdata_tree_t *tdatas, prof_tdata_t *tdata, void *arg) |
| 1958 | { |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 1959 | tsdn_t *tsdn = (tsdn_t *)arg; |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1960 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 1961 | return (prof_tdata_expire(tsdn, tdata) ? tdata : NULL); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1962 | } |
| 1963 | |
| 1964 | void |
Jason Evans | b54d160 | 2016-10-20 23:59:12 -0700 | [diff] [blame^] | 1965 | prof_reset(tsd_t *tsd, size_t lg_sample) |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1966 | { |
Jason Evans | 20c31de | 2014-10-02 23:01:10 -0700 | [diff] [blame] | 1967 | prof_tdata_t *next; |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1968 | |
| 1969 | assert(lg_sample < (sizeof(uint64_t) << 3)); |
| 1970 | |
Jason Evans | b54d160 | 2016-10-20 23:59:12 -0700 | [diff] [blame^] | 1971 | malloc_mutex_lock(tsd_tsdn(tsd), &prof_dump_mtx); |
| 1972 | malloc_mutex_lock(tsd_tsdn(tsd), &tdatas_mtx); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1973 | |
| 1974 | lg_prof_sample = lg_sample; |
Jason Evans | 20c31de | 2014-10-02 23:01:10 -0700 | [diff] [blame] | 1975 | |
| 1976 | next = NULL; |
| 1977 | do { |
| 1978 | prof_tdata_t *to_destroy = tdata_tree_iter(&tdatas, next, |
Jason Evans | b54d160 | 2016-10-20 23:59:12 -0700 | [diff] [blame^] | 1979 | prof_tdata_reset_iter, (void *)tsd); |
Jason Evans | 20c31de | 2014-10-02 23:01:10 -0700 | [diff] [blame] | 1980 | if (to_destroy != NULL) { |
| 1981 | next = tdata_tree_next(&tdatas, to_destroy); |
Jason Evans | b54d160 | 2016-10-20 23:59:12 -0700 | [diff] [blame^] | 1982 | prof_tdata_destroy_locked(tsd, to_destroy, false); |
Jason Evans | 20c31de | 2014-10-02 23:01:10 -0700 | [diff] [blame] | 1983 | } else |
| 1984 | next = NULL; |
| 1985 | } while (next != NULL); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1986 | |
Jason Evans | b54d160 | 2016-10-20 23:59:12 -0700 | [diff] [blame^] | 1987 | malloc_mutex_unlock(tsd_tsdn(tsd), &tdatas_mtx); |
| 1988 | malloc_mutex_unlock(tsd_tsdn(tsd), &prof_dump_mtx); |
Jason Evans | 4d6a134 | 2010-10-20 19:05:59 -0700 | [diff] [blame] | 1989 | } |
| 1990 | |
Jason Evans | cd9a134 | 2012-03-21 18:33:03 -0700 | [diff] [blame] | 1991 | void |
Jason Evans | 5460aa6 | 2014-09-22 21:09:23 -0700 | [diff] [blame] | 1992 | prof_tdata_cleanup(tsd_t *tsd) |
Jason Evans | 4d6a134 | 2010-10-20 19:05:59 -0700 | [diff] [blame] | 1993 | { |
Jason Evans | 5460aa6 | 2014-09-22 21:09:23 -0700 | [diff] [blame] | 1994 | prof_tdata_t *tdata; |
Jason Evans | 4d6a134 | 2010-10-20 19:05:59 -0700 | [diff] [blame] | 1995 | |
Jason Evans | 5460aa6 | 2014-09-22 21:09:23 -0700 | [diff] [blame] | 1996 | if (!config_prof) |
| 1997 | return; |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 1998 | |
Jason Evans | 5460aa6 | 2014-09-22 21:09:23 -0700 | [diff] [blame] | 1999 | tdata = tsd_prof_tdata_get(tsd); |
| 2000 | if (tdata != NULL) |
| 2001 | prof_tdata_detach(tsd, tdata); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 2002 | } |
| 2003 | |
Jason Evans | fc12c0b | 2014-10-03 23:25:30 -0700 | [diff] [blame] | 2004 | bool |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 2005 | prof_active_get(tsdn_t *tsdn) |
Jason Evans | fc12c0b | 2014-10-03 23:25:30 -0700 | [diff] [blame] | 2006 | { |
| 2007 | bool prof_active_current; |
| 2008 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 2009 | malloc_mutex_lock(tsdn, &prof_active_mtx); |
Jason Evans | fc12c0b | 2014-10-03 23:25:30 -0700 | [diff] [blame] | 2010 | prof_active_current = prof_active; |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 2011 | malloc_mutex_unlock(tsdn, &prof_active_mtx); |
Jason Evans | fc12c0b | 2014-10-03 23:25:30 -0700 | [diff] [blame] | 2012 | return (prof_active_current); |
| 2013 | } |
| 2014 | |
| 2015 | bool |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 2016 | prof_active_set(tsdn_t *tsdn, bool active) |
Jason Evans | fc12c0b | 2014-10-03 23:25:30 -0700 | [diff] [blame] | 2017 | { |
| 2018 | bool prof_active_old; |
| 2019 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 2020 | malloc_mutex_lock(tsdn, &prof_active_mtx); |
Jason Evans | fc12c0b | 2014-10-03 23:25:30 -0700 | [diff] [blame] | 2021 | prof_active_old = prof_active; |
| 2022 | prof_active = active; |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 2023 | malloc_mutex_unlock(tsdn, &prof_active_mtx); |
Jason Evans | fc12c0b | 2014-10-03 23:25:30 -0700 | [diff] [blame] | 2024 | return (prof_active_old); |
| 2025 | } |
| 2026 | |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 2027 | const char * |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 2028 | prof_thread_name_get(tsd_t *tsd) |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 2029 | { |
Jason Evans | 5460aa6 | 2014-09-22 21:09:23 -0700 | [diff] [blame] | 2030 | prof_tdata_t *tdata; |
| 2031 | |
Jason Evans | 5460aa6 | 2014-09-22 21:09:23 -0700 | [diff] [blame] | 2032 | tdata = prof_tdata_get(tsd, true); |
| 2033 | if (tdata == NULL) |
Jason Evans | fc12c0b | 2014-10-03 23:25:30 -0700 | [diff] [blame] | 2034 | return (""); |
| 2035 | return (tdata->thread_name != NULL ? tdata->thread_name : ""); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 2036 | } |
| 2037 | |
Jason Evans | fc12c0b | 2014-10-03 23:25:30 -0700 | [diff] [blame] | 2038 | static char * |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 2039 | prof_thread_name_alloc(tsdn_t *tsdn, const char *thread_name) |
Jason Evans | fc12c0b | 2014-10-03 23:25:30 -0700 | [diff] [blame] | 2040 | { |
| 2041 | char *ret; |
| 2042 | size_t size; |
| 2043 | |
| 2044 | if (thread_name == NULL) |
| 2045 | return (NULL); |
| 2046 | |
| 2047 | size = strlen(thread_name) + 1; |
| 2048 | if (size == 1) |
| 2049 | return (""); |
| 2050 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 2051 | ret = iallocztm(tsdn, size, size2index(size), false, NULL, true, |
| 2052 | arena_get(TSDN_NULL, 0, true), true); |
Jason Evans | fc12c0b | 2014-10-03 23:25:30 -0700 | [diff] [blame] | 2053 | if (ret == NULL) |
| 2054 | return (NULL); |
| 2055 | memcpy(ret, thread_name, size); |
| 2056 | return (ret); |
| 2057 | } |
| 2058 | |
| 2059 | int |
Jason Evans | 5460aa6 | 2014-09-22 21:09:23 -0700 | [diff] [blame] | 2060 | prof_thread_name_set(tsd_t *tsd, const char *thread_name) |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 2061 | { |
| 2062 | prof_tdata_t *tdata; |
Jason Evans | fc12c0b | 2014-10-03 23:25:30 -0700 | [diff] [blame] | 2063 | unsigned i; |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 2064 | char *s; |
| 2065 | |
Jason Evans | 5460aa6 | 2014-09-22 21:09:23 -0700 | [diff] [blame] | 2066 | tdata = prof_tdata_get(tsd, true); |
| 2067 | if (tdata == NULL) |
Jason Evans | fc12c0b | 2014-10-03 23:25:30 -0700 | [diff] [blame] | 2068 | return (EAGAIN); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 2069 | |
Jason Evans | fc12c0b | 2014-10-03 23:25:30 -0700 | [diff] [blame] | 2070 | /* Validate input. */ |
| 2071 | if (thread_name == NULL) |
| 2072 | return (EFAULT); |
| 2073 | for (i = 0; thread_name[i] != '\0'; i++) { |
| 2074 | char c = thread_name[i]; |
| 2075 | if (!isgraph(c) && !isblank(c)) |
| 2076 | return (EFAULT); |
| 2077 | } |
| 2078 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 2079 | s = prof_thread_name_alloc(tsd_tsdn(tsd), thread_name); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 2080 | if (s == NULL) |
Jason Evans | fc12c0b | 2014-10-03 23:25:30 -0700 | [diff] [blame] | 2081 | return (EAGAIN); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 2082 | |
Jason Evans | fc12c0b | 2014-10-03 23:25:30 -0700 | [diff] [blame] | 2083 | if (tdata->thread_name != NULL) { |
Jason Evans | 8c9be3e | 2016-04-16 00:36:11 -0700 | [diff] [blame] | 2084 | idalloctm(tsd_tsdn(tsd), iealloc(tsd_tsdn(tsd), |
| 2085 | tdata->thread_name), tdata->thread_name, NULL, true, true); |
Jason Evans | fc12c0b | 2014-10-03 23:25:30 -0700 | [diff] [blame] | 2086 | tdata->thread_name = NULL; |
| 2087 | } |
| 2088 | if (strlen(s) > 0) |
| 2089 | tdata->thread_name = s; |
| 2090 | return (0); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 2091 | } |
| 2092 | |
| 2093 | bool |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 2094 | prof_thread_active_get(tsd_t *tsd) |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 2095 | { |
Jason Evans | 5460aa6 | 2014-09-22 21:09:23 -0700 | [diff] [blame] | 2096 | prof_tdata_t *tdata; |
| 2097 | |
Jason Evans | 5460aa6 | 2014-09-22 21:09:23 -0700 | [diff] [blame] | 2098 | tdata = prof_tdata_get(tsd, true); |
| 2099 | if (tdata == NULL) |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 2100 | return (false); |
| 2101 | return (tdata->active); |
| 2102 | } |
| 2103 | |
| 2104 | bool |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 2105 | prof_thread_active_set(tsd_t *tsd, bool active) |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 2106 | { |
| 2107 | prof_tdata_t *tdata; |
| 2108 | |
Jason Evans | 5460aa6 | 2014-09-22 21:09:23 -0700 | [diff] [blame] | 2109 | tdata = prof_tdata_get(tsd, true); |
| 2110 | if (tdata == NULL) |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 2111 | return (true); |
| 2112 | tdata->active = active; |
| 2113 | return (false); |
| 2114 | } |
| 2115 | |
Jason Evans | fc12c0b | 2014-10-03 23:25:30 -0700 | [diff] [blame] | 2116 | bool |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 2117 | prof_thread_active_init_get(tsdn_t *tsdn) |
Jason Evans | fc12c0b | 2014-10-03 23:25:30 -0700 | [diff] [blame] | 2118 | { |
| 2119 | bool active_init; |
| 2120 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 2121 | malloc_mutex_lock(tsdn, &prof_thread_active_init_mtx); |
Jason Evans | fc12c0b | 2014-10-03 23:25:30 -0700 | [diff] [blame] | 2122 | active_init = prof_thread_active_init; |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 2123 | malloc_mutex_unlock(tsdn, &prof_thread_active_init_mtx); |
Jason Evans | fc12c0b | 2014-10-03 23:25:30 -0700 | [diff] [blame] | 2124 | return (active_init); |
| 2125 | } |
| 2126 | |
| 2127 | bool |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 2128 | prof_thread_active_init_set(tsdn_t *tsdn, bool active_init) |
Jason Evans | fc12c0b | 2014-10-03 23:25:30 -0700 | [diff] [blame] | 2129 | { |
| 2130 | bool active_init_old; |
| 2131 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 2132 | malloc_mutex_lock(tsdn, &prof_thread_active_init_mtx); |
Jason Evans | fc12c0b | 2014-10-03 23:25:30 -0700 | [diff] [blame] | 2133 | active_init_old = prof_thread_active_init; |
| 2134 | prof_thread_active_init = active_init; |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 2135 | malloc_mutex_unlock(tsdn, &prof_thread_active_init_mtx); |
Jason Evans | fc12c0b | 2014-10-03 23:25:30 -0700 | [diff] [blame] | 2136 | return (active_init_old); |
| 2137 | } |
| 2138 | |
Jason Evans | 5b8ed5b | 2015-01-25 21:16:57 -0800 | [diff] [blame] | 2139 | bool |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 2140 | prof_gdump_get(tsdn_t *tsdn) |
Jason Evans | 5b8ed5b | 2015-01-25 21:16:57 -0800 | [diff] [blame] | 2141 | { |
| 2142 | bool prof_gdump_current; |
| 2143 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 2144 | malloc_mutex_lock(tsdn, &prof_gdump_mtx); |
Jason Evans | 5b8ed5b | 2015-01-25 21:16:57 -0800 | [diff] [blame] | 2145 | prof_gdump_current = prof_gdump_val; |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 2146 | malloc_mutex_unlock(tsdn, &prof_gdump_mtx); |
Jason Evans | 5b8ed5b | 2015-01-25 21:16:57 -0800 | [diff] [blame] | 2147 | return (prof_gdump_current); |
| 2148 | } |
| 2149 | |
| 2150 | bool |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 2151 | prof_gdump_set(tsdn_t *tsdn, bool gdump) |
Jason Evans | 5b8ed5b | 2015-01-25 21:16:57 -0800 | [diff] [blame] | 2152 | { |
| 2153 | bool prof_gdump_old; |
| 2154 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 2155 | malloc_mutex_lock(tsdn, &prof_gdump_mtx); |
Jason Evans | 5b8ed5b | 2015-01-25 21:16:57 -0800 | [diff] [blame] | 2156 | prof_gdump_old = prof_gdump_val; |
| 2157 | prof_gdump_val = gdump; |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 2158 | malloc_mutex_unlock(tsdn, &prof_gdump_mtx); |
Jason Evans | 5b8ed5b | 2015-01-25 21:16:57 -0800 | [diff] [blame] | 2159 | return (prof_gdump_old); |
| 2160 | } |
| 2161 | |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 2162 | void |
| 2163 | prof_boot0(void) |
| 2164 | { |
| 2165 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 2166 | cassert(config_prof); |
| 2167 | |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 2168 | memcpy(opt_prof_prefix, PROF_PREFIX_DEFAULT, |
| 2169 | sizeof(PROF_PREFIX_DEFAULT)); |
| 2170 | } |
| 2171 | |
| 2172 | void |
| 2173 | prof_boot1(void) |
| 2174 | { |
| 2175 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 2176 | cassert(config_prof); |
| 2177 | |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 2178 | /* |
Jason Evans | 9b0cbf0 | 2014-04-11 14:24:51 -0700 | [diff] [blame] | 2179 | * opt_prof must be in its final state before any arenas are |
| 2180 | * initialized, so this function must be executed early. |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 2181 | */ |
| 2182 | |
Jason Evans | 551ebc4 | 2014-10-03 10:16:09 -0700 | [diff] [blame] | 2183 | if (opt_prof_leak && !opt_prof) { |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 2184 | /* |
| 2185 | * Enable opt_prof, but in such a way that profiles are never |
| 2186 | * automatically dumped. |
| 2187 | */ |
| 2188 | opt_prof = true; |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 2189 | opt_prof_gdump = false; |
Jason Evans | a02fc08 | 2010-03-31 17:35:51 -0700 | [diff] [blame] | 2190 | } else if (opt_prof) { |
| 2191 | if (opt_lg_prof_interval >= 0) { |
| 2192 | prof_interval = (((uint64_t)1U) << |
| 2193 | opt_lg_prof_interval); |
Jason Evans | a3b3386 | 2012-11-13 12:56:27 -0800 | [diff] [blame] | 2194 | } |
Jason Evans | a02fc08 | 2010-03-31 17:35:51 -0700 | [diff] [blame] | 2195 | } |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 2196 | } |
| 2197 | |
| 2198 | bool |
Jason Evans | b54d160 | 2016-10-20 23:59:12 -0700 | [diff] [blame^] | 2199 | prof_boot2(tsd_t *tsd) |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 2200 | { |
| 2201 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 2202 | cassert(config_prof); |
| 2203 | |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 2204 | if (opt_prof) { |
Jason Evans | 6da5418 | 2012-03-23 18:05:51 -0700 | [diff] [blame] | 2205 | unsigned i; |
| 2206 | |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 2207 | lg_prof_sample = opt_lg_prof_sample; |
| 2208 | |
Jason Evans | fc12c0b | 2014-10-03 23:25:30 -0700 | [diff] [blame] | 2209 | prof_active = opt_prof_active; |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 2210 | if (malloc_mutex_init(&prof_active_mtx, "prof_active", |
| 2211 | WITNESS_RANK_PROF_ACTIVE)) |
Jason Evans | fc12c0b | 2014-10-03 23:25:30 -0700 | [diff] [blame] | 2212 | return (true); |
| 2213 | |
Jason Evans | 5b8ed5b | 2015-01-25 21:16:57 -0800 | [diff] [blame] | 2214 | prof_gdump_val = opt_prof_gdump; |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 2215 | if (malloc_mutex_init(&prof_gdump_mtx, "prof_gdump", |
| 2216 | WITNESS_RANK_PROF_GDUMP)) |
Jason Evans | 5b8ed5b | 2015-01-25 21:16:57 -0800 | [diff] [blame] | 2217 | return (true); |
| 2218 | |
Jason Evans | fc12c0b | 2014-10-03 23:25:30 -0700 | [diff] [blame] | 2219 | prof_thread_active_init = opt_prof_thread_active_init; |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 2220 | if (malloc_mutex_init(&prof_thread_active_init_mtx, |
| 2221 | "prof_thread_active_init", |
| 2222 | WITNESS_RANK_PROF_THREAD_ACTIVE_INIT)) |
Jason Evans | fc12c0b | 2014-10-03 23:25:30 -0700 | [diff] [blame] | 2223 | return (true); |
| 2224 | |
Jason Evans | b54d160 | 2016-10-20 23:59:12 -0700 | [diff] [blame^] | 2225 | if (ckh_new(tsd, &bt2gctx, PROF_CKH_MINITEMS, prof_bt_hash, |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 2226 | prof_bt_keycomp)) |
| 2227 | return (true); |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 2228 | if (malloc_mutex_init(&bt2gctx_mtx, "prof_bt2gctx", |
| 2229 | WITNESS_RANK_PROF_BT2GCTX)) |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 2230 | return (true); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 2231 | |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 2232 | tdata_tree_new(&tdatas); |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 2233 | if (malloc_mutex_init(&tdatas_mtx, "prof_tdatas", |
| 2234 | WITNESS_RANK_PROF_TDATAS)) |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 2235 | return (true); |
| 2236 | |
| 2237 | next_thr_uid = 0; |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 2238 | if (malloc_mutex_init(&next_thr_uid_mtx, "prof_next_thr_uid", |
| 2239 | WITNESS_RANK_PROF_NEXT_THR_UID)) |
Jason Evans | 9d8f3d2 | 2014-09-11 18:06:30 -0700 | [diff] [blame] | 2240 | return (true); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 2241 | |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 2242 | if (malloc_mutex_init(&prof_dump_seq_mtx, "prof_dump_seq", |
| 2243 | WITNESS_RANK_PROF_DUMP_SEQ)) |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 2244 | return (true); |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 2245 | if (malloc_mutex_init(&prof_dump_mtx, "prof_dump", |
| 2246 | WITNESS_RANK_PROF_DUMP)) |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 2247 | return (true); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 2248 | |
Jason Evans | 57efa7b | 2014-10-08 17:57:19 -0700 | [diff] [blame] | 2249 | if (opt_prof_final && opt_prof_prefix[0] != '\0' && |
| 2250 | atexit(prof_fdump) != 0) { |
Jason Evans | 698805c | 2010-03-03 17:45:38 -0800 | [diff] [blame] | 2251 | malloc_write("<jemalloc>: Error in atexit()\n"); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 2252 | if (opt_abort) |
| 2253 | abort(); |
| 2254 | } |
Jason Evans | 6da5418 | 2012-03-23 18:05:51 -0700 | [diff] [blame] | 2255 | |
Jason Evans | b54d160 | 2016-10-20 23:59:12 -0700 | [diff] [blame^] | 2256 | gctx_locks = (malloc_mutex_t *)base_alloc(tsd_tsdn(tsd), |
| 2257 | PROF_NCTX_LOCKS * sizeof(malloc_mutex_t)); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 2258 | if (gctx_locks == NULL) |
Jason Evans | 6da5418 | 2012-03-23 18:05:51 -0700 | [diff] [blame] | 2259 | return (true); |
| 2260 | for (i = 0; i < PROF_NCTX_LOCKS; i++) { |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 2261 | if (malloc_mutex_init(&gctx_locks[i], "prof_gctx", |
| 2262 | WITNESS_RANK_PROF_GCTX)) |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 2263 | return (true); |
| 2264 | } |
| 2265 | |
Jason Evans | b54d160 | 2016-10-20 23:59:12 -0700 | [diff] [blame^] | 2266 | tdata_locks = (malloc_mutex_t *)base_alloc(tsd_tsdn(tsd), |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 2267 | PROF_NTDATA_LOCKS * sizeof(malloc_mutex_t)); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 2268 | if (tdata_locks == NULL) |
| 2269 | return (true); |
| 2270 | for (i = 0; i < PROF_NTDATA_LOCKS; i++) { |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 2271 | if (malloc_mutex_init(&tdata_locks[i], "prof_tdata", |
| 2272 | WITNESS_RANK_PROF_TDATA)) |
Jason Evans | 6da5418 | 2012-03-23 18:05:51 -0700 | [diff] [blame] | 2273 | return (true); |
| 2274 | } |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 2275 | } |
| 2276 | |
Jason Evans | b27805b | 2010-02-10 18:15:53 -0800 | [diff] [blame] | 2277 | #ifdef JEMALLOC_PROF_LIBGCC |
| 2278 | /* |
| 2279 | * Cause the backtracing machinery to allocate its internal state |
| 2280 | * before enabling profiling. |
| 2281 | */ |
| 2282 | _Unwind_Backtrace(prof_unwind_init_callback, NULL); |
| 2283 | #endif |
| 2284 | |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 2285 | prof_booted = true; |
| 2286 | |
| 2287 | return (false); |
| 2288 | } |
| 2289 | |
Jason Evans | 20f1fc9 | 2012-10-09 14:46:22 -0700 | [diff] [blame] | 2290 | void |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 2291 | prof_prefork0(tsdn_t *tsdn) |
Jason Evans | 20f1fc9 | 2012-10-09 14:46:22 -0700 | [diff] [blame] | 2292 | { |
| 2293 | |
| 2294 | if (opt_prof) { |
| 2295 | unsigned i; |
| 2296 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 2297 | malloc_mutex_prefork(tsdn, &prof_dump_mtx); |
| 2298 | malloc_mutex_prefork(tsdn, &bt2gctx_mtx); |
| 2299 | malloc_mutex_prefork(tsdn, &tdatas_mtx); |
Jason Evans | 9d8f3d2 | 2014-09-11 18:06:30 -0700 | [diff] [blame] | 2300 | for (i = 0; i < PROF_NTDATA_LOCKS; i++) |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 2301 | malloc_mutex_prefork(tsdn, &tdata_locks[i]); |
Jason Evans | 174c0c3 | 2016-04-25 23:14:40 -0700 | [diff] [blame] | 2302 | for (i = 0; i < PROF_NCTX_LOCKS; i++) |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 2303 | malloc_mutex_prefork(tsdn, &gctx_locks[i]); |
Jason Evans | 174c0c3 | 2016-04-25 23:14:40 -0700 | [diff] [blame] | 2304 | } |
| 2305 | } |
| 2306 | |
| 2307 | void |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 2308 | prof_prefork1(tsdn_t *tsdn) |
Jason Evans | 174c0c3 | 2016-04-25 23:14:40 -0700 | [diff] [blame] | 2309 | { |
| 2310 | |
| 2311 | if (opt_prof) { |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 2312 | malloc_mutex_prefork(tsdn, &prof_active_mtx); |
| 2313 | malloc_mutex_prefork(tsdn, &prof_dump_seq_mtx); |
| 2314 | malloc_mutex_prefork(tsdn, &prof_gdump_mtx); |
| 2315 | malloc_mutex_prefork(tsdn, &next_thr_uid_mtx); |
| 2316 | malloc_mutex_prefork(tsdn, &prof_thread_active_init_mtx); |
Jason Evans | 20f1fc9 | 2012-10-09 14:46:22 -0700 | [diff] [blame] | 2317 | } |
| 2318 | } |
| 2319 | |
| 2320 | void |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 2321 | prof_postfork_parent(tsdn_t *tsdn) |
Jason Evans | 20f1fc9 | 2012-10-09 14:46:22 -0700 | [diff] [blame] | 2322 | { |
| 2323 | |
| 2324 | if (opt_prof) { |
| 2325 | unsigned i; |
| 2326 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 2327 | malloc_mutex_postfork_parent(tsdn, |
| 2328 | &prof_thread_active_init_mtx); |
| 2329 | malloc_mutex_postfork_parent(tsdn, &next_thr_uid_mtx); |
| 2330 | malloc_mutex_postfork_parent(tsdn, &prof_gdump_mtx); |
| 2331 | malloc_mutex_postfork_parent(tsdn, &prof_dump_seq_mtx); |
| 2332 | malloc_mutex_postfork_parent(tsdn, &prof_active_mtx); |
Jason Evans | 20f1fc9 | 2012-10-09 14:46:22 -0700 | [diff] [blame] | 2333 | for (i = 0; i < PROF_NCTX_LOCKS; i++) |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 2334 | malloc_mutex_postfork_parent(tsdn, &gctx_locks[i]); |
Jason Evans | 174c0c3 | 2016-04-25 23:14:40 -0700 | [diff] [blame] | 2335 | for (i = 0; i < PROF_NTDATA_LOCKS; i++) |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 2336 | malloc_mutex_postfork_parent(tsdn, &tdata_locks[i]); |
| 2337 | malloc_mutex_postfork_parent(tsdn, &tdatas_mtx); |
| 2338 | malloc_mutex_postfork_parent(tsdn, &bt2gctx_mtx); |
| 2339 | malloc_mutex_postfork_parent(tsdn, &prof_dump_mtx); |
Jason Evans | 20f1fc9 | 2012-10-09 14:46:22 -0700 | [diff] [blame] | 2340 | } |
| 2341 | } |
| 2342 | |
| 2343 | void |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 2344 | prof_postfork_child(tsdn_t *tsdn) |
Jason Evans | 20f1fc9 | 2012-10-09 14:46:22 -0700 | [diff] [blame] | 2345 | { |
| 2346 | |
| 2347 | if (opt_prof) { |
| 2348 | unsigned i; |
| 2349 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 2350 | malloc_mutex_postfork_child(tsdn, &prof_thread_active_init_mtx); |
| 2351 | malloc_mutex_postfork_child(tsdn, &next_thr_uid_mtx); |
| 2352 | malloc_mutex_postfork_child(tsdn, &prof_gdump_mtx); |
| 2353 | malloc_mutex_postfork_child(tsdn, &prof_dump_seq_mtx); |
| 2354 | malloc_mutex_postfork_child(tsdn, &prof_active_mtx); |
Jason Evans | 20f1fc9 | 2012-10-09 14:46:22 -0700 | [diff] [blame] | 2355 | for (i = 0; i < PROF_NCTX_LOCKS; i++) |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 2356 | malloc_mutex_postfork_child(tsdn, &gctx_locks[i]); |
Jason Evans | 174c0c3 | 2016-04-25 23:14:40 -0700 | [diff] [blame] | 2357 | for (i = 0; i < PROF_NTDATA_LOCKS; i++) |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame] | 2358 | malloc_mutex_postfork_child(tsdn, &tdata_locks[i]); |
| 2359 | malloc_mutex_postfork_child(tsdn, &tdatas_mtx); |
| 2360 | malloc_mutex_postfork_child(tsdn, &bt2gctx_mtx); |
| 2361 | malloc_mutex_postfork_child(tsdn, &prof_dump_mtx); |
Jason Evans | 20f1fc9 | 2012-10-09 14:46:22 -0700 | [diff] [blame] | 2362 | } |
| 2363 | } |
| 2364 | |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 2365 | /******************************************************************************/ |