Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1 | #define JEMALLOC_PROF_C_ |
Jason Evans | 376b152 | 2010-02-11 14:45:59 -0800 | [diff] [blame] | 2 | #include "jemalloc/internal/jemalloc_internal.h" |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 3 | /******************************************************************************/ |
| 4 | |
| 5 | #ifdef JEMALLOC_PROF_LIBUNWIND |
| 6 | #define UNW_LOCAL_ONLY |
| 7 | #include <libunwind.h> |
| 8 | #endif |
| 9 | |
Jason Evans | 77f350b | 2011-03-15 22:23:12 -0700 | [diff] [blame] | 10 | #ifdef JEMALLOC_PROF_LIBGCC |
| 11 | #include <unwind.h> |
| 12 | #endif |
| 13 | |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 14 | /******************************************************************************/ |
| 15 | /* Data. */ |
| 16 | |
| 17 | bool opt_prof = false; |
Jason Evans | f18c982 | 2010-03-31 18:43:24 -0700 | [diff] [blame] | 18 | bool opt_prof_active = true; |
Jason Evans | fc12c0b | 2014-10-03 23:25:30 -0700 | [diff] [blame] | 19 | bool opt_prof_thread_active_init = true; |
Jason Evans | b9477e7 | 2010-03-01 20:15:26 -0800 | [diff] [blame] | 20 | size_t opt_lg_prof_sample = LG_PROF_SAMPLE_DEFAULT; |
Jason Evans | a02fc08 | 2010-03-31 17:35:51 -0700 | [diff] [blame] | 21 | ssize_t opt_lg_prof_interval = LG_PROF_INTERVAL_DEFAULT; |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 22 | bool opt_prof_gdump = false; |
Jason Evans | 57efa7b | 2014-10-08 17:57:19 -0700 | [diff] [blame] | 23 | bool opt_prof_final = false; |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 24 | bool opt_prof_leak = false; |
Jason Evans | 0b25fe7 | 2012-04-17 16:39:33 -0700 | [diff] [blame] | 25 | bool opt_prof_accum = false; |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 26 | char opt_prof_prefix[ |
| 27 | /* Minimize memory bloat for non-prof builds. */ |
| 28 | #ifdef JEMALLOC_PROF |
| 29 | PATH_MAX + |
| 30 | #endif |
Jason Evans | eefdd02 | 2014-01-16 18:04:30 -0800 | [diff] [blame] | 31 | 1]; |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 32 | |
Jason Evans | fc12c0b | 2014-10-03 23:25:30 -0700 | [diff] [blame] | 33 | /* |
| 34 | * Initialized as opt_prof_active, and accessed via |
| 35 | * prof_active_[gs]et{_unlocked,}(). |
| 36 | */ |
| 37 | bool prof_active; |
| 38 | static malloc_mutex_t prof_active_mtx; |
| 39 | |
| 40 | /* |
| 41 | * Initialized as opt_prof_thread_active_init, and accessed via |
| 42 | * prof_thread_active_init_[gs]et(). |
| 43 | */ |
| 44 | static bool prof_thread_active_init; |
| 45 | static malloc_mutex_t prof_thread_active_init_mtx; |
| 46 | |
Jason Evans | 5b8ed5b | 2015-01-25 21:16:57 -0800 | [diff] [blame] | 47 | /* |
| 48 | * Initialized as opt_prof_gdump, and accessed via |
| 49 | * prof_gdump_[gs]et{_unlocked,}(). |
| 50 | */ |
| 51 | bool prof_gdump_val; |
| 52 | static malloc_mutex_t prof_gdump_mtx; |
| 53 | |
Jason Evans | a3b3386 | 2012-11-13 12:56:27 -0800 | [diff] [blame] | 54 | uint64_t prof_interval = 0; |
Jason Evans | d34f9e7 | 2010-02-11 13:19:21 -0800 | [diff] [blame] | 55 | |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 56 | size_t lg_prof_sample; |
| 57 | |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 58 | /* |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 59 | * Table of mutexes that are shared among gctx's. These are leaf locks, so |
| 60 | * there is no problem with using them for more than one gctx at the same time. |
| 61 | * The primary motivation for this sharing though is that gctx's are ephemeral, |
Jason Evans | 6da5418 | 2012-03-23 18:05:51 -0700 | [diff] [blame] | 62 | * and destroying mutexes causes complications for systems that allocate when |
| 63 | * creating/destroying mutexes. |
| 64 | */ |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 65 | static malloc_mutex_t *gctx_locks; |
| 66 | static unsigned cum_gctxs; /* Atomic counter. */ |
Jason Evans | 6da5418 | 2012-03-23 18:05:51 -0700 | [diff] [blame] | 67 | |
| 68 | /* |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 69 | * Table of mutexes that are shared among tdata's. No operations require |
| 70 | * holding multiple tdata locks, so there is no problem with using them for more |
| 71 | * than one tdata at the same time, even though a gctx lock may be acquired |
| 72 | * while holding a tdata lock. |
| 73 | */ |
| 74 | static malloc_mutex_t *tdata_locks; |
| 75 | |
| 76 | /* |
| 77 | * Global hash of (prof_bt_t *)-->(prof_gctx_t *). This is the master data |
Jason Evans | a881cd2 | 2010-10-02 15:18:50 -0700 | [diff] [blame] | 78 | * structure that knows about all backtraces currently captured. |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 79 | */ |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 80 | static ckh_t bt2gctx; |
| 81 | static malloc_mutex_t bt2gctx_mtx; |
| 82 | |
| 83 | /* |
| 84 | * Tree of all extant prof_tdata_t structures, regardless of state, |
| 85 | * {attached,detached,expired}. |
| 86 | */ |
| 87 | static prof_tdata_tree_t tdatas; |
| 88 | static malloc_mutex_t tdatas_mtx; |
| 89 | |
| 90 | static uint64_t next_thr_uid; |
Jason Evans | 9d8f3d2 | 2014-09-11 18:06:30 -0700 | [diff] [blame] | 91 | static malloc_mutex_t next_thr_uid_mtx; |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 92 | |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 93 | static malloc_mutex_t prof_dump_seq_mtx; |
| 94 | static uint64_t prof_dump_seq; |
| 95 | static uint64_t prof_dump_iseq; |
| 96 | static uint64_t prof_dump_mseq; |
| 97 | static uint64_t prof_dump_useq; |
| 98 | |
| 99 | /* |
| 100 | * This buffer is rather large for stack allocation, so use a single buffer for |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 101 | * all profile dumps. |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 102 | */ |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 103 | static malloc_mutex_t prof_dump_mtx; |
| 104 | static char prof_dump_buf[ |
| 105 | /* Minimize memory bloat for non-prof builds. */ |
| 106 | #ifdef JEMALLOC_PROF |
| 107 | PROF_DUMP_BUFSIZE |
| 108 | #else |
| 109 | 1 |
| 110 | #endif |
| 111 | ]; |
Jason Evans | 42ce80e | 2016-02-25 20:51:00 -0800 | [diff] [blame] | 112 | static size_t prof_dump_buf_end; |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 113 | static int prof_dump_fd; |
| 114 | |
| 115 | /* Do not dump any profiles until bootstrapping is complete. */ |
| 116 | static bool prof_booted = false; |
| 117 | |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 118 | /******************************************************************************/ |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 119 | /* |
| 120 | * Function prototypes for static functions that are referenced prior to |
| 121 | * definition. |
| 122 | */ |
| 123 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 124 | static bool prof_tctx_should_destroy(tsdn_t *tsdn, prof_tctx_t *tctx); |
Jason Evans | 5460aa6 | 2014-09-22 21:09:23 -0700 | [diff] [blame] | 125 | static void prof_tctx_destroy(tsd_t *tsd, prof_tctx_t *tctx); |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 126 | static bool prof_tdata_should_destroy(tsdn_t *tsdn, prof_tdata_t *tdata, |
Jason Evans | f04a0be | 2014-10-04 15:03:49 -0700 | [diff] [blame] | 127 | bool even_if_attached); |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 128 | static void prof_tdata_destroy(tsdn_t *tsdn, prof_tdata_t *tdata, |
Jason Evans | f04a0be | 2014-10-04 15:03:49 -0700 | [diff] [blame] | 129 | bool even_if_attached); |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 130 | static char *prof_thread_name_alloc(tsdn_t *tsdn, const char *thread_name); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 131 | |
| 132 | /******************************************************************************/ |
| 133 | /* Red-black trees. */ |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 134 | |
Jason Evans | 3a81cbd | 2014-08-16 12:58:55 -0700 | [diff] [blame] | 135 | JEMALLOC_INLINE_C int |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 136 | prof_tctx_comp(const prof_tctx_t *a, const prof_tctx_t *b) |
Jason Evans | 3a81cbd | 2014-08-16 12:58:55 -0700 | [diff] [blame] | 137 | { |
Jason Evans | 04211e2 | 2015-03-16 15:11:06 -0700 | [diff] [blame] | 138 | uint64_t a_thr_uid = a->thr_uid; |
| 139 | uint64_t b_thr_uid = b->thr_uid; |
| 140 | int ret = (a_thr_uid > b_thr_uid) - (a_thr_uid < b_thr_uid); |
Jason Evans | d69964b | 2015-03-12 16:25:18 -0700 | [diff] [blame] | 141 | if (ret == 0) { |
Jason Evans | a00b107 | 2015-09-09 23:16:10 -0700 | [diff] [blame] | 142 | uint64_t a_thr_discrim = a->thr_discrim; |
| 143 | uint64_t b_thr_discrim = b->thr_discrim; |
| 144 | ret = (a_thr_discrim > b_thr_discrim) - (a_thr_discrim < |
| 145 | b_thr_discrim); |
| 146 | if (ret == 0) { |
| 147 | uint64_t a_tctx_uid = a->tctx_uid; |
| 148 | uint64_t b_tctx_uid = b->tctx_uid; |
| 149 | ret = (a_tctx_uid > b_tctx_uid) - (a_tctx_uid < |
| 150 | b_tctx_uid); |
| 151 | } |
Jason Evans | d69964b | 2015-03-12 16:25:18 -0700 | [diff] [blame] | 152 | } |
| 153 | return (ret); |
Jason Evans | 3a81cbd | 2014-08-16 12:58:55 -0700 | [diff] [blame] | 154 | } |
| 155 | |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 156 | rb_gen(static UNUSED, tctx_tree_, prof_tctx_tree_t, prof_tctx_t, |
| 157 | tctx_link, prof_tctx_comp) |
Jason Evans | 3a81cbd | 2014-08-16 12:58:55 -0700 | [diff] [blame] | 158 | |
| 159 | JEMALLOC_INLINE_C int |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 160 | prof_gctx_comp(const prof_gctx_t *a, const prof_gctx_t *b) |
Jason Evans | 3a81cbd | 2014-08-16 12:58:55 -0700 | [diff] [blame] | 161 | { |
| 162 | unsigned a_len = a->bt.len; |
| 163 | unsigned b_len = b->bt.len; |
| 164 | unsigned comp_len = (a_len < b_len) ? a_len : b_len; |
| 165 | int ret = memcmp(a->bt.vec, b->bt.vec, comp_len * sizeof(void *)); |
| 166 | if (ret == 0) |
| 167 | ret = (a_len > b_len) - (a_len < b_len); |
| 168 | return (ret); |
| 169 | } |
| 170 | |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 171 | rb_gen(static UNUSED, gctx_tree_, prof_gctx_tree_t, prof_gctx_t, dump_link, |
| 172 | prof_gctx_comp) |
| 173 | |
| 174 | JEMALLOC_INLINE_C int |
| 175 | prof_tdata_comp(const prof_tdata_t *a, const prof_tdata_t *b) |
| 176 | { |
Jason Evans | 20c31de | 2014-10-02 23:01:10 -0700 | [diff] [blame] | 177 | int ret; |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 178 | uint64_t a_uid = a->thr_uid; |
| 179 | uint64_t b_uid = b->thr_uid; |
| 180 | |
Jason Evans | 20c31de | 2014-10-02 23:01:10 -0700 | [diff] [blame] | 181 | ret = ((a_uid > b_uid) - (a_uid < b_uid)); |
| 182 | if (ret == 0) { |
| 183 | uint64_t a_discrim = a->thr_discrim; |
| 184 | uint64_t b_discrim = b->thr_discrim; |
| 185 | |
| 186 | ret = ((a_discrim > b_discrim) - (a_discrim < b_discrim)); |
| 187 | } |
| 188 | return (ret); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 189 | } |
| 190 | |
| 191 | rb_gen(static UNUSED, tdata_tree_, prof_tdata_tree_t, prof_tdata_t, tdata_link, |
| 192 | prof_tdata_comp) |
| 193 | |
| 194 | /******************************************************************************/ |
| 195 | |
| 196 | void |
Jason Evans | 5460aa6 | 2014-09-22 21:09:23 -0700 | [diff] [blame] | 197 | prof_alloc_rollback(tsd_t *tsd, prof_tctx_t *tctx, bool updated) |
Jason Evans | 6e73dc1 | 2014-09-09 19:37:26 -0700 | [diff] [blame] | 198 | { |
| 199 | prof_tdata_t *tdata; |
| 200 | |
| 201 | cassert(config_prof); |
| 202 | |
| 203 | if (updated) { |
| 204 | /* |
| 205 | * Compute a new sample threshold. This isn't very important in |
| 206 | * practice, because this function is rarely executed, so the |
| 207 | * potential for sample bias is minimal except in contrived |
| 208 | * programs. |
| 209 | */ |
Jason Evans | 5460aa6 | 2014-09-22 21:09:23 -0700 | [diff] [blame] | 210 | tdata = prof_tdata_get(tsd, true); |
| 211 | if (tdata != NULL) |
Jason Evans | 3ca0cf6 | 2015-09-17 14:47:39 -0700 | [diff] [blame] | 212 | prof_sample_threshold_update(tdata); |
Jason Evans | 6e73dc1 | 2014-09-09 19:37:26 -0700 | [diff] [blame] | 213 | } |
| 214 | |
| 215 | if ((uintptr_t)tctx > (uintptr_t)1U) { |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 216 | malloc_mutex_lock(tsd_tsdn(tsd), tctx->tdata->lock); |
Jason Evans | 6e73dc1 | 2014-09-09 19:37:26 -0700 | [diff] [blame] | 217 | tctx->prepared = false; |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 218 | if (prof_tctx_should_destroy(tsd_tsdn(tsd), tctx)) |
Jason Evans | 5460aa6 | 2014-09-22 21:09:23 -0700 | [diff] [blame] | 219 | prof_tctx_destroy(tsd, tctx); |
Jason Evans | 6e73dc1 | 2014-09-09 19:37:26 -0700 | [diff] [blame] | 220 | else |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 221 | malloc_mutex_unlock(tsd_tsdn(tsd), tctx->tdata->lock); |
Jason Evans | 6e73dc1 | 2014-09-09 19:37:26 -0700 | [diff] [blame] | 222 | } |
| 223 | } |
| 224 | |
| 225 | void |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 226 | prof_malloc_sample_object(tsdn_t *tsdn, const void *ptr, size_t usize, |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 227 | prof_tctx_t *tctx) |
Jason Evans | cfc5706 | 2014-10-30 23:18:45 -0700 | [diff] [blame] | 228 | { |
| 229 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 230 | prof_tctx_set(tsdn, ptr, usize, tctx); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 231 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 232 | malloc_mutex_lock(tsdn, tctx->tdata->lock); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 233 | tctx->cnts.curobjs++; |
| 234 | tctx->cnts.curbytes += usize; |
| 235 | if (opt_prof_accum) { |
| 236 | tctx->cnts.accumobjs++; |
| 237 | tctx->cnts.accumbytes += usize; |
| 238 | } |
Jason Evans | 6e73dc1 | 2014-09-09 19:37:26 -0700 | [diff] [blame] | 239 | tctx->prepared = false; |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 240 | malloc_mutex_unlock(tsdn, tctx->tdata->lock); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 241 | } |
| 242 | |
| 243 | void |
Jason Evans | 5460aa6 | 2014-09-22 21:09:23 -0700 | [diff] [blame] | 244 | prof_free_sampled_object(tsd_t *tsd, size_t usize, prof_tctx_t *tctx) |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 245 | { |
| 246 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 247 | malloc_mutex_lock(tsd_tsdn(tsd), tctx->tdata->lock); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 248 | assert(tctx->cnts.curobjs > 0); |
| 249 | assert(tctx->cnts.curbytes >= usize); |
| 250 | tctx->cnts.curobjs--; |
| 251 | tctx->cnts.curbytes -= usize; |
| 252 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 253 | if (prof_tctx_should_destroy(tsd_tsdn(tsd), tctx)) |
Jason Evans | 5460aa6 | 2014-09-22 21:09:23 -0700 | [diff] [blame] | 254 | prof_tctx_destroy(tsd, tctx); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 255 | else |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 256 | malloc_mutex_unlock(tsd_tsdn(tsd), tctx->tdata->lock); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 257 | } |
Jason Evans | 3a81cbd | 2014-08-16 12:58:55 -0700 | [diff] [blame] | 258 | |
Jason Evans | 4d6a134 | 2010-10-20 19:05:59 -0700 | [diff] [blame] | 259 | void |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 260 | bt_init(prof_bt_t *bt, void **vec) |
| 261 | { |
| 262 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 263 | cassert(config_prof); |
| 264 | |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 265 | bt->vec = vec; |
| 266 | bt->len = 0; |
| 267 | } |
| 268 | |
Jason Evans | af1f592 | 2014-10-30 16:38:08 -0700 | [diff] [blame] | 269 | JEMALLOC_INLINE_C void |
Jason Evans | c93ed81 | 2014-10-30 16:50:33 -0700 | [diff] [blame] | 270 | prof_enter(tsd_t *tsd, prof_tdata_t *tdata) |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 271 | { |
| 272 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 273 | cassert(config_prof); |
Jason Evans | c93ed81 | 2014-10-30 16:50:33 -0700 | [diff] [blame] | 274 | assert(tdata == prof_tdata_get(tsd, false)); |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 275 | |
Jason Evans | 82cb603 | 2014-11-01 00:20:28 -0700 | [diff] [blame] | 276 | if (tdata != NULL) { |
| 277 | assert(!tdata->enq); |
| 278 | tdata->enq = true; |
| 279 | } |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 280 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 281 | malloc_mutex_lock(tsd_tsdn(tsd), &bt2gctx_mtx); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 282 | } |
| 283 | |
Jason Evans | af1f592 | 2014-10-30 16:38:08 -0700 | [diff] [blame] | 284 | JEMALLOC_INLINE_C void |
Jason Evans | c93ed81 | 2014-10-30 16:50:33 -0700 | [diff] [blame] | 285 | prof_leave(tsd_t *tsd, prof_tdata_t *tdata) |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 286 | { |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 287 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 288 | cassert(config_prof); |
Jason Evans | c93ed81 | 2014-10-30 16:50:33 -0700 | [diff] [blame] | 289 | assert(tdata == prof_tdata_get(tsd, false)); |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 290 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 291 | malloc_mutex_unlock(tsd_tsdn(tsd), &bt2gctx_mtx); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 292 | |
Jason Evans | 82cb603 | 2014-11-01 00:20:28 -0700 | [diff] [blame] | 293 | if (tdata != NULL) { |
| 294 | bool idump, gdump; |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 295 | |
Jason Evans | 82cb603 | 2014-11-01 00:20:28 -0700 | [diff] [blame] | 296 | assert(tdata->enq); |
| 297 | tdata->enq = false; |
| 298 | idump = tdata->enq_idump; |
| 299 | tdata->enq_idump = false; |
| 300 | gdump = tdata->enq_gdump; |
| 301 | tdata->enq_gdump = false; |
| 302 | |
| 303 | if (idump) |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 304 | prof_idump(tsd_tsdn(tsd)); |
Jason Evans | 82cb603 | 2014-11-01 00:20:28 -0700 | [diff] [blame] | 305 | if (gdump) |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 306 | prof_gdump(tsd_tsdn(tsd)); |
Jason Evans | 82cb603 | 2014-11-01 00:20:28 -0700 | [diff] [blame] | 307 | } |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 308 | } |
| 309 | |
Jason Evans | 77f350b | 2011-03-15 22:23:12 -0700 | [diff] [blame] | 310 | #ifdef JEMALLOC_PROF_LIBUNWIND |
Jason Evans | 4d6a134 | 2010-10-20 19:05:59 -0700 | [diff] [blame] | 311 | void |
Jason Evans | 6f00105 | 2014-04-22 18:41:15 -0700 | [diff] [blame] | 312 | prof_backtrace(prof_bt_t *bt) |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 313 | { |
Jason Evans | 6f00105 | 2014-04-22 18:41:15 -0700 | [diff] [blame] | 314 | int nframes; |
| 315 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 316 | cassert(config_prof); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 317 | assert(bt->len == 0); |
| 318 | assert(bt->vec != NULL); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 319 | |
Jason Evans | 6f00105 | 2014-04-22 18:41:15 -0700 | [diff] [blame] | 320 | nframes = unw_backtrace(bt->vec, PROF_BT_MAX); |
| 321 | if (nframes <= 0) |
Lucian Adrian Grijincu | 9d4e13f | 2014-04-21 20:52:35 -0700 | [diff] [blame] | 322 | return; |
Jason Evans | 6f00105 | 2014-04-22 18:41:15 -0700 | [diff] [blame] | 323 | bt->len = nframes; |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 324 | } |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 325 | #elif (defined(JEMALLOC_PROF_LIBGCC)) |
Jason Evans | 77f350b | 2011-03-15 22:23:12 -0700 | [diff] [blame] | 326 | static _Unwind_Reason_Code |
| 327 | prof_unwind_init_callback(struct _Unwind_Context *context, void *arg) |
| 328 | { |
| 329 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 330 | cassert(config_prof); |
| 331 | |
Jason Evans | 77f350b | 2011-03-15 22:23:12 -0700 | [diff] [blame] | 332 | return (_URC_NO_REASON); |
| 333 | } |
| 334 | |
| 335 | static _Unwind_Reason_Code |
| 336 | prof_unwind_callback(struct _Unwind_Context *context, void *arg) |
| 337 | { |
| 338 | prof_unwind_data_t *data = (prof_unwind_data_t *)arg; |
Jason Evans | 6f00105 | 2014-04-22 18:41:15 -0700 | [diff] [blame] | 339 | void *ip; |
Jason Evans | 77f350b | 2011-03-15 22:23:12 -0700 | [diff] [blame] | 340 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 341 | cassert(config_prof); |
| 342 | |
Jason Evans | 6f00105 | 2014-04-22 18:41:15 -0700 | [diff] [blame] | 343 | ip = (void *)_Unwind_GetIP(context); |
| 344 | if (ip == NULL) |
| 345 | return (_URC_END_OF_STACK); |
| 346 | data->bt->vec[data->bt->len] = ip; |
| 347 | data->bt->len++; |
| 348 | if (data->bt->len == data->max) |
| 349 | return (_URC_END_OF_STACK); |
Jason Evans | 77f350b | 2011-03-15 22:23:12 -0700 | [diff] [blame] | 350 | |
| 351 | return (_URC_NO_REASON); |
| 352 | } |
| 353 | |
| 354 | void |
Jason Evans | 6f00105 | 2014-04-22 18:41:15 -0700 | [diff] [blame] | 355 | prof_backtrace(prof_bt_t *bt) |
Jason Evans | 77f350b | 2011-03-15 22:23:12 -0700 | [diff] [blame] | 356 | { |
Jason Evans | 6f00105 | 2014-04-22 18:41:15 -0700 | [diff] [blame] | 357 | prof_unwind_data_t data = {bt, PROF_BT_MAX}; |
Jason Evans | 77f350b | 2011-03-15 22:23:12 -0700 | [diff] [blame] | 358 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 359 | cassert(config_prof); |
| 360 | |
Jason Evans | 77f350b | 2011-03-15 22:23:12 -0700 | [diff] [blame] | 361 | _Unwind_Backtrace(prof_unwind_callback, &data); |
| 362 | } |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 363 | #elif (defined(JEMALLOC_PROF_GCC)) |
Jason Evans | 4d6a134 | 2010-10-20 19:05:59 -0700 | [diff] [blame] | 364 | void |
Jason Evans | 6f00105 | 2014-04-22 18:41:15 -0700 | [diff] [blame] | 365 | prof_backtrace(prof_bt_t *bt) |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 366 | { |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 367 | #define BT_FRAME(i) \ |
Jason Evans | 6f00105 | 2014-04-22 18:41:15 -0700 | [diff] [blame] | 368 | if ((i) < PROF_BT_MAX) { \ |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 369 | void *p; \ |
| 370 | if (__builtin_frame_address(i) == 0) \ |
Jason Evans | b27805b | 2010-02-10 18:15:53 -0800 | [diff] [blame] | 371 | return; \ |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 372 | p = __builtin_return_address(i); \ |
| 373 | if (p == NULL) \ |
Jason Evans | b27805b | 2010-02-10 18:15:53 -0800 | [diff] [blame] | 374 | return; \ |
Jason Evans | 6f00105 | 2014-04-22 18:41:15 -0700 | [diff] [blame] | 375 | bt->vec[(i)] = p; \ |
| 376 | bt->len = (i) + 1; \ |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 377 | } else \ |
Jason Evans | b27805b | 2010-02-10 18:15:53 -0800 | [diff] [blame] | 378 | return; |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 379 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 380 | cassert(config_prof); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 381 | |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 382 | BT_FRAME(0) |
| 383 | BT_FRAME(1) |
| 384 | BT_FRAME(2) |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 385 | BT_FRAME(3) |
| 386 | BT_FRAME(4) |
| 387 | BT_FRAME(5) |
| 388 | BT_FRAME(6) |
| 389 | BT_FRAME(7) |
| 390 | BT_FRAME(8) |
| 391 | BT_FRAME(9) |
| 392 | |
| 393 | BT_FRAME(10) |
| 394 | BT_FRAME(11) |
| 395 | BT_FRAME(12) |
| 396 | BT_FRAME(13) |
| 397 | BT_FRAME(14) |
| 398 | BT_FRAME(15) |
| 399 | BT_FRAME(16) |
| 400 | BT_FRAME(17) |
| 401 | BT_FRAME(18) |
| 402 | BT_FRAME(19) |
| 403 | |
| 404 | BT_FRAME(20) |
| 405 | BT_FRAME(21) |
| 406 | BT_FRAME(22) |
| 407 | BT_FRAME(23) |
| 408 | BT_FRAME(24) |
| 409 | BT_FRAME(25) |
| 410 | BT_FRAME(26) |
| 411 | BT_FRAME(27) |
| 412 | BT_FRAME(28) |
| 413 | BT_FRAME(29) |
| 414 | |
| 415 | BT_FRAME(30) |
| 416 | BT_FRAME(31) |
| 417 | BT_FRAME(32) |
| 418 | BT_FRAME(33) |
| 419 | BT_FRAME(34) |
| 420 | BT_FRAME(35) |
| 421 | BT_FRAME(36) |
| 422 | BT_FRAME(37) |
| 423 | BT_FRAME(38) |
| 424 | BT_FRAME(39) |
| 425 | |
| 426 | BT_FRAME(40) |
| 427 | BT_FRAME(41) |
| 428 | BT_FRAME(42) |
| 429 | BT_FRAME(43) |
| 430 | BT_FRAME(44) |
| 431 | BT_FRAME(45) |
| 432 | BT_FRAME(46) |
| 433 | BT_FRAME(47) |
| 434 | BT_FRAME(48) |
| 435 | BT_FRAME(49) |
| 436 | |
| 437 | BT_FRAME(50) |
| 438 | BT_FRAME(51) |
| 439 | BT_FRAME(52) |
| 440 | BT_FRAME(53) |
| 441 | BT_FRAME(54) |
| 442 | BT_FRAME(55) |
| 443 | BT_FRAME(56) |
| 444 | BT_FRAME(57) |
| 445 | BT_FRAME(58) |
| 446 | BT_FRAME(59) |
| 447 | |
| 448 | BT_FRAME(60) |
| 449 | BT_FRAME(61) |
| 450 | BT_FRAME(62) |
| 451 | BT_FRAME(63) |
| 452 | BT_FRAME(64) |
| 453 | BT_FRAME(65) |
| 454 | BT_FRAME(66) |
| 455 | BT_FRAME(67) |
| 456 | BT_FRAME(68) |
| 457 | BT_FRAME(69) |
| 458 | |
| 459 | BT_FRAME(70) |
| 460 | BT_FRAME(71) |
| 461 | BT_FRAME(72) |
| 462 | BT_FRAME(73) |
| 463 | BT_FRAME(74) |
| 464 | BT_FRAME(75) |
| 465 | BT_FRAME(76) |
| 466 | BT_FRAME(77) |
| 467 | BT_FRAME(78) |
| 468 | BT_FRAME(79) |
| 469 | |
| 470 | BT_FRAME(80) |
| 471 | BT_FRAME(81) |
| 472 | BT_FRAME(82) |
| 473 | BT_FRAME(83) |
| 474 | BT_FRAME(84) |
| 475 | BT_FRAME(85) |
| 476 | BT_FRAME(86) |
| 477 | BT_FRAME(87) |
| 478 | BT_FRAME(88) |
| 479 | BT_FRAME(89) |
| 480 | |
| 481 | BT_FRAME(90) |
| 482 | BT_FRAME(91) |
| 483 | BT_FRAME(92) |
| 484 | BT_FRAME(93) |
| 485 | BT_FRAME(94) |
| 486 | BT_FRAME(95) |
| 487 | BT_FRAME(96) |
| 488 | BT_FRAME(97) |
| 489 | BT_FRAME(98) |
| 490 | BT_FRAME(99) |
| 491 | |
| 492 | BT_FRAME(100) |
| 493 | BT_FRAME(101) |
| 494 | BT_FRAME(102) |
| 495 | BT_FRAME(103) |
| 496 | BT_FRAME(104) |
| 497 | BT_FRAME(105) |
| 498 | BT_FRAME(106) |
| 499 | BT_FRAME(107) |
| 500 | BT_FRAME(108) |
| 501 | BT_FRAME(109) |
| 502 | |
| 503 | BT_FRAME(110) |
| 504 | BT_FRAME(111) |
| 505 | BT_FRAME(112) |
| 506 | BT_FRAME(113) |
| 507 | BT_FRAME(114) |
| 508 | BT_FRAME(115) |
| 509 | BT_FRAME(116) |
| 510 | BT_FRAME(117) |
| 511 | BT_FRAME(118) |
| 512 | BT_FRAME(119) |
| 513 | |
| 514 | BT_FRAME(120) |
| 515 | BT_FRAME(121) |
| 516 | BT_FRAME(122) |
| 517 | BT_FRAME(123) |
| 518 | BT_FRAME(124) |
| 519 | BT_FRAME(125) |
| 520 | BT_FRAME(126) |
| 521 | BT_FRAME(127) |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 522 | #undef BT_FRAME |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 523 | } |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 524 | #else |
| 525 | void |
Jason Evans | 6f00105 | 2014-04-22 18:41:15 -0700 | [diff] [blame] | 526 | prof_backtrace(prof_bt_t *bt) |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 527 | { |
| 528 | |
| 529 | cassert(config_prof); |
Jason Evans | 6556e28 | 2013-10-21 14:56:27 -0700 | [diff] [blame] | 530 | not_reached(); |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 531 | } |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 532 | #endif |
| 533 | |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 534 | static malloc_mutex_t * |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 535 | prof_gctx_mutex_choose(void) |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 536 | { |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 537 | unsigned ngctxs = atomic_add_u(&cum_gctxs, 1); |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 538 | |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 539 | return (&gctx_locks[(ngctxs - 1) % PROF_NCTX_LOCKS]); |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 540 | } |
| 541 | |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 542 | static malloc_mutex_t * |
| 543 | prof_tdata_mutex_choose(uint64_t thr_uid) |
| 544 | { |
| 545 | |
| 546 | return (&tdata_locks[thr_uid % PROF_NTDATA_LOCKS]); |
| 547 | } |
| 548 | |
| 549 | static prof_gctx_t * |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 550 | prof_gctx_create(tsdn_t *tsdn, prof_bt_t *bt) |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 551 | { |
Jason Evans | ab532e9 | 2014-08-15 15:05:12 -0700 | [diff] [blame] | 552 | /* |
| 553 | * Create a single allocation that has space for vec of length bt->len. |
| 554 | */ |
Qi Wang | f4a0f32 | 2015-10-27 15:12:10 -0700 | [diff] [blame] | 555 | size_t size = offsetof(prof_gctx_t, vec) + (bt->len * sizeof(void *)); |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 556 | prof_gctx_t *gctx = (prof_gctx_t *)iallocztm(tsdn, size, |
| 557 | size2index(size), false, NULL, true, arena_get(TSDN_NULL, 0, true), |
Jason Evans | 66cd953 | 2016-04-22 14:34:14 -0700 | [diff] [blame] | 558 | true); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 559 | if (gctx == NULL) |
Jason Evans | ab532e9 | 2014-08-15 15:05:12 -0700 | [diff] [blame] | 560 | return (NULL); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 561 | gctx->lock = prof_gctx_mutex_choose(); |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 562 | /* |
| 563 | * Set nlimbo to 1, in order to avoid a race condition with |
Jason Evans | 20c31de | 2014-10-02 23:01:10 -0700 | [diff] [blame] | 564 | * prof_tctx_destroy()/prof_gctx_try_destroy(). |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 565 | */ |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 566 | gctx->nlimbo = 1; |
| 567 | tctx_tree_new(&gctx->tctxs); |
Jason Evans | ab532e9 | 2014-08-15 15:05:12 -0700 | [diff] [blame] | 568 | /* Duplicate bt. */ |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 569 | memcpy(gctx->vec, bt->vec, bt->len * sizeof(void *)); |
| 570 | gctx->bt.vec = gctx->vec; |
| 571 | gctx->bt.len = bt->len; |
| 572 | return (gctx); |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 573 | } |
| 574 | |
| 575 | static void |
Jason Evans | c93ed81 | 2014-10-30 16:50:33 -0700 | [diff] [blame] | 576 | prof_gctx_try_destroy(tsd_t *tsd, prof_tdata_t *tdata_self, prof_gctx_t *gctx, |
| 577 | prof_tdata_t *tdata) |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 578 | { |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 579 | |
| 580 | cassert(config_prof); |
| 581 | |
| 582 | /* |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 583 | * Check that gctx is still unused by any thread cache before destroying |
| 584 | * it. prof_lookup() increments gctx->nlimbo in order to avoid a race |
| 585 | * condition with this function, as does prof_tctx_destroy() in order to |
| 586 | * avoid a race between the main body of prof_tctx_destroy() and entry |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 587 | * into this function. |
| 588 | */ |
Jason Evans | c93ed81 | 2014-10-30 16:50:33 -0700 | [diff] [blame] | 589 | prof_enter(tsd, tdata_self); |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 590 | malloc_mutex_lock(tsd_tsdn(tsd), gctx->lock); |
Jason Evans | 20c31de | 2014-10-02 23:01:10 -0700 | [diff] [blame] | 591 | assert(gctx->nlimbo != 0); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 592 | if (tctx_tree_empty(&gctx->tctxs) && gctx->nlimbo == 1) { |
| 593 | /* Remove gctx from bt2gctx. */ |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 594 | if (ckh_remove(tsd_tsdn(tsd), &bt2gctx, &gctx->bt, NULL, NULL)) |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 595 | not_reached(); |
Jason Evans | c93ed81 | 2014-10-30 16:50:33 -0700 | [diff] [blame] | 596 | prof_leave(tsd, tdata_self); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 597 | /* Destroy gctx. */ |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 598 | malloc_mutex_unlock(tsd_tsdn(tsd), gctx->lock); |
| 599 | idalloctm(tsd_tsdn(tsd), gctx, NULL, true, true); |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 600 | } else { |
| 601 | /* |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 602 | * Compensate for increment in prof_tctx_destroy() or |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 603 | * prof_lookup(). |
| 604 | */ |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 605 | gctx->nlimbo--; |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 606 | malloc_mutex_unlock(tsd_tsdn(tsd), gctx->lock); |
Jason Evans | c93ed81 | 2014-10-30 16:50:33 -0700 | [diff] [blame] | 607 | prof_leave(tsd, tdata_self); |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 608 | } |
| 609 | } |
| 610 | |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 611 | static bool |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 612 | prof_tctx_should_destroy(tsdn_t *tsdn, prof_tctx_t *tctx) |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 613 | { |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 614 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 615 | malloc_mutex_assert_owner(tsdn, tctx->tdata->lock); |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 616 | |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 617 | if (opt_prof_accum) |
| 618 | return (false); |
| 619 | if (tctx->cnts.curobjs != 0) |
| 620 | return (false); |
Jason Evans | 6e73dc1 | 2014-09-09 19:37:26 -0700 | [diff] [blame] | 621 | if (tctx->prepared) |
| 622 | return (false); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 623 | return (true); |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 624 | } |
| 625 | |
Jason Evans | fb1775e | 2014-01-14 17:04:34 -0800 | [diff] [blame] | 626 | static bool |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 627 | prof_gctx_should_destroy(prof_gctx_t *gctx) |
| 628 | { |
| 629 | |
| 630 | if (opt_prof_accum) |
| 631 | return (false); |
Jason Evans | 551ebc4 | 2014-10-03 10:16:09 -0700 | [diff] [blame] | 632 | if (!tctx_tree_empty(&gctx->tctxs)) |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 633 | return (false); |
| 634 | if (gctx->nlimbo != 0) |
| 635 | return (false); |
| 636 | return (true); |
| 637 | } |
| 638 | |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 639 | static void |
Jason Evans | 5460aa6 | 2014-09-22 21:09:23 -0700 | [diff] [blame] | 640 | prof_tctx_destroy(tsd_t *tsd, prof_tctx_t *tctx) |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 641 | { |
Jason Evans | 6fd53da | 2014-09-09 12:45:53 -0700 | [diff] [blame] | 642 | prof_tdata_t *tdata = tctx->tdata; |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 643 | prof_gctx_t *gctx = tctx->gctx; |
Jason Evans | bf40641 | 2014-10-06 16:35:11 -0700 | [diff] [blame] | 644 | bool destroy_tdata, destroy_tctx, destroy_gctx; |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 645 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 646 | malloc_mutex_assert_owner(tsd_tsdn(tsd), tctx->tdata->lock); |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 647 | |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 648 | assert(tctx->cnts.curobjs == 0); |
| 649 | assert(tctx->cnts.curbytes == 0); |
Jason Evans | 551ebc4 | 2014-10-03 10:16:09 -0700 | [diff] [blame] | 650 | assert(!opt_prof_accum); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 651 | assert(tctx->cnts.accumobjs == 0); |
| 652 | assert(tctx->cnts.accumbytes == 0); |
| 653 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 654 | ckh_remove(tsd_tsdn(tsd), &tdata->bt2tctx, &gctx->bt, NULL, NULL); |
| 655 | destroy_tdata = prof_tdata_should_destroy(tsd_tsdn(tsd), tdata, false); |
| 656 | malloc_mutex_unlock(tsd_tsdn(tsd), tdata->lock); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 657 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 658 | malloc_mutex_lock(tsd_tsdn(tsd), gctx->lock); |
Jason Evans | 764b000 | 2015-03-14 14:01:35 -0700 | [diff] [blame] | 659 | switch (tctx->state) { |
Jason Evans | 04211e2 | 2015-03-16 15:11:06 -0700 | [diff] [blame] | 660 | case prof_tctx_state_nominal: |
Jason Evans | bf40641 | 2014-10-06 16:35:11 -0700 | [diff] [blame] | 661 | tctx_tree_remove(&gctx->tctxs, tctx); |
| 662 | destroy_tctx = true; |
| 663 | if (prof_gctx_should_destroy(gctx)) { |
| 664 | /* |
| 665 | * Increment gctx->nlimbo in order to keep another |
| 666 | * thread from winning the race to destroy gctx while |
| 667 | * this one has gctx->lock dropped. Without this, it |
| 668 | * would be possible for another thread to: |
| 669 | * |
| 670 | * 1) Sample an allocation associated with gctx. |
| 671 | * 2) Deallocate the sampled object. |
| 672 | * 3) Successfully prof_gctx_try_destroy(gctx). |
| 673 | * |
| 674 | * The result would be that gctx no longer exists by the |
| 675 | * time this thread accesses it in |
| 676 | * prof_gctx_try_destroy(). |
| 677 | */ |
| 678 | gctx->nlimbo++; |
| 679 | destroy_gctx = true; |
| 680 | } else |
| 681 | destroy_gctx = false; |
Jason Evans | 764b000 | 2015-03-14 14:01:35 -0700 | [diff] [blame] | 682 | break; |
| 683 | case prof_tctx_state_dumping: |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 684 | /* |
Jason Evans | bf40641 | 2014-10-06 16:35:11 -0700 | [diff] [blame] | 685 | * A dumping thread needs tctx to remain valid until dumping |
| 686 | * has finished. Change state such that the dumping thread will |
| 687 | * complete destruction during a late dump iteration phase. |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 688 | */ |
Jason Evans | bf40641 | 2014-10-06 16:35:11 -0700 | [diff] [blame] | 689 | tctx->state = prof_tctx_state_purgatory; |
| 690 | destroy_tctx = false; |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 691 | destroy_gctx = false; |
Jason Evans | 764b000 | 2015-03-14 14:01:35 -0700 | [diff] [blame] | 692 | break; |
| 693 | default: |
| 694 | not_reached(); |
Jason Evans | 262146d | 2015-03-14 14:34:16 -0700 | [diff] [blame] | 695 | destroy_tctx = false; |
| 696 | destroy_gctx = false; |
Jason Evans | bf40641 | 2014-10-06 16:35:11 -0700 | [diff] [blame] | 697 | } |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 698 | malloc_mutex_unlock(tsd_tsdn(tsd), gctx->lock); |
Jason Evans | c93ed81 | 2014-10-30 16:50:33 -0700 | [diff] [blame] | 699 | if (destroy_gctx) { |
| 700 | prof_gctx_try_destroy(tsd, prof_tdata_get(tsd, false), gctx, |
| 701 | tdata); |
| 702 | } |
Jason Evans | 6fd53da | 2014-09-09 12:45:53 -0700 | [diff] [blame] | 703 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 704 | malloc_mutex_assert_not_owner(tsd_tsdn(tsd), tctx->tdata->lock); |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 705 | |
Jason Evans | 6fd53da | 2014-09-09 12:45:53 -0700 | [diff] [blame] | 706 | if (destroy_tdata) |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 707 | prof_tdata_destroy(tsd_tsdn(tsd), tdata, false); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 708 | |
Jason Evans | bf40641 | 2014-10-06 16:35:11 -0700 | [diff] [blame] | 709 | if (destroy_tctx) |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 710 | idalloctm(tsd_tsdn(tsd), tctx, NULL, true, true); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 711 | } |
| 712 | |
| 713 | static bool |
Jason Evans | 5460aa6 | 2014-09-22 21:09:23 -0700 | [diff] [blame] | 714 | prof_lookup_global(tsd_t *tsd, prof_bt_t *bt, prof_tdata_t *tdata, |
| 715 | void **p_btkey, prof_gctx_t **p_gctx, bool *p_new_gctx) |
Jason Evans | fb1775e | 2014-01-14 17:04:34 -0800 | [diff] [blame] | 716 | { |
| 717 | union { |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 718 | prof_gctx_t *p; |
Jason Evans | fb1775e | 2014-01-14 17:04:34 -0800 | [diff] [blame] | 719 | void *v; |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 720 | } gctx; |
Jason Evans | fb1775e | 2014-01-14 17:04:34 -0800 | [diff] [blame] | 721 | union { |
| 722 | prof_bt_t *p; |
| 723 | void *v; |
| 724 | } btkey; |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 725 | bool new_gctx; |
Jason Evans | fb1775e | 2014-01-14 17:04:34 -0800 | [diff] [blame] | 726 | |
Jason Evans | c93ed81 | 2014-10-30 16:50:33 -0700 | [diff] [blame] | 727 | prof_enter(tsd, tdata); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 728 | if (ckh_search(&bt2gctx, bt, &btkey.v, &gctx.v)) { |
Jason Evans | fb1775e | 2014-01-14 17:04:34 -0800 | [diff] [blame] | 729 | /* bt has never been seen before. Insert it. */ |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 730 | gctx.p = prof_gctx_create(tsd_tsdn(tsd), bt); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 731 | if (gctx.v == NULL) { |
Jason Evans | c93ed81 | 2014-10-30 16:50:33 -0700 | [diff] [blame] | 732 | prof_leave(tsd, tdata); |
Jason Evans | fb1775e | 2014-01-14 17:04:34 -0800 | [diff] [blame] | 733 | return (true); |
| 734 | } |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 735 | btkey.p = &gctx.p->bt; |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 736 | if (ckh_insert(tsd_tsdn(tsd), &bt2gctx, btkey.v, gctx.v)) { |
Jason Evans | fb1775e | 2014-01-14 17:04:34 -0800 | [diff] [blame] | 737 | /* OOM. */ |
Jason Evans | c93ed81 | 2014-10-30 16:50:33 -0700 | [diff] [blame] | 738 | prof_leave(tsd, tdata); |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 739 | idalloctm(tsd_tsdn(tsd), gctx.v, NULL, true, true); |
Jason Evans | fb1775e | 2014-01-14 17:04:34 -0800 | [diff] [blame] | 740 | return (true); |
| 741 | } |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 742 | new_gctx = true; |
Jason Evans | fb1775e | 2014-01-14 17:04:34 -0800 | [diff] [blame] | 743 | } else { |
| 744 | /* |
| 745 | * Increment nlimbo, in order to avoid a race condition with |
Jason Evans | 20c31de | 2014-10-02 23:01:10 -0700 | [diff] [blame] | 746 | * prof_tctx_destroy()/prof_gctx_try_destroy(). |
Jason Evans | fb1775e | 2014-01-14 17:04:34 -0800 | [diff] [blame] | 747 | */ |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 748 | malloc_mutex_lock(tsd_tsdn(tsd), gctx.p->lock); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 749 | gctx.p->nlimbo++; |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 750 | malloc_mutex_unlock(tsd_tsdn(tsd), gctx.p->lock); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 751 | new_gctx = false; |
Jason Evans | fb1775e | 2014-01-14 17:04:34 -0800 | [diff] [blame] | 752 | } |
Jason Evans | c93ed81 | 2014-10-30 16:50:33 -0700 | [diff] [blame] | 753 | prof_leave(tsd, tdata); |
Jason Evans | fb1775e | 2014-01-14 17:04:34 -0800 | [diff] [blame] | 754 | |
| 755 | *p_btkey = btkey.v; |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 756 | *p_gctx = gctx.p; |
| 757 | *p_new_gctx = new_gctx; |
Jason Evans | fb1775e | 2014-01-14 17:04:34 -0800 | [diff] [blame] | 758 | return (false); |
| 759 | } |
| 760 | |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 761 | prof_tctx_t * |
Jason Evans | 5460aa6 | 2014-09-22 21:09:23 -0700 | [diff] [blame] | 762 | prof_lookup(tsd_t *tsd, prof_bt_t *bt) |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 763 | { |
Jason Evans | 075e77c | 2010-09-20 19:53:25 -0700 | [diff] [blame] | 764 | union { |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 765 | prof_tctx_t *p; |
Jason Evans | 075e77c | 2010-09-20 19:53:25 -0700 | [diff] [blame] | 766 | void *v; |
| 767 | } ret; |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 768 | prof_tdata_t *tdata; |
| 769 | bool not_found; |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 770 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 771 | cassert(config_prof); |
| 772 | |
Jason Evans | 5460aa6 | 2014-09-22 21:09:23 -0700 | [diff] [blame] | 773 | tdata = prof_tdata_get(tsd, false); |
| 774 | if (tdata == NULL) |
Jason Evans | 52386b2 | 2012-04-22 16:00:11 -0700 | [diff] [blame] | 775 | return (NULL); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 776 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 777 | malloc_mutex_lock(tsd_tsdn(tsd), tdata->lock); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 778 | not_found = ckh_search(&tdata->bt2tctx, bt, NULL, &ret.v); |
Jason Evans | 6e73dc1 | 2014-09-09 19:37:26 -0700 | [diff] [blame] | 779 | if (!not_found) /* Note double negative! */ |
| 780 | ret.p->prepared = true; |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 781 | malloc_mutex_unlock(tsd_tsdn(tsd), tdata->lock); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 782 | if (not_found) { |
Jason Evans | fb1775e | 2014-01-14 17:04:34 -0800 | [diff] [blame] | 783 | void *btkey; |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 784 | prof_gctx_t *gctx; |
| 785 | bool new_gctx, error; |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 786 | |
| 787 | /* |
| 788 | * This thread's cache lacks bt. Look for it in the global |
| 789 | * cache. |
| 790 | */ |
Jason Evans | 5460aa6 | 2014-09-22 21:09:23 -0700 | [diff] [blame] | 791 | if (prof_lookup_global(tsd, bt, tdata, &btkey, &gctx, |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 792 | &new_gctx)) |
Jason Evans | fb1775e | 2014-01-14 17:04:34 -0800 | [diff] [blame] | 793 | return (NULL); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 794 | |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 795 | /* Link a prof_tctx_t into gctx for this thread. */ |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 796 | ret.v = iallocztm(tsd_tsdn(tsd), sizeof(prof_tctx_t), |
Jason Evans | 66cd953 | 2016-04-22 14:34:14 -0700 | [diff] [blame] | 797 | size2index(sizeof(prof_tctx_t)), false, NULL, true, |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 798 | arena_ichoose(tsd_tsdn(tsd), NULL), true); |
Jason Evans | b41ccdb | 2014-08-15 15:01:15 -0700 | [diff] [blame] | 799 | if (ret.p == NULL) { |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 800 | if (new_gctx) |
Jason Evans | c93ed81 | 2014-10-30 16:50:33 -0700 | [diff] [blame] | 801 | prof_gctx_try_destroy(tsd, tdata, gctx, tdata); |
Jason Evans | b41ccdb | 2014-08-15 15:01:15 -0700 | [diff] [blame] | 802 | return (NULL); |
Jason Evans | a881cd2 | 2010-10-02 15:18:50 -0700 | [diff] [blame] | 803 | } |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 804 | ret.p->tdata = tdata; |
Jason Evans | 44c97b7 | 2014-10-12 13:03:20 -0700 | [diff] [blame] | 805 | ret.p->thr_uid = tdata->thr_uid; |
Jason Evans | a00b107 | 2015-09-09 23:16:10 -0700 | [diff] [blame] | 806 | ret.p->thr_discrim = tdata->thr_discrim; |
Jason Evans | 075e77c | 2010-09-20 19:53:25 -0700 | [diff] [blame] | 807 | memset(&ret.p->cnts, 0, sizeof(prof_cnt_t)); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 808 | ret.p->gctx = gctx; |
Jason Evans | 04211e2 | 2015-03-16 15:11:06 -0700 | [diff] [blame] | 809 | ret.p->tctx_uid = tdata->tctx_uid_next++; |
Jason Evans | 6e73dc1 | 2014-09-09 19:37:26 -0700 | [diff] [blame] | 810 | ret.p->prepared = true; |
Jason Evans | 6ef80d6 | 2014-09-24 22:14:21 -0700 | [diff] [blame] | 811 | ret.p->state = prof_tctx_state_initializing; |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 812 | malloc_mutex_lock(tsd_tsdn(tsd), tdata->lock); |
| 813 | error = ckh_insert(tsd_tsdn(tsd), &tdata->bt2tctx, btkey, |
| 814 | ret.v); |
| 815 | malloc_mutex_unlock(tsd_tsdn(tsd), tdata->lock); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 816 | if (error) { |
| 817 | if (new_gctx) |
Jason Evans | c93ed81 | 2014-10-30 16:50:33 -0700 | [diff] [blame] | 818 | prof_gctx_try_destroy(tsd, tdata, gctx, tdata); |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 819 | idalloctm(tsd_tsdn(tsd), ret.v, NULL, true, true); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 820 | return (NULL); |
| 821 | } |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 822 | malloc_mutex_lock(tsd_tsdn(tsd), gctx->lock); |
Jason Evans | 6ef80d6 | 2014-09-24 22:14:21 -0700 | [diff] [blame] | 823 | ret.p->state = prof_tctx_state_nominal; |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 824 | tctx_tree_insert(&gctx->tctxs, ret.p); |
| 825 | gctx->nlimbo--; |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 826 | malloc_mutex_unlock(tsd_tsdn(tsd), gctx->lock); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 827 | } |
| 828 | |
Jason Evans | 075e77c | 2010-09-20 19:53:25 -0700 | [diff] [blame] | 829 | return (ret.p); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 830 | } |
| 831 | |
Jason Evans | dc391ad | 2016-05-04 12:14:36 -0700 | [diff] [blame] | 832 | /* |
| 833 | * The bodies of this function and prof_leakcheck() are compiled out unless heap |
| 834 | * profiling is enabled, so that it is possible to compile jemalloc with |
| 835 | * floating point support completely disabled. Avoiding floating point code is |
| 836 | * important on memory-constrained systems, but it also enables a workaround for |
| 837 | * versions of glibc that don't properly save/restore floating point registers |
| 838 | * during dynamic lazy symbol loading (which internally calls into whatever |
| 839 | * malloc implementation happens to be integrated into the application). Note |
| 840 | * that some compilers (e.g. gcc 4.8) may use floating point registers for fast |
| 841 | * memory moves, so jemalloc must be compiled with such optimizations disabled |
| 842 | * (e.g. |
| 843 | * -mno-sse) in order for the workaround to be complete. |
| 844 | */ |
Ben Maurer | 6c39f9e | 2014-04-15 13:47:13 -0700 | [diff] [blame] | 845 | void |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 846 | prof_sample_threshold_update(prof_tdata_t *tdata) |
Ben Maurer | 6c39f9e | 2014-04-15 13:47:13 -0700 | [diff] [blame] | 847 | { |
Ben Maurer | 6c39f9e | 2014-04-15 13:47:13 -0700 | [diff] [blame] | 848 | #ifdef JEMALLOC_PROF |
| 849 | uint64_t r; |
| 850 | double u; |
| 851 | |
| 852 | if (!config_prof) |
| 853 | return; |
| 854 | |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 855 | if (lg_prof_sample == 0) { |
| 856 | tdata->bytes_until_sample = 0; |
Ben Maurer | 6c39f9e | 2014-04-15 13:47:13 -0700 | [diff] [blame] | 857 | return; |
| 858 | } |
| 859 | |
| 860 | /* |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 861 | * Compute sample interval as a geometrically distributed random |
| 862 | * variable with mean (2^lg_prof_sample). |
Ben Maurer | 6c39f9e | 2014-04-15 13:47:13 -0700 | [diff] [blame] | 863 | * |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 864 | * __ __ |
| 865 | * | log(u) | 1 |
| 866 | * tdata->bytes_until_sample = | -------- |, where p = --------------- |
| 867 | * | log(1-p) | lg_prof_sample |
| 868 | * 2 |
Ben Maurer | 6c39f9e | 2014-04-15 13:47:13 -0700 | [diff] [blame] | 869 | * |
| 870 | * For more information on the math, see: |
| 871 | * |
| 872 | * Non-Uniform Random Variate Generation |
| 873 | * Luc Devroye |
| 874 | * Springer-Verlag, New York, 1986 |
| 875 | * pp 500 |
| 876 | * (http://luc.devroye.org/rnbookindex.html) |
| 877 | */ |
Jason Evans | 34676d3 | 2016-02-09 16:28:40 -0800 | [diff] [blame] | 878 | r = prng_lg_range(&tdata->prng_state, 53); |
Ben Maurer | 6c39f9e | 2014-04-15 13:47:13 -0700 | [diff] [blame] | 879 | u = (double)r * (1.0/9007199254740992.0L); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 880 | tdata->bytes_until_sample = (uint64_t)(log(u) / |
| 881 | log(1.0 - (1.0 / (double)((uint64_t)1U << lg_prof_sample)))) |
Ben Maurer | 6c39f9e | 2014-04-15 13:47:13 -0700 | [diff] [blame] | 882 | + (uint64_t)1U; |
| 883 | #endif |
| 884 | } |
| 885 | |
Jason Evans | 772163b | 2014-01-17 15:40:52 -0800 | [diff] [blame] | 886 | #ifdef JEMALLOC_JET |
Jason Evans | 20c31de | 2014-10-02 23:01:10 -0700 | [diff] [blame] | 887 | static prof_tdata_t * |
| 888 | prof_tdata_count_iter(prof_tdata_tree_t *tdatas, prof_tdata_t *tdata, void *arg) |
| 889 | { |
| 890 | size_t *tdata_count = (size_t *)arg; |
| 891 | |
| 892 | (*tdata_count)++; |
| 893 | |
| 894 | return (NULL); |
| 895 | } |
| 896 | |
| 897 | size_t |
| 898 | prof_tdata_count(void) |
| 899 | { |
| 900 | size_t tdata_count = 0; |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 901 | tsdn_t *tsdn; |
Jason Evans | 20c31de | 2014-10-02 23:01:10 -0700 | [diff] [blame] | 902 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 903 | tsdn = tsdn_fetch(); |
| 904 | malloc_mutex_lock(tsdn, &tdatas_mtx); |
Jason Evans | 20c31de | 2014-10-02 23:01:10 -0700 | [diff] [blame] | 905 | tdata_tree_iter(&tdatas, NULL, prof_tdata_count_iter, |
| 906 | (void *)&tdata_count); |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 907 | malloc_mutex_unlock(tsdn, &tdatas_mtx); |
Jason Evans | 20c31de | 2014-10-02 23:01:10 -0700 | [diff] [blame] | 908 | |
| 909 | return (tdata_count); |
| 910 | } |
| 911 | #endif |
| 912 | |
| 913 | #ifdef JEMALLOC_JET |
Jason Evans | 772163b | 2014-01-17 15:40:52 -0800 | [diff] [blame] | 914 | size_t |
| 915 | prof_bt_count(void) |
| 916 | { |
| 917 | size_t bt_count; |
Jason Evans | 5460aa6 | 2014-09-22 21:09:23 -0700 | [diff] [blame] | 918 | tsd_t *tsd; |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 919 | prof_tdata_t *tdata; |
Jason Evans | 772163b | 2014-01-17 15:40:52 -0800 | [diff] [blame] | 920 | |
Jason Evans | 029d44c | 2014-10-04 11:12:53 -0700 | [diff] [blame] | 921 | tsd = tsd_fetch(); |
Jason Evans | 5460aa6 | 2014-09-22 21:09:23 -0700 | [diff] [blame] | 922 | tdata = prof_tdata_get(tsd, false); |
| 923 | if (tdata == NULL) |
Jason Evans | 772163b | 2014-01-17 15:40:52 -0800 | [diff] [blame] | 924 | return (0); |
| 925 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 926 | malloc_mutex_lock(tsd_tsdn(tsd), &bt2gctx_mtx); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 927 | bt_count = ckh_count(&bt2gctx); |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 928 | malloc_mutex_unlock(tsd_tsdn(tsd), &bt2gctx_mtx); |
Jason Evans | 772163b | 2014-01-17 15:40:52 -0800 | [diff] [blame] | 929 | |
| 930 | return (bt_count); |
| 931 | } |
| 932 | #endif |
| 933 | |
| 934 | #ifdef JEMALLOC_JET |
| 935 | #undef prof_dump_open |
| 936 | #define prof_dump_open JEMALLOC_N(prof_dump_open_impl) |
| 937 | #endif |
| 938 | static int |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 939 | prof_dump_open(bool propagate_err, const char *filename) |
| 940 | { |
Jason Evans | 772163b | 2014-01-17 15:40:52 -0800 | [diff] [blame] | 941 | int fd; |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 942 | |
Jason Evans | 772163b | 2014-01-17 15:40:52 -0800 | [diff] [blame] | 943 | fd = creat(filename, 0644); |
Jason Evans | 551ebc4 | 2014-10-03 10:16:09 -0700 | [diff] [blame] | 944 | if (fd == -1 && !propagate_err) { |
Jason Evans | 772163b | 2014-01-17 15:40:52 -0800 | [diff] [blame] | 945 | malloc_printf("<jemalloc>: creat(\"%s\"), 0644) failed\n", |
| 946 | filename); |
| 947 | if (opt_abort) |
| 948 | abort(); |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 949 | } |
| 950 | |
Jason Evans | 772163b | 2014-01-17 15:40:52 -0800 | [diff] [blame] | 951 | return (fd); |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 952 | } |
Jason Evans | 772163b | 2014-01-17 15:40:52 -0800 | [diff] [blame] | 953 | #ifdef JEMALLOC_JET |
| 954 | #undef prof_dump_open |
| 955 | #define prof_dump_open JEMALLOC_N(prof_dump_open) |
| 956 | prof_dump_open_t *prof_dump_open = JEMALLOC_N(prof_dump_open_impl); |
| 957 | #endif |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 958 | |
| 959 | static bool |
| 960 | prof_dump_flush(bool propagate_err) |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 961 | { |
Jason Evans | 22ca855 | 2010-03-02 11:57:30 -0800 | [diff] [blame] | 962 | bool ret = false; |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 963 | ssize_t err; |
| 964 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 965 | cassert(config_prof); |
| 966 | |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 967 | err = write(prof_dump_fd, prof_dump_buf, prof_dump_buf_end); |
| 968 | if (err == -1) { |
Jason Evans | 551ebc4 | 2014-10-03 10:16:09 -0700 | [diff] [blame] | 969 | if (!propagate_err) { |
Jason Evans | 698805c | 2010-03-03 17:45:38 -0800 | [diff] [blame] | 970 | malloc_write("<jemalloc>: write() failed during heap " |
| 971 | "profile flush\n"); |
Jason Evans | 22ca855 | 2010-03-02 11:57:30 -0800 | [diff] [blame] | 972 | if (opt_abort) |
| 973 | abort(); |
| 974 | } |
| 975 | ret = true; |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 976 | } |
| 977 | prof_dump_buf_end = 0; |
Jason Evans | 22ca855 | 2010-03-02 11:57:30 -0800 | [diff] [blame] | 978 | |
| 979 | return (ret); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 980 | } |
| 981 | |
Jason Evans | 22ca855 | 2010-03-02 11:57:30 -0800 | [diff] [blame] | 982 | static bool |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 983 | prof_dump_close(bool propagate_err) |
| 984 | { |
| 985 | bool ret; |
| 986 | |
| 987 | assert(prof_dump_fd != -1); |
| 988 | ret = prof_dump_flush(propagate_err); |
| 989 | close(prof_dump_fd); |
| 990 | prof_dump_fd = -1; |
| 991 | |
| 992 | return (ret); |
| 993 | } |
| 994 | |
| 995 | static bool |
| 996 | prof_dump_write(bool propagate_err, const char *s) |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 997 | { |
Jason Evans | ca8fffb | 2016-02-24 13:16:51 -0800 | [diff] [blame] | 998 | size_t i, slen, n; |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 999 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 1000 | cassert(config_prof); |
| 1001 | |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1002 | i = 0; |
| 1003 | slen = strlen(s); |
| 1004 | while (i < slen) { |
| 1005 | /* Flush the buffer if it is full. */ |
Jason Evans | cd9a134 | 2012-03-21 18:33:03 -0700 | [diff] [blame] | 1006 | if (prof_dump_buf_end == PROF_DUMP_BUFSIZE) |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 1007 | if (prof_dump_flush(propagate_err) && propagate_err) |
Jason Evans | 22ca855 | 2010-03-02 11:57:30 -0800 | [diff] [blame] | 1008 | return (true); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1009 | |
Jason Evans | cd9a134 | 2012-03-21 18:33:03 -0700 | [diff] [blame] | 1010 | if (prof_dump_buf_end + slen <= PROF_DUMP_BUFSIZE) { |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1011 | /* Finish writing. */ |
| 1012 | n = slen - i; |
| 1013 | } else { |
| 1014 | /* Write as much of s as will fit. */ |
Jason Evans | cd9a134 | 2012-03-21 18:33:03 -0700 | [diff] [blame] | 1015 | n = PROF_DUMP_BUFSIZE - prof_dump_buf_end; |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1016 | } |
| 1017 | memcpy(&prof_dump_buf[prof_dump_buf_end], &s[i], n); |
| 1018 | prof_dump_buf_end += n; |
| 1019 | i += n; |
| 1020 | } |
Jason Evans | 22ca855 | 2010-03-02 11:57:30 -0800 | [diff] [blame] | 1021 | |
| 1022 | return (false); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1023 | } |
| 1024 | |
Jason Evans | e42c309 | 2015-07-22 15:44:47 -0700 | [diff] [blame] | 1025 | JEMALLOC_FORMAT_PRINTF(2, 3) |
Jason Evans | d81e4bd | 2012-03-06 14:57:45 -0800 | [diff] [blame] | 1026 | static bool |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 1027 | prof_dump_printf(bool propagate_err, const char *format, ...) |
Jason Evans | d81e4bd | 2012-03-06 14:57:45 -0800 | [diff] [blame] | 1028 | { |
| 1029 | bool ret; |
| 1030 | va_list ap; |
Jason Evans | cd9a134 | 2012-03-21 18:33:03 -0700 | [diff] [blame] | 1031 | char buf[PROF_PRINTF_BUFSIZE]; |
Jason Evans | d81e4bd | 2012-03-06 14:57:45 -0800 | [diff] [blame] | 1032 | |
| 1033 | va_start(ap, format); |
Jason Evans | 6da5418 | 2012-03-23 18:05:51 -0700 | [diff] [blame] | 1034 | malloc_vsnprintf(buf, sizeof(buf), format, ap); |
Jason Evans | d81e4bd | 2012-03-06 14:57:45 -0800 | [diff] [blame] | 1035 | va_end(ap); |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 1036 | ret = prof_dump_write(propagate_err, buf); |
Jason Evans | d81e4bd | 2012-03-06 14:57:45 -0800 | [diff] [blame] | 1037 | |
| 1038 | return (ret); |
| 1039 | } |
| 1040 | |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1041 | static void |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1042 | prof_tctx_merge_tdata(tsdn_t *tsdn, prof_tctx_t *tctx, prof_tdata_t *tdata) |
Jason Evans | 3a81cbd | 2014-08-16 12:58:55 -0700 | [diff] [blame] | 1043 | { |
Jason Evans | 3a81cbd | 2014-08-16 12:58:55 -0700 | [diff] [blame] | 1044 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1045 | malloc_mutex_assert_owner(tsdn, tctx->tdata->lock); |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 1046 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1047 | malloc_mutex_lock(tsdn, tctx->gctx->lock); |
Jason Evans | 764b000 | 2015-03-14 14:01:35 -0700 | [diff] [blame] | 1048 | |
| 1049 | switch (tctx->state) { |
| 1050 | case prof_tctx_state_initializing: |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1051 | malloc_mutex_unlock(tsdn, tctx->gctx->lock); |
Jason Evans | 6ef80d6 | 2014-09-24 22:14:21 -0700 | [diff] [blame] | 1052 | return; |
Jason Evans | 764b000 | 2015-03-14 14:01:35 -0700 | [diff] [blame] | 1053 | case prof_tctx_state_nominal: |
| 1054 | tctx->state = prof_tctx_state_dumping; |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1055 | malloc_mutex_unlock(tsdn, tctx->gctx->lock); |
Jason Evans | 6ef80d6 | 2014-09-24 22:14:21 -0700 | [diff] [blame] | 1056 | |
Jason Evans | 764b000 | 2015-03-14 14:01:35 -0700 | [diff] [blame] | 1057 | memcpy(&tctx->dump_cnts, &tctx->cnts, sizeof(prof_cnt_t)); |
Jason Evans | 3a81cbd | 2014-08-16 12:58:55 -0700 | [diff] [blame] | 1058 | |
Jason Evans | 764b000 | 2015-03-14 14:01:35 -0700 | [diff] [blame] | 1059 | tdata->cnt_summed.curobjs += tctx->dump_cnts.curobjs; |
| 1060 | tdata->cnt_summed.curbytes += tctx->dump_cnts.curbytes; |
| 1061 | if (opt_prof_accum) { |
| 1062 | tdata->cnt_summed.accumobjs += |
| 1063 | tctx->dump_cnts.accumobjs; |
| 1064 | tdata->cnt_summed.accumbytes += |
| 1065 | tctx->dump_cnts.accumbytes; |
| 1066 | } |
| 1067 | break; |
| 1068 | case prof_tctx_state_dumping: |
| 1069 | case prof_tctx_state_purgatory: |
| 1070 | not_reached(); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1071 | } |
| 1072 | } |
| 1073 | |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1074 | static void |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1075 | prof_tctx_merge_gctx(tsdn_t *tsdn, prof_tctx_t *tctx, prof_gctx_t *gctx) |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1076 | { |
| 1077 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1078 | malloc_mutex_assert_owner(tsdn, gctx->lock); |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 1079 | |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1080 | gctx->cnt_summed.curobjs += tctx->dump_cnts.curobjs; |
| 1081 | gctx->cnt_summed.curbytes += tctx->dump_cnts.curbytes; |
| 1082 | if (opt_prof_accum) { |
| 1083 | gctx->cnt_summed.accumobjs += tctx->dump_cnts.accumobjs; |
| 1084 | gctx->cnt_summed.accumbytes += tctx->dump_cnts.accumbytes; |
| 1085 | } |
| 1086 | } |
| 1087 | |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1088 | static prof_tctx_t * |
| 1089 | prof_tctx_merge_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *arg) |
| 1090 | { |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1091 | tsdn_t *tsdn = (tsdn_t *)arg; |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 1092 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1093 | malloc_mutex_assert_owner(tsdn, tctx->gctx->lock); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1094 | |
| 1095 | switch (tctx->state) { |
| 1096 | case prof_tctx_state_nominal: |
| 1097 | /* New since dumping started; ignore. */ |
| 1098 | break; |
| 1099 | case prof_tctx_state_dumping: |
| 1100 | case prof_tctx_state_purgatory: |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1101 | prof_tctx_merge_gctx(tsdn, tctx, tctx->gctx); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1102 | break; |
| 1103 | default: |
| 1104 | not_reached(); |
Jason Evans | 3a81cbd | 2014-08-16 12:58:55 -0700 | [diff] [blame] | 1105 | } |
| 1106 | |
| 1107 | return (NULL); |
| 1108 | } |
| 1109 | |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 1110 | struct prof_tctx_dump_iter_arg_s { |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1111 | tsdn_t *tsdn; |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 1112 | bool propagate_err; |
| 1113 | }; |
| 1114 | |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1115 | static prof_tctx_t * |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 1116 | prof_tctx_dump_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *opaque) |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1117 | { |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 1118 | struct prof_tctx_dump_iter_arg_s *arg = |
| 1119 | (struct prof_tctx_dump_iter_arg_s *)opaque; |
| 1120 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1121 | malloc_mutex_assert_owner(arg->tsdn, tctx->gctx->lock); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1122 | |
Jason Evans | fb64ec2 | 2015-09-21 18:37:18 -0700 | [diff] [blame] | 1123 | switch (tctx->state) { |
| 1124 | case prof_tctx_state_initializing: |
| 1125 | case prof_tctx_state_nominal: |
| 1126 | /* Not captured by this dump. */ |
| 1127 | break; |
| 1128 | case prof_tctx_state_dumping: |
| 1129 | case prof_tctx_state_purgatory: |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 1130 | if (prof_dump_printf(arg->propagate_err, |
Jason Evans | fb64ec2 | 2015-09-21 18:37:18 -0700 | [diff] [blame] | 1131 | " t%"FMTu64": %"FMTu64": %"FMTu64" [%"FMTu64": " |
| 1132 | "%"FMTu64"]\n", tctx->thr_uid, tctx->dump_cnts.curobjs, |
| 1133 | tctx->dump_cnts.curbytes, tctx->dump_cnts.accumobjs, |
| 1134 | tctx->dump_cnts.accumbytes)) |
| 1135 | return (tctx); |
| 1136 | break; |
| 1137 | default: |
| 1138 | not_reached(); |
| 1139 | } |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1140 | return (NULL); |
| 1141 | } |
| 1142 | |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1143 | static prof_tctx_t * |
| 1144 | prof_tctx_finish_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *arg) |
| 1145 | { |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1146 | tsdn_t *tsdn = (tsdn_t *)arg; |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1147 | prof_tctx_t *ret; |
| 1148 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1149 | malloc_mutex_assert_owner(tsdn, tctx->gctx->lock); |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 1150 | |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1151 | switch (tctx->state) { |
| 1152 | case prof_tctx_state_nominal: |
| 1153 | /* New since dumping started; ignore. */ |
| 1154 | break; |
| 1155 | case prof_tctx_state_dumping: |
| 1156 | tctx->state = prof_tctx_state_nominal; |
| 1157 | break; |
| 1158 | case prof_tctx_state_purgatory: |
Jason Evans | 20c31de | 2014-10-02 23:01:10 -0700 | [diff] [blame] | 1159 | ret = tctx; |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1160 | goto label_return; |
| 1161 | default: |
| 1162 | not_reached(); |
| 1163 | } |
| 1164 | |
| 1165 | ret = NULL; |
| 1166 | label_return: |
| 1167 | return (ret); |
| 1168 | } |
| 1169 | |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1170 | static void |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1171 | prof_dump_gctx_prep(tsdn_t *tsdn, prof_gctx_t *gctx, prof_gctx_tree_t *gctxs) |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1172 | { |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1173 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 1174 | cassert(config_prof); |
| 1175 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1176 | malloc_mutex_lock(tsdn, gctx->lock); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1177 | |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 1178 | /* |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1179 | * Increment nlimbo so that gctx won't go away before dump. |
| 1180 | * Additionally, link gctx into the dump list so that it is included in |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 1181 | * prof_dump()'s second pass. |
| 1182 | */ |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1183 | gctx->nlimbo++; |
| 1184 | gctx_tree_insert(gctxs, gctx); |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 1185 | |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1186 | memset(&gctx->cnt_summed, 0, sizeof(prof_cnt_t)); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1187 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1188 | malloc_mutex_unlock(tsdn, gctx->lock); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1189 | } |
Jason Evans | 9ce3bfd | 2010-10-02 22:39:59 -0700 | [diff] [blame] | 1190 | |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 1191 | struct prof_gctx_merge_iter_arg_s { |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1192 | tsdn_t *tsdn; |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 1193 | size_t leak_ngctx; |
| 1194 | }; |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1195 | |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 1196 | static prof_gctx_t * |
| 1197 | prof_gctx_merge_iter(prof_gctx_tree_t *gctxs, prof_gctx_t *gctx, void *opaque) |
| 1198 | { |
| 1199 | struct prof_gctx_merge_iter_arg_s *arg = |
| 1200 | (struct prof_gctx_merge_iter_arg_s *)opaque; |
| 1201 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1202 | malloc_mutex_lock(arg->tsdn, gctx->lock); |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 1203 | tctx_tree_iter(&gctx->tctxs, NULL, prof_tctx_merge_iter, |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1204 | (void *)arg->tsdn); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1205 | if (gctx->cnt_summed.curobjs != 0) |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 1206 | arg->leak_ngctx++; |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1207 | malloc_mutex_unlock(arg->tsdn, gctx->lock); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1208 | |
| 1209 | return (NULL); |
| 1210 | } |
| 1211 | |
Jason Evans | 20c31de | 2014-10-02 23:01:10 -0700 | [diff] [blame] | 1212 | static void |
| 1213 | prof_gctx_finish(tsd_t *tsd, prof_gctx_tree_t *gctxs) |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1214 | { |
Jason Evans | 5460aa6 | 2014-09-22 21:09:23 -0700 | [diff] [blame] | 1215 | prof_tdata_t *tdata = prof_tdata_get(tsd, false); |
Jason Evans | 20c31de | 2014-10-02 23:01:10 -0700 | [diff] [blame] | 1216 | prof_gctx_t *gctx; |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1217 | |
Jason Evans | 20c31de | 2014-10-02 23:01:10 -0700 | [diff] [blame] | 1218 | /* |
| 1219 | * Standard tree iteration won't work here, because as soon as we |
| 1220 | * decrement gctx->nlimbo and unlock gctx, another thread can |
| 1221 | * concurrently destroy it, which will corrupt the tree. Therefore, |
| 1222 | * tear down the tree one node at a time during iteration. |
| 1223 | */ |
| 1224 | while ((gctx = gctx_tree_first(gctxs)) != NULL) { |
| 1225 | gctx_tree_remove(gctxs, gctx); |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1226 | malloc_mutex_lock(tsd_tsdn(tsd), gctx->lock); |
Jason Evans | 20c31de | 2014-10-02 23:01:10 -0700 | [diff] [blame] | 1227 | { |
| 1228 | prof_tctx_t *next; |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1229 | |
Jason Evans | 20c31de | 2014-10-02 23:01:10 -0700 | [diff] [blame] | 1230 | next = NULL; |
| 1231 | do { |
| 1232 | prof_tctx_t *to_destroy = |
| 1233 | tctx_tree_iter(&gctx->tctxs, next, |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1234 | prof_tctx_finish_iter, |
| 1235 | (void *)tsd_tsdn(tsd)); |
Jason Evans | 20c31de | 2014-10-02 23:01:10 -0700 | [diff] [blame] | 1236 | if (to_destroy != NULL) { |
| 1237 | next = tctx_tree_next(&gctx->tctxs, |
| 1238 | to_destroy); |
| 1239 | tctx_tree_remove(&gctx->tctxs, |
| 1240 | to_destroy); |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1241 | idalloctm(tsd_tsdn(tsd), to_destroy, |
| 1242 | NULL, true, true); |
Jason Evans | 20c31de | 2014-10-02 23:01:10 -0700 | [diff] [blame] | 1243 | } else |
| 1244 | next = NULL; |
| 1245 | } while (next != NULL); |
| 1246 | } |
| 1247 | gctx->nlimbo--; |
| 1248 | if (prof_gctx_should_destroy(gctx)) { |
| 1249 | gctx->nlimbo++; |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1250 | malloc_mutex_unlock(tsd_tsdn(tsd), gctx->lock); |
Jason Evans | c93ed81 | 2014-10-30 16:50:33 -0700 | [diff] [blame] | 1251 | prof_gctx_try_destroy(tsd, tdata, gctx, tdata); |
Jason Evans | 20c31de | 2014-10-02 23:01:10 -0700 | [diff] [blame] | 1252 | } else |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1253 | malloc_mutex_unlock(tsd_tsdn(tsd), gctx->lock); |
Jason Evans | 20c31de | 2014-10-02 23:01:10 -0700 | [diff] [blame] | 1254 | } |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1255 | } |
| 1256 | |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 1257 | struct prof_tdata_merge_iter_arg_s { |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1258 | tsdn_t *tsdn; |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 1259 | prof_cnt_t cnt_all; |
| 1260 | }; |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1261 | |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 1262 | static prof_tdata_t * |
| 1263 | prof_tdata_merge_iter(prof_tdata_tree_t *tdatas, prof_tdata_t *tdata, |
| 1264 | void *opaque) |
| 1265 | { |
| 1266 | struct prof_tdata_merge_iter_arg_s *arg = |
| 1267 | (struct prof_tdata_merge_iter_arg_s *)opaque; |
| 1268 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1269 | malloc_mutex_lock(arg->tsdn, tdata->lock); |
Jason Evans | 20c31de | 2014-10-02 23:01:10 -0700 | [diff] [blame] | 1270 | if (!tdata->expired) { |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1271 | size_t tabind; |
| 1272 | union { |
| 1273 | prof_tctx_t *p; |
| 1274 | void *v; |
| 1275 | } tctx; |
| 1276 | |
| 1277 | tdata->dumping = true; |
| 1278 | memset(&tdata->cnt_summed, 0, sizeof(prof_cnt_t)); |
Jason Evans | 551ebc4 | 2014-10-03 10:16:09 -0700 | [diff] [blame] | 1279 | for (tabind = 0; !ckh_iter(&tdata->bt2tctx, &tabind, NULL, |
| 1280 | &tctx.v);) |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1281 | prof_tctx_merge_tdata(arg->tsdn, tctx.p, tdata); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1282 | |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 1283 | arg->cnt_all.curobjs += tdata->cnt_summed.curobjs; |
| 1284 | arg->cnt_all.curbytes += tdata->cnt_summed.curbytes; |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1285 | if (opt_prof_accum) { |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 1286 | arg->cnt_all.accumobjs += tdata->cnt_summed.accumobjs; |
| 1287 | arg->cnt_all.accumbytes += tdata->cnt_summed.accumbytes; |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1288 | } |
| 1289 | } else |
| 1290 | tdata->dumping = false; |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1291 | malloc_mutex_unlock(arg->tsdn, tdata->lock); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1292 | |
| 1293 | return (NULL); |
| 1294 | } |
| 1295 | |
| 1296 | static prof_tdata_t * |
| 1297 | prof_tdata_dump_iter(prof_tdata_tree_t *tdatas, prof_tdata_t *tdata, void *arg) |
| 1298 | { |
| 1299 | bool propagate_err = *(bool *)arg; |
| 1300 | |
Jason Evans | 551ebc4 | 2014-10-03 10:16:09 -0700 | [diff] [blame] | 1301 | if (!tdata->dumping) |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1302 | return (NULL); |
| 1303 | |
| 1304 | if (prof_dump_printf(propagate_err, |
Jason Evans | 5fae7dc | 2015-07-23 13:56:25 -0700 | [diff] [blame] | 1305 | " t%"FMTu64": %"FMTu64": %"FMTu64" [%"FMTu64": %"FMTu64"]%s%s\n", |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1306 | tdata->thr_uid, tdata->cnt_summed.curobjs, |
| 1307 | tdata->cnt_summed.curbytes, tdata->cnt_summed.accumobjs, |
| 1308 | tdata->cnt_summed.accumbytes, |
| 1309 | (tdata->thread_name != NULL) ? " " : "", |
| 1310 | (tdata->thread_name != NULL) ? tdata->thread_name : "")) |
| 1311 | return (tdata); |
| 1312 | return (NULL); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1313 | } |
| 1314 | |
Jason Evans | 20c31de | 2014-10-02 23:01:10 -0700 | [diff] [blame] | 1315 | #ifdef JEMALLOC_JET |
| 1316 | #undef prof_dump_header |
| 1317 | #define prof_dump_header JEMALLOC_N(prof_dump_header_impl) |
| 1318 | #endif |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 1319 | static bool |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1320 | prof_dump_header(tsdn_t *tsdn, bool propagate_err, const prof_cnt_t *cnt_all) |
Jason Evans | a881cd2 | 2010-10-02 15:18:50 -0700 | [diff] [blame] | 1321 | { |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1322 | bool ret; |
Jason Evans | a881cd2 | 2010-10-02 15:18:50 -0700 | [diff] [blame] | 1323 | |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1324 | if (prof_dump_printf(propagate_err, |
Jason Evans | 5fae7dc | 2015-07-23 13:56:25 -0700 | [diff] [blame] | 1325 | "heap_v2/%"FMTu64"\n" |
| 1326 | " t*: %"FMTu64": %"FMTu64" [%"FMTu64": %"FMTu64"]\n", |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1327 | ((uint64_t)1U << lg_prof_sample), cnt_all->curobjs, |
| 1328 | cnt_all->curbytes, cnt_all->accumobjs, cnt_all->accumbytes)) |
| 1329 | return (true); |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 1330 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1331 | malloc_mutex_lock(tsdn, &tdatas_mtx); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1332 | ret = (tdata_tree_iter(&tdatas, NULL, prof_tdata_dump_iter, |
| 1333 | (void *)&propagate_err) != NULL); |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1334 | malloc_mutex_unlock(tsdn, &tdatas_mtx); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1335 | return (ret); |
Jason Evans | a881cd2 | 2010-10-02 15:18:50 -0700 | [diff] [blame] | 1336 | } |
Jason Evans | 20c31de | 2014-10-02 23:01:10 -0700 | [diff] [blame] | 1337 | #ifdef JEMALLOC_JET |
| 1338 | #undef prof_dump_header |
| 1339 | #define prof_dump_header JEMALLOC_N(prof_dump_header) |
| 1340 | prof_dump_header_t *prof_dump_header = JEMALLOC_N(prof_dump_header_impl); |
| 1341 | #endif |
Jason Evans | a881cd2 | 2010-10-02 15:18:50 -0700 | [diff] [blame] | 1342 | |
Jason Evans | 22ca855 | 2010-03-02 11:57:30 -0800 | [diff] [blame] | 1343 | static bool |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1344 | prof_dump_gctx(tsdn_t *tsdn, bool propagate_err, prof_gctx_t *gctx, |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 1345 | const prof_bt_t *bt, prof_gctx_tree_t *gctxs) |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1346 | { |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 1347 | bool ret; |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1348 | unsigned i; |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 1349 | struct prof_tctx_dump_iter_arg_s prof_tctx_dump_iter_arg; |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1350 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 1351 | cassert(config_prof); |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1352 | malloc_mutex_assert_owner(tsdn, gctx->lock); |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 1353 | |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1354 | /* Avoid dumping such gctx's that have no useful data. */ |
Jason Evans | 551ebc4 | 2014-10-03 10:16:09 -0700 | [diff] [blame] | 1355 | if ((!opt_prof_accum && gctx->cnt_summed.curobjs == 0) || |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1356 | (opt_prof_accum && gctx->cnt_summed.accumobjs == 0)) { |
| 1357 | assert(gctx->cnt_summed.curobjs == 0); |
| 1358 | assert(gctx->cnt_summed.curbytes == 0); |
| 1359 | assert(gctx->cnt_summed.accumobjs == 0); |
| 1360 | assert(gctx->cnt_summed.accumbytes == 0); |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 1361 | ret = false; |
| 1362 | goto label_return; |
Jason Evans | a881cd2 | 2010-10-02 15:18:50 -0700 | [diff] [blame] | 1363 | } |
| 1364 | |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1365 | if (prof_dump_printf(propagate_err, "@")) { |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 1366 | ret = true; |
| 1367 | goto label_return; |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1368 | } |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 1369 | for (i = 0; i < bt->len; i++) { |
Jason Evans | 5fae7dc | 2015-07-23 13:56:25 -0700 | [diff] [blame] | 1370 | if (prof_dump_printf(propagate_err, " %#"FMTxPTR, |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 1371 | (uintptr_t)bt->vec[i])) { |
| 1372 | ret = true; |
| 1373 | goto label_return; |
| 1374 | } |
| 1375 | } |
Jason Evans | 22ca855 | 2010-03-02 11:57:30 -0800 | [diff] [blame] | 1376 | |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1377 | if (prof_dump_printf(propagate_err, |
| 1378 | "\n" |
Jason Evans | 5fae7dc | 2015-07-23 13:56:25 -0700 | [diff] [blame] | 1379 | " t*: %"FMTu64": %"FMTu64" [%"FMTu64": %"FMTu64"]\n", |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1380 | gctx->cnt_summed.curobjs, gctx->cnt_summed.curbytes, |
| 1381 | gctx->cnt_summed.accumobjs, gctx->cnt_summed.accumbytes)) { |
| 1382 | ret = true; |
| 1383 | goto label_return; |
| 1384 | } |
| 1385 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1386 | prof_tctx_dump_iter_arg.tsdn = tsdn; |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 1387 | prof_tctx_dump_iter_arg.propagate_err = propagate_err; |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1388 | if (tctx_tree_iter(&gctx->tctxs, NULL, prof_tctx_dump_iter, |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 1389 | (void *)&prof_tctx_dump_iter_arg) != NULL) { |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 1390 | ret = true; |
| 1391 | goto label_return; |
| 1392 | } |
| 1393 | |
Jason Evans | 772163b | 2014-01-17 15:40:52 -0800 | [diff] [blame] | 1394 | ret = false; |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 1395 | label_return: |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 1396 | return (ret); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1397 | } |
| 1398 | |
Jason Evans | 788d29d | 2016-02-20 23:46:14 -0800 | [diff] [blame] | 1399 | #ifndef _WIN32 |
Jason Evans | e42c309 | 2015-07-22 15:44:47 -0700 | [diff] [blame] | 1400 | JEMALLOC_FORMAT_PRINTF(1, 2) |
Jason Evans | 8e33c21 | 2015-05-01 09:03:20 -0700 | [diff] [blame] | 1401 | static int |
| 1402 | prof_open_maps(const char *format, ...) |
| 1403 | { |
| 1404 | int mfd; |
| 1405 | va_list ap; |
| 1406 | char filename[PATH_MAX + 1]; |
| 1407 | |
| 1408 | va_start(ap, format); |
| 1409 | malloc_vsnprintf(filename, sizeof(filename), format, ap); |
| 1410 | va_end(ap); |
| 1411 | mfd = open(filename, O_RDONLY); |
| 1412 | |
| 1413 | return (mfd); |
| 1414 | } |
Jason Evans | 788d29d | 2016-02-20 23:46:14 -0800 | [diff] [blame] | 1415 | #endif |
| 1416 | |
| 1417 | static int |
| 1418 | prof_getpid(void) |
| 1419 | { |
| 1420 | |
| 1421 | #ifdef _WIN32 |
| 1422 | return (GetCurrentProcessId()); |
| 1423 | #else |
| 1424 | return (getpid()); |
| 1425 | #endif |
| 1426 | } |
Jason Evans | 8e33c21 | 2015-05-01 09:03:20 -0700 | [diff] [blame] | 1427 | |
Jason Evans | 22ca855 | 2010-03-02 11:57:30 -0800 | [diff] [blame] | 1428 | static bool |
| 1429 | prof_dump_maps(bool propagate_err) |
Jason Evans | c717718 | 2010-02-11 09:25:56 -0800 | [diff] [blame] | 1430 | { |
Jason Evans | 93f39f8 | 2013-10-21 15:07:40 -0700 | [diff] [blame] | 1431 | bool ret; |
Jason Evans | c717718 | 2010-02-11 09:25:56 -0800 | [diff] [blame] | 1432 | int mfd; |
Jason Evans | c717718 | 2010-02-11 09:25:56 -0800 | [diff] [blame] | 1433 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 1434 | cassert(config_prof); |
Harald Weppner | c2da259 | 2014-03-18 00:00:14 -0700 | [diff] [blame] | 1435 | #ifdef __FreeBSD__ |
Jason Evans | 8e33c21 | 2015-05-01 09:03:20 -0700 | [diff] [blame] | 1436 | mfd = prof_open_maps("/proc/curproc/map"); |
rustyx | 7f28398 | 2016-01-30 14:51:16 +0100 | [diff] [blame] | 1437 | #elif defined(_WIN32) |
| 1438 | mfd = -1; // Not implemented |
Harald Weppner | c2da259 | 2014-03-18 00:00:14 -0700 | [diff] [blame] | 1439 | #else |
Jason Evans | 8e33c21 | 2015-05-01 09:03:20 -0700 | [diff] [blame] | 1440 | { |
Jason Evans | 788d29d | 2016-02-20 23:46:14 -0800 | [diff] [blame] | 1441 | int pid = prof_getpid(); |
Jason Evans | 8e33c21 | 2015-05-01 09:03:20 -0700 | [diff] [blame] | 1442 | |
| 1443 | mfd = prof_open_maps("/proc/%d/task/%d/maps", pid, pid); |
| 1444 | if (mfd == -1) |
| 1445 | mfd = prof_open_maps("/proc/%d/maps", pid); |
| 1446 | } |
Harald Weppner | c2da259 | 2014-03-18 00:00:14 -0700 | [diff] [blame] | 1447 | #endif |
Jason Evans | c717718 | 2010-02-11 09:25:56 -0800 | [diff] [blame] | 1448 | if (mfd != -1) { |
| 1449 | ssize_t nread; |
| 1450 | |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 1451 | if (prof_dump_write(propagate_err, "\nMAPPED_LIBRARIES:\n") && |
Jason Evans | 93f39f8 | 2013-10-21 15:07:40 -0700 | [diff] [blame] | 1452 | propagate_err) { |
| 1453 | ret = true; |
| 1454 | goto label_return; |
| 1455 | } |
Jason Evans | c717718 | 2010-02-11 09:25:56 -0800 | [diff] [blame] | 1456 | nread = 0; |
| 1457 | do { |
| 1458 | prof_dump_buf_end += nread; |
Jason Evans | cd9a134 | 2012-03-21 18:33:03 -0700 | [diff] [blame] | 1459 | if (prof_dump_buf_end == PROF_DUMP_BUFSIZE) { |
Jason Evans | c717718 | 2010-02-11 09:25:56 -0800 | [diff] [blame] | 1460 | /* Make space in prof_dump_buf before read(). */ |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 1461 | if (prof_dump_flush(propagate_err) && |
Jason Evans | 93f39f8 | 2013-10-21 15:07:40 -0700 | [diff] [blame] | 1462 | propagate_err) { |
| 1463 | ret = true; |
| 1464 | goto label_return; |
| 1465 | } |
Jason Evans | c717718 | 2010-02-11 09:25:56 -0800 | [diff] [blame] | 1466 | } |
| 1467 | nread = read(mfd, &prof_dump_buf[prof_dump_buf_end], |
Jason Evans | cd9a134 | 2012-03-21 18:33:03 -0700 | [diff] [blame] | 1468 | PROF_DUMP_BUFSIZE - prof_dump_buf_end); |
Jason Evans | c717718 | 2010-02-11 09:25:56 -0800 | [diff] [blame] | 1469 | } while (nread > 0); |
Jason Evans | 93f39f8 | 2013-10-21 15:07:40 -0700 | [diff] [blame] | 1470 | } else { |
| 1471 | ret = true; |
| 1472 | goto label_return; |
| 1473 | } |
Jason Evans | 22ca855 | 2010-03-02 11:57:30 -0800 | [diff] [blame] | 1474 | |
Jason Evans | 93f39f8 | 2013-10-21 15:07:40 -0700 | [diff] [blame] | 1475 | ret = false; |
| 1476 | label_return: |
| 1477 | if (mfd != -1) |
| 1478 | close(mfd); |
| 1479 | return (ret); |
Jason Evans | c717718 | 2010-02-11 09:25:56 -0800 | [diff] [blame] | 1480 | } |
| 1481 | |
Jason Evans | dc391ad | 2016-05-04 12:14:36 -0700 | [diff] [blame] | 1482 | /* |
| 1483 | * See prof_sample_threshold_update() comment for why the body of this function |
| 1484 | * is conditionally compiled. |
| 1485 | */ |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 1486 | static void |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1487 | prof_leakcheck(const prof_cnt_t *cnt_all, size_t leak_ngctx, |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 1488 | const char *filename) |
| 1489 | { |
| 1490 | |
Jason Evans | dc391ad | 2016-05-04 12:14:36 -0700 | [diff] [blame] | 1491 | #ifdef JEMALLOC_PROF |
| 1492 | /* |
| 1493 | * Scaling is equivalent AdjustSamples() in jeprof, but the result may |
| 1494 | * differ slightly from what jeprof reports, because here we scale the |
| 1495 | * summary values, whereas jeprof scales each context individually and |
| 1496 | * reports the sums of the scaled values. |
| 1497 | */ |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 1498 | if (cnt_all->curbytes != 0) { |
Jason Evans | dc391ad | 2016-05-04 12:14:36 -0700 | [diff] [blame] | 1499 | double sample_period = (double)((uint64_t)1 << lg_prof_sample); |
| 1500 | double ratio = (((double)cnt_all->curbytes) / |
| 1501 | (double)cnt_all->curobjs) / sample_period; |
| 1502 | double scale_factor = 1.0 / (1.0 - exp(-ratio)); |
| 1503 | uint64_t curbytes = (uint64_t)round(((double)cnt_all->curbytes) |
| 1504 | * scale_factor); |
| 1505 | uint64_t curobjs = (uint64_t)round(((double)cnt_all->curobjs) * |
| 1506 | scale_factor); |
| 1507 | |
| 1508 | malloc_printf("<jemalloc>: Leak approximation summary: ~%"FMTu64 |
| 1509 | " byte%s, ~%"FMTu64" object%s, >= %zu context%s\n", |
| 1510 | curbytes, (curbytes != 1) ? "s" : "", curobjs, (curobjs != |
| 1511 | 1) ? "s" : "", leak_ngctx, (leak_ngctx != 1) ? "s" : ""); |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 1512 | malloc_printf( |
Jason Evans | 7041720 | 2015-05-01 12:31:12 -0700 | [diff] [blame] | 1513 | "<jemalloc>: Run jeprof on \"%s\" for leak detail\n", |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 1514 | filename); |
| 1515 | } |
Jason Evans | dc391ad | 2016-05-04 12:14:36 -0700 | [diff] [blame] | 1516 | #endif |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 1517 | } |
| 1518 | |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 1519 | struct prof_gctx_dump_iter_arg_s { |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1520 | tsdn_t *tsdn; |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 1521 | bool propagate_err; |
| 1522 | }; |
| 1523 | |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1524 | static prof_gctx_t * |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 1525 | prof_gctx_dump_iter(prof_gctx_tree_t *gctxs, prof_gctx_t *gctx, void *opaque) |
Jason Evans | 3a81cbd | 2014-08-16 12:58:55 -0700 | [diff] [blame] | 1526 | { |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1527 | prof_gctx_t *ret; |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 1528 | struct prof_gctx_dump_iter_arg_s *arg = |
| 1529 | (struct prof_gctx_dump_iter_arg_s *)opaque; |
Jason Evans | 3a81cbd | 2014-08-16 12:58:55 -0700 | [diff] [blame] | 1530 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1531 | malloc_mutex_lock(arg->tsdn, gctx->lock); |
Jason Evans | 3a81cbd | 2014-08-16 12:58:55 -0700 | [diff] [blame] | 1532 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1533 | if (prof_dump_gctx(arg->tsdn, arg->propagate_err, gctx, &gctx->bt, |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 1534 | gctxs)) { |
Jason Evans | 20c31de | 2014-10-02 23:01:10 -0700 | [diff] [blame] | 1535 | ret = gctx; |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1536 | goto label_return; |
| 1537 | } |
Jason Evans | 3a81cbd | 2014-08-16 12:58:55 -0700 | [diff] [blame] | 1538 | |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1539 | ret = NULL; |
| 1540 | label_return: |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1541 | malloc_mutex_unlock(arg->tsdn, gctx->lock); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1542 | return (ret); |
Jason Evans | 3a81cbd | 2014-08-16 12:58:55 -0700 | [diff] [blame] | 1543 | } |
| 1544 | |
Jason Evans | 22ca855 | 2010-03-02 11:57:30 -0800 | [diff] [blame] | 1545 | static bool |
Jason Evans | 5460aa6 | 2014-09-22 21:09:23 -0700 | [diff] [blame] | 1546 | prof_dump(tsd_t *tsd, bool propagate_err, const char *filename, bool leakcheck) |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1547 | { |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1548 | prof_tdata_t *tdata; |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 1549 | struct prof_tdata_merge_iter_arg_s prof_tdata_merge_iter_arg; |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1550 | size_t tabind; |
Jason Evans | 075e77c | 2010-09-20 19:53:25 -0700 | [diff] [blame] | 1551 | union { |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1552 | prof_gctx_t *p; |
Jason Evans | 075e77c | 2010-09-20 19:53:25 -0700 | [diff] [blame] | 1553 | void *v; |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1554 | } gctx; |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 1555 | struct prof_gctx_merge_iter_arg_s prof_gctx_merge_iter_arg; |
| 1556 | struct prof_gctx_dump_iter_arg_s prof_gctx_dump_iter_arg; |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1557 | prof_gctx_tree_t gctxs; |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1558 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 1559 | cassert(config_prof); |
| 1560 | |
Jason Evans | 20c31de | 2014-10-02 23:01:10 -0700 | [diff] [blame] | 1561 | tdata = prof_tdata_get(tsd, true); |
Jason Evans | 5460aa6 | 2014-09-22 21:09:23 -0700 | [diff] [blame] | 1562 | if (tdata == NULL) |
Jason Evans | 52386b2 | 2012-04-22 16:00:11 -0700 | [diff] [blame] | 1563 | return (true); |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 1564 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1565 | malloc_mutex_lock(tsd_tsdn(tsd), &prof_dump_mtx); |
Jason Evans | c93ed81 | 2014-10-30 16:50:33 -0700 | [diff] [blame] | 1566 | prof_enter(tsd, tdata); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1567 | |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1568 | /* |
| 1569 | * Put gctx's in limbo and clear their counters in preparation for |
| 1570 | * summing. |
| 1571 | */ |
| 1572 | gctx_tree_new(&gctxs); |
Jason Evans | 551ebc4 | 2014-10-03 10:16:09 -0700 | [diff] [blame] | 1573 | for (tabind = 0; !ckh_iter(&bt2gctx, &tabind, NULL, &gctx.v);) |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1574 | prof_dump_gctx_prep(tsd_tsdn(tsd), gctx.p, &gctxs); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1575 | |
| 1576 | /* |
| 1577 | * Iterate over tdatas, and for the non-expired ones snapshot their tctx |
| 1578 | * stats and merge them into the associated gctx's. |
| 1579 | */ |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1580 | prof_tdata_merge_iter_arg.tsdn = tsd_tsdn(tsd); |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 1581 | memset(&prof_tdata_merge_iter_arg.cnt_all, 0, sizeof(prof_cnt_t)); |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1582 | malloc_mutex_lock(tsd_tsdn(tsd), &tdatas_mtx); |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 1583 | tdata_tree_iter(&tdatas, NULL, prof_tdata_merge_iter, |
| 1584 | (void *)&prof_tdata_merge_iter_arg); |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1585 | malloc_mutex_unlock(tsd_tsdn(tsd), &tdatas_mtx); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1586 | |
| 1587 | /* Merge tctx stats into gctx's. */ |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1588 | prof_gctx_merge_iter_arg.tsdn = tsd_tsdn(tsd); |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 1589 | prof_gctx_merge_iter_arg.leak_ngctx = 0; |
| 1590 | gctx_tree_iter(&gctxs, NULL, prof_gctx_merge_iter, |
| 1591 | (void *)&prof_gctx_merge_iter_arg); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1592 | |
Jason Evans | c93ed81 | 2014-10-30 16:50:33 -0700 | [diff] [blame] | 1593 | prof_leave(tsd, tdata); |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 1594 | |
| 1595 | /* Create dump file. */ |
Jason Evans | 772163b | 2014-01-17 15:40:52 -0800 | [diff] [blame] | 1596 | if ((prof_dump_fd = prof_dump_open(propagate_err, filename)) == -1) |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 1597 | goto label_open_close_error; |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1598 | |
| 1599 | /* Dump profile header. */ |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1600 | if (prof_dump_header(tsd_tsdn(tsd), propagate_err, |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 1601 | &prof_tdata_merge_iter_arg.cnt_all)) |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 1602 | goto label_write_error; |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1603 | |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1604 | /* Dump per gctx profile stats. */ |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1605 | prof_gctx_dump_iter_arg.tsdn = tsd_tsdn(tsd); |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 1606 | prof_gctx_dump_iter_arg.propagate_err = propagate_err; |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1607 | if (gctx_tree_iter(&gctxs, NULL, prof_gctx_dump_iter, |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 1608 | (void *)&prof_gctx_dump_iter_arg) != NULL) |
Jason Evans | 3a81cbd | 2014-08-16 12:58:55 -0700 | [diff] [blame] | 1609 | goto label_write_error; |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1610 | |
Jason Evans | c717718 | 2010-02-11 09:25:56 -0800 | [diff] [blame] | 1611 | /* Dump /proc/<pid>/maps if possible. */ |
Jason Evans | 22ca855 | 2010-03-02 11:57:30 -0800 | [diff] [blame] | 1612 | if (prof_dump_maps(propagate_err)) |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 1613 | goto label_write_error; |
Jason Evans | c717718 | 2010-02-11 09:25:56 -0800 | [diff] [blame] | 1614 | |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 1615 | if (prof_dump_close(propagate_err)) |
| 1616 | goto label_open_close_error; |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1617 | |
Jason Evans | 20c31de | 2014-10-02 23:01:10 -0700 | [diff] [blame] | 1618 | prof_gctx_finish(tsd, &gctxs); |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1619 | malloc_mutex_unlock(tsd_tsdn(tsd), &prof_dump_mtx); |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 1620 | |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 1621 | if (leakcheck) { |
| 1622 | prof_leakcheck(&prof_tdata_merge_iter_arg.cnt_all, |
| 1623 | prof_gctx_merge_iter_arg.leak_ngctx, filename); |
| 1624 | } |
Jason Evans | 22ca855 | 2010-03-02 11:57:30 -0800 | [diff] [blame] | 1625 | return (false); |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 1626 | label_write_error: |
| 1627 | prof_dump_close(propagate_err); |
| 1628 | label_open_close_error: |
Jason Evans | 20c31de | 2014-10-02 23:01:10 -0700 | [diff] [blame] | 1629 | prof_gctx_finish(tsd, &gctxs); |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1630 | malloc_mutex_unlock(tsd_tsdn(tsd), &prof_dump_mtx); |
Jason Evans | 22ca855 | 2010-03-02 11:57:30 -0800 | [diff] [blame] | 1631 | return (true); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1632 | } |
| 1633 | |
Jason Evans | d81e4bd | 2012-03-06 14:57:45 -0800 | [diff] [blame] | 1634 | #define DUMP_FILENAME_BUFSIZE (PATH_MAX + 1) |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 1635 | #define VSEQ_INVALID UINT64_C(0xffffffffffffffff) |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1636 | static void |
Chris Peterson | 3e310b3 | 2014-05-28 19:04:06 -0700 | [diff] [blame] | 1637 | prof_dump_filename(char *filename, char v, uint64_t vseq) |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1638 | { |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1639 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 1640 | cassert(config_prof); |
| 1641 | |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 1642 | if (vseq != VSEQ_INVALID) { |
Jason Evans | d81e4bd | 2012-03-06 14:57:45 -0800 | [diff] [blame] | 1643 | /* "<prefix>.<pid>.<seq>.v<vseq>.heap" */ |
| 1644 | malloc_snprintf(filename, DUMP_FILENAME_BUFSIZE, |
Jason Evans | 5fae7dc | 2015-07-23 13:56:25 -0700 | [diff] [blame] | 1645 | "%s.%d.%"FMTu64".%c%"FMTu64".heap", |
Jason Evans | 788d29d | 2016-02-20 23:46:14 -0800 | [diff] [blame] | 1646 | opt_prof_prefix, prof_getpid(), prof_dump_seq, v, vseq); |
Jason Evans | d81e4bd | 2012-03-06 14:57:45 -0800 | [diff] [blame] | 1647 | } else { |
| 1648 | /* "<prefix>.<pid>.<seq>.<v>.heap" */ |
| 1649 | malloc_snprintf(filename, DUMP_FILENAME_BUFSIZE, |
Jason Evans | 5fae7dc | 2015-07-23 13:56:25 -0700 | [diff] [blame] | 1650 | "%s.%d.%"FMTu64".%c.heap", |
Jason Evans | 788d29d | 2016-02-20 23:46:14 -0800 | [diff] [blame] | 1651 | opt_prof_prefix, prof_getpid(), prof_dump_seq, v); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1652 | } |
Jason Evans | 52386b2 | 2012-04-22 16:00:11 -0700 | [diff] [blame] | 1653 | prof_dump_seq++; |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1654 | } |
| 1655 | |
| 1656 | static void |
| 1657 | prof_fdump(void) |
| 1658 | { |
Jason Evans | 5460aa6 | 2014-09-22 21:09:23 -0700 | [diff] [blame] | 1659 | tsd_t *tsd; |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1660 | char filename[DUMP_FILENAME_BUFSIZE]; |
| 1661 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 1662 | cassert(config_prof); |
Jason Evans | 57efa7b | 2014-10-08 17:57:19 -0700 | [diff] [blame] | 1663 | assert(opt_prof_final); |
| 1664 | assert(opt_prof_prefix[0] != '\0'); |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 1665 | |
Jason Evans | 551ebc4 | 2014-10-03 10:16:09 -0700 | [diff] [blame] | 1666 | if (!prof_booted) |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1667 | return; |
Jason Evans | 029d44c | 2014-10-04 11:12:53 -0700 | [diff] [blame] | 1668 | tsd = tsd_fetch(); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1669 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1670 | malloc_mutex_lock(tsd_tsdn(tsd), &prof_dump_seq_mtx); |
Jason Evans | 57efa7b | 2014-10-08 17:57:19 -0700 | [diff] [blame] | 1671 | prof_dump_filename(filename, 'f', VSEQ_INVALID); |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1672 | malloc_mutex_unlock(tsd_tsdn(tsd), &prof_dump_seq_mtx); |
Jason Evans | 57efa7b | 2014-10-08 17:57:19 -0700 | [diff] [blame] | 1673 | prof_dump(tsd, false, filename, opt_prof_leak); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1674 | } |
| 1675 | |
| 1676 | void |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1677 | prof_idump(tsdn_t *tsdn) |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1678 | { |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1679 | tsd_t *tsd; |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1680 | prof_tdata_t *tdata; |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1681 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 1682 | cassert(config_prof); |
| 1683 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1684 | if (!prof_booted || tsdn_null(tsdn)) |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1685 | return; |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1686 | tsd = tsdn_tsd(tsdn); |
Jason Evans | 5460aa6 | 2014-09-22 21:09:23 -0700 | [diff] [blame] | 1687 | tdata = prof_tdata_get(tsd, false); |
| 1688 | if (tdata == NULL) |
Jason Evans | 52386b2 | 2012-04-22 16:00:11 -0700 | [diff] [blame] | 1689 | return; |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1690 | if (tdata->enq) { |
| 1691 | tdata->enq_idump = true; |
Jason Evans | d34f9e7 | 2010-02-11 13:19:21 -0800 | [diff] [blame] | 1692 | return; |
| 1693 | } |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1694 | |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 1695 | if (opt_prof_prefix[0] != '\0') { |
Dmitry-Me | 78ae1ac | 2015-09-08 15:09:20 +0300 | [diff] [blame] | 1696 | char filename[PATH_MAX + 1]; |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1697 | malloc_mutex_lock(tsd_tsdn(tsd), &prof_dump_seq_mtx); |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 1698 | prof_dump_filename(filename, 'i', prof_dump_iseq); |
| 1699 | prof_dump_iseq++; |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1700 | malloc_mutex_unlock(tsd_tsdn(tsd), &prof_dump_seq_mtx); |
Jason Evans | 5460aa6 | 2014-09-22 21:09:23 -0700 | [diff] [blame] | 1701 | prof_dump(tsd, false, filename, false); |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 1702 | } |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1703 | } |
| 1704 | |
Jason Evans | 22ca855 | 2010-03-02 11:57:30 -0800 | [diff] [blame] | 1705 | bool |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 1706 | prof_mdump(tsd_t *tsd, const char *filename) |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1707 | { |
Jason Evans | 22ca855 | 2010-03-02 11:57:30 -0800 | [diff] [blame] | 1708 | char filename_buf[DUMP_FILENAME_BUFSIZE]; |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1709 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 1710 | cassert(config_prof); |
| 1711 | |
Jason Evans | 551ebc4 | 2014-10-03 10:16:09 -0700 | [diff] [blame] | 1712 | if (!opt_prof || !prof_booted) |
Jason Evans | 22ca855 | 2010-03-02 11:57:30 -0800 | [diff] [blame] | 1713 | return (true); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1714 | |
Jason Evans | 22ca855 | 2010-03-02 11:57:30 -0800 | [diff] [blame] | 1715 | if (filename == NULL) { |
| 1716 | /* No filename specified, so automatically generate one. */ |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 1717 | if (opt_prof_prefix[0] == '\0') |
| 1718 | return (true); |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1719 | malloc_mutex_lock(tsd_tsdn(tsd), &prof_dump_seq_mtx); |
Jason Evans | 22ca855 | 2010-03-02 11:57:30 -0800 | [diff] [blame] | 1720 | prof_dump_filename(filename_buf, 'm', prof_dump_mseq); |
| 1721 | prof_dump_mseq++; |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1722 | malloc_mutex_unlock(tsd_tsdn(tsd), &prof_dump_seq_mtx); |
Jason Evans | 22ca855 | 2010-03-02 11:57:30 -0800 | [diff] [blame] | 1723 | filename = filename_buf; |
| 1724 | } |
Jason Evans | 5460aa6 | 2014-09-22 21:09:23 -0700 | [diff] [blame] | 1725 | return (prof_dump(tsd, true, filename, false)); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1726 | } |
| 1727 | |
| 1728 | void |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1729 | prof_gdump(tsdn_t *tsdn) |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1730 | { |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1731 | tsd_t *tsd; |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1732 | prof_tdata_t *tdata; |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1733 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 1734 | cassert(config_prof); |
| 1735 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1736 | if (!prof_booted || tsdn_null(tsdn)) |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1737 | return; |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1738 | tsd = tsdn_tsd(tsdn); |
Jason Evans | 5460aa6 | 2014-09-22 21:09:23 -0700 | [diff] [blame] | 1739 | tdata = prof_tdata_get(tsd, false); |
| 1740 | if (tdata == NULL) |
Jason Evans | 52386b2 | 2012-04-22 16:00:11 -0700 | [diff] [blame] | 1741 | return; |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1742 | if (tdata->enq) { |
| 1743 | tdata->enq_gdump = true; |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1744 | return; |
| 1745 | } |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1746 | |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 1747 | if (opt_prof_prefix[0] != '\0') { |
Dmitry-Me | 78ae1ac | 2015-09-08 15:09:20 +0300 | [diff] [blame] | 1748 | char filename[DUMP_FILENAME_BUFSIZE]; |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1749 | malloc_mutex_lock(tsdn, &prof_dump_seq_mtx); |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 1750 | prof_dump_filename(filename, 'u', prof_dump_useq); |
| 1751 | prof_dump_useq++; |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1752 | malloc_mutex_unlock(tsdn, &prof_dump_seq_mtx); |
Jason Evans | 5460aa6 | 2014-09-22 21:09:23 -0700 | [diff] [blame] | 1753 | prof_dump(tsd, false, filename, false); |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 1754 | } |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1755 | } |
| 1756 | |
| 1757 | static void |
Jason Evans | ae03bf6 | 2013-01-22 12:02:08 -0800 | [diff] [blame] | 1758 | prof_bt_hash(const void *key, size_t r_hash[2]) |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1759 | { |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1760 | prof_bt_t *bt = (prof_bt_t *)key; |
| 1761 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 1762 | cassert(config_prof); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1763 | |
Jason Evans | ae03bf6 | 2013-01-22 12:02:08 -0800 | [diff] [blame] | 1764 | hash(bt->vec, bt->len * sizeof(void *), 0x94122f33U, r_hash); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1765 | } |
| 1766 | |
| 1767 | static bool |
| 1768 | prof_bt_keycomp(const void *k1, const void *k2) |
| 1769 | { |
| 1770 | const prof_bt_t *bt1 = (prof_bt_t *)k1; |
| 1771 | const prof_bt_t *bt2 = (prof_bt_t *)k2; |
| 1772 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 1773 | cassert(config_prof); |
| 1774 | |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1775 | if (bt1->len != bt2->len) |
| 1776 | return (false); |
| 1777 | return (memcmp(bt1->vec, bt2->vec, bt1->len * sizeof(void *)) == 0); |
| 1778 | } |
| 1779 | |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1780 | JEMALLOC_INLINE_C uint64_t |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1781 | prof_thr_uid_alloc(tsdn_t *tsdn) |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1782 | { |
Jason Evans | 9d8f3d2 | 2014-09-11 18:06:30 -0700 | [diff] [blame] | 1783 | uint64_t thr_uid; |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1784 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1785 | malloc_mutex_lock(tsdn, &next_thr_uid_mtx); |
Jason Evans | 9d8f3d2 | 2014-09-11 18:06:30 -0700 | [diff] [blame] | 1786 | thr_uid = next_thr_uid; |
| 1787 | next_thr_uid++; |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1788 | malloc_mutex_unlock(tsdn, &next_thr_uid_mtx); |
Jason Evans | 9d8f3d2 | 2014-09-11 18:06:30 -0700 | [diff] [blame] | 1789 | |
| 1790 | return (thr_uid); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1791 | } |
| 1792 | |
| 1793 | static prof_tdata_t * |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1794 | prof_tdata_init_impl(tsdn_t *tsdn, uint64_t thr_uid, uint64_t thr_discrim, |
Jason Evans | fc12c0b | 2014-10-03 23:25:30 -0700 | [diff] [blame] | 1795 | char *thread_name, bool active) |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1796 | { |
| 1797 | prof_tdata_t *tdata; |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1798 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 1799 | cassert(config_prof); |
| 1800 | |
Jason Evans | 4d6a134 | 2010-10-20 19:05:59 -0700 | [diff] [blame] | 1801 | /* Initialize an empty cache for this thread. */ |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1802 | tdata = (prof_tdata_t *)iallocztm(tsdn, sizeof(prof_tdata_t), |
| 1803 | size2index(sizeof(prof_tdata_t)), false, NULL, true, |
| 1804 | arena_get(TSDN_NULL, 0, true), true); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1805 | if (tdata == NULL) |
Jason Evans | 4d6a134 | 2010-10-20 19:05:59 -0700 | [diff] [blame] | 1806 | return (NULL); |
| 1807 | |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1808 | tdata->lock = prof_tdata_mutex_choose(thr_uid); |
| 1809 | tdata->thr_uid = thr_uid; |
Jason Evans | 20c31de | 2014-10-02 23:01:10 -0700 | [diff] [blame] | 1810 | tdata->thr_discrim = thr_discrim; |
Jason Evans | fc12c0b | 2014-10-03 23:25:30 -0700 | [diff] [blame] | 1811 | tdata->thread_name = thread_name; |
Jason Evans | 20c31de | 2014-10-02 23:01:10 -0700 | [diff] [blame] | 1812 | tdata->attached = true; |
| 1813 | tdata->expired = false; |
Jason Evans | 04211e2 | 2015-03-16 15:11:06 -0700 | [diff] [blame] | 1814 | tdata->tctx_uid_next = 0; |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1815 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1816 | if (ckh_new(tsdn, &tdata->bt2tctx, PROF_CKH_MINITEMS, |
Jason Evans | 4d6a134 | 2010-10-20 19:05:59 -0700 | [diff] [blame] | 1817 | prof_bt_hash, prof_bt_keycomp)) { |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1818 | idalloctm(tsdn, tdata, NULL, true, true); |
Jason Evans | 4d6a134 | 2010-10-20 19:05:59 -0700 | [diff] [blame] | 1819 | return (NULL); |
| 1820 | } |
Jason Evans | 4d6a134 | 2010-10-20 19:05:59 -0700 | [diff] [blame] | 1821 | |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1822 | tdata->prng_state = (uint64_t)(uintptr_t)tdata; |
| 1823 | prof_sample_threshold_update(tdata); |
Jason Evans | 4d6a134 | 2010-10-20 19:05:59 -0700 | [diff] [blame] | 1824 | |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1825 | tdata->enq = false; |
| 1826 | tdata->enq_idump = false; |
| 1827 | tdata->enq_gdump = false; |
Jason Evans | 52386b2 | 2012-04-22 16:00:11 -0700 | [diff] [blame] | 1828 | |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1829 | tdata->dumping = false; |
Jason Evans | fc12c0b | 2014-10-03 23:25:30 -0700 | [diff] [blame] | 1830 | tdata->active = active; |
Jason Evans | 4d6a134 | 2010-10-20 19:05:59 -0700 | [diff] [blame] | 1831 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1832 | malloc_mutex_lock(tsdn, &tdatas_mtx); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1833 | tdata_tree_insert(&tdatas, tdata); |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1834 | malloc_mutex_unlock(tsdn, &tdatas_mtx); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1835 | |
| 1836 | return (tdata); |
| 1837 | } |
| 1838 | |
| 1839 | prof_tdata_t * |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1840 | prof_tdata_init(tsdn_t *tsdn) |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1841 | { |
| 1842 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1843 | return (prof_tdata_init_impl(tsdn, prof_thr_uid_alloc(tsdn), 0, NULL, |
| 1844 | prof_thread_active_init_get(tsdn))); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1845 | } |
| 1846 | |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1847 | static bool |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1848 | prof_tdata_should_destroy_unlocked(prof_tdata_t *tdata, bool even_if_attached) |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1849 | { |
| 1850 | |
Jason Evans | f04a0be | 2014-10-04 15:03:49 -0700 | [diff] [blame] | 1851 | if (tdata->attached && !even_if_attached) |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1852 | return (false); |
| 1853 | if (ckh_count(&tdata->bt2tctx) != 0) |
| 1854 | return (false); |
| 1855 | return (true); |
| 1856 | } |
| 1857 | |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 1858 | static bool |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1859 | prof_tdata_should_destroy(tsdn_t *tsdn, prof_tdata_t *tdata, |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 1860 | bool even_if_attached) |
| 1861 | { |
| 1862 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1863 | malloc_mutex_assert_owner(tsdn, tdata->lock); |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 1864 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1865 | return (prof_tdata_should_destroy_unlocked(tdata, even_if_attached)); |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 1866 | } |
| 1867 | |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1868 | static void |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1869 | prof_tdata_destroy_locked(tsdn_t *tsdn, prof_tdata_t *tdata, |
Jason Evans | f04a0be | 2014-10-04 15:03:49 -0700 | [diff] [blame] | 1870 | bool even_if_attached) |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1871 | { |
| 1872 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1873 | malloc_mutex_assert_owner(tsdn, &tdatas_mtx); |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 1874 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1875 | assert(tsdn_null(tsdn) || tsd_prof_tdata_get(tsdn_tsd(tsdn)) != tdata); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1876 | |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1877 | tdata_tree_remove(&tdatas, tdata); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1878 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1879 | assert(prof_tdata_should_destroy_unlocked(tdata, even_if_attached)); |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 1880 | |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1881 | if (tdata->thread_name != NULL) |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1882 | idalloctm(tsdn, tdata->thread_name, NULL, true, true); |
| 1883 | ckh_delete(tsdn, &tdata->bt2tctx); |
| 1884 | idalloctm(tsdn, tdata, NULL, true, true); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1885 | } |
| 1886 | |
| 1887 | static void |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1888 | prof_tdata_destroy(tsdn_t *tsdn, prof_tdata_t *tdata, bool even_if_attached) |
Jason Evans | 20c31de | 2014-10-02 23:01:10 -0700 | [diff] [blame] | 1889 | { |
| 1890 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1891 | malloc_mutex_lock(tsdn, &tdatas_mtx); |
| 1892 | prof_tdata_destroy_locked(tsdn, tdata, even_if_attached); |
| 1893 | malloc_mutex_unlock(tsdn, &tdatas_mtx); |
Jason Evans | 20c31de | 2014-10-02 23:01:10 -0700 | [diff] [blame] | 1894 | } |
| 1895 | |
| 1896 | static void |
| 1897 | prof_tdata_detach(tsd_t *tsd, prof_tdata_t *tdata) |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1898 | { |
| 1899 | bool destroy_tdata; |
| 1900 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1901 | malloc_mutex_lock(tsd_tsdn(tsd), tdata->lock); |
Jason Evans | 20c31de | 2014-10-02 23:01:10 -0700 | [diff] [blame] | 1902 | if (tdata->attached) { |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1903 | destroy_tdata = prof_tdata_should_destroy(tsd_tsdn(tsd), tdata, |
| 1904 | true); |
Jason Evans | f04a0be | 2014-10-04 15:03:49 -0700 | [diff] [blame] | 1905 | /* |
| 1906 | * Only detach if !destroy_tdata, because detaching would allow |
| 1907 | * another thread to win the race to destroy tdata. |
| 1908 | */ |
| 1909 | if (!destroy_tdata) |
| 1910 | tdata->attached = false; |
Jason Evans | 029d44c | 2014-10-04 11:12:53 -0700 | [diff] [blame] | 1911 | tsd_prof_tdata_set(tsd, NULL); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1912 | } else |
| 1913 | destroy_tdata = false; |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1914 | malloc_mutex_unlock(tsd_tsdn(tsd), tdata->lock); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1915 | if (destroy_tdata) |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1916 | prof_tdata_destroy(tsd_tsdn(tsd), tdata, true); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1917 | } |
| 1918 | |
Jason Evans | 20c31de | 2014-10-02 23:01:10 -0700 | [diff] [blame] | 1919 | prof_tdata_t * |
| 1920 | prof_tdata_reinit(tsd_t *tsd, prof_tdata_t *tdata) |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1921 | { |
Jason Evans | 20c31de | 2014-10-02 23:01:10 -0700 | [diff] [blame] | 1922 | uint64_t thr_uid = tdata->thr_uid; |
| 1923 | uint64_t thr_discrim = tdata->thr_discrim + 1; |
Jason Evans | fc12c0b | 2014-10-03 23:25:30 -0700 | [diff] [blame] | 1924 | char *thread_name = (tdata->thread_name != NULL) ? |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1925 | prof_thread_name_alloc(tsd_tsdn(tsd), tdata->thread_name) : NULL; |
Jason Evans | fc12c0b | 2014-10-03 23:25:30 -0700 | [diff] [blame] | 1926 | bool active = tdata->active; |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1927 | |
Jason Evans | 20c31de | 2014-10-02 23:01:10 -0700 | [diff] [blame] | 1928 | prof_tdata_detach(tsd, tdata); |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1929 | return (prof_tdata_init_impl(tsd_tsdn(tsd), thr_uid, thr_discrim, |
| 1930 | thread_name, active)); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1931 | } |
| 1932 | |
Jason Evans | 20c31de | 2014-10-02 23:01:10 -0700 | [diff] [blame] | 1933 | static bool |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1934 | prof_tdata_expire(tsdn_t *tsdn, prof_tdata_t *tdata) |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1935 | { |
Jason Evans | 20c31de | 2014-10-02 23:01:10 -0700 | [diff] [blame] | 1936 | bool destroy_tdata; |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1937 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1938 | malloc_mutex_lock(tsdn, tdata->lock); |
Jason Evans | 20c31de | 2014-10-02 23:01:10 -0700 | [diff] [blame] | 1939 | if (!tdata->expired) { |
| 1940 | tdata->expired = true; |
| 1941 | destroy_tdata = tdata->attached ? false : |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1942 | prof_tdata_should_destroy(tsdn, tdata, false); |
Jason Evans | 20c31de | 2014-10-02 23:01:10 -0700 | [diff] [blame] | 1943 | } else |
| 1944 | destroy_tdata = false; |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1945 | malloc_mutex_unlock(tsdn, tdata->lock); |
Jason Evans | 20c31de | 2014-10-02 23:01:10 -0700 | [diff] [blame] | 1946 | |
| 1947 | return (destroy_tdata); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1948 | } |
| 1949 | |
| 1950 | static prof_tdata_t * |
| 1951 | prof_tdata_reset_iter(prof_tdata_tree_t *tdatas, prof_tdata_t *tdata, void *arg) |
| 1952 | { |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1953 | tsdn_t *tsdn = (tsdn_t *)arg; |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1954 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1955 | return (prof_tdata_expire(tsdn, tdata) ? tdata : NULL); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1956 | } |
| 1957 | |
| 1958 | void |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1959 | prof_reset(tsdn_t *tsdn, size_t lg_sample) |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1960 | { |
Jason Evans | 20c31de | 2014-10-02 23:01:10 -0700 | [diff] [blame] | 1961 | prof_tdata_t *next; |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1962 | |
| 1963 | assert(lg_sample < (sizeof(uint64_t) << 3)); |
| 1964 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1965 | malloc_mutex_lock(tsdn, &prof_dump_mtx); |
| 1966 | malloc_mutex_lock(tsdn, &tdatas_mtx); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1967 | |
| 1968 | lg_prof_sample = lg_sample; |
Jason Evans | 20c31de | 2014-10-02 23:01:10 -0700 | [diff] [blame] | 1969 | |
| 1970 | next = NULL; |
| 1971 | do { |
| 1972 | prof_tdata_t *to_destroy = tdata_tree_iter(&tdatas, next, |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1973 | prof_tdata_reset_iter, (void *)tsdn); |
Jason Evans | 20c31de | 2014-10-02 23:01:10 -0700 | [diff] [blame] | 1974 | if (to_destroy != NULL) { |
| 1975 | next = tdata_tree_next(&tdatas, to_destroy); |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1976 | prof_tdata_destroy_locked(tsdn, to_destroy, false); |
Jason Evans | 20c31de | 2014-10-02 23:01:10 -0700 | [diff] [blame] | 1977 | } else |
| 1978 | next = NULL; |
| 1979 | } while (next != NULL); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 1980 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1981 | malloc_mutex_unlock(tsdn, &tdatas_mtx); |
| 1982 | malloc_mutex_unlock(tsdn, &prof_dump_mtx); |
Jason Evans | 4d6a134 | 2010-10-20 19:05:59 -0700 | [diff] [blame] | 1983 | } |
| 1984 | |
Jason Evans | cd9a134 | 2012-03-21 18:33:03 -0700 | [diff] [blame] | 1985 | void |
Jason Evans | 5460aa6 | 2014-09-22 21:09:23 -0700 | [diff] [blame] | 1986 | prof_tdata_cleanup(tsd_t *tsd) |
Jason Evans | 4d6a134 | 2010-10-20 19:05:59 -0700 | [diff] [blame] | 1987 | { |
Jason Evans | 5460aa6 | 2014-09-22 21:09:23 -0700 | [diff] [blame] | 1988 | prof_tdata_t *tdata; |
Jason Evans | 4d6a134 | 2010-10-20 19:05:59 -0700 | [diff] [blame] | 1989 | |
Jason Evans | 5460aa6 | 2014-09-22 21:09:23 -0700 | [diff] [blame] | 1990 | if (!config_prof) |
| 1991 | return; |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 1992 | |
Jason Evans | 5460aa6 | 2014-09-22 21:09:23 -0700 | [diff] [blame] | 1993 | tdata = tsd_prof_tdata_get(tsd); |
| 1994 | if (tdata != NULL) |
| 1995 | prof_tdata_detach(tsd, tdata); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1996 | } |
| 1997 | |
Jason Evans | fc12c0b | 2014-10-03 23:25:30 -0700 | [diff] [blame] | 1998 | bool |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 1999 | prof_active_get(tsdn_t *tsdn) |
Jason Evans | fc12c0b | 2014-10-03 23:25:30 -0700 | [diff] [blame] | 2000 | { |
| 2001 | bool prof_active_current; |
| 2002 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 2003 | malloc_mutex_lock(tsdn, &prof_active_mtx); |
Jason Evans | fc12c0b | 2014-10-03 23:25:30 -0700 | [diff] [blame] | 2004 | prof_active_current = prof_active; |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 2005 | malloc_mutex_unlock(tsdn, &prof_active_mtx); |
Jason Evans | fc12c0b | 2014-10-03 23:25:30 -0700 | [diff] [blame] | 2006 | return (prof_active_current); |
| 2007 | } |
| 2008 | |
| 2009 | bool |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 2010 | prof_active_set(tsdn_t *tsdn, bool active) |
Jason Evans | fc12c0b | 2014-10-03 23:25:30 -0700 | [diff] [blame] | 2011 | { |
| 2012 | bool prof_active_old; |
| 2013 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 2014 | malloc_mutex_lock(tsdn, &prof_active_mtx); |
Jason Evans | fc12c0b | 2014-10-03 23:25:30 -0700 | [diff] [blame] | 2015 | prof_active_old = prof_active; |
| 2016 | prof_active = active; |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 2017 | malloc_mutex_unlock(tsdn, &prof_active_mtx); |
Jason Evans | fc12c0b | 2014-10-03 23:25:30 -0700 | [diff] [blame] | 2018 | return (prof_active_old); |
| 2019 | } |
| 2020 | |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 2021 | const char * |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 2022 | prof_thread_name_get(tsd_t *tsd) |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 2023 | { |
Jason Evans | 5460aa6 | 2014-09-22 21:09:23 -0700 | [diff] [blame] | 2024 | prof_tdata_t *tdata; |
| 2025 | |
Jason Evans | 5460aa6 | 2014-09-22 21:09:23 -0700 | [diff] [blame] | 2026 | tdata = prof_tdata_get(tsd, true); |
| 2027 | if (tdata == NULL) |
Jason Evans | fc12c0b | 2014-10-03 23:25:30 -0700 | [diff] [blame] | 2028 | return (""); |
| 2029 | return (tdata->thread_name != NULL ? tdata->thread_name : ""); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 2030 | } |
| 2031 | |
Jason Evans | fc12c0b | 2014-10-03 23:25:30 -0700 | [diff] [blame] | 2032 | static char * |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 2033 | prof_thread_name_alloc(tsdn_t *tsdn, const char *thread_name) |
Jason Evans | fc12c0b | 2014-10-03 23:25:30 -0700 | [diff] [blame] | 2034 | { |
| 2035 | char *ret; |
| 2036 | size_t size; |
| 2037 | |
| 2038 | if (thread_name == NULL) |
| 2039 | return (NULL); |
| 2040 | |
| 2041 | size = strlen(thread_name) + 1; |
| 2042 | if (size == 1) |
| 2043 | return (""); |
| 2044 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 2045 | ret = iallocztm(tsdn, size, size2index(size), false, NULL, true, |
| 2046 | arena_get(TSDN_NULL, 0, true), true); |
Jason Evans | fc12c0b | 2014-10-03 23:25:30 -0700 | [diff] [blame] | 2047 | if (ret == NULL) |
| 2048 | return (NULL); |
| 2049 | memcpy(ret, thread_name, size); |
| 2050 | return (ret); |
| 2051 | } |
| 2052 | |
| 2053 | int |
Jason Evans | 5460aa6 | 2014-09-22 21:09:23 -0700 | [diff] [blame] | 2054 | prof_thread_name_set(tsd_t *tsd, const char *thread_name) |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 2055 | { |
| 2056 | prof_tdata_t *tdata; |
Jason Evans | fc12c0b | 2014-10-03 23:25:30 -0700 | [diff] [blame] | 2057 | unsigned i; |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 2058 | char *s; |
| 2059 | |
Jason Evans | 5460aa6 | 2014-09-22 21:09:23 -0700 | [diff] [blame] | 2060 | tdata = prof_tdata_get(tsd, true); |
| 2061 | if (tdata == NULL) |
Jason Evans | fc12c0b | 2014-10-03 23:25:30 -0700 | [diff] [blame] | 2062 | return (EAGAIN); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 2063 | |
Jason Evans | fc12c0b | 2014-10-03 23:25:30 -0700 | [diff] [blame] | 2064 | /* Validate input. */ |
| 2065 | if (thread_name == NULL) |
| 2066 | return (EFAULT); |
| 2067 | for (i = 0; thread_name[i] != '\0'; i++) { |
| 2068 | char c = thread_name[i]; |
| 2069 | if (!isgraph(c) && !isblank(c)) |
| 2070 | return (EFAULT); |
| 2071 | } |
| 2072 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 2073 | s = prof_thread_name_alloc(tsd_tsdn(tsd), thread_name); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 2074 | if (s == NULL) |
Jason Evans | fc12c0b | 2014-10-03 23:25:30 -0700 | [diff] [blame] | 2075 | return (EAGAIN); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 2076 | |
Jason Evans | fc12c0b | 2014-10-03 23:25:30 -0700 | [diff] [blame] | 2077 | if (tdata->thread_name != NULL) { |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 2078 | idalloctm(tsd_tsdn(tsd), tdata->thread_name, NULL, true, true); |
Jason Evans | fc12c0b | 2014-10-03 23:25:30 -0700 | [diff] [blame] | 2079 | tdata->thread_name = NULL; |
| 2080 | } |
| 2081 | if (strlen(s) > 0) |
| 2082 | tdata->thread_name = s; |
| 2083 | return (0); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 2084 | } |
| 2085 | |
| 2086 | bool |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 2087 | prof_thread_active_get(tsd_t *tsd) |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 2088 | { |
Jason Evans | 5460aa6 | 2014-09-22 21:09:23 -0700 | [diff] [blame] | 2089 | prof_tdata_t *tdata; |
| 2090 | |
Jason Evans | 5460aa6 | 2014-09-22 21:09:23 -0700 | [diff] [blame] | 2091 | tdata = prof_tdata_get(tsd, true); |
| 2092 | if (tdata == NULL) |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 2093 | return (false); |
| 2094 | return (tdata->active); |
| 2095 | } |
| 2096 | |
| 2097 | bool |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 2098 | prof_thread_active_set(tsd_t *tsd, bool active) |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 2099 | { |
| 2100 | prof_tdata_t *tdata; |
| 2101 | |
Jason Evans | 5460aa6 | 2014-09-22 21:09:23 -0700 | [diff] [blame] | 2102 | tdata = prof_tdata_get(tsd, true); |
| 2103 | if (tdata == NULL) |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 2104 | return (true); |
| 2105 | tdata->active = active; |
| 2106 | return (false); |
| 2107 | } |
| 2108 | |
Jason Evans | fc12c0b | 2014-10-03 23:25:30 -0700 | [diff] [blame] | 2109 | bool |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 2110 | prof_thread_active_init_get(tsdn_t *tsdn) |
Jason Evans | fc12c0b | 2014-10-03 23:25:30 -0700 | [diff] [blame] | 2111 | { |
| 2112 | bool active_init; |
| 2113 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 2114 | malloc_mutex_lock(tsdn, &prof_thread_active_init_mtx); |
Jason Evans | fc12c0b | 2014-10-03 23:25:30 -0700 | [diff] [blame] | 2115 | active_init = prof_thread_active_init; |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 2116 | malloc_mutex_unlock(tsdn, &prof_thread_active_init_mtx); |
Jason Evans | fc12c0b | 2014-10-03 23:25:30 -0700 | [diff] [blame] | 2117 | return (active_init); |
| 2118 | } |
| 2119 | |
| 2120 | bool |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 2121 | prof_thread_active_init_set(tsdn_t *tsdn, bool active_init) |
Jason Evans | fc12c0b | 2014-10-03 23:25:30 -0700 | [diff] [blame] | 2122 | { |
| 2123 | bool active_init_old; |
| 2124 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 2125 | malloc_mutex_lock(tsdn, &prof_thread_active_init_mtx); |
Jason Evans | fc12c0b | 2014-10-03 23:25:30 -0700 | [diff] [blame] | 2126 | active_init_old = prof_thread_active_init; |
| 2127 | prof_thread_active_init = active_init; |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 2128 | malloc_mutex_unlock(tsdn, &prof_thread_active_init_mtx); |
Jason Evans | fc12c0b | 2014-10-03 23:25:30 -0700 | [diff] [blame] | 2129 | return (active_init_old); |
| 2130 | } |
| 2131 | |
Jason Evans | 5b8ed5b | 2015-01-25 21:16:57 -0800 | [diff] [blame] | 2132 | bool |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 2133 | prof_gdump_get(tsdn_t *tsdn) |
Jason Evans | 5b8ed5b | 2015-01-25 21:16:57 -0800 | [diff] [blame] | 2134 | { |
| 2135 | bool prof_gdump_current; |
| 2136 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 2137 | malloc_mutex_lock(tsdn, &prof_gdump_mtx); |
Jason Evans | 5b8ed5b | 2015-01-25 21:16:57 -0800 | [diff] [blame] | 2138 | prof_gdump_current = prof_gdump_val; |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 2139 | malloc_mutex_unlock(tsdn, &prof_gdump_mtx); |
Jason Evans | 5b8ed5b | 2015-01-25 21:16:57 -0800 | [diff] [blame] | 2140 | return (prof_gdump_current); |
| 2141 | } |
| 2142 | |
| 2143 | bool |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 2144 | prof_gdump_set(tsdn_t *tsdn, bool gdump) |
Jason Evans | 5b8ed5b | 2015-01-25 21:16:57 -0800 | [diff] [blame] | 2145 | { |
| 2146 | bool prof_gdump_old; |
| 2147 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 2148 | malloc_mutex_lock(tsdn, &prof_gdump_mtx); |
Jason Evans | 5b8ed5b | 2015-01-25 21:16:57 -0800 | [diff] [blame] | 2149 | prof_gdump_old = prof_gdump_val; |
| 2150 | prof_gdump_val = gdump; |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 2151 | malloc_mutex_unlock(tsdn, &prof_gdump_mtx); |
Jason Evans | 5b8ed5b | 2015-01-25 21:16:57 -0800 | [diff] [blame] | 2152 | return (prof_gdump_old); |
| 2153 | } |
| 2154 | |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 2155 | void |
| 2156 | prof_boot0(void) |
| 2157 | { |
| 2158 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 2159 | cassert(config_prof); |
| 2160 | |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 2161 | memcpy(opt_prof_prefix, PROF_PREFIX_DEFAULT, |
| 2162 | sizeof(PROF_PREFIX_DEFAULT)); |
| 2163 | } |
| 2164 | |
| 2165 | void |
| 2166 | prof_boot1(void) |
| 2167 | { |
| 2168 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 2169 | cassert(config_prof); |
| 2170 | |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 2171 | /* |
Jason Evans | 9b0cbf0 | 2014-04-11 14:24:51 -0700 | [diff] [blame] | 2172 | * opt_prof must be in its final state before any arenas are |
| 2173 | * initialized, so this function must be executed early. |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 2174 | */ |
| 2175 | |
Jason Evans | 551ebc4 | 2014-10-03 10:16:09 -0700 | [diff] [blame] | 2176 | if (opt_prof_leak && !opt_prof) { |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 2177 | /* |
| 2178 | * Enable opt_prof, but in such a way that profiles are never |
| 2179 | * automatically dumped. |
| 2180 | */ |
| 2181 | opt_prof = true; |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 2182 | opt_prof_gdump = false; |
Jason Evans | a02fc08 | 2010-03-31 17:35:51 -0700 | [diff] [blame] | 2183 | } else if (opt_prof) { |
| 2184 | if (opt_lg_prof_interval >= 0) { |
| 2185 | prof_interval = (((uint64_t)1U) << |
| 2186 | opt_lg_prof_interval); |
Jason Evans | a3b3386 | 2012-11-13 12:56:27 -0800 | [diff] [blame] | 2187 | } |
Jason Evans | a02fc08 | 2010-03-31 17:35:51 -0700 | [diff] [blame] | 2188 | } |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 2189 | } |
| 2190 | |
| 2191 | bool |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 2192 | prof_boot2(tsdn_t *tsdn) |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 2193 | { |
| 2194 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 2195 | cassert(config_prof); |
| 2196 | |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 2197 | if (opt_prof) { |
Jason Evans | 6da5418 | 2012-03-23 18:05:51 -0700 | [diff] [blame] | 2198 | unsigned i; |
| 2199 | |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 2200 | lg_prof_sample = opt_lg_prof_sample; |
| 2201 | |
Jason Evans | fc12c0b | 2014-10-03 23:25:30 -0700 | [diff] [blame] | 2202 | prof_active = opt_prof_active; |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 2203 | if (malloc_mutex_init(&prof_active_mtx, "prof_active", |
| 2204 | WITNESS_RANK_PROF_ACTIVE)) |
Jason Evans | fc12c0b | 2014-10-03 23:25:30 -0700 | [diff] [blame] | 2205 | return (true); |
| 2206 | |
Jason Evans | 5b8ed5b | 2015-01-25 21:16:57 -0800 | [diff] [blame] | 2207 | prof_gdump_val = opt_prof_gdump; |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 2208 | if (malloc_mutex_init(&prof_gdump_mtx, "prof_gdump", |
| 2209 | WITNESS_RANK_PROF_GDUMP)) |
Jason Evans | 5b8ed5b | 2015-01-25 21:16:57 -0800 | [diff] [blame] | 2210 | return (true); |
| 2211 | |
Jason Evans | fc12c0b | 2014-10-03 23:25:30 -0700 | [diff] [blame] | 2212 | prof_thread_active_init = opt_prof_thread_active_init; |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 2213 | if (malloc_mutex_init(&prof_thread_active_init_mtx, |
| 2214 | "prof_thread_active_init", |
| 2215 | WITNESS_RANK_PROF_THREAD_ACTIVE_INIT)) |
Jason Evans | fc12c0b | 2014-10-03 23:25:30 -0700 | [diff] [blame] | 2216 | return (true); |
| 2217 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 2218 | if (ckh_new(tsdn, &bt2gctx, PROF_CKH_MINITEMS, prof_bt_hash, |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 2219 | prof_bt_keycomp)) |
| 2220 | return (true); |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 2221 | if (malloc_mutex_init(&bt2gctx_mtx, "prof_bt2gctx", |
| 2222 | WITNESS_RANK_PROF_BT2GCTX)) |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 2223 | return (true); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 2224 | |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 2225 | tdata_tree_new(&tdatas); |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 2226 | if (malloc_mutex_init(&tdatas_mtx, "prof_tdatas", |
| 2227 | WITNESS_RANK_PROF_TDATAS)) |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 2228 | return (true); |
| 2229 | |
| 2230 | next_thr_uid = 0; |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 2231 | if (malloc_mutex_init(&next_thr_uid_mtx, "prof_next_thr_uid", |
| 2232 | WITNESS_RANK_PROF_NEXT_THR_UID)) |
Jason Evans | 9d8f3d2 | 2014-09-11 18:06:30 -0700 | [diff] [blame] | 2233 | return (true); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 2234 | |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 2235 | if (malloc_mutex_init(&prof_dump_seq_mtx, "prof_dump_seq", |
| 2236 | WITNESS_RANK_PROF_DUMP_SEQ)) |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 2237 | return (true); |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 2238 | if (malloc_mutex_init(&prof_dump_mtx, "prof_dump", |
| 2239 | WITNESS_RANK_PROF_DUMP)) |
Jason Evans | 4f37ef6 | 2014-01-16 13:23:56 -0800 | [diff] [blame] | 2240 | return (true); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 2241 | |
Jason Evans | 57efa7b | 2014-10-08 17:57:19 -0700 | [diff] [blame] | 2242 | if (opt_prof_final && opt_prof_prefix[0] != '\0' && |
| 2243 | atexit(prof_fdump) != 0) { |
Jason Evans | 698805c | 2010-03-03 17:45:38 -0800 | [diff] [blame] | 2244 | malloc_write("<jemalloc>: Error in atexit()\n"); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 2245 | if (opt_abort) |
| 2246 | abort(); |
| 2247 | } |
Jason Evans | 6da5418 | 2012-03-23 18:05:51 -0700 | [diff] [blame] | 2248 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 2249 | gctx_locks = (malloc_mutex_t *)base_alloc(tsdn, PROF_NCTX_LOCKS |
| 2250 | * sizeof(malloc_mutex_t)); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 2251 | if (gctx_locks == NULL) |
Jason Evans | 6da5418 | 2012-03-23 18:05:51 -0700 | [diff] [blame] | 2252 | return (true); |
| 2253 | for (i = 0; i < PROF_NCTX_LOCKS; i++) { |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 2254 | if (malloc_mutex_init(&gctx_locks[i], "prof_gctx", |
| 2255 | WITNESS_RANK_PROF_GCTX)) |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 2256 | return (true); |
| 2257 | } |
| 2258 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 2259 | tdata_locks = (malloc_mutex_t *)base_alloc(tsdn, |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 2260 | PROF_NTDATA_LOCKS * sizeof(malloc_mutex_t)); |
Jason Evans | 602c8e0 | 2014-08-18 16:22:13 -0700 | [diff] [blame] | 2261 | if (tdata_locks == NULL) |
| 2262 | return (true); |
| 2263 | for (i = 0; i < PROF_NTDATA_LOCKS; i++) { |
Jason Evans | b2c0d63 | 2016-04-13 23:36:15 -0700 | [diff] [blame] | 2264 | if (malloc_mutex_init(&tdata_locks[i], "prof_tdata", |
| 2265 | WITNESS_RANK_PROF_TDATA)) |
Jason Evans | 6da5418 | 2012-03-23 18:05:51 -0700 | [diff] [blame] | 2266 | return (true); |
| 2267 | } |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 2268 | } |
| 2269 | |
Jason Evans | b27805b | 2010-02-10 18:15:53 -0800 | [diff] [blame] | 2270 | #ifdef JEMALLOC_PROF_LIBGCC |
| 2271 | /* |
| 2272 | * Cause the backtracing machinery to allocate its internal state |
| 2273 | * before enabling profiling. |
| 2274 | */ |
| 2275 | _Unwind_Backtrace(prof_unwind_init_callback, NULL); |
| 2276 | #endif |
| 2277 | |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 2278 | prof_booted = true; |
| 2279 | |
| 2280 | return (false); |
| 2281 | } |
| 2282 | |
Jason Evans | 20f1fc9 | 2012-10-09 14:46:22 -0700 | [diff] [blame] | 2283 | void |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 2284 | prof_prefork0(tsdn_t *tsdn) |
Jason Evans | 20f1fc9 | 2012-10-09 14:46:22 -0700 | [diff] [blame] | 2285 | { |
| 2286 | |
| 2287 | if (opt_prof) { |
| 2288 | unsigned i; |
| 2289 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 2290 | malloc_mutex_prefork(tsdn, &prof_dump_mtx); |
| 2291 | malloc_mutex_prefork(tsdn, &bt2gctx_mtx); |
| 2292 | malloc_mutex_prefork(tsdn, &tdatas_mtx); |
Jason Evans | 9d8f3d2 | 2014-09-11 18:06:30 -0700 | [diff] [blame] | 2293 | for (i = 0; i < PROF_NTDATA_LOCKS; i++) |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 2294 | malloc_mutex_prefork(tsdn, &tdata_locks[i]); |
Jason Evans | 174c0c3 | 2016-04-25 23:14:40 -0700 | [diff] [blame] | 2295 | for (i = 0; i < PROF_NCTX_LOCKS; i++) |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 2296 | malloc_mutex_prefork(tsdn, &gctx_locks[i]); |
Jason Evans | 174c0c3 | 2016-04-25 23:14:40 -0700 | [diff] [blame] | 2297 | } |
| 2298 | } |
| 2299 | |
| 2300 | void |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 2301 | prof_prefork1(tsdn_t *tsdn) |
Jason Evans | 174c0c3 | 2016-04-25 23:14:40 -0700 | [diff] [blame] | 2302 | { |
| 2303 | |
| 2304 | if (opt_prof) { |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 2305 | malloc_mutex_prefork(tsdn, &prof_active_mtx); |
| 2306 | malloc_mutex_prefork(tsdn, &prof_dump_seq_mtx); |
| 2307 | malloc_mutex_prefork(tsdn, &prof_gdump_mtx); |
| 2308 | malloc_mutex_prefork(tsdn, &next_thr_uid_mtx); |
| 2309 | malloc_mutex_prefork(tsdn, &prof_thread_active_init_mtx); |
Jason Evans | 20f1fc9 | 2012-10-09 14:46:22 -0700 | [diff] [blame] | 2310 | } |
| 2311 | } |
| 2312 | |
| 2313 | void |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 2314 | prof_postfork_parent(tsdn_t *tsdn) |
Jason Evans | 20f1fc9 | 2012-10-09 14:46:22 -0700 | [diff] [blame] | 2315 | { |
| 2316 | |
| 2317 | if (opt_prof) { |
| 2318 | unsigned i; |
| 2319 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 2320 | malloc_mutex_postfork_parent(tsdn, |
| 2321 | &prof_thread_active_init_mtx); |
| 2322 | malloc_mutex_postfork_parent(tsdn, &next_thr_uid_mtx); |
| 2323 | malloc_mutex_postfork_parent(tsdn, &prof_gdump_mtx); |
| 2324 | malloc_mutex_postfork_parent(tsdn, &prof_dump_seq_mtx); |
| 2325 | malloc_mutex_postfork_parent(tsdn, &prof_active_mtx); |
Jason Evans | 20f1fc9 | 2012-10-09 14:46:22 -0700 | [diff] [blame] | 2326 | for (i = 0; i < PROF_NCTX_LOCKS; i++) |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 2327 | malloc_mutex_postfork_parent(tsdn, &gctx_locks[i]); |
Jason Evans | 174c0c3 | 2016-04-25 23:14:40 -0700 | [diff] [blame] | 2328 | for (i = 0; i < PROF_NTDATA_LOCKS; i++) |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 2329 | malloc_mutex_postfork_parent(tsdn, &tdata_locks[i]); |
| 2330 | malloc_mutex_postfork_parent(tsdn, &tdatas_mtx); |
| 2331 | malloc_mutex_postfork_parent(tsdn, &bt2gctx_mtx); |
| 2332 | malloc_mutex_postfork_parent(tsdn, &prof_dump_mtx); |
Jason Evans | 20f1fc9 | 2012-10-09 14:46:22 -0700 | [diff] [blame] | 2333 | } |
| 2334 | } |
| 2335 | |
| 2336 | void |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 2337 | prof_postfork_child(tsdn_t *tsdn) |
Jason Evans | 20f1fc9 | 2012-10-09 14:46:22 -0700 | [diff] [blame] | 2338 | { |
| 2339 | |
| 2340 | if (opt_prof) { |
| 2341 | unsigned i; |
| 2342 | |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 2343 | malloc_mutex_postfork_child(tsdn, &prof_thread_active_init_mtx); |
| 2344 | malloc_mutex_postfork_child(tsdn, &next_thr_uid_mtx); |
| 2345 | malloc_mutex_postfork_child(tsdn, &prof_gdump_mtx); |
| 2346 | malloc_mutex_postfork_child(tsdn, &prof_dump_seq_mtx); |
| 2347 | malloc_mutex_postfork_child(tsdn, &prof_active_mtx); |
Jason Evans | 20f1fc9 | 2012-10-09 14:46:22 -0700 | [diff] [blame] | 2348 | for (i = 0; i < PROF_NCTX_LOCKS; i++) |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 2349 | malloc_mutex_postfork_child(tsdn, &gctx_locks[i]); |
Jason Evans | 174c0c3 | 2016-04-25 23:14:40 -0700 | [diff] [blame] | 2350 | for (i = 0; i < PROF_NTDATA_LOCKS; i++) |
Jason Evans | c1e00ef | 2016-05-10 22:21:10 -0700 | [diff] [blame^] | 2351 | malloc_mutex_postfork_child(tsdn, &tdata_locks[i]); |
| 2352 | malloc_mutex_postfork_child(tsdn, &tdatas_mtx); |
| 2353 | malloc_mutex_postfork_child(tsdn, &bt2gctx_mtx); |
| 2354 | malloc_mutex_postfork_child(tsdn, &prof_dump_mtx); |
Jason Evans | 20f1fc9 | 2012-10-09 14:46:22 -0700 | [diff] [blame] | 2355 | } |
| 2356 | } |
| 2357 | |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 2358 | /******************************************************************************/ |