Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1 | #define JEMALLOC_PROF_C_ |
Jason Evans | 376b152 | 2010-02-11 14:45:59 -0800 | [diff] [blame] | 2 | #include "jemalloc/internal/jemalloc_internal.h" |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 3 | /******************************************************************************/ |
| 4 | |
| 5 | #ifdef JEMALLOC_PROF_LIBUNWIND |
| 6 | #define UNW_LOCAL_ONLY |
| 7 | #include <libunwind.h> |
| 8 | #endif |
| 9 | |
Jason Evans | 77f350b | 2011-03-15 22:23:12 -0700 | [diff] [blame] | 10 | #ifdef JEMALLOC_PROF_LIBGCC |
| 11 | #include <unwind.h> |
| 12 | #endif |
| 13 | |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 14 | /******************************************************************************/ |
| 15 | /* Data. */ |
| 16 | |
| 17 | bool opt_prof = false; |
Jason Evans | f18c982 | 2010-03-31 18:43:24 -0700 | [diff] [blame] | 18 | bool opt_prof_active = true; |
Jason Evans | b9477e7 | 2010-03-01 20:15:26 -0800 | [diff] [blame] | 19 | size_t opt_lg_prof_bt_max = LG_PROF_BT_MAX_DEFAULT; |
| 20 | size_t opt_lg_prof_sample = LG_PROF_SAMPLE_DEFAULT; |
Jason Evans | a02fc08 | 2010-03-31 17:35:51 -0700 | [diff] [blame] | 21 | ssize_t opt_lg_prof_interval = LG_PROF_INTERVAL_DEFAULT; |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 22 | bool opt_prof_gdump = false; |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 23 | bool opt_prof_leak = false; |
Jason Evans | a881cd2 | 2010-10-02 15:18:50 -0700 | [diff] [blame] | 24 | bool opt_prof_accum = true; |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 25 | char opt_prof_prefix[PATH_MAX + 1]; |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 26 | |
Jason Evans | d34f9e7 | 2010-02-11 13:19:21 -0800 | [diff] [blame] | 27 | uint64_t prof_interval; |
Jason Evans | 0b270a9 | 2010-03-31 16:45:04 -0700 | [diff] [blame] | 28 | bool prof_promote; |
Jason Evans | d34f9e7 | 2010-02-11 13:19:21 -0800 | [diff] [blame] | 29 | |
Jason Evans | 4d6a134 | 2010-10-20 19:05:59 -0700 | [diff] [blame] | 30 | unsigned prof_bt_max; |
| 31 | |
| 32 | #ifndef NO_TLS |
| 33 | __thread prof_tdata_t *prof_tdata_tls |
| 34 | JEMALLOC_ATTR(tls_model("initial-exec")); |
| 35 | #endif |
| 36 | pthread_key_t prof_tdata_tsd; |
| 37 | |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 38 | /* |
| 39 | * Global hash of (prof_bt_t *)-->(prof_ctx_t *). This is the master data |
Jason Evans | a881cd2 | 2010-10-02 15:18:50 -0700 | [diff] [blame] | 40 | * structure that knows about all backtraces currently captured. |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 41 | */ |
| 42 | static ckh_t bt2ctx; |
| 43 | static malloc_mutex_t bt2ctx_mtx; |
| 44 | |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 45 | static malloc_mutex_t prof_dump_seq_mtx; |
| 46 | static uint64_t prof_dump_seq; |
| 47 | static uint64_t prof_dump_iseq; |
| 48 | static uint64_t prof_dump_mseq; |
| 49 | static uint64_t prof_dump_useq; |
| 50 | |
| 51 | /* |
| 52 | * This buffer is rather large for stack allocation, so use a single buffer for |
| 53 | * all profile dumps. The buffer is implicitly protected by bt2ctx_mtx, since |
| 54 | * it must be locked anyway during dumping. |
| 55 | */ |
| 56 | static char prof_dump_buf[PROF_DUMP_BUF_SIZE]; |
| 57 | static unsigned prof_dump_buf_end; |
| 58 | static int prof_dump_fd; |
| 59 | |
| 60 | /* Do not dump any profiles until bootstrapping is complete. */ |
| 61 | static bool prof_booted = false; |
| 62 | |
| 63 | static malloc_mutex_t enq_mtx; |
| 64 | static bool enq; |
Jason Evans | d34f9e7 | 2010-02-11 13:19:21 -0800 | [diff] [blame] | 65 | static bool enq_idump; |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 66 | static bool enq_gdump; |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 67 | |
| 68 | /******************************************************************************/ |
| 69 | /* Function prototypes for non-inline static functions. */ |
| 70 | |
| 71 | static prof_bt_t *bt_dup(prof_bt_t *bt); |
Jason Evans | a881cd2 | 2010-10-02 15:18:50 -0700 | [diff] [blame] | 72 | static void bt_destroy(prof_bt_t *bt); |
Jason Evans | b27805b | 2010-02-10 18:15:53 -0800 | [diff] [blame] | 73 | #ifdef JEMALLOC_PROF_LIBGCC |
| 74 | static _Unwind_Reason_Code prof_unwind_init_callback( |
| 75 | struct _Unwind_Context *context, void *arg); |
| 76 | static _Unwind_Reason_Code prof_unwind_callback( |
| 77 | struct _Unwind_Context *context, void *arg); |
| 78 | #endif |
Jason Evans | 22ca855 | 2010-03-02 11:57:30 -0800 | [diff] [blame] | 79 | static bool prof_flush(bool propagate_err); |
| 80 | static bool prof_write(const char *s, bool propagate_err); |
Jason Evans | a881cd2 | 2010-10-02 15:18:50 -0700 | [diff] [blame] | 81 | static void prof_ctx_sum(prof_ctx_t *ctx, prof_cnt_t *cnt_all, |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 82 | size_t *leak_nctx); |
Jason Evans | a881cd2 | 2010-10-02 15:18:50 -0700 | [diff] [blame] | 83 | static void prof_ctx_destroy(prof_ctx_t *ctx); |
| 84 | static void prof_ctx_merge(prof_ctx_t *ctx, prof_thr_cnt_t *cnt); |
Jason Evans | 22ca855 | 2010-03-02 11:57:30 -0800 | [diff] [blame] | 85 | static bool prof_dump_ctx(prof_ctx_t *ctx, prof_bt_t *bt, |
| 86 | bool propagate_err); |
| 87 | static bool prof_dump_maps(bool propagate_err); |
| 88 | static bool prof_dump(const char *filename, bool leakcheck, |
| 89 | bool propagate_err); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 90 | static void prof_dump_filename(char *filename, char v, int64_t vseq); |
| 91 | static void prof_fdump(void); |
| 92 | static void prof_bt_hash(const void *key, unsigned minbits, size_t *hash1, |
| 93 | size_t *hash2); |
| 94 | static bool prof_bt_keycomp(const void *k1, const void *k2); |
Jason Evans | 4d6a134 | 2010-10-20 19:05:59 -0700 | [diff] [blame] | 95 | static void prof_tdata_cleanup(void *arg); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 96 | |
| 97 | /******************************************************************************/ |
| 98 | |
Jason Evans | 4d6a134 | 2010-10-20 19:05:59 -0700 | [diff] [blame] | 99 | void |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 100 | bt_init(prof_bt_t *bt, void **vec) |
| 101 | { |
| 102 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 103 | cassert(config_prof); |
| 104 | |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 105 | bt->vec = vec; |
| 106 | bt->len = 0; |
| 107 | } |
| 108 | |
Jason Evans | a881cd2 | 2010-10-02 15:18:50 -0700 | [diff] [blame] | 109 | static void |
| 110 | bt_destroy(prof_bt_t *bt) |
| 111 | { |
| 112 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 113 | cassert(config_prof); |
| 114 | |
Jason Evans | a881cd2 | 2010-10-02 15:18:50 -0700 | [diff] [blame] | 115 | idalloc(bt); |
| 116 | } |
| 117 | |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 118 | static prof_bt_t * |
| 119 | bt_dup(prof_bt_t *bt) |
| 120 | { |
| 121 | prof_bt_t *ret; |
| 122 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 123 | cassert(config_prof); |
| 124 | |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 125 | /* |
| 126 | * Create a single allocation that has space for vec immediately |
| 127 | * following the prof_bt_t structure. The backtraces that get |
| 128 | * stored in the backtrace caches are copied from stack-allocated |
| 129 | * temporary variables, so size is known at creation time. Making this |
| 130 | * a contiguous object improves cache locality. |
| 131 | */ |
| 132 | ret = (prof_bt_t *)imalloc(QUANTUM_CEILING(sizeof(prof_bt_t)) + |
| 133 | (bt->len * sizeof(void *))); |
| 134 | if (ret == NULL) |
| 135 | return (NULL); |
| 136 | ret->vec = (void **)((uintptr_t)ret + |
| 137 | QUANTUM_CEILING(sizeof(prof_bt_t))); |
| 138 | memcpy(ret->vec, bt->vec, bt->len * sizeof(void *)); |
| 139 | ret->len = bt->len; |
| 140 | |
| 141 | return (ret); |
| 142 | } |
| 143 | |
| 144 | static inline void |
| 145 | prof_enter(void) |
| 146 | { |
| 147 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 148 | cassert(config_prof); |
| 149 | |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 150 | malloc_mutex_lock(&enq_mtx); |
| 151 | enq = true; |
| 152 | malloc_mutex_unlock(&enq_mtx); |
| 153 | |
| 154 | malloc_mutex_lock(&bt2ctx_mtx); |
| 155 | } |
| 156 | |
| 157 | static inline void |
| 158 | prof_leave(void) |
| 159 | { |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 160 | bool idump, gdump; |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 161 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 162 | cassert(config_prof); |
| 163 | |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 164 | malloc_mutex_unlock(&bt2ctx_mtx); |
| 165 | |
| 166 | malloc_mutex_lock(&enq_mtx); |
| 167 | enq = false; |
Jason Evans | d34f9e7 | 2010-02-11 13:19:21 -0800 | [diff] [blame] | 168 | idump = enq_idump; |
| 169 | enq_idump = false; |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 170 | gdump = enq_gdump; |
| 171 | enq_gdump = false; |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 172 | malloc_mutex_unlock(&enq_mtx); |
| 173 | |
Jason Evans | d34f9e7 | 2010-02-11 13:19:21 -0800 | [diff] [blame] | 174 | if (idump) |
| 175 | prof_idump(); |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 176 | if (gdump) |
| 177 | prof_gdump(); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 178 | } |
| 179 | |
Jason Evans | 77f350b | 2011-03-15 22:23:12 -0700 | [diff] [blame] | 180 | #ifdef JEMALLOC_PROF_LIBUNWIND |
Jason Evans | 4d6a134 | 2010-10-20 19:05:59 -0700 | [diff] [blame] | 181 | void |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 182 | prof_backtrace(prof_bt_t *bt, unsigned nignore, unsigned max) |
| 183 | { |
| 184 | unw_context_t uc; |
| 185 | unw_cursor_t cursor; |
| 186 | unsigned i; |
| 187 | int err; |
| 188 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 189 | cassert(config_prof); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 190 | assert(bt->len == 0); |
| 191 | assert(bt->vec != NULL); |
| 192 | assert(max <= (1U << opt_lg_prof_bt_max)); |
| 193 | |
| 194 | unw_getcontext(&uc); |
| 195 | unw_init_local(&cursor, &uc); |
| 196 | |
Jason Evans | 9f949f9 | 2011-03-22 20:44:40 -0700 | [diff] [blame] | 197 | /* Throw away (nignore+1) stack frames, if that many exist. */ |
| 198 | for (i = 0; i < nignore + 1; i++) { |
| 199 | err = unw_step(&cursor); |
| 200 | if (err <= 0) |
| 201 | return; |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 202 | } |
| 203 | |
Jason Evans | 9f949f9 | 2011-03-22 20:44:40 -0700 | [diff] [blame] | 204 | /* |
| 205 | * Iterate over stack frames until there are no more, or until no space |
| 206 | * remains in bt. |
| 207 | */ |
| 208 | for (i = 0; i < max; i++) { |
| 209 | unw_get_reg(&cursor, UNW_REG_IP, (unw_word_t *)&bt->vec[i]); |
| 210 | bt->len++; |
| 211 | err = unw_step(&cursor); |
| 212 | if (err <= 0) |
| 213 | break; |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 214 | } |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 215 | } |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 216 | #elif (defined(JEMALLOC_PROF_LIBGCC)) |
Jason Evans | 77f350b | 2011-03-15 22:23:12 -0700 | [diff] [blame] | 217 | static _Unwind_Reason_Code |
| 218 | prof_unwind_init_callback(struct _Unwind_Context *context, void *arg) |
| 219 | { |
| 220 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 221 | cassert(config_prof); |
| 222 | |
Jason Evans | 77f350b | 2011-03-15 22:23:12 -0700 | [diff] [blame] | 223 | return (_URC_NO_REASON); |
| 224 | } |
| 225 | |
| 226 | static _Unwind_Reason_Code |
| 227 | prof_unwind_callback(struct _Unwind_Context *context, void *arg) |
| 228 | { |
| 229 | prof_unwind_data_t *data = (prof_unwind_data_t *)arg; |
| 230 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 231 | cassert(config_prof); |
| 232 | |
Jason Evans | 77f350b | 2011-03-15 22:23:12 -0700 | [diff] [blame] | 233 | if (data->nignore > 0) |
| 234 | data->nignore--; |
| 235 | else { |
| 236 | data->bt->vec[data->bt->len] = (void *)_Unwind_GetIP(context); |
| 237 | data->bt->len++; |
| 238 | if (data->bt->len == data->max) |
| 239 | return (_URC_END_OF_STACK); |
| 240 | } |
| 241 | |
| 242 | return (_URC_NO_REASON); |
| 243 | } |
| 244 | |
| 245 | void |
| 246 | prof_backtrace(prof_bt_t *bt, unsigned nignore, unsigned max) |
| 247 | { |
| 248 | prof_unwind_data_t data = {bt, nignore, max}; |
| 249 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 250 | cassert(config_prof); |
| 251 | |
Jason Evans | 77f350b | 2011-03-15 22:23:12 -0700 | [diff] [blame] | 252 | _Unwind_Backtrace(prof_unwind_callback, &data); |
| 253 | } |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 254 | #elif (defined(JEMALLOC_PROF_GCC)) |
Jason Evans | 4d6a134 | 2010-10-20 19:05:59 -0700 | [diff] [blame] | 255 | void |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 256 | prof_backtrace(prof_bt_t *bt, unsigned nignore, unsigned max) |
| 257 | { |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 258 | #define BT_FRAME(i) \ |
Jason Evans | e4f7846 | 2010-10-22 10:45:59 -0700 | [diff] [blame] | 259 | if ((i) < nignore + max) { \ |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 260 | void *p; \ |
| 261 | if (__builtin_frame_address(i) == 0) \ |
Jason Evans | b27805b | 2010-02-10 18:15:53 -0800 | [diff] [blame] | 262 | return; \ |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 263 | p = __builtin_return_address(i); \ |
| 264 | if (p == NULL) \ |
Jason Evans | b27805b | 2010-02-10 18:15:53 -0800 | [diff] [blame] | 265 | return; \ |
Jason Evans | e4f7846 | 2010-10-22 10:45:59 -0700 | [diff] [blame] | 266 | if (i >= nignore) { \ |
| 267 | bt->vec[(i) - nignore] = p; \ |
| 268 | bt->len = (i) - nignore + 1; \ |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 269 | } \ |
| 270 | } else \ |
Jason Evans | b27805b | 2010-02-10 18:15:53 -0800 | [diff] [blame] | 271 | return; |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 272 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 273 | cassert(config_prof); |
Jason Evans | b04a940 | 2010-10-27 19:47:40 -0700 | [diff] [blame] | 274 | assert(nignore <= 3); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 275 | assert(max <= (1U << opt_lg_prof_bt_max)); |
| 276 | |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 277 | BT_FRAME(0) |
| 278 | BT_FRAME(1) |
| 279 | BT_FRAME(2) |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 280 | BT_FRAME(3) |
| 281 | BT_FRAME(4) |
| 282 | BT_FRAME(5) |
| 283 | BT_FRAME(6) |
| 284 | BT_FRAME(7) |
| 285 | BT_FRAME(8) |
| 286 | BT_FRAME(9) |
| 287 | |
| 288 | BT_FRAME(10) |
| 289 | BT_FRAME(11) |
| 290 | BT_FRAME(12) |
| 291 | BT_FRAME(13) |
| 292 | BT_FRAME(14) |
| 293 | BT_FRAME(15) |
| 294 | BT_FRAME(16) |
| 295 | BT_FRAME(17) |
| 296 | BT_FRAME(18) |
| 297 | BT_FRAME(19) |
| 298 | |
| 299 | BT_FRAME(20) |
| 300 | BT_FRAME(21) |
| 301 | BT_FRAME(22) |
| 302 | BT_FRAME(23) |
| 303 | BT_FRAME(24) |
| 304 | BT_FRAME(25) |
| 305 | BT_FRAME(26) |
| 306 | BT_FRAME(27) |
| 307 | BT_FRAME(28) |
| 308 | BT_FRAME(29) |
| 309 | |
| 310 | BT_FRAME(30) |
| 311 | BT_FRAME(31) |
| 312 | BT_FRAME(32) |
| 313 | BT_FRAME(33) |
| 314 | BT_FRAME(34) |
| 315 | BT_FRAME(35) |
| 316 | BT_FRAME(36) |
| 317 | BT_FRAME(37) |
| 318 | BT_FRAME(38) |
| 319 | BT_FRAME(39) |
| 320 | |
| 321 | BT_FRAME(40) |
| 322 | BT_FRAME(41) |
| 323 | BT_FRAME(42) |
| 324 | BT_FRAME(43) |
| 325 | BT_FRAME(44) |
| 326 | BT_FRAME(45) |
| 327 | BT_FRAME(46) |
| 328 | BT_FRAME(47) |
| 329 | BT_FRAME(48) |
| 330 | BT_FRAME(49) |
| 331 | |
| 332 | BT_FRAME(50) |
| 333 | BT_FRAME(51) |
| 334 | BT_FRAME(52) |
| 335 | BT_FRAME(53) |
| 336 | BT_FRAME(54) |
| 337 | BT_FRAME(55) |
| 338 | BT_FRAME(56) |
| 339 | BT_FRAME(57) |
| 340 | BT_FRAME(58) |
| 341 | BT_FRAME(59) |
| 342 | |
| 343 | BT_FRAME(60) |
| 344 | BT_FRAME(61) |
| 345 | BT_FRAME(62) |
| 346 | BT_FRAME(63) |
| 347 | BT_FRAME(64) |
| 348 | BT_FRAME(65) |
| 349 | BT_FRAME(66) |
| 350 | BT_FRAME(67) |
| 351 | BT_FRAME(68) |
| 352 | BT_FRAME(69) |
| 353 | |
| 354 | BT_FRAME(70) |
| 355 | BT_FRAME(71) |
| 356 | BT_FRAME(72) |
| 357 | BT_FRAME(73) |
| 358 | BT_FRAME(74) |
| 359 | BT_FRAME(75) |
| 360 | BT_FRAME(76) |
| 361 | BT_FRAME(77) |
| 362 | BT_FRAME(78) |
| 363 | BT_FRAME(79) |
| 364 | |
| 365 | BT_FRAME(80) |
| 366 | BT_FRAME(81) |
| 367 | BT_FRAME(82) |
| 368 | BT_FRAME(83) |
| 369 | BT_FRAME(84) |
| 370 | BT_FRAME(85) |
| 371 | BT_FRAME(86) |
| 372 | BT_FRAME(87) |
| 373 | BT_FRAME(88) |
| 374 | BT_FRAME(89) |
| 375 | |
| 376 | BT_FRAME(90) |
| 377 | BT_FRAME(91) |
| 378 | BT_FRAME(92) |
| 379 | BT_FRAME(93) |
| 380 | BT_FRAME(94) |
| 381 | BT_FRAME(95) |
| 382 | BT_FRAME(96) |
| 383 | BT_FRAME(97) |
| 384 | BT_FRAME(98) |
| 385 | BT_FRAME(99) |
| 386 | |
| 387 | BT_FRAME(100) |
| 388 | BT_FRAME(101) |
| 389 | BT_FRAME(102) |
| 390 | BT_FRAME(103) |
| 391 | BT_FRAME(104) |
| 392 | BT_FRAME(105) |
| 393 | BT_FRAME(106) |
| 394 | BT_FRAME(107) |
| 395 | BT_FRAME(108) |
| 396 | BT_FRAME(109) |
| 397 | |
| 398 | BT_FRAME(110) |
| 399 | BT_FRAME(111) |
| 400 | BT_FRAME(112) |
| 401 | BT_FRAME(113) |
| 402 | BT_FRAME(114) |
| 403 | BT_FRAME(115) |
| 404 | BT_FRAME(116) |
| 405 | BT_FRAME(117) |
| 406 | BT_FRAME(118) |
| 407 | BT_FRAME(119) |
| 408 | |
| 409 | BT_FRAME(120) |
| 410 | BT_FRAME(121) |
| 411 | BT_FRAME(122) |
| 412 | BT_FRAME(123) |
| 413 | BT_FRAME(124) |
| 414 | BT_FRAME(125) |
| 415 | BT_FRAME(126) |
| 416 | BT_FRAME(127) |
| 417 | |
Jason Evans | b04a940 | 2010-10-27 19:47:40 -0700 | [diff] [blame] | 418 | /* Extras to compensate for nignore. */ |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 419 | BT_FRAME(128) |
| 420 | BT_FRAME(129) |
| 421 | BT_FRAME(130) |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 422 | #undef BT_FRAME |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 423 | } |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 424 | #else |
| 425 | void |
| 426 | prof_backtrace(prof_bt_t *bt, unsigned nignore, unsigned max) |
| 427 | { |
| 428 | |
| 429 | cassert(config_prof); |
| 430 | assert(false); |
| 431 | } |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 432 | #endif |
| 433 | |
Jason Evans | 4d6a134 | 2010-10-20 19:05:59 -0700 | [diff] [blame] | 434 | prof_thr_cnt_t * |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 435 | prof_lookup(prof_bt_t *bt) |
| 436 | { |
Jason Evans | 075e77c | 2010-09-20 19:53:25 -0700 | [diff] [blame] | 437 | union { |
| 438 | prof_thr_cnt_t *p; |
| 439 | void *v; |
| 440 | } ret; |
Jason Evans | 4d6a134 | 2010-10-20 19:05:59 -0700 | [diff] [blame] | 441 | prof_tdata_t *prof_tdata; |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 442 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 443 | cassert(config_prof); |
| 444 | |
Jason Evans | 4d6a134 | 2010-10-20 19:05:59 -0700 | [diff] [blame] | 445 | prof_tdata = PROF_TCACHE_GET(); |
| 446 | if (prof_tdata == NULL) { |
| 447 | prof_tdata = prof_tdata_init(); |
| 448 | if (prof_tdata == NULL) |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 449 | return (NULL); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 450 | } |
| 451 | |
Jason Evans | 4d6a134 | 2010-10-20 19:05:59 -0700 | [diff] [blame] | 452 | if (ckh_search(&prof_tdata->bt2cnt, bt, NULL, &ret.v)) { |
Jason Evans | 075e77c | 2010-09-20 19:53:25 -0700 | [diff] [blame] | 453 | union { |
| 454 | prof_bt_t *p; |
| 455 | void *v; |
| 456 | } btkey; |
| 457 | union { |
| 458 | prof_ctx_t *p; |
| 459 | void *v; |
| 460 | } ctx; |
Jason Evans | 10e4523 | 2011-01-14 17:27:44 -0800 | [diff] [blame] | 461 | bool new_ctx; |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 462 | |
| 463 | /* |
| 464 | * This thread's cache lacks bt. Look for it in the global |
| 465 | * cache. |
| 466 | */ |
| 467 | prof_enter(); |
Jason Evans | 075e77c | 2010-09-20 19:53:25 -0700 | [diff] [blame] | 468 | if (ckh_search(&bt2ctx, bt, &btkey.v, &ctx.v)) { |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 469 | /* bt has never been seen before. Insert it. */ |
Jason Evans | 075e77c | 2010-09-20 19:53:25 -0700 | [diff] [blame] | 470 | ctx.v = imalloc(sizeof(prof_ctx_t)); |
| 471 | if (ctx.v == NULL) { |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 472 | prof_leave(); |
| 473 | return (NULL); |
| 474 | } |
Jason Evans | 075e77c | 2010-09-20 19:53:25 -0700 | [diff] [blame] | 475 | btkey.p = bt_dup(bt); |
| 476 | if (btkey.v == NULL) { |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 477 | prof_leave(); |
Jason Evans | 075e77c | 2010-09-20 19:53:25 -0700 | [diff] [blame] | 478 | idalloc(ctx.v); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 479 | return (NULL); |
| 480 | } |
Jason Evans | 075e77c | 2010-09-20 19:53:25 -0700 | [diff] [blame] | 481 | ctx.p->bt = btkey.p; |
| 482 | if (malloc_mutex_init(&ctx.p->lock)) { |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 483 | prof_leave(); |
Jason Evans | 075e77c | 2010-09-20 19:53:25 -0700 | [diff] [blame] | 484 | idalloc(btkey.v); |
| 485 | idalloc(ctx.v); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 486 | return (NULL); |
| 487 | } |
Jason Evans | 075e77c | 2010-09-20 19:53:25 -0700 | [diff] [blame] | 488 | memset(&ctx.p->cnt_merged, 0, sizeof(prof_cnt_t)); |
| 489 | ql_new(&ctx.p->cnts_ql); |
| 490 | if (ckh_insert(&bt2ctx, btkey.v, ctx.v)) { |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 491 | /* OOM. */ |
| 492 | prof_leave(); |
Jason Evans | a881cd2 | 2010-10-02 15:18:50 -0700 | [diff] [blame] | 493 | malloc_mutex_destroy(&ctx.p->lock); |
Jason Evans | 075e77c | 2010-09-20 19:53:25 -0700 | [diff] [blame] | 494 | idalloc(btkey.v); |
| 495 | idalloc(ctx.v); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 496 | return (NULL); |
| 497 | } |
Jason Evans | 10e4523 | 2011-01-14 17:27:44 -0800 | [diff] [blame] | 498 | /* |
| 499 | * Artificially raise curobjs, in order to avoid a race |
| 500 | * condition with prof_ctx_merge()/prof_ctx_destroy(). |
Jason Evans | a9076c9 | 2011-08-30 23:40:11 -0700 | [diff] [blame] | 501 | * |
| 502 | * No locking is necessary for ctx here because no other |
| 503 | * threads have had the opportunity to fetch it from |
| 504 | * bt2ctx yet. |
Jason Evans | 10e4523 | 2011-01-14 17:27:44 -0800 | [diff] [blame] | 505 | */ |
| 506 | ctx.p->cnt_merged.curobjs++; |
| 507 | new_ctx = true; |
Jason Evans | a9076c9 | 2011-08-30 23:40:11 -0700 | [diff] [blame] | 508 | } else { |
| 509 | /* |
| 510 | * Artificially raise curobjs, in order to avoid a race |
| 511 | * condition with prof_ctx_merge()/prof_ctx_destroy(). |
| 512 | */ |
| 513 | malloc_mutex_lock(&ctx.p->lock); |
| 514 | ctx.p->cnt_merged.curobjs++; |
| 515 | malloc_mutex_unlock(&ctx.p->lock); |
Jason Evans | 10e4523 | 2011-01-14 17:27:44 -0800 | [diff] [blame] | 516 | new_ctx = false; |
Jason Evans | a9076c9 | 2011-08-30 23:40:11 -0700 | [diff] [blame] | 517 | } |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 518 | prof_leave(); |
| 519 | |
| 520 | /* Link a prof_thd_cnt_t into ctx for this thread. */ |
Jason Evans | 0b526ff | 2012-02-13 18:04:26 -0800 | [diff] [blame^] | 521 | if (ckh_count(&prof_tdata->bt2cnt) == PROF_TCMAX) { |
Jason Evans | 4d6a134 | 2010-10-20 19:05:59 -0700 | [diff] [blame] | 522 | assert(ckh_count(&prof_tdata->bt2cnt) > 0); |
Jason Evans | a881cd2 | 2010-10-02 15:18:50 -0700 | [diff] [blame] | 523 | /* |
Jason Evans | e4f7846 | 2010-10-22 10:45:59 -0700 | [diff] [blame] | 524 | * Flush the least recently used cnt in order to keep |
| 525 | * bt2cnt from becoming too large. |
Jason Evans | a881cd2 | 2010-10-02 15:18:50 -0700 | [diff] [blame] | 526 | */ |
Jason Evans | 4d6a134 | 2010-10-20 19:05:59 -0700 | [diff] [blame] | 527 | ret.p = ql_last(&prof_tdata->lru_ql, lru_link); |
Jason Evans | a881cd2 | 2010-10-02 15:18:50 -0700 | [diff] [blame] | 528 | assert(ret.v != NULL); |
Jason Evans | a9076c9 | 2011-08-30 23:40:11 -0700 | [diff] [blame] | 529 | if (ckh_remove(&prof_tdata->bt2cnt, ret.p->ctx->bt, |
| 530 | NULL, NULL)) |
| 531 | assert(false); |
Jason Evans | 4d6a134 | 2010-10-20 19:05:59 -0700 | [diff] [blame] | 532 | ql_remove(&prof_tdata->lru_ql, ret.p, lru_link); |
Jason Evans | a881cd2 | 2010-10-02 15:18:50 -0700 | [diff] [blame] | 533 | prof_ctx_merge(ret.p->ctx, ret.p); |
| 534 | /* ret can now be re-used. */ |
| 535 | } else { |
Jason Evans | 0b526ff | 2012-02-13 18:04:26 -0800 | [diff] [blame^] | 536 | assert(ckh_count(&prof_tdata->bt2cnt) < PROF_TCMAX); |
Jason Evans | a881cd2 | 2010-10-02 15:18:50 -0700 | [diff] [blame] | 537 | /* Allocate and partially initialize a new cnt. */ |
| 538 | ret.v = imalloc(sizeof(prof_thr_cnt_t)); |
Jason Evans | b04a940 | 2010-10-27 19:47:40 -0700 | [diff] [blame] | 539 | if (ret.p == NULL) { |
Jason Evans | 0cdd42e | 2011-08-09 19:06:06 -0700 | [diff] [blame] | 540 | if (new_ctx) |
| 541 | prof_ctx_destroy(ctx.p); |
Jason Evans | a881cd2 | 2010-10-02 15:18:50 -0700 | [diff] [blame] | 542 | return (NULL); |
Jason Evans | b04a940 | 2010-10-27 19:47:40 -0700 | [diff] [blame] | 543 | } |
Jason Evans | a881cd2 | 2010-10-02 15:18:50 -0700 | [diff] [blame] | 544 | ql_elm_new(ret.p, cnts_link); |
| 545 | ql_elm_new(ret.p, lru_link); |
| 546 | } |
| 547 | /* Finish initializing ret. */ |
Jason Evans | 075e77c | 2010-09-20 19:53:25 -0700 | [diff] [blame] | 548 | ret.p->ctx = ctx.p; |
| 549 | ret.p->epoch = 0; |
| 550 | memset(&ret.p->cnts, 0, sizeof(prof_cnt_t)); |
Jason Evans | 4d6a134 | 2010-10-20 19:05:59 -0700 | [diff] [blame] | 551 | if (ckh_insert(&prof_tdata->bt2cnt, btkey.v, ret.v)) { |
Jason Evans | 0cdd42e | 2011-08-09 19:06:06 -0700 | [diff] [blame] | 552 | if (new_ctx) |
| 553 | prof_ctx_destroy(ctx.p); |
Jason Evans | 075e77c | 2010-09-20 19:53:25 -0700 | [diff] [blame] | 554 | idalloc(ret.v); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 555 | return (NULL); |
| 556 | } |
Jason Evans | 4d6a134 | 2010-10-20 19:05:59 -0700 | [diff] [blame] | 557 | ql_head_insert(&prof_tdata->lru_ql, ret.p, lru_link); |
Jason Evans | 10e4523 | 2011-01-14 17:27:44 -0800 | [diff] [blame] | 558 | malloc_mutex_lock(&ctx.p->lock); |
Jason Evans | a881cd2 | 2010-10-02 15:18:50 -0700 | [diff] [blame] | 559 | ql_tail_insert(&ctx.p->cnts_ql, ret.p, cnts_link); |
Jason Evans | a9076c9 | 2011-08-30 23:40:11 -0700 | [diff] [blame] | 560 | ctx.p->cnt_merged.curobjs--; |
Jason Evans | 075e77c | 2010-09-20 19:53:25 -0700 | [diff] [blame] | 561 | malloc_mutex_unlock(&ctx.p->lock); |
Jason Evans | a881cd2 | 2010-10-02 15:18:50 -0700 | [diff] [blame] | 562 | } else { |
| 563 | /* Move ret to the front of the LRU. */ |
Jason Evans | 4d6a134 | 2010-10-20 19:05:59 -0700 | [diff] [blame] | 564 | ql_remove(&prof_tdata->lru_ql, ret.p, lru_link); |
| 565 | ql_head_insert(&prof_tdata->lru_ql, ret.p, lru_link); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 566 | } |
| 567 | |
Jason Evans | 075e77c | 2010-09-20 19:53:25 -0700 | [diff] [blame] | 568 | return (ret.p); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 569 | } |
| 570 | |
Jason Evans | 22ca855 | 2010-03-02 11:57:30 -0800 | [diff] [blame] | 571 | static bool |
| 572 | prof_flush(bool propagate_err) |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 573 | { |
Jason Evans | 22ca855 | 2010-03-02 11:57:30 -0800 | [diff] [blame] | 574 | bool ret = false; |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 575 | ssize_t err; |
| 576 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 577 | cassert(config_prof); |
| 578 | |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 579 | err = write(prof_dump_fd, prof_dump_buf, prof_dump_buf_end); |
| 580 | if (err == -1) { |
Jason Evans | 22ca855 | 2010-03-02 11:57:30 -0800 | [diff] [blame] | 581 | if (propagate_err == false) { |
Jason Evans | 698805c | 2010-03-03 17:45:38 -0800 | [diff] [blame] | 582 | malloc_write("<jemalloc>: write() failed during heap " |
| 583 | "profile flush\n"); |
Jason Evans | 22ca855 | 2010-03-02 11:57:30 -0800 | [diff] [blame] | 584 | if (opt_abort) |
| 585 | abort(); |
| 586 | } |
| 587 | ret = true; |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 588 | } |
| 589 | prof_dump_buf_end = 0; |
Jason Evans | 22ca855 | 2010-03-02 11:57:30 -0800 | [diff] [blame] | 590 | |
| 591 | return (ret); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 592 | } |
| 593 | |
Jason Evans | 22ca855 | 2010-03-02 11:57:30 -0800 | [diff] [blame] | 594 | static bool |
| 595 | prof_write(const char *s, bool propagate_err) |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 596 | { |
| 597 | unsigned i, slen, n; |
| 598 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 599 | cassert(config_prof); |
| 600 | |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 601 | i = 0; |
| 602 | slen = strlen(s); |
| 603 | while (i < slen) { |
| 604 | /* Flush the buffer if it is full. */ |
| 605 | if (prof_dump_buf_end == PROF_DUMP_BUF_SIZE) |
Jason Evans | 22ca855 | 2010-03-02 11:57:30 -0800 | [diff] [blame] | 606 | if (prof_flush(propagate_err) && propagate_err) |
| 607 | return (true); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 608 | |
| 609 | if (prof_dump_buf_end + slen <= PROF_DUMP_BUF_SIZE) { |
| 610 | /* Finish writing. */ |
| 611 | n = slen - i; |
| 612 | } else { |
| 613 | /* Write as much of s as will fit. */ |
| 614 | n = PROF_DUMP_BUF_SIZE - prof_dump_buf_end; |
| 615 | } |
| 616 | memcpy(&prof_dump_buf[prof_dump_buf_end], &s[i], n); |
| 617 | prof_dump_buf_end += n; |
| 618 | i += n; |
| 619 | } |
Jason Evans | 22ca855 | 2010-03-02 11:57:30 -0800 | [diff] [blame] | 620 | |
| 621 | return (false); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 622 | } |
| 623 | |
| 624 | static void |
Jason Evans | a881cd2 | 2010-10-02 15:18:50 -0700 | [diff] [blame] | 625 | prof_ctx_sum(prof_ctx_t *ctx, prof_cnt_t *cnt_all, size_t *leak_nctx) |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 626 | { |
| 627 | prof_thr_cnt_t *thr_cnt; |
| 628 | prof_cnt_t tcnt; |
| 629 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 630 | cassert(config_prof); |
| 631 | |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 632 | malloc_mutex_lock(&ctx->lock); |
| 633 | |
Jason Evans | a881cd2 | 2010-10-02 15:18:50 -0700 | [diff] [blame] | 634 | memcpy(&ctx->cnt_summed, &ctx->cnt_merged, sizeof(prof_cnt_t)); |
| 635 | ql_foreach(thr_cnt, &ctx->cnts_ql, cnts_link) { |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 636 | volatile unsigned *epoch = &thr_cnt->epoch; |
| 637 | |
| 638 | while (true) { |
| 639 | unsigned epoch0 = *epoch; |
| 640 | |
| 641 | /* Make sure epoch is even. */ |
| 642 | if (epoch0 & 1U) |
| 643 | continue; |
| 644 | |
| 645 | memcpy(&tcnt, &thr_cnt->cnts, sizeof(prof_cnt_t)); |
| 646 | |
| 647 | /* Terminate if epoch didn't change while reading. */ |
| 648 | if (*epoch == epoch0) |
| 649 | break; |
| 650 | } |
| 651 | |
Jason Evans | a881cd2 | 2010-10-02 15:18:50 -0700 | [diff] [blame] | 652 | ctx->cnt_summed.curobjs += tcnt.curobjs; |
| 653 | ctx->cnt_summed.curbytes += tcnt.curbytes; |
| 654 | if (opt_prof_accum) { |
| 655 | ctx->cnt_summed.accumobjs += tcnt.accumobjs; |
| 656 | ctx->cnt_summed.accumbytes += tcnt.accumbytes; |
| 657 | } |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 658 | } |
| 659 | |
Jason Evans | 9ce3bfd | 2010-10-02 22:39:59 -0700 | [diff] [blame] | 660 | if (ctx->cnt_summed.curobjs != 0) |
| 661 | (*leak_nctx)++; |
| 662 | |
Jason Evans | a881cd2 | 2010-10-02 15:18:50 -0700 | [diff] [blame] | 663 | /* Add to cnt_all. */ |
| 664 | cnt_all->curobjs += ctx->cnt_summed.curobjs; |
| 665 | cnt_all->curbytes += ctx->cnt_summed.curbytes; |
| 666 | if (opt_prof_accum) { |
| 667 | cnt_all->accumobjs += ctx->cnt_summed.accumobjs; |
| 668 | cnt_all->accumbytes += ctx->cnt_summed.accumbytes; |
| 669 | } |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 670 | |
| 671 | malloc_mutex_unlock(&ctx->lock); |
| 672 | } |
| 673 | |
Jason Evans | a881cd2 | 2010-10-02 15:18:50 -0700 | [diff] [blame] | 674 | static void |
| 675 | prof_ctx_destroy(prof_ctx_t *ctx) |
| 676 | { |
| 677 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 678 | cassert(config_prof); |
| 679 | |
Jason Evans | a881cd2 | 2010-10-02 15:18:50 -0700 | [diff] [blame] | 680 | /* |
| 681 | * Check that ctx is still unused by any thread cache before destroying |
Jason Evans | 0cdd42e | 2011-08-09 19:06:06 -0700 | [diff] [blame] | 682 | * it. prof_lookup() artificially raises ctx->cnt_merge.curobjs in |
| 683 | * order to avoid a race condition with this function, as does |
| 684 | * prof_ctx_merge() in order to avoid a race between the main body of |
| 685 | * prof_ctx_merge() and entry into this function. |
Jason Evans | a881cd2 | 2010-10-02 15:18:50 -0700 | [diff] [blame] | 686 | */ |
| 687 | prof_enter(); |
| 688 | malloc_mutex_lock(&ctx->lock); |
Jason Evans | b04a940 | 2010-10-27 19:47:40 -0700 | [diff] [blame] | 689 | if (ql_first(&ctx->cnts_ql) == NULL && ctx->cnt_merged.curobjs == 1) { |
Jason Evans | a881cd2 | 2010-10-02 15:18:50 -0700 | [diff] [blame] | 690 | assert(ctx->cnt_merged.curbytes == 0); |
| 691 | assert(ctx->cnt_merged.accumobjs == 0); |
| 692 | assert(ctx->cnt_merged.accumbytes == 0); |
| 693 | /* Remove ctx from bt2ctx. */ |
Jason Evans | a9076c9 | 2011-08-30 23:40:11 -0700 | [diff] [blame] | 694 | if (ckh_remove(&bt2ctx, ctx->bt, NULL, NULL)) |
| 695 | assert(false); |
Jason Evans | a881cd2 | 2010-10-02 15:18:50 -0700 | [diff] [blame] | 696 | prof_leave(); |
| 697 | /* Destroy ctx. */ |
| 698 | malloc_mutex_unlock(&ctx->lock); |
| 699 | bt_destroy(ctx->bt); |
| 700 | malloc_mutex_destroy(&ctx->lock); |
| 701 | idalloc(ctx); |
| 702 | } else { |
Jason Evans | 0cdd42e | 2011-08-09 19:06:06 -0700 | [diff] [blame] | 703 | /* |
| 704 | * Compensate for increment in prof_ctx_merge() or |
| 705 | * prof_lookup(). |
| 706 | */ |
Jason Evans | b04a940 | 2010-10-27 19:47:40 -0700 | [diff] [blame] | 707 | ctx->cnt_merged.curobjs--; |
Jason Evans | a881cd2 | 2010-10-02 15:18:50 -0700 | [diff] [blame] | 708 | malloc_mutex_unlock(&ctx->lock); |
| 709 | prof_leave(); |
| 710 | } |
| 711 | } |
| 712 | |
| 713 | static void |
| 714 | prof_ctx_merge(prof_ctx_t *ctx, prof_thr_cnt_t *cnt) |
| 715 | { |
| 716 | bool destroy; |
| 717 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 718 | cassert(config_prof); |
| 719 | |
Jason Evans | a881cd2 | 2010-10-02 15:18:50 -0700 | [diff] [blame] | 720 | /* Merge cnt stats and detach from ctx. */ |
| 721 | malloc_mutex_lock(&ctx->lock); |
| 722 | ctx->cnt_merged.curobjs += cnt->cnts.curobjs; |
| 723 | ctx->cnt_merged.curbytes += cnt->cnts.curbytes; |
| 724 | ctx->cnt_merged.accumobjs += cnt->cnts.accumobjs; |
| 725 | ctx->cnt_merged.accumbytes += cnt->cnts.accumbytes; |
| 726 | ql_remove(&ctx->cnts_ql, cnt, cnts_link); |
| 727 | if (opt_prof_accum == false && ql_first(&ctx->cnts_ql) == NULL && |
Jason Evans | b04a940 | 2010-10-27 19:47:40 -0700 | [diff] [blame] | 728 | ctx->cnt_merged.curobjs == 0) { |
| 729 | /* |
| 730 | * Artificially raise ctx->cnt_merged.curobjs in order to keep |
| 731 | * another thread from winning the race to destroy ctx while |
| 732 | * this one has ctx->lock dropped. Without this, it would be |
| 733 | * possible for another thread to: |
| 734 | * |
| 735 | * 1) Sample an allocation associated with ctx. |
| 736 | * 2) Deallocate the sampled object. |
| 737 | * 3) Successfully prof_ctx_destroy(ctx). |
| 738 | * |
| 739 | * The result would be that ctx no longer exists by the time |
| 740 | * this thread accesses it in prof_ctx_destroy(). |
| 741 | */ |
| 742 | ctx->cnt_merged.curobjs++; |
Jason Evans | a881cd2 | 2010-10-02 15:18:50 -0700 | [diff] [blame] | 743 | destroy = true; |
Jason Evans | b04a940 | 2010-10-27 19:47:40 -0700 | [diff] [blame] | 744 | } else |
Jason Evans | a881cd2 | 2010-10-02 15:18:50 -0700 | [diff] [blame] | 745 | destroy = false; |
| 746 | malloc_mutex_unlock(&ctx->lock); |
| 747 | if (destroy) |
| 748 | prof_ctx_destroy(ctx); |
| 749 | } |
| 750 | |
Jason Evans | 22ca855 | 2010-03-02 11:57:30 -0800 | [diff] [blame] | 751 | static bool |
| 752 | prof_dump_ctx(prof_ctx_t *ctx, prof_bt_t *bt, bool propagate_err) |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 753 | { |
| 754 | char buf[UMAX2S_BUFSIZE]; |
| 755 | unsigned i; |
| 756 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 757 | cassert(config_prof); |
| 758 | |
Jason Evans | a881cd2 | 2010-10-02 15:18:50 -0700 | [diff] [blame] | 759 | if (opt_prof_accum == false && ctx->cnt_summed.curobjs == 0) { |
| 760 | assert(ctx->cnt_summed.curbytes == 0); |
| 761 | assert(ctx->cnt_summed.accumobjs == 0); |
| 762 | assert(ctx->cnt_summed.accumbytes == 0); |
| 763 | return (false); |
| 764 | } |
| 765 | |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 766 | if (prof_write(u2s(ctx->cnt_summed.curobjs, 10, buf), propagate_err) |
Jason Evans | 22ca855 | 2010-03-02 11:57:30 -0800 | [diff] [blame] | 767 | || prof_write(": ", propagate_err) |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 768 | || prof_write(u2s(ctx->cnt_summed.curbytes, 10, buf), |
Jason Evans | 22ca855 | 2010-03-02 11:57:30 -0800 | [diff] [blame] | 769 | propagate_err) |
| 770 | || prof_write(" [", propagate_err) |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 771 | || prof_write(u2s(ctx->cnt_summed.accumobjs, 10, buf), |
Jason Evans | 22ca855 | 2010-03-02 11:57:30 -0800 | [diff] [blame] | 772 | propagate_err) |
| 773 | || prof_write(": ", propagate_err) |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 774 | || prof_write(u2s(ctx->cnt_summed.accumbytes, 10, buf), |
Jason Evans | 22ca855 | 2010-03-02 11:57:30 -0800 | [diff] [blame] | 775 | propagate_err) |
| 776 | || prof_write("] @", propagate_err)) |
| 777 | return (true); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 778 | |
| 779 | for (i = 0; i < bt->len; i++) { |
Jason Evans | 22ca855 | 2010-03-02 11:57:30 -0800 | [diff] [blame] | 780 | if (prof_write(" 0x", propagate_err) |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 781 | || prof_write(u2s((uintptr_t)bt->vec[i], 16, buf), |
Jason Evans | 22ca855 | 2010-03-02 11:57:30 -0800 | [diff] [blame] | 782 | propagate_err)) |
| 783 | return (true); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 784 | } |
| 785 | |
Jason Evans | 22ca855 | 2010-03-02 11:57:30 -0800 | [diff] [blame] | 786 | if (prof_write("\n", propagate_err)) |
| 787 | return (true); |
| 788 | |
| 789 | return (false); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 790 | } |
| 791 | |
Jason Evans | 22ca855 | 2010-03-02 11:57:30 -0800 | [diff] [blame] | 792 | static bool |
| 793 | prof_dump_maps(bool propagate_err) |
Jason Evans | c717718 | 2010-02-11 09:25:56 -0800 | [diff] [blame] | 794 | { |
| 795 | int mfd; |
| 796 | char buf[UMAX2S_BUFSIZE]; |
| 797 | char *s; |
| 798 | unsigned i, slen; |
| 799 | /* /proc/<pid>/maps\0 */ |
| 800 | char mpath[6 + UMAX2S_BUFSIZE |
| 801 | + 5 + 1]; |
| 802 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 803 | cassert(config_prof); |
| 804 | |
Jason Evans | c717718 | 2010-02-11 09:25:56 -0800 | [diff] [blame] | 805 | i = 0; |
| 806 | |
| 807 | s = "/proc/"; |
| 808 | slen = strlen(s); |
| 809 | memcpy(&mpath[i], s, slen); |
| 810 | i += slen; |
| 811 | |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 812 | s = u2s(getpid(), 10, buf); |
Jason Evans | c717718 | 2010-02-11 09:25:56 -0800 | [diff] [blame] | 813 | slen = strlen(s); |
| 814 | memcpy(&mpath[i], s, slen); |
| 815 | i += slen; |
| 816 | |
| 817 | s = "/maps"; |
| 818 | slen = strlen(s); |
| 819 | memcpy(&mpath[i], s, slen); |
| 820 | i += slen; |
| 821 | |
| 822 | mpath[i] = '\0'; |
| 823 | |
| 824 | mfd = open(mpath, O_RDONLY); |
| 825 | if (mfd != -1) { |
| 826 | ssize_t nread; |
| 827 | |
Jason Evans | 22ca855 | 2010-03-02 11:57:30 -0800 | [diff] [blame] | 828 | if (prof_write("\nMAPPED_LIBRARIES:\n", propagate_err) && |
| 829 | propagate_err) |
| 830 | return (true); |
Jason Evans | c717718 | 2010-02-11 09:25:56 -0800 | [diff] [blame] | 831 | nread = 0; |
| 832 | do { |
| 833 | prof_dump_buf_end += nread; |
| 834 | if (prof_dump_buf_end == PROF_DUMP_BUF_SIZE) { |
| 835 | /* Make space in prof_dump_buf before read(). */ |
Jason Evans | 22ca855 | 2010-03-02 11:57:30 -0800 | [diff] [blame] | 836 | if (prof_flush(propagate_err) && propagate_err) |
| 837 | return (true); |
Jason Evans | c717718 | 2010-02-11 09:25:56 -0800 | [diff] [blame] | 838 | } |
| 839 | nread = read(mfd, &prof_dump_buf[prof_dump_buf_end], |
| 840 | PROF_DUMP_BUF_SIZE - prof_dump_buf_end); |
| 841 | } while (nread > 0); |
Jason Evans | d34f9e7 | 2010-02-11 13:19:21 -0800 | [diff] [blame] | 842 | close(mfd); |
Jason Evans | 22ca855 | 2010-03-02 11:57:30 -0800 | [diff] [blame] | 843 | } else |
| 844 | return (true); |
| 845 | |
| 846 | return (false); |
Jason Evans | c717718 | 2010-02-11 09:25:56 -0800 | [diff] [blame] | 847 | } |
| 848 | |
Jason Evans | 22ca855 | 2010-03-02 11:57:30 -0800 | [diff] [blame] | 849 | static bool |
| 850 | prof_dump(const char *filename, bool leakcheck, bool propagate_err) |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 851 | { |
| 852 | prof_cnt_t cnt_all; |
| 853 | size_t tabind; |
Jason Evans | 075e77c | 2010-09-20 19:53:25 -0700 | [diff] [blame] | 854 | union { |
| 855 | prof_bt_t *p; |
| 856 | void *v; |
| 857 | } bt; |
| 858 | union { |
| 859 | prof_ctx_t *p; |
| 860 | void *v; |
| 861 | } ctx; |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 862 | char buf[UMAX2S_BUFSIZE]; |
| 863 | size_t leak_nctx; |
| 864 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 865 | cassert(config_prof); |
| 866 | |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 867 | prof_enter(); |
| 868 | prof_dump_fd = creat(filename, 0644); |
| 869 | if (prof_dump_fd == -1) { |
Jason Evans | 22ca855 | 2010-03-02 11:57:30 -0800 | [diff] [blame] | 870 | if (propagate_err == false) { |
Jason Evans | 698805c | 2010-03-03 17:45:38 -0800 | [diff] [blame] | 871 | malloc_write("<jemalloc>: creat(\""); |
| 872 | malloc_write(filename); |
| 873 | malloc_write("\", 0644) failed\n"); |
Jason Evans | 22ca855 | 2010-03-02 11:57:30 -0800 | [diff] [blame] | 874 | if (opt_abort) |
| 875 | abort(); |
| 876 | } |
Jason Evans | 22ca855 | 2010-03-02 11:57:30 -0800 | [diff] [blame] | 877 | goto ERROR; |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 878 | } |
| 879 | |
| 880 | /* Merge per thread profile stats, and sum them in cnt_all. */ |
| 881 | memset(&cnt_all, 0, sizeof(prof_cnt_t)); |
| 882 | leak_nctx = 0; |
Jason Evans | 588a32c | 2010-10-02 22:38:14 -0700 | [diff] [blame] | 883 | for (tabind = 0; ckh_iter(&bt2ctx, &tabind, NULL, &ctx.v) == false;) |
Jason Evans | a881cd2 | 2010-10-02 15:18:50 -0700 | [diff] [blame] | 884 | prof_ctx_sum(ctx.p, &cnt_all, &leak_nctx); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 885 | |
| 886 | /* Dump profile header. */ |
Jason Evans | 22ca855 | 2010-03-02 11:57:30 -0800 | [diff] [blame] | 887 | if (prof_write("heap profile: ", propagate_err) |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 888 | || prof_write(u2s(cnt_all.curobjs, 10, buf), propagate_err) |
Jason Evans | 22ca855 | 2010-03-02 11:57:30 -0800 | [diff] [blame] | 889 | || prof_write(": ", propagate_err) |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 890 | || prof_write(u2s(cnt_all.curbytes, 10, buf), propagate_err) |
Jason Evans | 22ca855 | 2010-03-02 11:57:30 -0800 | [diff] [blame] | 891 | || prof_write(" [", propagate_err) |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 892 | || prof_write(u2s(cnt_all.accumobjs, 10, buf), propagate_err) |
Jason Evans | 22ca855 | 2010-03-02 11:57:30 -0800 | [diff] [blame] | 893 | || prof_write(": ", propagate_err) |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 894 | || prof_write(u2s(cnt_all.accumbytes, 10, buf), propagate_err)) |
Jason Evans | 22ca855 | 2010-03-02 11:57:30 -0800 | [diff] [blame] | 895 | goto ERROR; |
| 896 | |
| 897 | if (opt_lg_prof_sample == 0) { |
| 898 | if (prof_write("] @ heapprofile\n", propagate_err)) |
| 899 | goto ERROR; |
| 900 | } else { |
| 901 | if (prof_write("] @ heap_v2/", propagate_err) |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 902 | || prof_write(u2s((uint64_t)1U << opt_lg_prof_sample, 10, |
Jason Evans | 22ca855 | 2010-03-02 11:57:30 -0800 | [diff] [blame] | 903 | buf), propagate_err) |
| 904 | || prof_write("\n", propagate_err)) |
| 905 | goto ERROR; |
Jason Evans | b9477e7 | 2010-03-01 20:15:26 -0800 | [diff] [blame] | 906 | } |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 907 | |
| 908 | /* Dump per ctx profile stats. */ |
Jason Evans | 075e77c | 2010-09-20 19:53:25 -0700 | [diff] [blame] | 909 | for (tabind = 0; ckh_iter(&bt2ctx, &tabind, &bt.v, &ctx.v) |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 910 | == false;) { |
Jason Evans | 075e77c | 2010-09-20 19:53:25 -0700 | [diff] [blame] | 911 | if (prof_dump_ctx(ctx.p, bt.p, propagate_err)) |
Jason Evans | 22ca855 | 2010-03-02 11:57:30 -0800 | [diff] [blame] | 912 | goto ERROR; |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 913 | } |
| 914 | |
Jason Evans | c717718 | 2010-02-11 09:25:56 -0800 | [diff] [blame] | 915 | /* Dump /proc/<pid>/maps if possible. */ |
Jason Evans | 22ca855 | 2010-03-02 11:57:30 -0800 | [diff] [blame] | 916 | if (prof_dump_maps(propagate_err)) |
| 917 | goto ERROR; |
Jason Evans | c717718 | 2010-02-11 09:25:56 -0800 | [diff] [blame] | 918 | |
Jason Evans | 22ca855 | 2010-03-02 11:57:30 -0800 | [diff] [blame] | 919 | if (prof_flush(propagate_err)) |
| 920 | goto ERROR; |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 921 | close(prof_dump_fd); |
| 922 | prof_leave(); |
| 923 | |
| 924 | if (leakcheck && cnt_all.curbytes != 0) { |
Jason Evans | 698805c | 2010-03-03 17:45:38 -0800 | [diff] [blame] | 925 | malloc_write("<jemalloc>: Leak summary: "); |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 926 | malloc_write(u2s(cnt_all.curbytes, 10, buf)); |
Jason Evans | 698805c | 2010-03-03 17:45:38 -0800 | [diff] [blame] | 927 | malloc_write((cnt_all.curbytes != 1) ? " bytes, " : " byte, "); |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 928 | malloc_write(u2s(cnt_all.curobjs, 10, buf)); |
Jason Evans | 698805c | 2010-03-03 17:45:38 -0800 | [diff] [blame] | 929 | malloc_write((cnt_all.curobjs != 1) ? " objects, " : |
| 930 | " object, "); |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 931 | malloc_write(u2s(leak_nctx, 10, buf)); |
Jason Evans | 698805c | 2010-03-03 17:45:38 -0800 | [diff] [blame] | 932 | malloc_write((leak_nctx != 1) ? " contexts\n" : " context\n"); |
| 933 | malloc_write("<jemalloc>: Run pprof on \""); |
| 934 | malloc_write(filename); |
| 935 | malloc_write("\" for leak detail\n"); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 936 | } |
Jason Evans | 22ca855 | 2010-03-02 11:57:30 -0800 | [diff] [blame] | 937 | |
| 938 | return (false); |
| 939 | ERROR: |
| 940 | prof_leave(); |
| 941 | return (true); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 942 | } |
| 943 | |
Jason Evans | b01a6c2 | 2010-02-11 10:25:36 -0800 | [diff] [blame] | 944 | #define DUMP_FILENAME_BUFSIZE (PATH_MAX+ UMAX2S_BUFSIZE \ |
| 945 | + 1 \ |
| 946 | + UMAX2S_BUFSIZE \ |
| 947 | + 2 \ |
| 948 | + UMAX2S_BUFSIZE \ |
| 949 | + 5 + 1) |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 950 | static void |
| 951 | prof_dump_filename(char *filename, char v, int64_t vseq) |
| 952 | { |
| 953 | char buf[UMAX2S_BUFSIZE]; |
| 954 | char *s; |
| 955 | unsigned i, slen; |
| 956 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 957 | cassert(config_prof); |
| 958 | |
Jason Evans | b01a6c2 | 2010-02-11 10:25:36 -0800 | [diff] [blame] | 959 | /* |
| 960 | * Construct a filename of the form: |
| 961 | * |
| 962 | * <prefix>.<pid>.<seq>.v<vseq>.heap\0 |
Jason Evans | b01a6c2 | 2010-02-11 10:25:36 -0800 | [diff] [blame] | 963 | */ |
| 964 | |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 965 | i = 0; |
| 966 | |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 967 | s = opt_prof_prefix; |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 968 | slen = strlen(s); |
| 969 | memcpy(&filename[i], s, slen); |
| 970 | i += slen; |
| 971 | |
| 972 | s = "."; |
| 973 | slen = strlen(s); |
| 974 | memcpy(&filename[i], s, slen); |
| 975 | i += slen; |
| 976 | |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 977 | s = u2s(getpid(), 10, buf); |
| 978 | slen = strlen(s); |
| 979 | memcpy(&filename[i], s, slen); |
| 980 | i += slen; |
| 981 | |
| 982 | s = "."; |
| 983 | slen = strlen(s); |
| 984 | memcpy(&filename[i], s, slen); |
| 985 | i += slen; |
| 986 | |
| 987 | s = u2s(prof_dump_seq, 10, buf); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 988 | prof_dump_seq++; |
| 989 | slen = strlen(s); |
| 990 | memcpy(&filename[i], s, slen); |
| 991 | i += slen; |
| 992 | |
| 993 | s = "."; |
| 994 | slen = strlen(s); |
| 995 | memcpy(&filename[i], s, slen); |
| 996 | i += slen; |
| 997 | |
| 998 | filename[i] = v; |
| 999 | i++; |
| 1000 | |
| 1001 | if (vseq != 0xffffffffffffffffLLU) { |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 1002 | s = u2s(vseq, 10, buf); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1003 | slen = strlen(s); |
| 1004 | memcpy(&filename[i], s, slen); |
| 1005 | i += slen; |
| 1006 | } |
| 1007 | |
| 1008 | s = ".heap"; |
| 1009 | slen = strlen(s); |
| 1010 | memcpy(&filename[i], s, slen); |
| 1011 | i += slen; |
| 1012 | |
| 1013 | filename[i] = '\0'; |
| 1014 | } |
| 1015 | |
| 1016 | static void |
| 1017 | prof_fdump(void) |
| 1018 | { |
| 1019 | char filename[DUMP_FILENAME_BUFSIZE]; |
| 1020 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 1021 | cassert(config_prof); |
| 1022 | |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1023 | if (prof_booted == false) |
| 1024 | return; |
| 1025 | |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 1026 | if (opt_prof_prefix[0] != '\0') { |
| 1027 | malloc_mutex_lock(&prof_dump_seq_mtx); |
| 1028 | prof_dump_filename(filename, 'f', 0xffffffffffffffffLLU); |
| 1029 | malloc_mutex_unlock(&prof_dump_seq_mtx); |
| 1030 | prof_dump(filename, opt_prof_leak, false); |
| 1031 | } |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1032 | } |
| 1033 | |
| 1034 | void |
| 1035 | prof_idump(void) |
| 1036 | { |
| 1037 | char filename[DUMP_FILENAME_BUFSIZE]; |
| 1038 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 1039 | cassert(config_prof); |
| 1040 | |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1041 | if (prof_booted == false) |
| 1042 | return; |
Jason Evans | d34f9e7 | 2010-02-11 13:19:21 -0800 | [diff] [blame] | 1043 | malloc_mutex_lock(&enq_mtx); |
| 1044 | if (enq) { |
| 1045 | enq_idump = true; |
| 1046 | malloc_mutex_unlock(&enq_mtx); |
| 1047 | return; |
| 1048 | } |
| 1049 | malloc_mutex_unlock(&enq_mtx); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1050 | |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 1051 | if (opt_prof_prefix[0] != '\0') { |
| 1052 | malloc_mutex_lock(&prof_dump_seq_mtx); |
| 1053 | prof_dump_filename(filename, 'i', prof_dump_iseq); |
| 1054 | prof_dump_iseq++; |
| 1055 | malloc_mutex_unlock(&prof_dump_seq_mtx); |
| 1056 | prof_dump(filename, false, false); |
| 1057 | } |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1058 | } |
| 1059 | |
Jason Evans | 22ca855 | 2010-03-02 11:57:30 -0800 | [diff] [blame] | 1060 | bool |
| 1061 | prof_mdump(const char *filename) |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1062 | { |
Jason Evans | 22ca855 | 2010-03-02 11:57:30 -0800 | [diff] [blame] | 1063 | char filename_buf[DUMP_FILENAME_BUFSIZE]; |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1064 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 1065 | cassert(config_prof); |
| 1066 | |
Jason Evans | 22ca855 | 2010-03-02 11:57:30 -0800 | [diff] [blame] | 1067 | if (opt_prof == false || prof_booted == false) |
| 1068 | return (true); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1069 | |
Jason Evans | 22ca855 | 2010-03-02 11:57:30 -0800 | [diff] [blame] | 1070 | if (filename == NULL) { |
| 1071 | /* No filename specified, so automatically generate one. */ |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 1072 | if (opt_prof_prefix[0] == '\0') |
| 1073 | return (true); |
Jason Evans | 22ca855 | 2010-03-02 11:57:30 -0800 | [diff] [blame] | 1074 | malloc_mutex_lock(&prof_dump_seq_mtx); |
| 1075 | prof_dump_filename(filename_buf, 'm', prof_dump_mseq); |
| 1076 | prof_dump_mseq++; |
| 1077 | malloc_mutex_unlock(&prof_dump_seq_mtx); |
| 1078 | filename = filename_buf; |
| 1079 | } |
| 1080 | return (prof_dump(filename, false, true)); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1081 | } |
| 1082 | |
| 1083 | void |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 1084 | prof_gdump(void) |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1085 | { |
| 1086 | char filename[DUMP_FILENAME_BUFSIZE]; |
| 1087 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 1088 | cassert(config_prof); |
| 1089 | |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1090 | if (prof_booted == false) |
| 1091 | return; |
| 1092 | malloc_mutex_lock(&enq_mtx); |
| 1093 | if (enq) { |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 1094 | enq_gdump = true; |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1095 | malloc_mutex_unlock(&enq_mtx); |
| 1096 | return; |
| 1097 | } |
| 1098 | malloc_mutex_unlock(&enq_mtx); |
| 1099 | |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 1100 | if (opt_prof_prefix[0] != '\0') { |
| 1101 | malloc_mutex_lock(&prof_dump_seq_mtx); |
| 1102 | prof_dump_filename(filename, 'u', prof_dump_useq); |
| 1103 | prof_dump_useq++; |
| 1104 | malloc_mutex_unlock(&prof_dump_seq_mtx); |
| 1105 | prof_dump(filename, false, false); |
| 1106 | } |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1107 | } |
| 1108 | |
| 1109 | static void |
| 1110 | prof_bt_hash(const void *key, unsigned minbits, size_t *hash1, size_t *hash2) |
| 1111 | { |
| 1112 | size_t ret1, ret2; |
| 1113 | uint64_t h; |
| 1114 | prof_bt_t *bt = (prof_bt_t *)key; |
| 1115 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 1116 | cassert(config_prof); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1117 | assert(minbits <= 32 || (SIZEOF_PTR == 8 && minbits <= 64)); |
| 1118 | assert(hash1 != NULL); |
| 1119 | assert(hash2 != NULL); |
| 1120 | |
| 1121 | h = hash(bt->vec, bt->len * sizeof(void *), 0x94122f335b332aeaLLU); |
| 1122 | if (minbits <= 32) { |
| 1123 | /* |
| 1124 | * Avoid doing multiple hashes, since a single hash provides |
| 1125 | * enough bits. |
| 1126 | */ |
| 1127 | ret1 = h & ZU(0xffffffffU); |
| 1128 | ret2 = h >> 32; |
| 1129 | } else { |
| 1130 | ret1 = h; |
| 1131 | ret2 = hash(bt->vec, bt->len * sizeof(void *), |
Jason Evans | f0b22cf | 2011-05-22 10:49:44 -0700 | [diff] [blame] | 1132 | 0x8432a476666bbc13LLU); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1133 | } |
| 1134 | |
| 1135 | *hash1 = ret1; |
| 1136 | *hash2 = ret2; |
| 1137 | } |
| 1138 | |
| 1139 | static bool |
| 1140 | prof_bt_keycomp(const void *k1, const void *k2) |
| 1141 | { |
| 1142 | const prof_bt_t *bt1 = (prof_bt_t *)k1; |
| 1143 | const prof_bt_t *bt2 = (prof_bt_t *)k2; |
| 1144 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 1145 | cassert(config_prof); |
| 1146 | |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1147 | if (bt1->len != bt2->len) |
| 1148 | return (false); |
| 1149 | return (memcmp(bt1->vec, bt2->vec, bt1->len * sizeof(void *)) == 0); |
| 1150 | } |
| 1151 | |
Jason Evans | 4d6a134 | 2010-10-20 19:05:59 -0700 | [diff] [blame] | 1152 | prof_tdata_t * |
| 1153 | prof_tdata_init(void) |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1154 | { |
Jason Evans | 4d6a134 | 2010-10-20 19:05:59 -0700 | [diff] [blame] | 1155 | prof_tdata_t *prof_tdata; |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1156 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 1157 | cassert(config_prof); |
| 1158 | |
Jason Evans | 4d6a134 | 2010-10-20 19:05:59 -0700 | [diff] [blame] | 1159 | /* Initialize an empty cache for this thread. */ |
| 1160 | prof_tdata = (prof_tdata_t *)imalloc(sizeof(prof_tdata_t)); |
| 1161 | if (prof_tdata == NULL) |
| 1162 | return (NULL); |
| 1163 | |
| 1164 | if (ckh_new(&prof_tdata->bt2cnt, PROF_CKH_MINITEMS, |
| 1165 | prof_bt_hash, prof_bt_keycomp)) { |
| 1166 | idalloc(prof_tdata); |
| 1167 | return (NULL); |
| 1168 | } |
| 1169 | ql_new(&prof_tdata->lru_ql); |
| 1170 | |
| 1171 | prof_tdata->vec = imalloc(sizeof(void *) * prof_bt_max); |
| 1172 | if (prof_tdata->vec == NULL) { |
Jason Evans | 4d6a134 | 2010-10-20 19:05:59 -0700 | [diff] [blame] | 1173 | ckh_delete(&prof_tdata->bt2cnt); |
| 1174 | idalloc(prof_tdata); |
| 1175 | return (NULL); |
| 1176 | } |
| 1177 | |
| 1178 | prof_tdata->prn_state = 0; |
| 1179 | prof_tdata->threshold = 0; |
| 1180 | prof_tdata->accum = 0; |
| 1181 | |
| 1182 | PROF_TCACHE_SET(prof_tdata); |
| 1183 | |
| 1184 | return (prof_tdata); |
| 1185 | } |
| 1186 | |
| 1187 | static void |
| 1188 | prof_tdata_cleanup(void *arg) |
| 1189 | { |
Jason Evans | 41b954e | 2011-08-08 17:10:07 -0700 | [diff] [blame] | 1190 | prof_thr_cnt_t *cnt; |
| 1191 | prof_tdata_t *prof_tdata = (prof_tdata_t *)arg; |
Jason Evans | 4d6a134 | 2010-10-20 19:05:59 -0700 | [diff] [blame] | 1192 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 1193 | cassert(config_prof); |
| 1194 | |
Jason Evans | 41b954e | 2011-08-08 17:10:07 -0700 | [diff] [blame] | 1195 | /* |
Jason Evans | 0cdd42e | 2011-08-09 19:06:06 -0700 | [diff] [blame] | 1196 | * Delete the hash table. All of its contents can still be iterated |
| 1197 | * over via the LRU. |
Jason Evans | 41b954e | 2011-08-08 17:10:07 -0700 | [diff] [blame] | 1198 | */ |
| 1199 | ckh_delete(&prof_tdata->bt2cnt); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1200 | |
Jason Evans | 0cdd42e | 2011-08-09 19:06:06 -0700 | [diff] [blame] | 1201 | /* Iteratively merge cnt's into the global stats and delete them. */ |
Jason Evans | 41b954e | 2011-08-08 17:10:07 -0700 | [diff] [blame] | 1202 | while ((cnt = ql_last(&prof_tdata->lru_ql, lru_link)) != NULL) { |
Jason Evans | 41b954e | 2011-08-08 17:10:07 -0700 | [diff] [blame] | 1203 | ql_remove(&prof_tdata->lru_ql, cnt, lru_link); |
Jason Evans | 0cdd42e | 2011-08-09 19:06:06 -0700 | [diff] [blame] | 1204 | prof_ctx_merge(cnt->ctx, cnt); |
Jason Evans | 41b954e | 2011-08-08 17:10:07 -0700 | [diff] [blame] | 1205 | idalloc(cnt); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1206 | } |
Jason Evans | 41b954e | 2011-08-08 17:10:07 -0700 | [diff] [blame] | 1207 | |
| 1208 | idalloc(prof_tdata->vec); |
| 1209 | |
| 1210 | idalloc(prof_tdata); |
| 1211 | PROF_TCACHE_SET(NULL); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1212 | } |
| 1213 | |
| 1214 | void |
| 1215 | prof_boot0(void) |
| 1216 | { |
| 1217 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 1218 | cassert(config_prof); |
| 1219 | |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 1220 | memcpy(opt_prof_prefix, PROF_PREFIX_DEFAULT, |
| 1221 | sizeof(PROF_PREFIX_DEFAULT)); |
| 1222 | } |
| 1223 | |
| 1224 | void |
| 1225 | prof_boot1(void) |
| 1226 | { |
| 1227 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 1228 | cassert(config_prof); |
| 1229 | |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1230 | /* |
Jason Evans | 0b270a9 | 2010-03-31 16:45:04 -0700 | [diff] [blame] | 1231 | * opt_prof and prof_promote must be in their final state before any |
| 1232 | * arenas are initialized, so this function must be executed early. |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1233 | */ |
| 1234 | |
| 1235 | if (opt_prof_leak && opt_prof == false) { |
| 1236 | /* |
| 1237 | * Enable opt_prof, but in such a way that profiles are never |
| 1238 | * automatically dumped. |
| 1239 | */ |
| 1240 | opt_prof = true; |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 1241 | opt_prof_gdump = false; |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1242 | prof_interval = 0; |
Jason Evans | a02fc08 | 2010-03-31 17:35:51 -0700 | [diff] [blame] | 1243 | } else if (opt_prof) { |
| 1244 | if (opt_lg_prof_interval >= 0) { |
| 1245 | prof_interval = (((uint64_t)1U) << |
| 1246 | opt_lg_prof_interval); |
| 1247 | } else |
| 1248 | prof_interval = 0; |
| 1249 | } |
Jason Evans | 0b270a9 | 2010-03-31 16:45:04 -0700 | [diff] [blame] | 1250 | |
| 1251 | prof_promote = (opt_prof && opt_lg_prof_sample > PAGE_SHIFT); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1252 | } |
| 1253 | |
| 1254 | bool |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 1255 | prof_boot2(void) |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1256 | { |
| 1257 | |
Jason Evans | 7372b15 | 2012-02-10 20:22:09 -0800 | [diff] [blame] | 1258 | cassert(config_prof); |
| 1259 | |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1260 | if (opt_prof) { |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1261 | if (ckh_new(&bt2ctx, PROF_CKH_MINITEMS, prof_bt_hash, |
| 1262 | prof_bt_keycomp)) |
| 1263 | return (true); |
| 1264 | if (malloc_mutex_init(&bt2ctx_mtx)) |
| 1265 | return (true); |
Jason Evans | 4d6a134 | 2010-10-20 19:05:59 -0700 | [diff] [blame] | 1266 | if (pthread_key_create(&prof_tdata_tsd, prof_tdata_cleanup) |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1267 | != 0) { |
Jason Evans | 698805c | 2010-03-03 17:45:38 -0800 | [diff] [blame] | 1268 | malloc_write( |
| 1269 | "<jemalloc>: Error in pthread_key_create()\n"); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1270 | abort(); |
| 1271 | } |
| 1272 | |
| 1273 | prof_bt_max = (1U << opt_lg_prof_bt_max); |
| 1274 | if (malloc_mutex_init(&prof_dump_seq_mtx)) |
| 1275 | return (true); |
| 1276 | |
| 1277 | if (malloc_mutex_init(&enq_mtx)) |
| 1278 | return (true); |
| 1279 | enq = false; |
Jason Evans | d34f9e7 | 2010-02-11 13:19:21 -0800 | [diff] [blame] | 1280 | enq_idump = false; |
Jason Evans | e733970 | 2010-10-23 18:37:06 -0700 | [diff] [blame] | 1281 | enq_gdump = false; |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1282 | |
| 1283 | if (atexit(prof_fdump) != 0) { |
Jason Evans | 698805c | 2010-03-03 17:45:38 -0800 | [diff] [blame] | 1284 | malloc_write("<jemalloc>: Error in atexit()\n"); |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1285 | if (opt_abort) |
| 1286 | abort(); |
| 1287 | } |
| 1288 | } |
| 1289 | |
Jason Evans | b27805b | 2010-02-10 18:15:53 -0800 | [diff] [blame] | 1290 | #ifdef JEMALLOC_PROF_LIBGCC |
| 1291 | /* |
| 1292 | * Cause the backtracing machinery to allocate its internal state |
| 1293 | * before enabling profiling. |
| 1294 | */ |
| 1295 | _Unwind_Backtrace(prof_unwind_init_callback, NULL); |
| 1296 | #endif |
| 1297 | |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1298 | prof_booted = true; |
| 1299 | |
| 1300 | return (false); |
| 1301 | } |
| 1302 | |
Jason Evans | 6109fe0 | 2010-02-10 10:37:56 -0800 | [diff] [blame] | 1303 | /******************************************************************************/ |