Jason Evans | bbe29d3 | 2013-01-30 15:03:11 -0800 | [diff] [blame] | 1 | #define JEMALLOC_QUARANTINE_C_ |
Jason Evans | 122449b | 2012-04-06 00:35:09 -0700 | [diff] [blame] | 2 | #include "jemalloc/internal/jemalloc_internal.h" |
| 3 | |
Jason Evans | 577dd84 | 2012-04-23 21:14:26 -0700 | [diff] [blame] | 4 | /* |
| 5 | * quarantine pointers close to NULL are used to encode state information that |
| 6 | * is used for cleaning up during thread shutdown. |
| 7 | */ |
| 8 | #define QUARANTINE_STATE_REINCARNATED ((quarantine_t *)(uintptr_t)1) |
| 9 | #define QUARANTINE_STATE_PURGATORY ((quarantine_t *)(uintptr_t)2) |
| 10 | #define QUARANTINE_STATE_MAX QUARANTINE_STATE_PURGATORY |
| 11 | |
Jason Evans | 122449b | 2012-04-06 00:35:09 -0700 | [diff] [blame] | 12 | /******************************************************************************/ |
Jason Evans | 122449b | 2012-04-06 00:35:09 -0700 | [diff] [blame] | 13 | /* Function prototypes for non-inline static functions. */ |
| 14 | |
Jason Evans | 5460aa6 | 2014-09-22 21:09:23 -0700 | [diff] [blame] | 15 | static quarantine_t *quarantine_grow(tsd_t *tsd, quarantine_t *quarantine); |
| 16 | static void quarantine_drain_one(tsd_t *tsd, quarantine_t *quarantine); |
| 17 | static void quarantine_drain(tsd_t *tsd, quarantine_t *quarantine, |
| 18 | size_t upper_bound); |
Jason Evans | 122449b | 2012-04-06 00:35:09 -0700 | [diff] [blame] | 19 | |
| 20 | /******************************************************************************/ |
| 21 | |
Jason Evans | 9cf2be0 | 2014-11-07 14:50:38 -0800 | [diff] [blame] | 22 | static quarantine_t * |
Jason Evans | 5460aa6 | 2014-09-22 21:09:23 -0700 | [diff] [blame] | 23 | quarantine_init(tsd_t *tsd, size_t lg_maxobjs) |
Jason Evans | 122449b | 2012-04-06 00:35:09 -0700 | [diff] [blame] | 24 | { |
| 25 | quarantine_t *quarantine; |
| 26 | |
Jason Evans | c002a5c | 2014-11-04 18:03:11 -0800 | [diff] [blame] | 27 | assert(tsd_nominal(tsd)); |
| 28 | |
Jason Evans | 5460aa6 | 2014-09-22 21:09:23 -0700 | [diff] [blame] | 29 | quarantine = (quarantine_t *)imalloc(tsd, offsetof(quarantine_t, objs) + |
Jason Evans | 9cd351d | 2012-04-23 21:43:18 -0700 | [diff] [blame] | 30 | ((ZU(1) << lg_maxobjs) * sizeof(quarantine_obj_t))); |
Jason Evans | 122449b | 2012-04-06 00:35:09 -0700 | [diff] [blame] | 31 | if (quarantine == NULL) |
| 32 | return (NULL); |
| 33 | quarantine->curbytes = 0; |
| 34 | quarantine->curobjs = 0; |
| 35 | quarantine->first = 0; |
| 36 | quarantine->lg_maxobjs = lg_maxobjs; |
| 37 | |
Jason Evans | 122449b | 2012-04-06 00:35:09 -0700 | [diff] [blame] | 38 | return (quarantine); |
| 39 | } |
| 40 | |
Jason Evans | c002a5c | 2014-11-04 18:03:11 -0800 | [diff] [blame] | 41 | void |
| 42 | quarantine_alloc_hook_work(tsd_t *tsd) |
| 43 | { |
| 44 | quarantine_t *quarantine; |
| 45 | |
| 46 | if (!tsd_nominal(tsd)) |
| 47 | return; |
| 48 | |
| 49 | quarantine = quarantine_init(tsd, LG_MAXOBJS_INIT); |
| 50 | /* |
| 51 | * Check again whether quarantine has been initialized, because |
| 52 | * qurantine_init() may have triggered recursive initialization. |
| 53 | */ |
| 54 | if (tsd_quarantine_get(tsd) == NULL) |
| 55 | tsd_quarantine_set(tsd, quarantine); |
| 56 | else |
| 57 | idalloc(tsd, quarantine); |
| 58 | } |
| 59 | |
Jason Evans | 122449b | 2012-04-06 00:35:09 -0700 | [diff] [blame] | 60 | static quarantine_t * |
Jason Evans | 5460aa6 | 2014-09-22 21:09:23 -0700 | [diff] [blame] | 61 | quarantine_grow(tsd_t *tsd, quarantine_t *quarantine) |
Jason Evans | 122449b | 2012-04-06 00:35:09 -0700 | [diff] [blame] | 62 | { |
| 63 | quarantine_t *ret; |
| 64 | |
Jason Evans | 5460aa6 | 2014-09-22 21:09:23 -0700 | [diff] [blame] | 65 | ret = quarantine_init(tsd, quarantine->lg_maxobjs + 1); |
Jason Evans | d0e942e | 2013-01-31 14:42:41 -0800 | [diff] [blame] | 66 | if (ret == NULL) { |
Jason Evans | 5460aa6 | 2014-09-22 21:09:23 -0700 | [diff] [blame] | 67 | quarantine_drain_one(tsd, quarantine); |
Jason Evans | 122449b | 2012-04-06 00:35:09 -0700 | [diff] [blame] | 68 | return (quarantine); |
Jason Evans | d0e942e | 2013-01-31 14:42:41 -0800 | [diff] [blame] | 69 | } |
Jason Evans | 122449b | 2012-04-06 00:35:09 -0700 | [diff] [blame] | 70 | |
| 71 | ret->curbytes = quarantine->curbytes; |
Jason Evans | 7e06039 | 2012-04-23 22:07:30 -0700 | [diff] [blame] | 72 | ret->curobjs = quarantine->curobjs; |
| 73 | if (quarantine->first + quarantine->curobjs <= (ZU(1) << |
Jason Evans | 122449b | 2012-04-06 00:35:09 -0700 | [diff] [blame] | 74 | quarantine->lg_maxobjs)) { |
| 75 | /* objs ring buffer data are contiguous. */ |
| 76 | memcpy(ret->objs, &quarantine->objs[quarantine->first], |
Jason Evans | 9cd351d | 2012-04-23 21:43:18 -0700 | [diff] [blame] | 77 | quarantine->curobjs * sizeof(quarantine_obj_t)); |
Jason Evans | 122449b | 2012-04-06 00:35:09 -0700 | [diff] [blame] | 78 | } else { |
| 79 | /* objs ring buffer data wrap around. */ |
Jason Evans | 7e06039 | 2012-04-23 22:07:30 -0700 | [diff] [blame] | 80 | size_t ncopy_a = (ZU(1) << quarantine->lg_maxobjs) - |
Jason Evans | 122449b | 2012-04-06 00:35:09 -0700 | [diff] [blame] | 81 | quarantine->first; |
Jason Evans | 7e06039 | 2012-04-23 22:07:30 -0700 | [diff] [blame] | 82 | size_t ncopy_b = quarantine->curobjs - ncopy_a; |
| 83 | |
| 84 | memcpy(ret->objs, &quarantine->objs[quarantine->first], ncopy_a |
| 85 | * sizeof(quarantine_obj_t)); |
| 86 | memcpy(&ret->objs[ncopy_a], quarantine->objs, ncopy_b * |
Jason Evans | 9cd351d | 2012-04-23 21:43:18 -0700 | [diff] [blame] | 87 | sizeof(quarantine_obj_t)); |
Jason Evans | 122449b | 2012-04-06 00:35:09 -0700 | [diff] [blame] | 88 | } |
Jason Evans | 5460aa6 | 2014-09-22 21:09:23 -0700 | [diff] [blame] | 89 | idalloc(tsd, quarantine); |
Jason Evans | 122449b | 2012-04-06 00:35:09 -0700 | [diff] [blame] | 90 | |
Jason Evans | c002a5c | 2014-11-04 18:03:11 -0800 | [diff] [blame] | 91 | tsd_quarantine_set(tsd, ret); |
Jason Evans | 122449b | 2012-04-06 00:35:09 -0700 | [diff] [blame] | 92 | return (ret); |
| 93 | } |
| 94 | |
| 95 | static void |
Jason Evans | 5460aa6 | 2014-09-22 21:09:23 -0700 | [diff] [blame] | 96 | quarantine_drain_one(tsd_t *tsd, quarantine_t *quarantine) |
Jason Evans | d0e942e | 2013-01-31 14:42:41 -0800 | [diff] [blame] | 97 | { |
| 98 | quarantine_obj_t *obj = &quarantine->objs[quarantine->first]; |
| 99 | assert(obj->usize == isalloc(obj->ptr, config_prof)); |
Jason Evans | 5460aa6 | 2014-09-22 21:09:23 -0700 | [diff] [blame] | 100 | idalloc(tsd, obj->ptr); |
Jason Evans | d0e942e | 2013-01-31 14:42:41 -0800 | [diff] [blame] | 101 | quarantine->curbytes -= obj->usize; |
| 102 | quarantine->curobjs--; |
| 103 | quarantine->first = (quarantine->first + 1) & ((ZU(1) << |
| 104 | quarantine->lg_maxobjs) - 1); |
| 105 | } |
| 106 | |
| 107 | static void |
Jason Evans | 5460aa6 | 2014-09-22 21:09:23 -0700 | [diff] [blame] | 108 | quarantine_drain(tsd_t *tsd, quarantine_t *quarantine, size_t upper_bound) |
Jason Evans | 122449b | 2012-04-06 00:35:09 -0700 | [diff] [blame] | 109 | { |
| 110 | |
Jason Evans | d0e942e | 2013-01-31 14:42:41 -0800 | [diff] [blame] | 111 | while (quarantine->curbytes > upper_bound && quarantine->curobjs > 0) |
Jason Evans | 5460aa6 | 2014-09-22 21:09:23 -0700 | [diff] [blame] | 112 | quarantine_drain_one(tsd, quarantine); |
Jason Evans | 122449b | 2012-04-06 00:35:09 -0700 | [diff] [blame] | 113 | } |
| 114 | |
| 115 | void |
Jason Evans | 5460aa6 | 2014-09-22 21:09:23 -0700 | [diff] [blame] | 116 | quarantine(tsd_t *tsd, void *ptr) |
Jason Evans | 122449b | 2012-04-06 00:35:09 -0700 | [diff] [blame] | 117 | { |
| 118 | quarantine_t *quarantine; |
| 119 | size_t usize = isalloc(ptr, config_prof); |
| 120 | |
Jason Evans | 78f7352 | 2012-04-18 13:38:40 -0700 | [diff] [blame] | 121 | cassert(config_fill); |
Jason Evans | 122449b | 2012-04-06 00:35:09 -0700 | [diff] [blame] | 122 | assert(opt_quarantine); |
| 123 | |
Jason Evans | 5460aa6 | 2014-09-22 21:09:23 -0700 | [diff] [blame] | 124 | if ((quarantine = tsd_quarantine_get(tsd)) == NULL) { |
| 125 | idalloc(tsd, ptr); |
Jason Evans | bbe29d3 | 2013-01-30 15:03:11 -0800 | [diff] [blame] | 126 | return; |
Jason Evans | 122449b | 2012-04-06 00:35:09 -0700 | [diff] [blame] | 127 | } |
| 128 | /* |
| 129 | * Drain one or more objects if the quarantine size limit would be |
| 130 | * exceeded by appending ptr. |
| 131 | */ |
| 132 | if (quarantine->curbytes + usize > opt_quarantine) { |
| 133 | size_t upper_bound = (opt_quarantine >= usize) ? opt_quarantine |
| 134 | - usize : 0; |
Jason Evans | 5460aa6 | 2014-09-22 21:09:23 -0700 | [diff] [blame] | 135 | quarantine_drain(tsd, quarantine, upper_bound); |
Jason Evans | 122449b | 2012-04-06 00:35:09 -0700 | [diff] [blame] | 136 | } |
| 137 | /* Grow the quarantine ring buffer if it's full. */ |
| 138 | if (quarantine->curobjs == (ZU(1) << quarantine->lg_maxobjs)) |
Jason Evans | 5460aa6 | 2014-09-22 21:09:23 -0700 | [diff] [blame] | 139 | quarantine = quarantine_grow(tsd, quarantine); |
Jason Evans | 122449b | 2012-04-06 00:35:09 -0700 | [diff] [blame] | 140 | /* quarantine_grow() must free a slot if it fails to grow. */ |
| 141 | assert(quarantine->curobjs < (ZU(1) << quarantine->lg_maxobjs)); |
| 142 | /* Append ptr if its size doesn't exceed the quarantine size. */ |
| 143 | if (quarantine->curbytes + usize <= opt_quarantine) { |
| 144 | size_t offset = (quarantine->first + quarantine->curobjs) & |
| 145 | ((ZU(1) << quarantine->lg_maxobjs) - 1); |
Jason Evans | 9cd351d | 2012-04-23 21:43:18 -0700 | [diff] [blame] | 146 | quarantine_obj_t *obj = &quarantine->objs[offset]; |
| 147 | obj->ptr = ptr; |
| 148 | obj->usize = usize; |
Jason Evans | 122449b | 2012-04-06 00:35:09 -0700 | [diff] [blame] | 149 | quarantine->curbytes += usize; |
| 150 | quarantine->curobjs++; |
Jason Evans | 9c640bf | 2014-09-11 16:20:44 -0700 | [diff] [blame] | 151 | if (config_fill && unlikely(opt_junk)) { |
Jason Evans | 0d6c5d8 | 2013-12-17 15:14:36 -0800 | [diff] [blame] | 152 | /* |
| 153 | * Only do redzone validation if Valgrind isn't in |
| 154 | * operation. |
| 155 | */ |
Jason Evans | 9c640bf | 2014-09-11 16:20:44 -0700 | [diff] [blame] | 156 | if ((!config_valgrind || likely(!in_valgrind)) |
Jason Evans | 0d6c5d8 | 2013-12-17 15:14:36 -0800 | [diff] [blame] | 157 | && usize <= SMALL_MAXCLASS) |
| 158 | arena_quarantine_junk_small(ptr, usize); |
| 159 | else |
| 160 | memset(ptr, 0x5a, usize); |
| 161 | } |
Jason Evans | 122449b | 2012-04-06 00:35:09 -0700 | [diff] [blame] | 162 | } else { |
| 163 | assert(quarantine->curbytes == 0); |
Jason Evans | 5460aa6 | 2014-09-22 21:09:23 -0700 | [diff] [blame] | 164 | idalloc(tsd, ptr); |
Jason Evans | 122449b | 2012-04-06 00:35:09 -0700 | [diff] [blame] | 165 | } |
| 166 | } |
| 167 | |
Jason Evans | bbe29d3 | 2013-01-30 15:03:11 -0800 | [diff] [blame] | 168 | void |
Jason Evans | 5460aa6 | 2014-09-22 21:09:23 -0700 | [diff] [blame] | 169 | quarantine_cleanup(tsd_t *tsd) |
Jason Evans | 122449b | 2012-04-06 00:35:09 -0700 | [diff] [blame] | 170 | { |
Jason Evans | 5460aa6 | 2014-09-22 21:09:23 -0700 | [diff] [blame] | 171 | quarantine_t *quarantine; |
Jason Evans | 122449b | 2012-04-06 00:35:09 -0700 | [diff] [blame] | 172 | |
Jason Evans | 5460aa6 | 2014-09-22 21:09:23 -0700 | [diff] [blame] | 173 | if (!config_fill) |
| 174 | return; |
| 175 | |
| 176 | quarantine = tsd_quarantine_get(tsd); |
| 177 | if (quarantine != NULL) { |
| 178 | quarantine_drain(tsd, quarantine, 0); |
| 179 | idalloc(tsd, quarantine); |
| 180 | tsd_quarantine_set(tsd, NULL); |
Jason Evans | 122449b | 2012-04-06 00:35:09 -0700 | [diff] [blame] | 181 | } |
| 182 | } |