Jason Evans | d81e4bd | 2012-03-06 14:57:45 -0800 | [diff] [blame] | 1 | /******************************************************************************/ |
| 2 | #ifdef JEMALLOC_H_TYPES |
| 3 | |
| 4 | /* Size of stack-allocated buffer passed to buferror(). */ |
| 5 | #define BUFERROR_BUF 64 |
| 6 | |
| 7 | /* |
Jason Evans | cd9a134 | 2012-03-21 18:33:03 -0700 | [diff] [blame] | 8 | * Size of stack-allocated buffer used by malloc_{,v,vc}printf(). This must be |
| 9 | * large enough for all possible uses within jemalloc. |
Jason Evans | 824d34e | 2012-03-13 13:19:04 -0700 | [diff] [blame] | 10 | */ |
| 11 | #define MALLOC_PRINTF_BUFSIZE 4096 |
| 12 | |
| 13 | /* |
Jason Evans | cd9a134 | 2012-03-21 18:33:03 -0700 | [diff] [blame] | 14 | * Wrap a cpp argument that contains commas such that it isn't broken up into |
| 15 | * multiple arguments. |
| 16 | */ |
Jason Evans | a4f124f | 2013-12-08 22:28:27 -0800 | [diff] [blame] | 17 | #define JEMALLOC_ARG_CONCAT(...) __VA_ARGS__ |
Jason Evans | cd9a134 | 2012-03-21 18:33:03 -0700 | [diff] [blame] | 18 | |
| 19 | /* |
Jason Evans | 9225a19 | 2012-03-23 15:39:07 -0700 | [diff] [blame] | 20 | * Silence compiler warnings due to uninitialized values. This is used |
| 21 | * wherever the compiler fails to recognize that the variable is never used |
| 22 | * uninitialized. |
| 23 | */ |
| 24 | #ifdef JEMALLOC_CC_SILENCE |
| 25 | # define JEMALLOC_CC_SILENCE_INIT(v) = v |
| 26 | #else |
| 27 | # define JEMALLOC_CC_SILENCE_INIT(v) |
| 28 | #endif |
| 29 | |
Daniel Micay | 6b5609d | 2014-09-08 22:18:49 -0400 | [diff] [blame] | 30 | #ifdef __GNUC__ |
| 31 | #define likely(x) __builtin_expect(!!(x), 1) |
| 32 | #define unlikely(x) __builtin_expect(!!(x), 0) |
| 33 | #else |
| 34 | #define likely(x) !!(x) |
| 35 | #define unlikely(x) !!(x) |
| 36 | #endif |
| 37 | |
Jason Evans | 9225a19 | 2012-03-23 15:39:07 -0700 | [diff] [blame] | 38 | /* |
Jason Evans | d81e4bd | 2012-03-06 14:57:45 -0800 | [diff] [blame] | 39 | * Define a custom assert() in order to reduce the chances of deadlock during |
| 40 | * assertion failure. |
| 41 | */ |
| 42 | #ifndef assert |
| 43 | #define assert(e) do { \ |
Daniel Micay | 23fdf8b | 2014-09-09 15:26:05 -0400 | [diff] [blame] | 44 | if (unlikely(config_debug && !(e))) { \ |
Jason Evans | d81e4bd | 2012-03-06 14:57:45 -0800 | [diff] [blame] | 45 | malloc_printf( \ |
| 46 | "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", \ |
| 47 | __FILE__, __LINE__, #e); \ |
| 48 | abort(); \ |
| 49 | } \ |
| 50 | } while (0) |
| 51 | #endif |
| 52 | |
Jason Evans | d81e4bd | 2012-03-06 14:57:45 -0800 | [diff] [blame] | 53 | #ifndef not_reached |
| 54 | #define not_reached() do { \ |
| 55 | if (config_debug) { \ |
| 56 | malloc_printf( \ |
| 57 | "<jemalloc>: %s:%d: Unreachable code reached\n", \ |
| 58 | __FILE__, __LINE__); \ |
| 59 | abort(); \ |
| 60 | } \ |
| 61 | } while (0) |
| 62 | #endif |
| 63 | |
| 64 | #ifndef not_implemented |
| 65 | #define not_implemented() do { \ |
| 66 | if (config_debug) { \ |
| 67 | malloc_printf("<jemalloc>: %s:%d: Not implemented\n", \ |
| 68 | __FILE__, __LINE__); \ |
| 69 | abort(); \ |
| 70 | } \ |
| 71 | } while (0) |
| 72 | #endif |
| 73 | |
Jason Evans | 86abd0d | 2013-11-30 15:25:42 -0800 | [diff] [blame] | 74 | #ifndef assert_not_implemented |
Jason Evans | d81e4bd | 2012-03-06 14:57:45 -0800 | [diff] [blame] | 75 | #define assert_not_implemented(e) do { \ |
Daniel Micay | 23fdf8b | 2014-09-09 15:26:05 -0400 | [diff] [blame] | 76 | if (unlikely(config_debug && !(e))) \ |
Jason Evans | d81e4bd | 2012-03-06 14:57:45 -0800 | [diff] [blame] | 77 | not_implemented(); \ |
| 78 | } while (0) |
Jason Evans | 86abd0d | 2013-11-30 15:25:42 -0800 | [diff] [blame] | 79 | #endif |
Jason Evans | d81e4bd | 2012-03-06 14:57:45 -0800 | [diff] [blame] | 80 | |
Jason Evans | 6556e28 | 2013-10-21 14:56:27 -0700 | [diff] [blame] | 81 | /* Use to assert a particular configuration, e.g., cassert(config_debug). */ |
| 82 | #define cassert(c) do { \ |
Daniel Micay | 23fdf8b | 2014-09-09 15:26:05 -0400 | [diff] [blame] | 83 | if (unlikely(!(c))) \ |
Jason Evans | 6556e28 | 2013-10-21 14:56:27 -0700 | [diff] [blame] | 84 | not_reached(); \ |
| 85 | } while (0) |
| 86 | |
Jason Evans | d81e4bd | 2012-03-06 14:57:45 -0800 | [diff] [blame] | 87 | #endif /* JEMALLOC_H_TYPES */ |
| 88 | /******************************************************************************/ |
| 89 | #ifdef JEMALLOC_H_STRUCTS |
| 90 | |
| 91 | #endif /* JEMALLOC_H_STRUCTS */ |
| 92 | /******************************************************************************/ |
| 93 | #ifdef JEMALLOC_H_EXTERNS |
| 94 | |
Jason Evans | 2a83ed0 | 2013-12-08 20:52:21 -0800 | [diff] [blame] | 95 | int buferror(int err, char *buf, size_t buflen); |
Jason Evans | e18c25d | 2014-01-06 20:33:48 -0800 | [diff] [blame] | 96 | uintmax_t malloc_strtoumax(const char *restrict nptr, |
| 97 | char **restrict endptr, int base); |
Jason Evans | 889ec59 | 2012-05-02 02:08:03 -0700 | [diff] [blame] | 98 | void malloc_write(const char *s); |
Jason Evans | d81e4bd | 2012-03-06 14:57:45 -0800 | [diff] [blame] | 99 | |
| 100 | /* |
| 101 | * malloc_vsnprintf() supports a subset of snprintf(3) that avoids floating |
| 102 | * point math. |
| 103 | */ |
| 104 | int malloc_vsnprintf(char *str, size_t size, const char *format, |
| 105 | va_list ap); |
| 106 | int malloc_snprintf(char *str, size_t size, const char *format, ...) |
| 107 | JEMALLOC_ATTR(format(printf, 3, 4)); |
Jason Evans | d81e4bd | 2012-03-06 14:57:45 -0800 | [diff] [blame] | 108 | void malloc_vcprintf(void (*write_cb)(void *, const char *), void *cbopaque, |
| 109 | const char *format, va_list ap); |
| 110 | void malloc_cprintf(void (*write)(void *, const char *), void *cbopaque, |
| 111 | const char *format, ...) JEMALLOC_ATTR(format(printf, 3, 4)); |
| 112 | void malloc_printf(const char *format, ...) |
| 113 | JEMALLOC_ATTR(format(printf, 1, 2)); |
| 114 | |
| 115 | #endif /* JEMALLOC_H_EXTERNS */ |
| 116 | /******************************************************************************/ |
| 117 | #ifdef JEMALLOC_H_INLINES |
| 118 | |
| 119 | #ifndef JEMALLOC_ENABLE_INLINE |
Richard Diamond | 9c3a10f | 2014-05-28 21:37:02 -0500 | [diff] [blame] | 120 | int jemalloc_ffsl(long bitmap); |
| 121 | int jemalloc_ffs(int bitmap); |
Jason Evans | d81e4bd | 2012-03-06 14:57:45 -0800 | [diff] [blame] | 122 | size_t pow2_ceil(size_t x); |
Jason Evans | d04047c | 2014-05-28 16:11:55 -0700 | [diff] [blame] | 123 | size_t lg_floor(size_t x); |
Mike Hommey | a14bce8 | 2012-04-30 12:38:26 +0200 | [diff] [blame] | 124 | void set_errno(int errnum); |
| 125 | int get_errno(void); |
Jason Evans | d81e4bd | 2012-03-06 14:57:45 -0800 | [diff] [blame] | 126 | #endif |
| 127 | |
| 128 | #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_UTIL_C_)) |
Richard Diamond | 9c3a10f | 2014-05-28 21:37:02 -0500 | [diff] [blame] | 129 | |
| 130 | /* Sanity check: */ |
| 131 | #if !defined(JEMALLOC_INTERNAL_FFSL) || !defined(JEMALLOC_INTERNAL_FFS) |
| 132 | # error Both JEMALLOC_INTERNAL_FFSL && JEMALLOC_INTERNAL_FFS should have been defined by configure |
| 133 | #endif |
| 134 | |
| 135 | JEMALLOC_ALWAYS_INLINE int |
| 136 | jemalloc_ffsl(long bitmap) |
| 137 | { |
| 138 | |
| 139 | return (JEMALLOC_INTERNAL_FFSL(bitmap)); |
| 140 | } |
| 141 | |
| 142 | JEMALLOC_ALWAYS_INLINE int |
| 143 | jemalloc_ffs(int bitmap) |
| 144 | { |
| 145 | |
| 146 | return (JEMALLOC_INTERNAL_FFS(bitmap)); |
| 147 | } |
| 148 | |
Jason Evans | d81e4bd | 2012-03-06 14:57:45 -0800 | [diff] [blame] | 149 | /* Compute the smallest power of 2 that is >= x. */ |
| 150 | JEMALLOC_INLINE size_t |
| 151 | pow2_ceil(size_t x) |
| 152 | { |
| 153 | |
| 154 | x--; |
| 155 | x |= x >> 1; |
| 156 | x |= x >> 2; |
| 157 | x |= x >> 4; |
| 158 | x |= x >> 8; |
| 159 | x |= x >> 16; |
| 160 | #if (LG_SIZEOF_PTR == 3) |
| 161 | x |= x >> 32; |
| 162 | #endif |
| 163 | x++; |
| 164 | return (x); |
| 165 | } |
| 166 | |
Jason Evans | d04047c | 2014-05-28 16:11:55 -0700 | [diff] [blame] | 167 | #if (defined(__i386__) || defined(__amd64__) || defined(__x86_64__)) |
| 168 | JEMALLOC_INLINE size_t |
| 169 | lg_floor(size_t x) |
| 170 | { |
| 171 | size_t ret; |
| 172 | |
| 173 | asm ("bsr %1, %0" |
| 174 | : "=r"(ret) // Outputs. |
| 175 | : "r"(x) // Inputs. |
| 176 | ); |
| 177 | return (ret); |
| 178 | } |
Dave Rigby | 112704c | 2014-09-22 15:54:33 +0100 | [diff] [blame] | 179 | #elif (defined(_MSC_VER)) |
| 180 | JEMALLOC_INLINE size_t |
| 181 | lg_floor(size_t x) |
| 182 | { |
| 183 | unsigned long ret; |
| 184 | |
| 185 | #if (LG_SIZEOF_PTR == 3) |
| 186 | _BitScanReverse64(&ret, x); |
| 187 | #elif (LG_SIZEOF_PTR == 2) |
| 188 | _BitScanReverse(&ret, x); |
| 189 | #else |
| 190 | # error "Unsupported type sizes for lg_floor()" |
| 191 | #endif |
| 192 | return (ret); |
| 193 | } |
Jason Evans | d04047c | 2014-05-28 16:11:55 -0700 | [diff] [blame] | 194 | #elif (defined(JEMALLOC_HAVE_BUILTIN_CLZ)) |
| 195 | JEMALLOC_INLINE size_t |
| 196 | lg_floor(size_t x) |
| 197 | { |
| 198 | |
| 199 | #if (LG_SIZEOF_PTR == LG_SIZEOF_INT) |
Jason Evans | 0b5c922 | 2014-06-01 22:05:08 -0700 | [diff] [blame] | 200 | return (((8 << LG_SIZEOF_PTR) - 1) - __builtin_clz(x)); |
Jason Evans | d04047c | 2014-05-28 16:11:55 -0700 | [diff] [blame] | 201 | #elif (LG_SIZEOF_PTR == LG_SIZEOF_LONG) |
Jason Evans | 0b5c922 | 2014-06-01 22:05:08 -0700 | [diff] [blame] | 202 | return (((8 << LG_SIZEOF_PTR) - 1) - __builtin_clzl(x)); |
Jason Evans | d04047c | 2014-05-28 16:11:55 -0700 | [diff] [blame] | 203 | #else |
| 204 | # error "Unsupported type sizes for lg_floor()" |
| 205 | #endif |
| 206 | } |
| 207 | #else |
| 208 | JEMALLOC_INLINE size_t |
| 209 | lg_floor(size_t x) |
| 210 | { |
| 211 | |
Jason Evans | 0b5c922 | 2014-06-01 22:05:08 -0700 | [diff] [blame] | 212 | x |= (x >> 1); |
| 213 | x |= (x >> 2); |
| 214 | x |= (x >> 4); |
| 215 | x |= (x >> 8); |
| 216 | x |= (x >> 16); |
Jason Evans | d04047c | 2014-05-28 16:11:55 -0700 | [diff] [blame] | 217 | #if (LG_SIZEOF_PTR == 3 && LG_SIZEOF_PTR == LG_SIZEOF_LONG) |
Jason Evans | 0b5c922 | 2014-06-01 22:05:08 -0700 | [diff] [blame] | 218 | x |= (x >> 32); |
| 219 | if (x == KZU(0xffffffffffffffff)) |
| 220 | return (63); |
| 221 | x++; |
Richard Diamond | 9c3a10f | 2014-05-28 21:37:02 -0500 | [diff] [blame] | 222 | return (jemalloc_ffsl(x) - 2); |
Jason Evans | d04047c | 2014-05-28 16:11:55 -0700 | [diff] [blame] | 223 | #elif (LG_SIZEOF_PTR == 2) |
Jason Evans | 0b5c922 | 2014-06-01 22:05:08 -0700 | [diff] [blame] | 224 | if (x == KZU(0xffffffff)) |
| 225 | return (31); |
| 226 | x++; |
Richard Diamond | 9c3a10f | 2014-05-28 21:37:02 -0500 | [diff] [blame] | 227 | return (jemalloc_ffs(x) - 2); |
Jason Evans | d04047c | 2014-05-28 16:11:55 -0700 | [diff] [blame] | 228 | #else |
| 229 | # error "Unsupported type sizes for lg_floor()" |
| 230 | #endif |
| 231 | } |
| 232 | #endif |
| 233 | |
Mike Hommey | a14bce8 | 2012-04-30 12:38:26 +0200 | [diff] [blame] | 234 | /* Sets error code */ |
| 235 | JEMALLOC_INLINE void |
| 236 | set_errno(int errnum) |
| 237 | { |
| 238 | |
| 239 | #ifdef _WIN32 |
| 240 | SetLastError(errnum); |
| 241 | #else |
| 242 | errno = errnum; |
| 243 | #endif |
| 244 | } |
| 245 | |
| 246 | /* Get last error code */ |
| 247 | JEMALLOC_INLINE int |
| 248 | get_errno(void) |
| 249 | { |
| 250 | |
| 251 | #ifdef _WIN32 |
Mike Hommey | 7cdea39 | 2012-04-30 12:38:27 +0200 | [diff] [blame] | 252 | return (GetLastError()); |
Mike Hommey | a14bce8 | 2012-04-30 12:38:26 +0200 | [diff] [blame] | 253 | #else |
Mike Hommey | 7cdea39 | 2012-04-30 12:38:27 +0200 | [diff] [blame] | 254 | return (errno); |
Mike Hommey | a14bce8 | 2012-04-30 12:38:26 +0200 | [diff] [blame] | 255 | #endif |
| 256 | } |
Jason Evans | d81e4bd | 2012-03-06 14:57:45 -0800 | [diff] [blame] | 257 | #endif |
| 258 | |
| 259 | #endif /* JEMALLOC_H_INLINES */ |
| 260 | /******************************************************************************/ |