blob: 5dae7e7c9b79928c2ef69aa792ac71277536fea4 [file] [log] [blame]
Jens Axboed24c33a2008-03-01 18:27:36 +01001/*
2 * simple memory allocator, backed by mmap() so that it hands out memory
3 * that can be shared across processes and threads
4 */
5#include <sys/mman.h>
6#include <stdio.h>
7#include <stdlib.h>
8#include <assert.h>
9#include <string.h>
10#include <unistd.h>
Bruce Crane43606c2012-02-20 09:34:24 +010011#include <inttypes.h>
Jens Axboed24c33a2008-03-01 18:27:36 +010012#include <sys/types.h>
13#include <limits.h>
Greg Edwards3a8600b2010-06-25 09:24:19 -060014#include <fcntl.h>
Jens Axboed24c33a2008-03-01 18:27:36 +010015
Jens Axboe6548f472008-06-13 08:38:31 +020016#include "mutex.h"
Jens Axboeb3268b92008-06-02 09:59:32 +020017#include "arch/arch.h"
Greg Edwards3a8600b2010-06-25 09:24:19 -060018#include "os/os.h"
Jens Axboed24c33a2008-03-01 18:27:36 +010019
Jens Axboe55f64912008-05-26 09:37:21 +020020#define SMALLOC_REDZONE /* define to detect memory corruption */
Jens Axboed24c33a2008-03-01 18:27:36 +010021
Jens Axboeec996e92008-05-27 11:39:39 +020022#define SMALLOC_BPB 32 /* block size, bytes-per-bit in bitmap */
23#define SMALLOC_BPI (sizeof(unsigned int) * 8)
24#define SMALLOC_BPL (SMALLOC_BPB * SMALLOC_BPI)
25
Jens Axboe5e012982009-07-31 11:23:19 +020026#define INITIAL_SIZE 8192*1024 /* new pool size */
Jens Axboe68857682008-07-31 19:50:54 +020027#define MAX_POOLS 128 /* maximum number of pools to setup */
Jens Axboed24c33a2008-03-01 18:27:36 +010028
Jens Axboe55f64912008-05-26 09:37:21 +020029#define SMALLOC_PRE_RED 0xdeadbeefU
30#define SMALLOC_POST_RED 0x5aa55aa5U
Jens Axboe55f64912008-05-26 09:37:21 +020031
Jens Axboe2b386d22008-03-26 10:32:57 +010032unsigned int smalloc_pool_size = INITIAL_SIZE;
Jens Axboe122426d2009-07-07 23:03:06 +020033const int int_mask = sizeof(int) - 1;
Jens Axboe2b386d22008-03-26 10:32:57 +010034
Jens Axboed24c33a2008-03-01 18:27:36 +010035struct pool {
Jens Axboe6548f472008-06-13 08:38:31 +020036 struct fio_mutex *lock; /* protects this pool */
Jens Axboed24c33a2008-03-01 18:27:36 +010037 void *map; /* map of blocks */
Jens Axboeec996e92008-05-27 11:39:39 +020038 unsigned int *bitmap; /* blocks free/busy map */
Jens Axboea3ebe7e2012-11-02 16:47:03 +010039 size_t free_blocks; /* free blocks */
40 size_t nr_blocks; /* total blocks */
41 size_t next_non_full;
42 size_t mmap_size;
Jens Axboeec996e92008-05-27 11:39:39 +020043};
44
45struct block_hdr {
Jens Axboea3ebe7e2012-11-02 16:47:03 +010046 size_t size;
Jens Axboeec996e92008-05-27 11:39:39 +020047#ifdef SMALLOC_REDZONE
48 unsigned int prered;
49#endif
Jens Axboed24c33a2008-03-01 18:27:36 +010050};
51
52static struct pool mp[MAX_POOLS];
53static unsigned int nr_pools;
54static unsigned int last_pool;
Jens Axboed7df1d12013-03-20 19:57:01 -060055static struct fio_rwlock *lock;
Jens Axboed24c33a2008-03-01 18:27:36 +010056
Jens Axboed24c33a2008-03-01 18:27:36 +010057static inline void pool_lock(struct pool *pool)
58{
Jens Axboe2e3e31e2009-01-06 11:30:08 +010059 fio_mutex_down(pool->lock);
Jens Axboed24c33a2008-03-01 18:27:36 +010060}
61
62static inline void pool_unlock(struct pool *pool)
63{
Jens Axboe2e3e31e2009-01-06 11:30:08 +010064 fio_mutex_up(pool->lock);
Jens Axboed24c33a2008-03-01 18:27:36 +010065}
66
Jens Axboe65864cf2008-03-03 10:36:36 +010067static inline void global_read_lock(void)
Jens Axboed24c33a2008-03-01 18:27:36 +010068{
Jens Axboed7df1d12013-03-20 19:57:01 -060069 fio_rwlock_read(lock);
Jens Axboed24c33a2008-03-01 18:27:36 +010070}
71
Jens Axboe65864cf2008-03-03 10:36:36 +010072static inline void global_read_unlock(void)
Jens Axboed24c33a2008-03-01 18:27:36 +010073{
Jens Axboed7df1d12013-03-20 19:57:01 -060074 fio_rwlock_unlock(lock);
Jens Axboe65864cf2008-03-03 10:36:36 +010075}
76
77static inline void global_write_lock(void)
78{
Jens Axboed7df1d12013-03-20 19:57:01 -060079 fio_rwlock_write(lock);
Jens Axboe65864cf2008-03-03 10:36:36 +010080}
81
82static inline void global_write_unlock(void)
83{
Jens Axboed7df1d12013-03-20 19:57:01 -060084 fio_rwlock_unlock(lock);
Jens Axboed24c33a2008-03-01 18:27:36 +010085}
86
Jens Axboed24c33a2008-03-01 18:27:36 +010087static inline int ptr_valid(struct pool *pool, void *ptr)
88{
Jens Axboedcb69092008-05-27 20:35:18 +020089 unsigned int pool_size = pool->nr_blocks * SMALLOC_BPL;
Jens Axboeec996e92008-05-27 11:39:39 +020090
91 return (ptr >= pool->map) && (ptr < pool->map + pool_size);
Jens Axboed24c33a2008-03-01 18:27:36 +010092}
93
Jens Axboea3ebe7e2012-11-02 16:47:03 +010094static inline size_t size_to_blocks(size_t size)
Jens Axboe808e9ea2008-05-27 14:12:45 +020095{
96 return (size + SMALLOC_BPB - 1) / SMALLOC_BPB;
97}
98
Jens Axboedcb69092008-05-27 20:35:18 +020099static int blocks_iter(struct pool *pool, unsigned int pool_idx,
Jens Axboea3ebe7e2012-11-02 16:47:03 +0100100 unsigned int idx, size_t nr_blocks,
Jens Axboeec996e92008-05-27 11:39:39 +0200101 int (*func)(unsigned int *map, unsigned int mask))
Jens Axboed24c33a2008-03-01 18:27:36 +0100102{
Jens Axboedcb69092008-05-27 20:35:18 +0200103
Jens Axboeec996e92008-05-27 11:39:39 +0200104 while (nr_blocks) {
105 unsigned int this_blocks, mask;
Jens Axboedcb69092008-05-27 20:35:18 +0200106 unsigned int *map;
107
108 if (pool_idx >= pool->nr_blocks)
109 return 0;
110
111 map = &pool->bitmap[pool_idx];
Jens Axboed24c33a2008-03-01 18:27:36 +0100112
Jens Axboeec996e92008-05-27 11:39:39 +0200113 this_blocks = nr_blocks;
114 if (this_blocks + idx > SMALLOC_BPI) {
115 this_blocks = SMALLOC_BPI - idx;
116 idx = SMALLOC_BPI - this_blocks;
Jens Axboed24c33a2008-03-01 18:27:36 +0100117 }
Jens Axboeec996e92008-05-27 11:39:39 +0200118
119 if (this_blocks == SMALLOC_BPI)
120 mask = -1U;
121 else
122 mask = ((1U << this_blocks) - 1) << idx;
123
124 if (!func(map, mask))
125 return 0;
126
127 nr_blocks -= this_blocks;
128 idx = 0;
Jens Axboedcb69092008-05-27 20:35:18 +0200129 pool_idx++;
Jens Axboed24c33a2008-03-01 18:27:36 +0100130 }
131
Jens Axboeec996e92008-05-27 11:39:39 +0200132 return 1;
Jens Axboeec996e92008-05-27 11:39:39 +0200133}
134
135static int mask_cmp(unsigned int *map, unsigned int mask)
136{
137 return !(*map & mask);
138}
139
140static int mask_clear(unsigned int *map, unsigned int mask)
141{
Jens Axboedcb69092008-05-27 20:35:18 +0200142 assert((*map & mask) == mask);
Jens Axboeec996e92008-05-27 11:39:39 +0200143 *map &= ~mask;
144 return 1;
145}
146
147static int mask_set(unsigned int *map, unsigned int mask)
148{
Jens Axboedcb69092008-05-27 20:35:18 +0200149 assert(!(*map & mask));
Jens Axboeec996e92008-05-27 11:39:39 +0200150 *map |= mask;
151 return 1;
152}
153
Jens Axboedcb69092008-05-27 20:35:18 +0200154static int blocks_free(struct pool *pool, unsigned int pool_idx,
Jens Axboea3ebe7e2012-11-02 16:47:03 +0100155 unsigned int idx, size_t nr_blocks)
Jens Axboeec996e92008-05-27 11:39:39 +0200156{
Jens Axboedcb69092008-05-27 20:35:18 +0200157 return blocks_iter(pool, pool_idx, idx, nr_blocks, mask_cmp);
Jens Axboeec996e92008-05-27 11:39:39 +0200158}
159
Jens Axboedcb69092008-05-27 20:35:18 +0200160static void set_blocks(struct pool *pool, unsigned int pool_idx,
Jens Axboea3ebe7e2012-11-02 16:47:03 +0100161 unsigned int idx, size_t nr_blocks)
Jens Axboeec996e92008-05-27 11:39:39 +0200162{
Jens Axboedcb69092008-05-27 20:35:18 +0200163 blocks_iter(pool, pool_idx, idx, nr_blocks, mask_set);
Jens Axboeec996e92008-05-27 11:39:39 +0200164}
165
Jens Axboedcb69092008-05-27 20:35:18 +0200166static void clear_blocks(struct pool *pool, unsigned int pool_idx,
Jens Axboea3ebe7e2012-11-02 16:47:03 +0100167 unsigned int idx, size_t nr_blocks)
Jens Axboeec996e92008-05-27 11:39:39 +0200168{
Jens Axboedcb69092008-05-27 20:35:18 +0200169 blocks_iter(pool, pool_idx, idx, nr_blocks, mask_clear);
Jens Axboeec996e92008-05-27 11:39:39 +0200170}
171
Jens Axboeec996e92008-05-27 11:39:39 +0200172static int find_next_zero(int word, int start)
173{
174 assert(word != -1U);
Jiri Horky271067a2011-08-15 15:18:03 +0200175 word >>= start;
176 return ffz(word) + start;
Jens Axboed24c33a2008-03-01 18:27:36 +0100177}
178
Jens Axboeadf57092008-04-16 19:43:17 +0200179static int add_pool(struct pool *pool, unsigned int alloc_size)
Jens Axboed24c33a2008-03-01 18:27:36 +0100180{
Jens Axboe8d5844e2011-06-29 09:50:08 +0200181 int bitmap_blocks;
Jens Axboeb8a65822008-12-04 14:33:41 +0100182 void *ptr;
Jens Axboeec996e92008-05-27 11:39:39 +0200183
Jens Axboe55f64912008-05-26 09:37:21 +0200184#ifdef SMALLOC_REDZONE
Jens Axboeec996e92008-05-27 11:39:39 +0200185 alloc_size += sizeof(unsigned int);
Jens Axboe55f64912008-05-26 09:37:21 +0200186#endif
Jens Axboeec996e92008-05-27 11:39:39 +0200187 alloc_size += sizeof(struct block_hdr);
188 if (alloc_size < INITIAL_SIZE)
189 alloc_size = INITIAL_SIZE;
Jens Axboeadf57092008-04-16 19:43:17 +0200190
Jens Axboeec996e92008-05-27 11:39:39 +0200191 /* round up to nearest full number of blocks */
192 alloc_size = (alloc_size + SMALLOC_BPL - 1) & ~(SMALLOC_BPL - 1);
193 bitmap_blocks = alloc_size / SMALLOC_BPL;
194 alloc_size += bitmap_blocks * sizeof(unsigned int);
195 pool->mmap_size = alloc_size;
Jens Axboe0b9d69e2009-09-11 22:29:54 +0200196
Jens Axboeec996e92008-05-27 11:39:39 +0200197 pool->nr_blocks = bitmap_blocks;
198 pool->free_blocks = bitmap_blocks * SMALLOC_BPB;
199
Jens Axboe8d5844e2011-06-29 09:50:08 +0200200 ptr = mmap(NULL, alloc_size, PROT_READ|PROT_WRITE,
201 MAP_SHARED | OS_MAP_ANON, -1, 0);
Jens Axboed24c33a2008-03-01 18:27:36 +0100202 if (ptr == MAP_FAILED)
Jens Axboe8d5844e2011-06-29 09:50:08 +0200203 goto out_fail;
Jens Axboed24c33a2008-03-01 18:27:36 +0100204
Jens Axboeec996e92008-05-27 11:39:39 +0200205 memset(ptr, 0, alloc_size);
206 pool->map = ptr;
207 pool->bitmap = (void *) ptr + (pool->nr_blocks * SMALLOC_BPL);
Jens Axboed24c33a2008-03-01 18:27:36 +0100208
Jens Axboe521da522012-08-02 11:21:36 +0200209 pool->lock = fio_mutex_init(FIO_MUTEX_UNLOCKED);
Jens Axboed24c33a2008-03-01 18:27:36 +0100210 if (!pool->lock)
Jens Axboe8d5844e2011-06-29 09:50:08 +0200211 goto out_fail;
Jens Axboed24c33a2008-03-01 18:27:36 +0100212
Jens Axboed24c33a2008-03-01 18:27:36 +0100213 nr_pools++;
214 return 0;
Jens Axboe8d5844e2011-06-29 09:50:08 +0200215out_fail:
Jens Axboeec996e92008-05-27 11:39:39 +0200216 fprintf(stderr, "smalloc: failed adding pool\n");
Jens Axboed24c33a2008-03-01 18:27:36 +0100217 if (pool->map)
Jens Axboeec996e92008-05-27 11:39:39 +0200218 munmap(pool->map, pool->mmap_size);
Jens Axboed24c33a2008-03-01 18:27:36 +0100219 return 1;
220}
221
222void sinit(void)
223{
Jens Axboe4d4e80f2008-03-04 10:18:56 +0100224 int ret;
Jens Axboed24c33a2008-03-01 18:27:36 +0100225
Jens Axboed7df1d12013-03-20 19:57:01 -0600226 lock = fio_rwlock_init();
Jens Axboeadf57092008-04-16 19:43:17 +0200227 ret = add_pool(&mp[0], INITIAL_SIZE);
Jens Axboed24c33a2008-03-01 18:27:36 +0100228 assert(!ret);
229}
230
231static void cleanup_pool(struct pool *pool)
232{
Jens Axboe443bb112008-12-04 14:30:13 +0100233 /*
234 * This will also remove the temporary file we used as a backing
235 * store, it was already unlinked
236 */
Jens Axboeec996e92008-05-27 11:39:39 +0200237 munmap(pool->map, pool->mmap_size);
Jens Axboe6548f472008-06-13 08:38:31 +0200238
239 if (pool->lock)
240 fio_mutex_remove(pool->lock);
Jens Axboed24c33a2008-03-01 18:27:36 +0100241}
242
243void scleanup(void)
244{
245 unsigned int i;
246
247 for (i = 0; i < nr_pools; i++)
248 cleanup_pool(&mp[i]);
249
Jens Axboe6548f472008-06-13 08:38:31 +0200250 if (lock)
Jens Axboed7df1d12013-03-20 19:57:01 -0600251 fio_rwlock_remove(lock);
Jens Axboed24c33a2008-03-01 18:27:36 +0100252}
253
Jens Axboe89da54e2008-05-27 20:49:29 +0200254#ifdef SMALLOC_REDZONE
Jens Axboecf987082009-07-07 17:43:11 +0200255static void *postred_ptr(struct block_hdr *hdr)
256{
Bruce Crane43606c2012-02-20 09:34:24 +0100257 uintptr_t ptr;
Jens Axboecf987082009-07-07 17:43:11 +0200258
Bruce Crane43606c2012-02-20 09:34:24 +0100259 ptr = (uintptr_t) hdr + hdr->size - sizeof(unsigned int);
Jens Axboecf987082009-07-07 17:43:11 +0200260 ptr = (ptr + int_mask) & ~int_mask;
261
262 return (void *) ptr;
263}
264
Jens Axboeec996e92008-05-27 11:39:39 +0200265static void fill_redzone(struct block_hdr *hdr)
Jens Axboe55f64912008-05-26 09:37:21 +0200266{
Jens Axboecf987082009-07-07 17:43:11 +0200267 unsigned int *postred = postred_ptr(hdr);
Jens Axboe55f64912008-05-26 09:37:21 +0200268
Jens Axboeec996e92008-05-27 11:39:39 +0200269 hdr->prered = SMALLOC_PRE_RED;
270 *postred = SMALLOC_POST_RED;
Jens Axboeec996e92008-05-27 11:39:39 +0200271}
Jens Axboe55f64912008-05-26 09:37:21 +0200272
Jens Axboeec996e92008-05-27 11:39:39 +0200273static void sfree_check_redzone(struct block_hdr *hdr)
274{
Jens Axboecf987082009-07-07 17:43:11 +0200275 unsigned int *postred = postred_ptr(hdr);
Jens Axboeec996e92008-05-27 11:39:39 +0200276
277 if (hdr->prered != SMALLOC_PRE_RED) {
Jens Axboe55f64912008-05-26 09:37:21 +0200278 fprintf(stderr, "smalloc pre redzone destroyed!\n");
279 fprintf(stderr, " ptr=%p, prered=%x, expected %x\n",
Jens Axboeec996e92008-05-27 11:39:39 +0200280 hdr, hdr->prered, SMALLOC_PRE_RED);
Jens Axboe55f64912008-05-26 09:37:21 +0200281 assert(0);
282 }
283 if (*postred != SMALLOC_POST_RED) {
284 fprintf(stderr, "smalloc post redzone destroyed!\n");
285 fprintf(stderr, " ptr=%p, postred=%x, expected %x\n",
Jens Axboeec996e92008-05-27 11:39:39 +0200286 hdr, *postred, SMALLOC_POST_RED);
Jens Axboe55f64912008-05-26 09:37:21 +0200287 assert(0);
288 }
Jens Axboe55f64912008-05-26 09:37:21 +0200289}
Jens Axboe89da54e2008-05-27 20:49:29 +0200290#else
291static void fill_redzone(struct block_hdr *hdr)
292{
293}
294
295static void sfree_check_redzone(struct block_hdr *hdr)
296{
297}
298#endif
Jens Axboe55f64912008-05-26 09:37:21 +0200299
Jens Axboed24c33a2008-03-01 18:27:36 +0100300static void sfree_pool(struct pool *pool, void *ptr)
301{
Jens Axboeec996e92008-05-27 11:39:39 +0200302 struct block_hdr *hdr;
Jens Axboe179446e2008-05-27 14:11:56 +0200303 unsigned int i, idx;
Jens Axboeec996e92008-05-27 11:39:39 +0200304 unsigned long offset;
Jens Axboed24c33a2008-03-01 18:27:36 +0100305
306 if (!ptr)
307 return;
308
Jens Axboeec996e92008-05-27 11:39:39 +0200309 ptr -= sizeof(*hdr);
310 hdr = ptr;
Jens Axboe55f64912008-05-26 09:37:21 +0200311
Jens Axboed24c33a2008-03-01 18:27:36 +0100312 assert(ptr_valid(pool, ptr));
313
Jens Axboeec996e92008-05-27 11:39:39 +0200314 sfree_check_redzone(hdr);
315
316 offset = ptr - pool->map;
317 i = offset / SMALLOC_BPL;
318 idx = (offset % SMALLOC_BPL) / SMALLOC_BPB;
319
Jens Axboed24c33a2008-03-01 18:27:36 +0100320 pool_lock(pool);
Jens Axboedcb69092008-05-27 20:35:18 +0200321 clear_blocks(pool, i, idx, size_to_blocks(hdr->size));
Jens Axboeec996e92008-05-27 11:39:39 +0200322 if (i < pool->next_non_full)
323 pool->next_non_full = i;
Jens Axboe179446e2008-05-27 14:11:56 +0200324 pool->free_blocks += size_to_blocks(hdr->size);
Jens Axboed24c33a2008-03-01 18:27:36 +0100325 pool_unlock(pool);
326}
327
328void sfree(void *ptr)
329{
330 struct pool *pool = NULL;
331 unsigned int i;
332
Jens Axboe8e5732e2008-04-16 19:51:46 +0200333 if (!ptr)
334 return;
335
Jens Axboe65864cf2008-03-03 10:36:36 +0100336 global_read_lock();
Jens Axboed24c33a2008-03-01 18:27:36 +0100337
338 for (i = 0; i < nr_pools; i++) {
339 if (ptr_valid(&mp[i], ptr)) {
340 pool = &mp[i];
341 break;
342 }
343 }
344
Jens Axboe65864cf2008-03-03 10:36:36 +0100345 global_read_unlock();
Jens Axboed24c33a2008-03-01 18:27:36 +0100346
347 assert(pool);
348 sfree_pool(pool, ptr);
349}
350
Jens Axboea3ebe7e2012-11-02 16:47:03 +0100351static void *__smalloc_pool(struct pool *pool, size_t size)
Jens Axboed24c33a2008-03-01 18:27:36 +0100352{
Jens Axboea3ebe7e2012-11-02 16:47:03 +0100353 size_t nr_blocks;
Jens Axboeec996e92008-05-27 11:39:39 +0200354 unsigned int i;
355 unsigned int offset;
356 unsigned int last_idx;
357 void *ret = NULL;
Jens Axboed24c33a2008-03-01 18:27:36 +0100358
Jens Axboed24c33a2008-03-01 18:27:36 +0100359 pool_lock(pool);
Jens Axboe179446e2008-05-27 14:11:56 +0200360
361 nr_blocks = size_to_blocks(size);
Jens Axboeec996e92008-05-27 11:39:39 +0200362 if (nr_blocks > pool->free_blocks)
Jens Axboed24c33a2008-03-01 18:27:36 +0100363 goto fail;
364
Jens Axboeec996e92008-05-27 11:39:39 +0200365 i = pool->next_non_full;
366 last_idx = 0;
367 offset = -1U;
368 while (i < pool->nr_blocks) {
369 unsigned int idx;
Jens Axboed24c33a2008-03-01 18:27:36 +0100370
Jens Axboeec996e92008-05-27 11:39:39 +0200371 if (pool->bitmap[i] == -1U) {
372 i++;
373 pool->next_non_full = i;
374 last_idx = 0;
375 continue;
Jens Axboed24c33a2008-03-01 18:27:36 +0100376 }
Jens Axboeec996e92008-05-27 11:39:39 +0200377
378 idx = find_next_zero(pool->bitmap[i], last_idx);
Jens Axboedcb69092008-05-27 20:35:18 +0200379 if (!blocks_free(pool, i, idx, nr_blocks)) {
Jens Axboeec996e92008-05-27 11:39:39 +0200380 idx += nr_blocks;
381 if (idx < SMALLOC_BPI)
382 last_idx = idx;
383 else {
384 last_idx = 0;
385 while (idx >= SMALLOC_BPI) {
386 i++;
387 idx -= SMALLOC_BPI;
388 }
389 }
390 continue;
391 }
Jens Axboedcb69092008-05-27 20:35:18 +0200392 set_blocks(pool, i, idx, nr_blocks);
Jens Axboeec996e92008-05-27 11:39:39 +0200393 offset = i * SMALLOC_BPL + idx * SMALLOC_BPB;
394 break;
Jens Axboed24c33a2008-03-01 18:27:36 +0100395 }
Jens Axboeec996e92008-05-27 11:39:39 +0200396
397 if (i < pool->nr_blocks) {
398 pool->free_blocks -= nr_blocks;
399 ret = pool->map + offset;
400 }
401fail:
Jens Axboed24c33a2008-03-01 18:27:36 +0100402 pool_unlock(pool);
Jens Axboeec996e92008-05-27 11:39:39 +0200403 return ret;
Jens Axboed24c33a2008-03-01 18:27:36 +0100404}
405
Jens Axboea3ebe7e2012-11-02 16:47:03 +0100406static void *smalloc_pool(struct pool *pool, size_t size)
Jens Axboe55f64912008-05-26 09:37:21 +0200407{
Jens Axboea3ebe7e2012-11-02 16:47:03 +0100408 size_t alloc_size = size + sizeof(struct block_hdr);
Jens Axboe55f64912008-05-26 09:37:21 +0200409 void *ptr;
410
Jens Axboecf987082009-07-07 17:43:11 +0200411 /*
Jens Axboe122426d2009-07-07 23:03:06 +0200412 * Round to int alignment, so that the postred pointer will
413 * be naturally aligned as well.
Jens Axboecf987082009-07-07 17:43:11 +0200414 */
Jens Axboeec996e92008-05-27 11:39:39 +0200415#ifdef SMALLOC_REDZONE
Jens Axboe122426d2009-07-07 23:03:06 +0200416 alloc_size += sizeof(unsigned int);
417 alloc_size = (alloc_size + int_mask) & ~int_mask;
Jens Axboe55f64912008-05-26 09:37:21 +0200418#endif
Jens Axboeec996e92008-05-27 11:39:39 +0200419
420 ptr = __smalloc_pool(pool, alloc_size);
Jens Axboe89da54e2008-05-27 20:49:29 +0200421 if (ptr) {
422 struct block_hdr *hdr = ptr;
Jens Axboeec996e92008-05-27 11:39:39 +0200423
Jens Axboe89da54e2008-05-27 20:49:29 +0200424 hdr->size = alloc_size;
425 fill_redzone(hdr);
Jens Axboeec996e92008-05-27 11:39:39 +0200426
Jens Axboe89da54e2008-05-27 20:49:29 +0200427 ptr += sizeof(*hdr);
428 memset(ptr, 0, size);
429 }
Jens Axboeec996e92008-05-27 11:39:39 +0200430
Jens Axboeec996e92008-05-27 11:39:39 +0200431 return ptr;
Jens Axboe55f64912008-05-26 09:37:21 +0200432}
433
Jens Axboe7982aa72012-11-02 16:35:18 +0100434void *smalloc(size_t size)
Jens Axboed24c33a2008-03-01 18:27:36 +0100435{
436 unsigned int i;
437
Jens Axboe7982aa72012-11-02 16:35:18 +0100438 if (size != (unsigned int) size)
439 return NULL;
440
Shaozhi Shawn Yed1271dc2008-09-10 09:02:51 +0200441 global_write_lock();
Jens Axboed24c33a2008-03-01 18:27:36 +0100442 i = last_pool;
443
444 do {
445 for (; i < nr_pools; i++) {
446 void *ptr = smalloc_pool(&mp[i], size);
447
448 if (ptr) {
449 last_pool = i;
Shaozhi Shawn Yed1271dc2008-09-10 09:02:51 +0200450 global_write_unlock();
Jens Axboed24c33a2008-03-01 18:27:36 +0100451 return ptr;
452 }
453 }
454 if (last_pool) {
455 last_pool = 0;
456 continue;
457 }
458
Jens Axboeec996e92008-05-27 11:39:39 +0200459 if (nr_pools + 1 > MAX_POOLS)
Jens Axboed24c33a2008-03-01 18:27:36 +0100460 break;
461 else {
462 i = nr_pools;
Jens Axboeadf57092008-04-16 19:43:17 +0200463 if (add_pool(&mp[nr_pools], size))
Jens Axboe65864cf2008-03-03 10:36:36 +0100464 goto out;
Jens Axboed24c33a2008-03-01 18:27:36 +0100465 }
466 } while (1);
467
Jens Axboe65864cf2008-03-03 10:36:36 +0100468out:
Shaozhi Shawn Yed1271dc2008-09-10 09:02:51 +0200469 global_write_unlock();
Jens Axboed24c33a2008-03-01 18:27:36 +0100470 return NULL;
471}
472
473char *smalloc_strdup(const char *str)
474{
475 char *ptr;
476
477 ptr = smalloc(strlen(str) + 1);
478 strcpy(ptr, str);
479 return ptr;
480}