Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1 | /* |
| 2 | * linux/mm/mempool.c |
| 3 | * |
| 4 | * memory buffer pool support. Such pools are mostly used |
| 5 | * for guaranteed, deadlock-free memory allocations during |
| 6 | * extreme VM load. |
| 7 | * |
| 8 | * started by Ingo Molnar, Copyright (C) 2001 |
| 9 | */ |
| 10 | |
| 11 | #include <linux/mm.h> |
| 12 | #include <linux/slab.h> |
| 13 | #include <linux/module.h> |
| 14 | #include <linux/mempool.h> |
| 15 | #include <linux/blkdev.h> |
| 16 | #include <linux/writeback.h> |
| 17 | |
| 18 | static void add_element(mempool_t *pool, void *element) |
| 19 | { |
| 20 | BUG_ON(pool->curr_nr >= pool->min_nr); |
| 21 | pool->elements[pool->curr_nr++] = element; |
| 22 | } |
| 23 | |
| 24 | static void *remove_element(mempool_t *pool) |
| 25 | { |
| 26 | BUG_ON(pool->curr_nr <= 0); |
| 27 | return pool->elements[--pool->curr_nr]; |
| 28 | } |
| 29 | |
| 30 | static void free_pool(mempool_t *pool) |
| 31 | { |
| 32 | while (pool->curr_nr) { |
| 33 | void *element = remove_element(pool); |
| 34 | pool->free(element, pool->pool_data); |
| 35 | } |
| 36 | kfree(pool->elements); |
| 37 | kfree(pool); |
| 38 | } |
| 39 | |
| 40 | /** |
| 41 | * mempool_create - create a memory pool |
| 42 | * @min_nr: the minimum number of elements guaranteed to be |
| 43 | * allocated for this pool. |
| 44 | * @alloc_fn: user-defined element-allocation function. |
| 45 | * @free_fn: user-defined element-freeing function. |
| 46 | * @pool_data: optional private data available to the user-defined functions. |
| 47 | * |
| 48 | * this function creates and allocates a guaranteed size, preallocated |
| 49 | * memory pool. The pool can be used from the mempool_alloc and mempool_free |
| 50 | * functions. This function might sleep. Both the alloc_fn() and the free_fn() |
| 51 | * functions might sleep - as long as the mempool_alloc function is not called |
| 52 | * from IRQ contexts. |
| 53 | */ |
Christoph Lameter | 1946089 | 2005-06-23 00:08:19 -0700 | [diff] [blame] | 54 | mempool_t *mempool_create(int min_nr, mempool_alloc_t *alloc_fn, |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 55 | mempool_free_t *free_fn, void *pool_data) |
| 56 | { |
Christoph Lameter | 1946089 | 2005-06-23 00:08:19 -0700 | [diff] [blame] | 57 | return mempool_create_node(min_nr,alloc_fn,free_fn, pool_data,-1); |
| 58 | } |
| 59 | EXPORT_SYMBOL(mempool_create); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 60 | |
Christoph Lameter | 1946089 | 2005-06-23 00:08:19 -0700 | [diff] [blame] | 61 | mempool_t *mempool_create_node(int min_nr, mempool_alloc_t *alloc_fn, |
| 62 | mempool_free_t *free_fn, void *pool_data, int node_id) |
| 63 | { |
| 64 | mempool_t *pool; |
| 65 | pool = kmalloc_node(sizeof(*pool), GFP_KERNEL, node_id); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 66 | if (!pool) |
| 67 | return NULL; |
| 68 | memset(pool, 0, sizeof(*pool)); |
Christoph Lameter | 1946089 | 2005-06-23 00:08:19 -0700 | [diff] [blame] | 69 | pool->elements = kmalloc_node(min_nr * sizeof(void *), |
| 70 | GFP_KERNEL, node_id); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 71 | if (!pool->elements) { |
| 72 | kfree(pool); |
| 73 | return NULL; |
| 74 | } |
| 75 | spin_lock_init(&pool->lock); |
| 76 | pool->min_nr = min_nr; |
| 77 | pool->pool_data = pool_data; |
| 78 | init_waitqueue_head(&pool->wait); |
| 79 | pool->alloc = alloc_fn; |
| 80 | pool->free = free_fn; |
| 81 | |
| 82 | /* |
| 83 | * First pre-allocate the guaranteed number of buffers. |
| 84 | */ |
| 85 | while (pool->curr_nr < pool->min_nr) { |
| 86 | void *element; |
| 87 | |
| 88 | element = pool->alloc(GFP_KERNEL, pool->pool_data); |
| 89 | if (unlikely(!element)) { |
| 90 | free_pool(pool); |
| 91 | return NULL; |
| 92 | } |
| 93 | add_element(pool, element); |
| 94 | } |
| 95 | return pool; |
| 96 | } |
Christoph Lameter | 1946089 | 2005-06-23 00:08:19 -0700 | [diff] [blame] | 97 | EXPORT_SYMBOL(mempool_create_node); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 98 | |
| 99 | /** |
| 100 | * mempool_resize - resize an existing memory pool |
| 101 | * @pool: pointer to the memory pool which was allocated via |
| 102 | * mempool_create(). |
| 103 | * @new_min_nr: the new minimum number of elements guaranteed to be |
| 104 | * allocated for this pool. |
| 105 | * @gfp_mask: the usual allocation bitmask. |
| 106 | * |
| 107 | * This function shrinks/grows the pool. In the case of growing, |
| 108 | * it cannot be guaranteed that the pool will be grown to the new |
| 109 | * size immediately, but new mempool_free() calls will refill it. |
| 110 | * |
| 111 | * Note, the caller must guarantee that no mempool_destroy is called |
| 112 | * while this function is running. mempool_alloc() & mempool_free() |
| 113 | * might be called (eg. from IRQ contexts) while this function executes. |
| 114 | */ |
Al Viro | dd0fc66 | 2005-10-07 07:46:04 +0100 | [diff] [blame] | 115 | int mempool_resize(mempool_t *pool, int new_min_nr, gfp_t gfp_mask) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 116 | { |
| 117 | void *element; |
| 118 | void **new_elements; |
| 119 | unsigned long flags; |
| 120 | |
| 121 | BUG_ON(new_min_nr <= 0); |
| 122 | |
| 123 | spin_lock_irqsave(&pool->lock, flags); |
| 124 | if (new_min_nr <= pool->min_nr) { |
| 125 | while (new_min_nr < pool->curr_nr) { |
| 126 | element = remove_element(pool); |
| 127 | spin_unlock_irqrestore(&pool->lock, flags); |
| 128 | pool->free(element, pool->pool_data); |
| 129 | spin_lock_irqsave(&pool->lock, flags); |
| 130 | } |
| 131 | pool->min_nr = new_min_nr; |
| 132 | goto out_unlock; |
| 133 | } |
| 134 | spin_unlock_irqrestore(&pool->lock, flags); |
| 135 | |
| 136 | /* Grow the pool */ |
| 137 | new_elements = kmalloc(new_min_nr * sizeof(*new_elements), gfp_mask); |
| 138 | if (!new_elements) |
| 139 | return -ENOMEM; |
| 140 | |
| 141 | spin_lock_irqsave(&pool->lock, flags); |
| 142 | if (unlikely(new_min_nr <= pool->min_nr)) { |
| 143 | /* Raced, other resize will do our work */ |
| 144 | spin_unlock_irqrestore(&pool->lock, flags); |
| 145 | kfree(new_elements); |
| 146 | goto out; |
| 147 | } |
| 148 | memcpy(new_elements, pool->elements, |
| 149 | pool->curr_nr * sizeof(*new_elements)); |
| 150 | kfree(pool->elements); |
| 151 | pool->elements = new_elements; |
| 152 | pool->min_nr = new_min_nr; |
| 153 | |
| 154 | while (pool->curr_nr < pool->min_nr) { |
| 155 | spin_unlock_irqrestore(&pool->lock, flags); |
| 156 | element = pool->alloc(gfp_mask, pool->pool_data); |
| 157 | if (!element) |
| 158 | goto out; |
| 159 | spin_lock_irqsave(&pool->lock, flags); |
| 160 | if (pool->curr_nr < pool->min_nr) { |
| 161 | add_element(pool, element); |
| 162 | } else { |
| 163 | spin_unlock_irqrestore(&pool->lock, flags); |
| 164 | pool->free(element, pool->pool_data); /* Raced */ |
| 165 | goto out; |
| 166 | } |
| 167 | } |
| 168 | out_unlock: |
| 169 | spin_unlock_irqrestore(&pool->lock, flags); |
| 170 | out: |
| 171 | return 0; |
| 172 | } |
| 173 | EXPORT_SYMBOL(mempool_resize); |
| 174 | |
| 175 | /** |
| 176 | * mempool_destroy - deallocate a memory pool |
| 177 | * @pool: pointer to the memory pool which was allocated via |
| 178 | * mempool_create(). |
| 179 | * |
| 180 | * this function only sleeps if the free_fn() function sleeps. The caller |
| 181 | * has to guarantee that all elements have been returned to the pool (ie: |
| 182 | * freed) prior to calling mempool_destroy(). |
| 183 | */ |
| 184 | void mempool_destroy(mempool_t *pool) |
| 185 | { |
Eric Sesterhenn | f02e1fa | 2006-03-26 18:31:56 +0200 | [diff] [blame] | 186 | /* Check for outstanding elements */ |
| 187 | BUG_ON(pool->curr_nr != pool->min_nr); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 188 | free_pool(pool); |
| 189 | } |
| 190 | EXPORT_SYMBOL(mempool_destroy); |
| 191 | |
| 192 | /** |
| 193 | * mempool_alloc - allocate an element from a specific memory pool |
| 194 | * @pool: pointer to the memory pool which was allocated via |
| 195 | * mempool_create(). |
| 196 | * @gfp_mask: the usual allocation bitmask. |
| 197 | * |
| 198 | * this function only sleeps if the alloc_fn function sleeps or |
| 199 | * returns NULL. Note that due to preallocation, this function |
| 200 | * *never* fails when called from process contexts. (it might |
| 201 | * fail if called from an IRQ context.) |
| 202 | */ |
Al Viro | dd0fc66 | 2005-10-07 07:46:04 +0100 | [diff] [blame] | 203 | void * mempool_alloc(mempool_t *pool, gfp_t gfp_mask) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 204 | { |
| 205 | void *element; |
| 206 | unsigned long flags; |
Benjamin LaHaise | 01890a4 | 2005-06-23 00:10:01 -0700 | [diff] [blame] | 207 | wait_queue_t wait; |
Al Viro | 6daa0e2 | 2005-10-21 03:18:50 -0400 | [diff] [blame] | 208 | gfp_t gfp_temp; |
Nick Piggin | 20a7777 | 2005-05-01 08:58:37 -0700 | [diff] [blame] | 209 | |
| 210 | might_sleep_if(gfp_mask & __GFP_WAIT); |
Nick Piggin | b84a35b | 2005-05-01 08:58:36 -0700 | [diff] [blame] | 211 | |
| 212 | gfp_mask |= __GFP_NOMEMALLOC; /* don't allocate emergency reserves */ |
| 213 | gfp_mask |= __GFP_NORETRY; /* don't loop in __alloc_pages */ |
| 214 | gfp_mask |= __GFP_NOWARN; /* failures are OK */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 215 | |
Nick Piggin | 20a7777 | 2005-05-01 08:58:37 -0700 | [diff] [blame] | 216 | gfp_temp = gfp_mask & ~(__GFP_WAIT|__GFP_IO); |
| 217 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 218 | repeat_alloc: |
Nick Piggin | 20a7777 | 2005-05-01 08:58:37 -0700 | [diff] [blame] | 219 | |
| 220 | element = pool->alloc(gfp_temp, pool->pool_data); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 221 | if (likely(element != NULL)) |
| 222 | return element; |
| 223 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 224 | spin_lock_irqsave(&pool->lock, flags); |
| 225 | if (likely(pool->curr_nr)) { |
| 226 | element = remove_element(pool); |
| 227 | spin_unlock_irqrestore(&pool->lock, flags); |
| 228 | return element; |
| 229 | } |
| 230 | spin_unlock_irqrestore(&pool->lock, flags); |
| 231 | |
| 232 | /* We must not sleep in the GFP_ATOMIC case */ |
| 233 | if (!(gfp_mask & __GFP_WAIT)) |
| 234 | return NULL; |
| 235 | |
Nick Piggin | 20a7777 | 2005-05-01 08:58:37 -0700 | [diff] [blame] | 236 | /* Now start performing page reclaim */ |
| 237 | gfp_temp = gfp_mask; |
Benjamin LaHaise | 01890a4 | 2005-06-23 00:10:01 -0700 | [diff] [blame] | 238 | init_wait(&wait); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 239 | prepare_to_wait(&pool->wait, &wait, TASK_UNINTERRUPTIBLE); |
akpm@osdl.org | d59dd46 | 2005-05-01 08:58:47 -0700 | [diff] [blame] | 240 | smp_mb(); |
Pavel Mironchik | 0b1d647 | 2006-08-31 21:27:47 -0700 | [diff] [blame^] | 241 | if (!pool->curr_nr) { |
| 242 | /* |
| 243 | * FIXME: this should be io_schedule(). The timeout is there |
| 244 | * as a workaround for some DM problems in 2.6.18. |
| 245 | */ |
| 246 | io_schedule_timeout(5*HZ); |
| 247 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 248 | finish_wait(&pool->wait, &wait); |
| 249 | |
| 250 | goto repeat_alloc; |
| 251 | } |
| 252 | EXPORT_SYMBOL(mempool_alloc); |
| 253 | |
| 254 | /** |
| 255 | * mempool_free - return an element to the pool. |
| 256 | * @element: pool element pointer. |
| 257 | * @pool: pointer to the memory pool which was allocated via |
| 258 | * mempool_create(). |
| 259 | * |
| 260 | * this function only sleeps if the free_fn() function sleeps. |
| 261 | */ |
| 262 | void mempool_free(void *element, mempool_t *pool) |
| 263 | { |
| 264 | unsigned long flags; |
| 265 | |
akpm@osdl.org | d59dd46 | 2005-05-01 08:58:47 -0700 | [diff] [blame] | 266 | smp_mb(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 267 | if (pool->curr_nr < pool->min_nr) { |
| 268 | spin_lock_irqsave(&pool->lock, flags); |
| 269 | if (pool->curr_nr < pool->min_nr) { |
| 270 | add_element(pool, element); |
| 271 | spin_unlock_irqrestore(&pool->lock, flags); |
| 272 | wake_up(&pool->wait); |
| 273 | return; |
| 274 | } |
| 275 | spin_unlock_irqrestore(&pool->lock, flags); |
| 276 | } |
| 277 | pool->free(element, pool->pool_data); |
| 278 | } |
| 279 | EXPORT_SYMBOL(mempool_free); |
| 280 | |
| 281 | /* |
| 282 | * A commonly used alloc and free fn. |
| 283 | */ |
Al Viro | dd0fc66 | 2005-10-07 07:46:04 +0100 | [diff] [blame] | 284 | void *mempool_alloc_slab(gfp_t gfp_mask, void *pool_data) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 285 | { |
Pekka Enberg | fcc234f | 2006-03-22 00:08:13 -0800 | [diff] [blame] | 286 | struct kmem_cache *mem = pool_data; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 287 | return kmem_cache_alloc(mem, gfp_mask); |
| 288 | } |
| 289 | EXPORT_SYMBOL(mempool_alloc_slab); |
| 290 | |
| 291 | void mempool_free_slab(void *element, void *pool_data) |
| 292 | { |
Pekka Enberg | fcc234f | 2006-03-22 00:08:13 -0800 | [diff] [blame] | 293 | struct kmem_cache *mem = pool_data; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 294 | kmem_cache_free(mem, element); |
| 295 | } |
| 296 | EXPORT_SYMBOL(mempool_free_slab); |
Matthew Dobson | 6e0678f | 2006-03-26 01:37:44 -0800 | [diff] [blame] | 297 | |
| 298 | /* |
Matthew Dobson | 5318408 | 2006-03-26 01:37:46 -0800 | [diff] [blame] | 299 | * A commonly used alloc and free fn that kmalloc/kfrees the amount of memory |
| 300 | * specfied by pool_data |
| 301 | */ |
| 302 | void *mempool_kmalloc(gfp_t gfp_mask, void *pool_data) |
| 303 | { |
Matthew Dobson | f183323 | 2006-03-26 01:37:48 -0800 | [diff] [blame] | 304 | size_t size = (size_t)(long)pool_data; |
Matthew Dobson | 5318408 | 2006-03-26 01:37:46 -0800 | [diff] [blame] | 305 | return kmalloc(size, gfp_mask); |
| 306 | } |
| 307 | EXPORT_SYMBOL(mempool_kmalloc); |
| 308 | |
Matthew Dobson | f183323 | 2006-03-26 01:37:48 -0800 | [diff] [blame] | 309 | void *mempool_kzalloc(gfp_t gfp_mask, void *pool_data) |
| 310 | { |
| 311 | size_t size = (size_t) pool_data; |
| 312 | return kzalloc(size, gfp_mask); |
| 313 | } |
| 314 | EXPORT_SYMBOL(mempool_kzalloc); |
| 315 | |
Matthew Dobson | 5318408 | 2006-03-26 01:37:46 -0800 | [diff] [blame] | 316 | void mempool_kfree(void *element, void *pool_data) |
| 317 | { |
| 318 | kfree(element); |
| 319 | } |
| 320 | EXPORT_SYMBOL(mempool_kfree); |
| 321 | |
| 322 | /* |
Matthew Dobson | 6e0678f | 2006-03-26 01:37:44 -0800 | [diff] [blame] | 323 | * A simple mempool-backed page allocator that allocates pages |
| 324 | * of the order specified by pool_data. |
| 325 | */ |
| 326 | void *mempool_alloc_pages(gfp_t gfp_mask, void *pool_data) |
| 327 | { |
| 328 | int order = (int)(long)pool_data; |
| 329 | return alloc_pages(gfp_mask, order); |
| 330 | } |
| 331 | EXPORT_SYMBOL(mempool_alloc_pages); |
| 332 | |
| 333 | void mempool_free_pages(void *element, void *pool_data) |
| 334 | { |
| 335 | int order = (int)(long)pool_data; |
| 336 | __free_pages(element, order); |
| 337 | } |
| 338 | EXPORT_SYMBOL(mempool_free_pages); |