blob: e2ea4543abb4067e966d63025db47b2b981e2747 [file] [log] [blame]
Matthew Wilcox6182a092007-12-03 12:16:57 -05001/*
2 * DMA Pool allocator
3 *
4 * Copyright 2001 David Brownell
5 * Copyright 2007 Intel Corporation
6 * Author: Matthew Wilcox <willy@linux.intel.com>
7 *
8 * This software may be redistributed and/or modified under the terms of
9 * the GNU General Public License ("GPL") version 2 as published by the
10 * Free Software Foundation.
11 *
12 * This allocator returns small blocks of a given size which are DMA-able by
13 * the given device. It uses the dma_alloc_coherent page allocator to get
14 * new pages, then splits them up into blocks of the required size.
15 * Many older drivers still have their own code to do this.
16 *
17 * The current design of this allocator is fairly simple. The pool is
18 * represented by the 'struct dma_pool' which keeps a doubly-linked list of
19 * allocated pages. Each page in the page_list is split into blocks of at
20 * least 'size' bytes.
21 */
Linus Torvalds1da177e2005-04-16 15:20:36 -070022
23#include <linux/device.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070024#include <linux/dma-mapping.h>
25#include <linux/dmapool.h>
Matthew Wilcox6182a092007-12-03 12:16:57 -050026#include <linux/kernel.h>
27#include <linux/list.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070028#include <linux/module.h>
Matthew Wilcox6182a092007-12-03 12:16:57 -050029#include <linux/mutex.h>
Randy Dunlapc9cf5522006-06-27 02:53:52 -070030#include <linux/poison.h>
Alexey Dobriyane8edc6e2007-05-21 01:22:52 +040031#include <linux/sched.h>
Matthew Wilcox6182a092007-12-03 12:16:57 -050032#include <linux/slab.h>
33#include <linux/spinlock.h>
34#include <linux/string.h>
35#include <linux/types.h>
36#include <linux/wait.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070037
Matthew Wilcoxe87aa772007-12-03 12:04:31 -050038struct dma_pool { /* the pool */
39 struct list_head page_list;
40 spinlock_t lock;
41 size_t blocks_per_page;
42 size_t size;
43 struct device *dev;
44 size_t allocation;
45 char name[32];
46 wait_queue_head_t waitq;
47 struct list_head pools;
Linus Torvalds1da177e2005-04-16 15:20:36 -070048};
49
Matthew Wilcoxe87aa772007-12-03 12:04:31 -050050struct dma_page { /* cacheable header for 'allocation' bytes */
51 struct list_head page_list;
52 void *vaddr;
53 dma_addr_t dma;
54 unsigned in_use;
55 unsigned long bitmap[0];
Linus Torvalds1da177e2005-04-16 15:20:36 -070056};
57
58#define POOL_TIMEOUT_JIFFIES ((100 /* msec */ * HZ) / 1000)
Linus Torvalds1da177e2005-04-16 15:20:36 -070059
Matthew Wilcoxe87aa772007-12-03 12:04:31 -050060static DEFINE_MUTEX(pools_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -070061
62static ssize_t
Matthew Wilcoxe87aa772007-12-03 12:04:31 -050063show_pools(struct device *dev, struct device_attribute *attr, char *buf)
Linus Torvalds1da177e2005-04-16 15:20:36 -070064{
65 unsigned temp;
66 unsigned size;
67 char *next;
68 struct dma_page *page;
69 struct dma_pool *pool;
70
71 next = buf;
72 size = PAGE_SIZE;
73
74 temp = scnprintf(next, size, "poolinfo - 0.1\n");
75 size -= temp;
76 next += temp;
77
Matthias Kaehlckeb2366d62007-04-24 22:45:25 +020078 mutex_lock(&pools_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -070079 list_for_each_entry(pool, &dev->dma_pools, pools) {
80 unsigned pages = 0;
81 unsigned blocks = 0;
82
83 list_for_each_entry(page, &pool->page_list, page_list) {
84 pages++;
85 blocks += page->in_use;
86 }
87
88 /* per-pool info, no real statistics yet */
89 temp = scnprintf(next, size, "%-16s %4u %4Zu %4Zu %2u\n",
Matthew Wilcoxe87aa772007-12-03 12:04:31 -050090 pool->name,
91 blocks, pages * pool->blocks_per_page,
92 pool->size, pages);
Linus Torvalds1da177e2005-04-16 15:20:36 -070093 size -= temp;
94 next += temp;
95 }
Matthias Kaehlckeb2366d62007-04-24 22:45:25 +020096 mutex_unlock(&pools_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -070097
98 return PAGE_SIZE - size;
99}
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500100
101static DEVICE_ATTR(pools, S_IRUGO, show_pools, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700102
103/**
104 * dma_pool_create - Creates a pool of consistent memory blocks, for dma.
105 * @name: name of pool, for diagnostics
106 * @dev: device that will be doing the DMA
107 * @size: size of the blocks in this pool.
108 * @align: alignment requirement for blocks; must be a power of two
109 * @allocation: returned blocks won't cross this boundary (or zero)
110 * Context: !in_interrupt()
111 *
112 * Returns a dma allocation pool with the requested characteristics, or
113 * null if one can't be created. Given one of these pools, dma_pool_alloc()
114 * may be used to allocate memory. Such memory will all have "consistent"
115 * DMA mappings, accessible by the device and its driver without using
116 * cache flushing primitives. The actual size of blocks allocated may be
117 * larger than requested because of alignment.
118 *
119 * If allocation is nonzero, objects returned from dma_pool_alloc() won't
120 * cross that size boundary. This is useful for devices which have
121 * addressing restrictions on individual DMA transfers, such as not crossing
122 * boundaries of 4KBytes.
123 */
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500124struct dma_pool *dma_pool_create(const char *name, struct device *dev,
125 size_t size, size_t align, size_t allocation)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700126{
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500127 struct dma_pool *retval;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700128
Matthew Wilcox399154b2007-12-03 12:10:24 -0500129 if (align == 0) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700130 align = 1;
Matthew Wilcox399154b2007-12-03 12:10:24 -0500131 } else if (align & (align - 1)) {
132 return NULL;
133 }
134
Linus Torvalds1da177e2005-04-16 15:20:36 -0700135 if (size == 0)
136 return NULL;
Matthew Wilcox399154b2007-12-03 12:10:24 -0500137
138 if ((size % align) != 0)
139 size = ALIGN(size, align);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700140
141 if (allocation == 0) {
142 if (PAGE_SIZE < size)
143 allocation = size;
144 else
145 allocation = PAGE_SIZE;
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500146 /* FIXME: round up for less fragmentation */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700147 } else if (allocation < size)
148 return NULL;
149
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500150 if (!
151 (retval =
152 kmalloc_node(sizeof *retval, GFP_KERNEL, dev_to_node(dev))))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700153 return retval;
154
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500155 strlcpy(retval->name, name, sizeof retval->name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700156
157 retval->dev = dev;
158
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500159 INIT_LIST_HEAD(&retval->page_list);
160 spin_lock_init(&retval->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700161 retval->size = size;
162 retval->allocation = allocation;
163 retval->blocks_per_page = allocation / size;
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500164 init_waitqueue_head(&retval->waitq);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700165
166 if (dev) {
Cornelia Huck141ecc52006-09-22 11:37:27 +0200167 int ret;
168
Matthias Kaehlckeb2366d62007-04-24 22:45:25 +0200169 mutex_lock(&pools_lock);
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500170 if (list_empty(&dev->dma_pools))
171 ret = device_create_file(dev, &dev_attr_pools);
Cornelia Huck141ecc52006-09-22 11:37:27 +0200172 else
173 ret = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700174 /* note: not currently insisting "name" be unique */
Cornelia Huck141ecc52006-09-22 11:37:27 +0200175 if (!ret)
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500176 list_add(&retval->pools, &dev->dma_pools);
Cornelia Huck141ecc52006-09-22 11:37:27 +0200177 else {
178 kfree(retval);
179 retval = NULL;
180 }
Matthias Kaehlckeb2366d62007-04-24 22:45:25 +0200181 mutex_unlock(&pools_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700182 } else
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500183 INIT_LIST_HEAD(&retval->pools);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700184
185 return retval;
186}
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500187EXPORT_SYMBOL(dma_pool_create);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700188
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500189static struct dma_page *pool_alloc_page(struct dma_pool *pool, gfp_t mem_flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700190{
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500191 struct dma_page *page;
192 int mapsize;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700193
194 mapsize = pool->blocks_per_page;
195 mapsize = (mapsize + BITS_PER_LONG - 1) / BITS_PER_LONG;
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500196 mapsize *= sizeof(long);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700197
Robert P. J. Day5cbded52006-12-13 00:35:56 -0800198 page = kmalloc(mapsize + sizeof *page, mem_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700199 if (!page)
200 return NULL;
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500201 page->vaddr = dma_alloc_coherent(pool->dev,
202 pool->allocation,
203 &page->dma, mem_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700204 if (page->vaddr) {
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500205 memset(page->bitmap, 0xff, mapsize); /* bit set == free */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700206#ifdef CONFIG_DEBUG_SLAB
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500207 memset(page->vaddr, POOL_POISON_FREED, pool->allocation);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700208#endif
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500209 list_add(&page->page_list, &pool->page_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700210 page->in_use = 0;
211 } else {
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500212 kfree(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700213 page = NULL;
214 }
215 return page;
216}
217
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500218static inline int is_page_busy(int blocks, unsigned long *bitmap)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700219{
220 while (blocks > 0) {
221 if (*bitmap++ != ~0UL)
222 return 1;
223 blocks -= BITS_PER_LONG;
224 }
225 return 0;
226}
227
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500228static void pool_free_page(struct dma_pool *pool, struct dma_page *page)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700229{
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500230 dma_addr_t dma = page->dma;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700231
232#ifdef CONFIG_DEBUG_SLAB
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500233 memset(page->vaddr, POOL_POISON_FREED, pool->allocation);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700234#endif
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500235 dma_free_coherent(pool->dev, pool->allocation, page->vaddr, dma);
236 list_del(&page->page_list);
237 kfree(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700238}
239
Linus Torvalds1da177e2005-04-16 15:20:36 -0700240/**
241 * dma_pool_destroy - destroys a pool of dma memory blocks.
242 * @pool: dma pool that will be destroyed
243 * Context: !in_interrupt()
244 *
245 * Caller guarantees that no more memory from the pool is in use,
246 * and that nothing will try to use the pool after this call.
247 */
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500248void dma_pool_destroy(struct dma_pool *pool)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700249{
Matthias Kaehlckeb2366d62007-04-24 22:45:25 +0200250 mutex_lock(&pools_lock);
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500251 list_del(&pool->pools);
252 if (pool->dev && list_empty(&pool->dev->dma_pools))
253 device_remove_file(pool->dev, &dev_attr_pools);
Matthias Kaehlckeb2366d62007-04-24 22:45:25 +0200254 mutex_unlock(&pools_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700255
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500256 while (!list_empty(&pool->page_list)) {
257 struct dma_page *page;
258 page = list_entry(pool->page_list.next,
259 struct dma_page, page_list);
260 if (is_page_busy(pool->blocks_per_page, page->bitmap)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700261 if (pool->dev)
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500262 dev_err(pool->dev,
263 "dma_pool_destroy %s, %p busy\n",
Linus Torvalds1da177e2005-04-16 15:20:36 -0700264 pool->name, page->vaddr);
265 else
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500266 printk(KERN_ERR
267 "dma_pool_destroy %s, %p busy\n",
268 pool->name, page->vaddr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700269 /* leak the still-in-use consistent memory */
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500270 list_del(&page->page_list);
271 kfree(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700272 } else
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500273 pool_free_page(pool, page);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700274 }
275
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500276 kfree(pool);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700277}
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500278EXPORT_SYMBOL(dma_pool_destroy);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700279
280/**
281 * dma_pool_alloc - get a block of consistent memory
282 * @pool: dma pool that will produce the block
283 * @mem_flags: GFP_* bitmask
284 * @handle: pointer to dma address of block
285 *
286 * This returns the kernel virtual address of a currently unused block,
287 * and reports its dma address through the handle.
Matthew Wilcox6182a092007-12-03 12:16:57 -0500288 * If such a memory block can't be allocated, %NULL is returned.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700289 */
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500290void *dma_pool_alloc(struct dma_pool *pool, gfp_t mem_flags,
291 dma_addr_t *handle)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700292{
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500293 unsigned long flags;
294 struct dma_page *page;
295 int map, block;
296 size_t offset;
297 void *retval;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700298
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500299 spin_lock_irqsave(&pool->lock, flags);
Matthew Wilcox2cae3672007-12-03 12:09:33 -0500300 restart:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700301 list_for_each_entry(page, &pool->page_list, page_list) {
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500302 int i;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700303 /* only cachable accesses here ... */
304 for (map = 0, i = 0;
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500305 i < pool->blocks_per_page; i += BITS_PER_LONG, map++) {
306 if (page->bitmap[map] == 0)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700307 continue;
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500308 block = ffz(~page->bitmap[map]);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700309 if ((i + block) < pool->blocks_per_page) {
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500310 clear_bit(block, &page->bitmap[map]);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700311 offset = (BITS_PER_LONG * map) + block;
312 offset *= pool->size;
313 goto ready;
314 }
315 }
316 }
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500317 page = pool_alloc_page(pool, GFP_ATOMIC);
318 if (!page) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700319 if (mem_flags & __GFP_WAIT) {
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500320 DECLARE_WAITQUEUE(wait, current);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700321
Arjan van de Vend9aaccc2007-10-16 23:29:32 -0700322 __set_current_state(TASK_INTERRUPTIBLE);
Matthew Wilcox2cae3672007-12-03 12:09:33 -0500323 __add_wait_queue(&pool->waitq, &wait);
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500324 spin_unlock_irqrestore(&pool->lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700325
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500326 schedule_timeout(POOL_TIMEOUT_JIFFIES);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700327
Matthew Wilcox2cae3672007-12-03 12:09:33 -0500328 spin_lock_irqsave(&pool->lock, flags);
329 __remove_wait_queue(&pool->waitq, &wait);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700330 goto restart;
331 }
332 retval = NULL;
333 goto done;
334 }
335
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500336 clear_bit(0, &page->bitmap[0]);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700337 offset = 0;
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500338 ready:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700339 page->in_use++;
340 retval = offset + page->vaddr;
341 *handle = offset + page->dma;
342#ifdef CONFIG_DEBUG_SLAB
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500343 memset(retval, POOL_POISON_ALLOCATED, pool->size);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700344#endif
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500345 done:
346 spin_unlock_irqrestore(&pool->lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700347 return retval;
348}
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500349EXPORT_SYMBOL(dma_pool_alloc);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700350
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500351static struct dma_page *pool_find_page(struct dma_pool *pool, dma_addr_t dma)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700352{
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500353 unsigned long flags;
354 struct dma_page *page;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700355
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500356 spin_lock_irqsave(&pool->lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700357 list_for_each_entry(page, &pool->page_list, page_list) {
358 if (dma < page->dma)
359 continue;
360 if (dma < (page->dma + pool->allocation))
361 goto done;
362 }
363 page = NULL;
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500364 done:
365 spin_unlock_irqrestore(&pool->lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700366 return page;
367}
368
Linus Torvalds1da177e2005-04-16 15:20:36 -0700369/**
370 * dma_pool_free - put block back into dma pool
371 * @pool: the dma pool holding the block
372 * @vaddr: virtual address of block
373 * @dma: dma address of block
374 *
375 * Caller promises neither device nor driver will again touch this block
376 * unless it is first re-allocated.
377 */
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500378void dma_pool_free(struct dma_pool *pool, void *vaddr, dma_addr_t dma)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700379{
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500380 struct dma_page *page;
381 unsigned long flags;
382 int map, block;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700383
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500384 page = pool_find_page(pool, dma);
385 if (!page) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700386 if (pool->dev)
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500387 dev_err(pool->dev,
388 "dma_pool_free %s, %p/%lx (bad dma)\n",
389 pool->name, vaddr, (unsigned long)dma);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700390 else
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500391 printk(KERN_ERR "dma_pool_free %s, %p/%lx (bad dma)\n",
392 pool->name, vaddr, (unsigned long)dma);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700393 return;
394 }
395
396 block = dma - page->dma;
397 block /= pool->size;
398 map = block / BITS_PER_LONG;
399 block %= BITS_PER_LONG;
400
401#ifdef CONFIG_DEBUG_SLAB
402 if (((dma - page->dma) + (void *)page->vaddr) != vaddr) {
403 if (pool->dev)
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500404 dev_err(pool->dev,
405 "dma_pool_free %s, %p (bad vaddr)/%Lx\n",
406 pool->name, vaddr, (unsigned long long)dma);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700407 else
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500408 printk(KERN_ERR
409 "dma_pool_free %s, %p (bad vaddr)/%Lx\n",
410 pool->name, vaddr, (unsigned long long)dma);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700411 return;
412 }
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500413 if (page->bitmap[map] & (1UL << block)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700414 if (pool->dev)
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500415 dev_err(pool->dev,
416 "dma_pool_free %s, dma %Lx already free\n",
Linus Torvalds1da177e2005-04-16 15:20:36 -0700417 pool->name, (unsigned long long)dma);
418 else
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500419 printk(KERN_ERR
420 "dma_pool_free %s, dma %Lx already free\n",
421 pool->name, (unsigned long long)dma);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700422 return;
423 }
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500424 memset(vaddr, POOL_POISON_FREED, pool->size);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700425#endif
426
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500427 spin_lock_irqsave(&pool->lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700428 page->in_use--;
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500429 set_bit(block, &page->bitmap[map]);
430 if (waitqueue_active(&pool->waitq))
Matthew Wilcox2cae3672007-12-03 12:09:33 -0500431 wake_up_locked(&pool->waitq);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700432 /*
433 * Resist a temptation to do
434 * if (!is_page_busy(bpp, page->bitmap)) pool_free_page(pool, page);
435 * Better have a few empty pages hang around.
436 */
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500437 spin_unlock_irqrestore(&pool->lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700438}
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500439EXPORT_SYMBOL(dma_pool_free);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700440
Tejun Heo9ac78492007-01-20 16:00:26 +0900441/*
442 * Managed DMA pool
443 */
444static void dmam_pool_release(struct device *dev, void *res)
445{
446 struct dma_pool *pool = *(struct dma_pool **)res;
447
448 dma_pool_destroy(pool);
449}
450
451static int dmam_pool_match(struct device *dev, void *res, void *match_data)
452{
453 return *(struct dma_pool **)res == match_data;
454}
455
456/**
457 * dmam_pool_create - Managed dma_pool_create()
458 * @name: name of pool, for diagnostics
459 * @dev: device that will be doing the DMA
460 * @size: size of the blocks in this pool.
461 * @align: alignment requirement for blocks; must be a power of two
462 * @allocation: returned blocks won't cross this boundary (or zero)
463 *
464 * Managed dma_pool_create(). DMA pool created with this function is
465 * automatically destroyed on driver detach.
466 */
467struct dma_pool *dmam_pool_create(const char *name, struct device *dev,
468 size_t size, size_t align, size_t allocation)
469{
470 struct dma_pool **ptr, *pool;
471
472 ptr = devres_alloc(dmam_pool_release, sizeof(*ptr), GFP_KERNEL);
473 if (!ptr)
474 return NULL;
475
476 pool = *ptr = dma_pool_create(name, dev, size, align, allocation);
477 if (pool)
478 devres_add(dev, ptr);
479 else
480 devres_free(ptr);
481
482 return pool;
483}
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500484EXPORT_SYMBOL(dmam_pool_create);
Tejun Heo9ac78492007-01-20 16:00:26 +0900485
486/**
487 * dmam_pool_destroy - Managed dma_pool_destroy()
488 * @pool: dma pool that will be destroyed
489 *
490 * Managed dma_pool_destroy().
491 */
492void dmam_pool_destroy(struct dma_pool *pool)
493{
494 struct device *dev = pool->dev;
495
496 dma_pool_destroy(pool);
497 WARN_ON(devres_destroy(dev, dmam_pool_release, dmam_pool_match, pool));
498}
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500499EXPORT_SYMBOL(dmam_pool_destroy);