blob: fbae8674e491f105269c30a1f10e1c9c4b9929cc [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001
2#include <linux/device.h>
3#include <linux/mm.h>
4#include <asm/io.h> /* Needed for i386 to build */
5#include <asm/scatterlist.h> /* Needed for i386 to build */
6#include <linux/dma-mapping.h>
7#include <linux/dmapool.h>
8#include <linux/slab.h>
9#include <linux/module.h>
Randy Dunlapc9cf5522006-06-27 02:53:52 -070010#include <linux/poison.h>
Alexey Dobriyane8edc6e2007-05-21 01:22:52 +040011#include <linux/sched.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070012
13/*
14 * Pool allocator ... wraps the dma_alloc_coherent page allocator, so
15 * small blocks are easily used by drivers for bus mastering controllers.
16 * This should probably be sharing the guts of the slab allocator.
17 */
18
19struct dma_pool { /* the pool */
20 struct list_head page_list;
21 spinlock_t lock;
22 size_t blocks_per_page;
23 size_t size;
24 struct device *dev;
25 size_t allocation;
26 char name [32];
27 wait_queue_head_t waitq;
28 struct list_head pools;
29};
30
31struct dma_page { /* cacheable header for 'allocation' bytes */
32 struct list_head page_list;
33 void *vaddr;
34 dma_addr_t dma;
35 unsigned in_use;
36 unsigned long bitmap [0];
37};
38
39#define POOL_TIMEOUT_JIFFIES ((100 /* msec */ * HZ) / 1000)
Linus Torvalds1da177e2005-04-16 15:20:36 -070040
Matthias Kaehlckeb2366d62007-04-24 22:45:25 +020041static DEFINE_MUTEX (pools_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -070042
43static ssize_t
Yani Ioannou74880c02005-05-17 06:41:12 -040044show_pools (struct device *dev, struct device_attribute *attr, char *buf)
Linus Torvalds1da177e2005-04-16 15:20:36 -070045{
46 unsigned temp;
47 unsigned size;
48 char *next;
49 struct dma_page *page;
50 struct dma_pool *pool;
51
52 next = buf;
53 size = PAGE_SIZE;
54
55 temp = scnprintf(next, size, "poolinfo - 0.1\n");
56 size -= temp;
57 next += temp;
58
Matthias Kaehlckeb2366d62007-04-24 22:45:25 +020059 mutex_lock(&pools_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -070060 list_for_each_entry(pool, &dev->dma_pools, pools) {
61 unsigned pages = 0;
62 unsigned blocks = 0;
63
64 list_for_each_entry(page, &pool->page_list, page_list) {
65 pages++;
66 blocks += page->in_use;
67 }
68
69 /* per-pool info, no real statistics yet */
70 temp = scnprintf(next, size, "%-16s %4u %4Zu %4Zu %2u\n",
71 pool->name,
72 blocks, pages * pool->blocks_per_page,
73 pool->size, pages);
74 size -= temp;
75 next += temp;
76 }
Matthias Kaehlckeb2366d62007-04-24 22:45:25 +020077 mutex_unlock(&pools_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -070078
79 return PAGE_SIZE - size;
80}
81static DEVICE_ATTR (pools, S_IRUGO, show_pools, NULL);
82
83/**
84 * dma_pool_create - Creates a pool of consistent memory blocks, for dma.
85 * @name: name of pool, for diagnostics
86 * @dev: device that will be doing the DMA
87 * @size: size of the blocks in this pool.
88 * @align: alignment requirement for blocks; must be a power of two
89 * @allocation: returned blocks won't cross this boundary (or zero)
90 * Context: !in_interrupt()
91 *
92 * Returns a dma allocation pool with the requested characteristics, or
93 * null if one can't be created. Given one of these pools, dma_pool_alloc()
94 * may be used to allocate memory. Such memory will all have "consistent"
95 * DMA mappings, accessible by the device and its driver without using
96 * cache flushing primitives. The actual size of blocks allocated may be
97 * larger than requested because of alignment.
98 *
99 * If allocation is nonzero, objects returned from dma_pool_alloc() won't
100 * cross that size boundary. This is useful for devices which have
101 * addressing restrictions on individual DMA transfers, such as not crossing
102 * boundaries of 4KBytes.
103 */
104struct dma_pool *
105dma_pool_create (const char *name, struct device *dev,
106 size_t size, size_t align, size_t allocation)
107{
108 struct dma_pool *retval;
109
110 if (align == 0)
111 align = 1;
112 if (size == 0)
113 return NULL;
114 else if (size < align)
115 size = align;
116 else if ((size % align) != 0) {
117 size += align + 1;
118 size &= ~(align - 1);
119 }
120
121 if (allocation == 0) {
122 if (PAGE_SIZE < size)
123 allocation = size;
124 else
125 allocation = PAGE_SIZE;
126 // FIXME: round up for less fragmentation
127 } else if (allocation < size)
128 return NULL;
129
Yinghai Lu2618c602007-07-15 23:41:41 -0700130 if (!(retval = kmalloc_node (sizeof *retval, GFP_KERNEL, dev_to_node(dev))))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700131 return retval;
132
133 strlcpy (retval->name, name, sizeof retval->name);
134
135 retval->dev = dev;
136
137 INIT_LIST_HEAD (&retval->page_list);
138 spin_lock_init (&retval->lock);
139 retval->size = size;
140 retval->allocation = allocation;
141 retval->blocks_per_page = allocation / size;
142 init_waitqueue_head (&retval->waitq);
143
144 if (dev) {
Cornelia Huck141ecc52006-09-22 11:37:27 +0200145 int ret;
146
Matthias Kaehlckeb2366d62007-04-24 22:45:25 +0200147 mutex_lock(&pools_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700148 if (list_empty (&dev->dma_pools))
Cornelia Huck141ecc52006-09-22 11:37:27 +0200149 ret = device_create_file (dev, &dev_attr_pools);
150 else
151 ret = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700152 /* note: not currently insisting "name" be unique */
Cornelia Huck141ecc52006-09-22 11:37:27 +0200153 if (!ret)
154 list_add (&retval->pools, &dev->dma_pools);
155 else {
156 kfree(retval);
157 retval = NULL;
158 }
Matthias Kaehlckeb2366d62007-04-24 22:45:25 +0200159 mutex_unlock(&pools_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700160 } else
161 INIT_LIST_HEAD (&retval->pools);
162
163 return retval;
164}
165
166
167static struct dma_page *
Al Virodd0fc662005-10-07 07:46:04 +0100168pool_alloc_page (struct dma_pool *pool, gfp_t mem_flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700169{
170 struct dma_page *page;
171 int mapsize;
172
173 mapsize = pool->blocks_per_page;
174 mapsize = (mapsize + BITS_PER_LONG - 1) / BITS_PER_LONG;
175 mapsize *= sizeof (long);
176
Robert P. J. Day5cbded52006-12-13 00:35:56 -0800177 page = kmalloc(mapsize + sizeof *page, mem_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700178 if (!page)
179 return NULL;
180 page->vaddr = dma_alloc_coherent (pool->dev,
181 pool->allocation,
182 &page->dma,
183 mem_flags);
184 if (page->vaddr) {
185 memset (page->bitmap, 0xff, mapsize); // bit set == free
186#ifdef CONFIG_DEBUG_SLAB
187 memset (page->vaddr, POOL_POISON_FREED, pool->allocation);
188#endif
189 list_add (&page->page_list, &pool->page_list);
190 page->in_use = 0;
191 } else {
192 kfree (page);
193 page = NULL;
194 }
195 return page;
196}
197
198
199static inline int
200is_page_busy (int blocks, unsigned long *bitmap)
201{
202 while (blocks > 0) {
203 if (*bitmap++ != ~0UL)
204 return 1;
205 blocks -= BITS_PER_LONG;
206 }
207 return 0;
208}
209
210static void
211pool_free_page (struct dma_pool *pool, struct dma_page *page)
212{
213 dma_addr_t dma = page->dma;
214
215#ifdef CONFIG_DEBUG_SLAB
216 memset (page->vaddr, POOL_POISON_FREED, pool->allocation);
217#endif
218 dma_free_coherent (pool->dev, pool->allocation, page->vaddr, dma);
219 list_del (&page->page_list);
220 kfree (page);
221}
222
223
224/**
225 * dma_pool_destroy - destroys a pool of dma memory blocks.
226 * @pool: dma pool that will be destroyed
227 * Context: !in_interrupt()
228 *
229 * Caller guarantees that no more memory from the pool is in use,
230 * and that nothing will try to use the pool after this call.
231 */
232void
233dma_pool_destroy (struct dma_pool *pool)
234{
Matthias Kaehlckeb2366d62007-04-24 22:45:25 +0200235 mutex_lock(&pools_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700236 list_del (&pool->pools);
237 if (pool->dev && list_empty (&pool->dev->dma_pools))
238 device_remove_file (pool->dev, &dev_attr_pools);
Matthias Kaehlckeb2366d62007-04-24 22:45:25 +0200239 mutex_unlock(&pools_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700240
241 while (!list_empty (&pool->page_list)) {
242 struct dma_page *page;
243 page = list_entry (pool->page_list.next,
244 struct dma_page, page_list);
245 if (is_page_busy (pool->blocks_per_page, page->bitmap)) {
246 if (pool->dev)
247 dev_err(pool->dev, "dma_pool_destroy %s, %p busy\n",
248 pool->name, page->vaddr);
249 else
250 printk (KERN_ERR "dma_pool_destroy %s, %p busy\n",
251 pool->name, page->vaddr);
252 /* leak the still-in-use consistent memory */
253 list_del (&page->page_list);
254 kfree (page);
255 } else
256 pool_free_page (pool, page);
257 }
258
259 kfree (pool);
260}
261
262
263/**
264 * dma_pool_alloc - get a block of consistent memory
265 * @pool: dma pool that will produce the block
266 * @mem_flags: GFP_* bitmask
267 * @handle: pointer to dma address of block
268 *
269 * This returns the kernel virtual address of a currently unused block,
270 * and reports its dma address through the handle.
271 * If such a memory block can't be allocated, null is returned.
272 */
273void *
Al Virodd0fc662005-10-07 07:46:04 +0100274dma_pool_alloc (struct dma_pool *pool, gfp_t mem_flags, dma_addr_t *handle)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700275{
276 unsigned long flags;
277 struct dma_page *page;
278 int map, block;
279 size_t offset;
280 void *retval;
281
282restart:
283 spin_lock_irqsave (&pool->lock, flags);
284 list_for_each_entry(page, &pool->page_list, page_list) {
285 int i;
286 /* only cachable accesses here ... */
287 for (map = 0, i = 0;
288 i < pool->blocks_per_page;
289 i += BITS_PER_LONG, map++) {
290 if (page->bitmap [map] == 0)
291 continue;
292 block = ffz (~ page->bitmap [map]);
293 if ((i + block) < pool->blocks_per_page) {
294 clear_bit (block, &page->bitmap [map]);
295 offset = (BITS_PER_LONG * map) + block;
296 offset *= pool->size;
297 goto ready;
298 }
299 }
300 }
Christoph Lameter54e6ecb2006-12-06 20:33:16 -0800301 if (!(page = pool_alloc_page (pool, GFP_ATOMIC))) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700302 if (mem_flags & __GFP_WAIT) {
303 DECLARE_WAITQUEUE (wait, current);
304
Arjan van de Vend9aaccc2007-10-16 23:29:32 -0700305 __set_current_state(TASK_INTERRUPTIBLE);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700306 add_wait_queue (&pool->waitq, &wait);
307 spin_unlock_irqrestore (&pool->lock, flags);
308
309 schedule_timeout (POOL_TIMEOUT_JIFFIES);
310
311 remove_wait_queue (&pool->waitq, &wait);
312 goto restart;
313 }
314 retval = NULL;
315 goto done;
316 }
317
318 clear_bit (0, &page->bitmap [0]);
319 offset = 0;
320ready:
321 page->in_use++;
322 retval = offset + page->vaddr;
323 *handle = offset + page->dma;
324#ifdef CONFIG_DEBUG_SLAB
325 memset (retval, POOL_POISON_ALLOCATED, pool->size);
326#endif
327done:
328 spin_unlock_irqrestore (&pool->lock, flags);
329 return retval;
330}
331
332
333static struct dma_page *
334pool_find_page (struct dma_pool *pool, dma_addr_t dma)
335{
336 unsigned long flags;
337 struct dma_page *page;
338
339 spin_lock_irqsave (&pool->lock, flags);
340 list_for_each_entry(page, &pool->page_list, page_list) {
341 if (dma < page->dma)
342 continue;
343 if (dma < (page->dma + pool->allocation))
344 goto done;
345 }
346 page = NULL;
347done:
348 spin_unlock_irqrestore (&pool->lock, flags);
349 return page;
350}
351
352
353/**
354 * dma_pool_free - put block back into dma pool
355 * @pool: the dma pool holding the block
356 * @vaddr: virtual address of block
357 * @dma: dma address of block
358 *
359 * Caller promises neither device nor driver will again touch this block
360 * unless it is first re-allocated.
361 */
362void
363dma_pool_free (struct dma_pool *pool, void *vaddr, dma_addr_t dma)
364{
365 struct dma_page *page;
366 unsigned long flags;
367 int map, block;
368
369 if ((page = pool_find_page (pool, dma)) == 0) {
370 if (pool->dev)
371 dev_err(pool->dev, "dma_pool_free %s, %p/%lx (bad dma)\n",
372 pool->name, vaddr, (unsigned long) dma);
373 else
374 printk (KERN_ERR "dma_pool_free %s, %p/%lx (bad dma)\n",
375 pool->name, vaddr, (unsigned long) dma);
376 return;
377 }
378
379 block = dma - page->dma;
380 block /= pool->size;
381 map = block / BITS_PER_LONG;
382 block %= BITS_PER_LONG;
383
384#ifdef CONFIG_DEBUG_SLAB
385 if (((dma - page->dma) + (void *)page->vaddr) != vaddr) {
386 if (pool->dev)
387 dev_err(pool->dev, "dma_pool_free %s, %p (bad vaddr)/%Lx\n",
388 pool->name, vaddr, (unsigned long long) dma);
389 else
390 printk (KERN_ERR "dma_pool_free %s, %p (bad vaddr)/%Lx\n",
391 pool->name, vaddr, (unsigned long long) dma);
392 return;
393 }
394 if (page->bitmap [map] & (1UL << block)) {
395 if (pool->dev)
396 dev_err(pool->dev, "dma_pool_free %s, dma %Lx already free\n",
397 pool->name, (unsigned long long)dma);
398 else
399 printk (KERN_ERR "dma_pool_free %s, dma %Lx already free\n",
400 pool->name, (unsigned long long)dma);
401 return;
402 }
403 memset (vaddr, POOL_POISON_FREED, pool->size);
404#endif
405
406 spin_lock_irqsave (&pool->lock, flags);
407 page->in_use--;
408 set_bit (block, &page->bitmap [map]);
409 if (waitqueue_active (&pool->waitq))
410 wake_up (&pool->waitq);
411 /*
412 * Resist a temptation to do
413 * if (!is_page_busy(bpp, page->bitmap)) pool_free_page(pool, page);
414 * Better have a few empty pages hang around.
415 */
416 spin_unlock_irqrestore (&pool->lock, flags);
417}
418
Tejun Heo9ac78492007-01-20 16:00:26 +0900419/*
420 * Managed DMA pool
421 */
422static void dmam_pool_release(struct device *dev, void *res)
423{
424 struct dma_pool *pool = *(struct dma_pool **)res;
425
426 dma_pool_destroy(pool);
427}
428
429static int dmam_pool_match(struct device *dev, void *res, void *match_data)
430{
431 return *(struct dma_pool **)res == match_data;
432}
433
434/**
435 * dmam_pool_create - Managed dma_pool_create()
436 * @name: name of pool, for diagnostics
437 * @dev: device that will be doing the DMA
438 * @size: size of the blocks in this pool.
439 * @align: alignment requirement for blocks; must be a power of two
440 * @allocation: returned blocks won't cross this boundary (or zero)
441 *
442 * Managed dma_pool_create(). DMA pool created with this function is
443 * automatically destroyed on driver detach.
444 */
445struct dma_pool *dmam_pool_create(const char *name, struct device *dev,
446 size_t size, size_t align, size_t allocation)
447{
448 struct dma_pool **ptr, *pool;
449
450 ptr = devres_alloc(dmam_pool_release, sizeof(*ptr), GFP_KERNEL);
451 if (!ptr)
452 return NULL;
453
454 pool = *ptr = dma_pool_create(name, dev, size, align, allocation);
455 if (pool)
456 devres_add(dev, ptr);
457 else
458 devres_free(ptr);
459
460 return pool;
461}
462
463/**
464 * dmam_pool_destroy - Managed dma_pool_destroy()
465 * @pool: dma pool that will be destroyed
466 *
467 * Managed dma_pool_destroy().
468 */
469void dmam_pool_destroy(struct dma_pool *pool)
470{
471 struct device *dev = pool->dev;
472
473 dma_pool_destroy(pool);
474 WARN_ON(devres_destroy(dev, dmam_pool_release, dmam_pool_match, pool));
475}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700476
477EXPORT_SYMBOL (dma_pool_create);
478EXPORT_SYMBOL (dma_pool_destroy);
479EXPORT_SYMBOL (dma_pool_alloc);
480EXPORT_SYMBOL (dma_pool_free);
Tejun Heo9ac78492007-01-20 16:00:26 +0900481EXPORT_SYMBOL (dmam_pool_create);
482EXPORT_SYMBOL (dmam_pool_destroy);