blob: 57312b5d6e12aaf2da3f6acd38a19bbe28959077 [file] [log] [blame]
Matthew Wilcox6182a092007-12-03 12:16:57 -05001/*
2 * DMA Pool allocator
3 *
4 * Copyright 2001 David Brownell
5 * Copyright 2007 Intel Corporation
6 * Author: Matthew Wilcox <willy@linux.intel.com>
7 *
8 * This software may be redistributed and/or modified under the terms of
9 * the GNU General Public License ("GPL") version 2 as published by the
10 * Free Software Foundation.
11 *
12 * This allocator returns small blocks of a given size which are DMA-able by
13 * the given device. It uses the dma_alloc_coherent page allocator to get
14 * new pages, then splits them up into blocks of the required size.
15 * Many older drivers still have their own code to do this.
16 *
17 * The current design of this allocator is fairly simple. The pool is
18 * represented by the 'struct dma_pool' which keeps a doubly-linked list of
19 * allocated pages. Each page in the page_list is split into blocks of at
Matthew Wilcoxa35a3452007-12-03 14:08:28 -050020 * least 'size' bytes. Free blocks are tracked in an unsorted singly-linked
21 * list of free blocks within the page. Used blocks aren't tracked, but we
22 * keep a count of how many are currently allocated from each page.
Matthew Wilcox6182a092007-12-03 12:16:57 -050023 */
Linus Torvalds1da177e2005-04-16 15:20:36 -070024
25#include <linux/device.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070026#include <linux/dma-mapping.h>
27#include <linux/dmapool.h>
Matthew Wilcox6182a092007-12-03 12:16:57 -050028#include <linux/kernel.h>
29#include <linux/list.h>
Paul Gortmakerb95f1b312011-10-16 02:01:52 -040030#include <linux/export.h>
Matthew Wilcox6182a092007-12-03 12:16:57 -050031#include <linux/mutex.h>
Randy Dunlapc9cf5522006-06-27 02:53:52 -070032#include <linux/poison.h>
Alexey Dobriyane8edc6e2007-05-21 01:22:52 +040033#include <linux/sched.h>
Matthew Wilcox6182a092007-12-03 12:16:57 -050034#include <linux/slab.h>
Paul Gortmaker7c775092011-10-16 02:03:46 -040035#include <linux/stat.h>
Matthew Wilcox6182a092007-12-03 12:16:57 -050036#include <linux/spinlock.h>
37#include <linux/string.h>
38#include <linux/types.h>
39#include <linux/wait.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070040
Andi Kleenb5ee5be2008-04-28 02:12:37 -070041#if defined(CONFIG_DEBUG_SLAB) || defined(CONFIG_SLUB_DEBUG_ON)
42#define DMAPOOL_DEBUG 1
43#endif
44
Matthew Wilcoxe87aa772007-12-03 12:04:31 -050045struct dma_pool { /* the pool */
46 struct list_head page_list;
47 spinlock_t lock;
Matthew Wilcoxe87aa772007-12-03 12:04:31 -050048 size_t size;
49 struct device *dev;
50 size_t allocation;
Matthew Wilcoxe34f44b2007-12-03 14:16:24 -050051 size_t boundary;
Matthew Wilcoxe87aa772007-12-03 12:04:31 -050052 char name[32];
Matthew Wilcoxe87aa772007-12-03 12:04:31 -050053 struct list_head pools;
Linus Torvalds1da177e2005-04-16 15:20:36 -070054};
55
Matthew Wilcoxe87aa772007-12-03 12:04:31 -050056struct dma_page { /* cacheable header for 'allocation' bytes */
57 struct list_head page_list;
58 void *vaddr;
59 dma_addr_t dma;
Matthew Wilcoxa35a3452007-12-03 14:08:28 -050060 unsigned int in_use;
61 unsigned int offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -070062};
63
Matthew Wilcoxe87aa772007-12-03 12:04:31 -050064static DEFINE_MUTEX(pools_lock);
Sebastian Andrzej Siewior01c29652014-10-09 15:28:50 -070065static DEFINE_MUTEX(pools_reg_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -070066
67static ssize_t
Matthew Wilcoxe87aa772007-12-03 12:04:31 -050068show_pools(struct device *dev, struct device_attribute *attr, char *buf)
Linus Torvalds1da177e2005-04-16 15:20:36 -070069{
70 unsigned temp;
71 unsigned size;
72 char *next;
73 struct dma_page *page;
74 struct dma_pool *pool;
75
76 next = buf;
77 size = PAGE_SIZE;
78
79 temp = scnprintf(next, size, "poolinfo - 0.1\n");
80 size -= temp;
81 next += temp;
82
Matthias Kaehlckeb2366d62007-04-24 22:45:25 +020083 mutex_lock(&pools_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -070084 list_for_each_entry(pool, &dev->dma_pools, pools) {
85 unsigned pages = 0;
86 unsigned blocks = 0;
87
Thomas Gleixnerc4956822009-06-30 11:41:25 -070088 spin_lock_irq(&pool->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -070089 list_for_each_entry(page, &pool->page_list, page_list) {
90 pages++;
91 blocks += page->in_use;
92 }
Thomas Gleixnerc4956822009-06-30 11:41:25 -070093 spin_unlock_irq(&pool->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -070094
95 /* per-pool info, no real statistics yet */
96 temp = scnprintf(next, size, "%-16s %4u %4Zu %4Zu %2u\n",
Matthew Wilcoxa35a3452007-12-03 14:08:28 -050097 pool->name, blocks,
98 pages * (pool->allocation / pool->size),
Matthew Wilcoxe87aa772007-12-03 12:04:31 -050099 pool->size, pages);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700100 size -= temp;
101 next += temp;
102 }
Matthias Kaehlckeb2366d62007-04-24 22:45:25 +0200103 mutex_unlock(&pools_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700104
105 return PAGE_SIZE - size;
106}
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500107
108static DEVICE_ATTR(pools, S_IRUGO, show_pools, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700109
110/**
111 * dma_pool_create - Creates a pool of consistent memory blocks, for dma.
112 * @name: name of pool, for diagnostics
113 * @dev: device that will be doing the DMA
114 * @size: size of the blocks in this pool.
115 * @align: alignment requirement for blocks; must be a power of two
Matthew Wilcoxe34f44b2007-12-03 14:16:24 -0500116 * @boundary: returned blocks won't cross this power of two boundary
Linus Torvalds1da177e2005-04-16 15:20:36 -0700117 * Context: !in_interrupt()
118 *
119 * Returns a dma allocation pool with the requested characteristics, or
120 * null if one can't be created. Given one of these pools, dma_pool_alloc()
121 * may be used to allocate memory. Such memory will all have "consistent"
122 * DMA mappings, accessible by the device and its driver without using
123 * cache flushing primitives. The actual size of blocks allocated may be
124 * larger than requested because of alignment.
125 *
Matthew Wilcoxe34f44b2007-12-03 14:16:24 -0500126 * If @boundary is nonzero, objects returned from dma_pool_alloc() won't
Linus Torvalds1da177e2005-04-16 15:20:36 -0700127 * cross that size boundary. This is useful for devices which have
128 * addressing restrictions on individual DMA transfers, such as not crossing
129 * boundaries of 4KBytes.
130 */
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500131struct dma_pool *dma_pool_create(const char *name, struct device *dev,
Matthew Wilcoxe34f44b2007-12-03 14:16:24 -0500132 size_t size, size_t align, size_t boundary)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700133{
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500134 struct dma_pool *retval;
Matthew Wilcoxe34f44b2007-12-03 14:16:24 -0500135 size_t allocation;
Sebastian Andrzej Siewior01c29652014-10-09 15:28:50 -0700136 bool empty = false;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700137
Paul McQuadebaa2ef82014-10-09 15:29:11 -0700138 if (align == 0)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700139 align = 1;
Paul McQuadebaa2ef82014-10-09 15:29:11 -0700140 else if (align & (align - 1))
Matthew Wilcox399154b2007-12-03 12:10:24 -0500141 return NULL;
Matthew Wilcox399154b2007-12-03 12:10:24 -0500142
Paul McQuadebaa2ef82014-10-09 15:29:11 -0700143 if (size == 0)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700144 return NULL;
Paul McQuadebaa2ef82014-10-09 15:29:11 -0700145 else if (size < 4)
Matthew Wilcoxa35a3452007-12-03 14:08:28 -0500146 size = 4;
Matthew Wilcox399154b2007-12-03 12:10:24 -0500147
148 if ((size % align) != 0)
149 size = ALIGN(size, align);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700150
Matthew Wilcoxe34f44b2007-12-03 14:16:24 -0500151 allocation = max_t(size_t, size, PAGE_SIZE);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700152
Paul McQuadebaa2ef82014-10-09 15:29:11 -0700153 if (!boundary)
Matthew Wilcoxe34f44b2007-12-03 14:16:24 -0500154 boundary = allocation;
Paul McQuadebaa2ef82014-10-09 15:29:11 -0700155 else if ((boundary < size) || (boundary & (boundary - 1)))
Matthew Wilcoxe34f44b2007-12-03 14:16:24 -0500156 return NULL;
Matthew Wilcoxe34f44b2007-12-03 14:16:24 -0500157
158 retval = kmalloc_node(sizeof(*retval), GFP_KERNEL, dev_to_node(dev));
159 if (!retval)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700160 return retval;
161
Matthew Wilcoxe34f44b2007-12-03 14:16:24 -0500162 strlcpy(retval->name, name, sizeof(retval->name));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700163
164 retval->dev = dev;
165
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500166 INIT_LIST_HEAD(&retval->page_list);
167 spin_lock_init(&retval->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700168 retval->size = size;
Matthew Wilcoxe34f44b2007-12-03 14:16:24 -0500169 retval->boundary = boundary;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700170 retval->allocation = allocation;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700171
Daeseok Youncc6b6642014-06-04 16:08:05 -0700172 INIT_LIST_HEAD(&retval->pools);
Cornelia Huck141ecc52006-09-22 11:37:27 +0200173
Sebastian Andrzej Siewior01c29652014-10-09 15:28:50 -0700174 /*
175 * pools_lock ensures that the ->dma_pools list does not get corrupted.
176 * pools_reg_lock ensures that there is not a race between
177 * dma_pool_create() and dma_pool_destroy() or within dma_pool_create()
178 * when the first invocation of dma_pool_create() failed on
179 * device_create_file() and the second assumes that it has been done (I
180 * know it is a short window).
181 */
182 mutex_lock(&pools_reg_lock);
Daeseok Youncc6b6642014-06-04 16:08:05 -0700183 mutex_lock(&pools_lock);
Sebastian Andrzej Siewior01c29652014-10-09 15:28:50 -0700184 if (list_empty(&dev->dma_pools))
185 empty = true;
186 list_add(&retval->pools, &dev->dma_pools);
Daeseok Youncc6b6642014-06-04 16:08:05 -0700187 mutex_unlock(&pools_lock);
Sebastian Andrzej Siewior01c29652014-10-09 15:28:50 -0700188 if (empty) {
189 int err;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700190
Sebastian Andrzej Siewior01c29652014-10-09 15:28:50 -0700191 err = device_create_file(dev, &dev_attr_pools);
192 if (err) {
193 mutex_lock(&pools_lock);
194 list_del(&retval->pools);
195 mutex_unlock(&pools_lock);
196 mutex_unlock(&pools_reg_lock);
197 kfree(retval);
198 return NULL;
199 }
200 }
201 mutex_unlock(&pools_reg_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700202 return retval;
203}
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500204EXPORT_SYMBOL(dma_pool_create);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700205
Matthew Wilcoxa35a3452007-12-03 14:08:28 -0500206static void pool_initialise_page(struct dma_pool *pool, struct dma_page *page)
207{
208 unsigned int offset = 0;
Matthew Wilcoxe34f44b2007-12-03 14:16:24 -0500209 unsigned int next_boundary = pool->boundary;
Matthew Wilcoxa35a3452007-12-03 14:08:28 -0500210
211 do {
212 unsigned int next = offset + pool->size;
Matthew Wilcoxe34f44b2007-12-03 14:16:24 -0500213 if (unlikely((next + pool->size) >= next_boundary)) {
214 next = next_boundary;
215 next_boundary += pool->boundary;
216 }
Matthew Wilcoxa35a3452007-12-03 14:08:28 -0500217 *(int *)(page->vaddr + offset) = next;
218 offset = next;
219 } while (offset < pool->allocation);
220}
221
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500222static struct dma_page *pool_alloc_page(struct dma_pool *pool, gfp_t mem_flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700223{
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500224 struct dma_page *page;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700225
Matthew Wilcoxa35a3452007-12-03 14:08:28 -0500226 page = kmalloc(sizeof(*page), mem_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700227 if (!page)
228 return NULL;
Matthew Wilcoxa35a3452007-12-03 14:08:28 -0500229 page->vaddr = dma_alloc_coherent(pool->dev, pool->allocation,
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500230 &page->dma, mem_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700231 if (page->vaddr) {
Andi Kleenb5ee5be2008-04-28 02:12:37 -0700232#ifdef DMAPOOL_DEBUG
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500233 memset(page->vaddr, POOL_POISON_FREED, pool->allocation);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700234#endif
Matthew Wilcoxa35a3452007-12-03 14:08:28 -0500235 pool_initialise_page(pool, page);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700236 page->in_use = 0;
Matthew Wilcoxa35a3452007-12-03 14:08:28 -0500237 page->offset = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700238 } else {
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500239 kfree(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700240 page = NULL;
241 }
242 return page;
243}
244
Nicholas Kraused9e7e372015-09-04 15:48:19 -0700245static inline bool is_page_busy(struct dma_page *page)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700246{
Matthew Wilcoxa35a3452007-12-03 14:08:28 -0500247 return page->in_use != 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700248}
249
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500250static void pool_free_page(struct dma_pool *pool, struct dma_page *page)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700251{
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500252 dma_addr_t dma = page->dma;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700253
Andi Kleenb5ee5be2008-04-28 02:12:37 -0700254#ifdef DMAPOOL_DEBUG
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500255 memset(page->vaddr, POOL_POISON_FREED, pool->allocation);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700256#endif
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500257 dma_free_coherent(pool->dev, pool->allocation, page->vaddr, dma);
258 list_del(&page->page_list);
259 kfree(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700260}
261
Linus Torvalds1da177e2005-04-16 15:20:36 -0700262/**
263 * dma_pool_destroy - destroys a pool of dma memory blocks.
264 * @pool: dma pool that will be destroyed
265 * Context: !in_interrupt()
266 *
267 * Caller guarantees that no more memory from the pool is in use,
268 * and that nothing will try to use the pool after this call.
269 */
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500270void dma_pool_destroy(struct dma_pool *pool)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700271{
Sebastian Andrzej Siewior01c29652014-10-09 15:28:50 -0700272 bool empty = false;
273
Sergey Senozhatsky44d71752015-09-08 15:00:56 -0700274 if (unlikely(!pool))
275 return;
276
Sebastian Andrzej Siewior01c29652014-10-09 15:28:50 -0700277 mutex_lock(&pools_reg_lock);
Matthias Kaehlckeb2366d62007-04-24 22:45:25 +0200278 mutex_lock(&pools_lock);
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500279 list_del(&pool->pools);
280 if (pool->dev && list_empty(&pool->dev->dma_pools))
Sebastian Andrzej Siewior01c29652014-10-09 15:28:50 -0700281 empty = true;
Matthias Kaehlckeb2366d62007-04-24 22:45:25 +0200282 mutex_unlock(&pools_lock);
Sebastian Andrzej Siewior01c29652014-10-09 15:28:50 -0700283 if (empty)
284 device_remove_file(pool->dev, &dev_attr_pools);
285 mutex_unlock(&pools_reg_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700286
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500287 while (!list_empty(&pool->page_list)) {
288 struct dma_page *page;
289 page = list_entry(pool->page_list.next,
290 struct dma_page, page_list);
Matthew Wilcoxa35a3452007-12-03 14:08:28 -0500291 if (is_page_busy(page)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700292 if (pool->dev)
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500293 dev_err(pool->dev,
294 "dma_pool_destroy %s, %p busy\n",
Linus Torvalds1da177e2005-04-16 15:20:36 -0700295 pool->name, page->vaddr);
296 else
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500297 printk(KERN_ERR
298 "dma_pool_destroy %s, %p busy\n",
299 pool->name, page->vaddr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700300 /* leak the still-in-use consistent memory */
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500301 list_del(&page->page_list);
302 kfree(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700303 } else
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500304 pool_free_page(pool, page);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700305 }
306
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500307 kfree(pool);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700308}
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500309EXPORT_SYMBOL(dma_pool_destroy);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700310
311/**
312 * dma_pool_alloc - get a block of consistent memory
313 * @pool: dma pool that will produce the block
314 * @mem_flags: GFP_* bitmask
315 * @handle: pointer to dma address of block
316 *
317 * This returns the kernel virtual address of a currently unused block,
318 * and reports its dma address through the handle.
Matthew Wilcox6182a092007-12-03 12:16:57 -0500319 * If such a memory block can't be allocated, %NULL is returned.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700320 */
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500321void *dma_pool_alloc(struct dma_pool *pool, gfp_t mem_flags,
322 dma_addr_t *handle)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700323{
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500324 unsigned long flags;
325 struct dma_page *page;
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500326 size_t offset;
327 void *retval;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700328
Mel Gormand0164ad2015-11-06 16:28:21 -0800329 might_sleep_if(gfpflags_allow_blocking(mem_flags));
Dima Zavinea05c842010-10-26 14:21:54 -0700330
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500331 spin_lock_irqsave(&pool->lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700332 list_for_each_entry(page, &pool->page_list, page_list) {
Matthew Wilcoxa35a3452007-12-03 14:08:28 -0500333 if (page->offset < pool->allocation)
334 goto ready;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700335 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700336
Marek Szyprowski387870f2012-11-07 15:37:07 +0100337 /* pool_alloc_page() might sleep, so temporarily drop &pool->lock */
338 spin_unlock_irqrestore(&pool->lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700339
Sean O. Stalleyfa23f562015-09-08 15:02:24 -0700340 page = pool_alloc_page(pool, mem_flags & (~__GFP_ZERO));
Marek Szyprowski387870f2012-11-07 15:37:07 +0100341 if (!page)
342 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700343
Marek Szyprowski387870f2012-11-07 15:37:07 +0100344 spin_lock_irqsave(&pool->lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700345
Marek Szyprowski387870f2012-11-07 15:37:07 +0100346 list_add(&page->page_list, &pool->page_list);
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500347 ready:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700348 page->in_use++;
Matthew Wilcoxa35a3452007-12-03 14:08:28 -0500349 offset = page->offset;
350 page->offset = *(int *)(page->vaddr + offset);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700351 retval = offset + page->vaddr;
352 *handle = offset + page->dma;
Andi Kleenb5ee5be2008-04-28 02:12:37 -0700353#ifdef DMAPOOL_DEBUG
Matthieu CASTET5de55b22012-12-11 16:01:31 -0800354 {
355 int i;
356 u8 *data = retval;
357 /* page->offset is stored in first 4 bytes */
358 for (i = sizeof(page->offset); i < pool->size; i++) {
359 if (data[i] == POOL_POISON_FREED)
360 continue;
361 if (pool->dev)
362 dev_err(pool->dev,
Hiroshige Sato5835f252014-04-16 21:28:34 +0900363 "dma_pool_alloc %s, %p (corrupted)\n",
Matthieu CASTET5de55b22012-12-11 16:01:31 -0800364 pool->name, retval);
365 else
Hiroshige Sato5835f252014-04-16 21:28:34 +0900366 pr_err("dma_pool_alloc %s, %p (corrupted)\n",
Matthieu CASTET5de55b22012-12-11 16:01:31 -0800367 pool->name, retval);
368
369 /*
370 * Dump the first 4 bytes even if they are not
371 * POOL_POISON_FREED
372 */
373 print_hex_dump(KERN_ERR, "", DUMP_PREFIX_OFFSET, 16, 1,
374 data, pool->size, 1);
375 break;
376 }
377 }
Sean O. Stalleyfa23f562015-09-08 15:02:24 -0700378 if (!(mem_flags & __GFP_ZERO))
379 memset(retval, POOL_POISON_ALLOCATED, pool->size);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700380#endif
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500381 spin_unlock_irqrestore(&pool->lock, flags);
Sean O. Stalleyfa23f562015-09-08 15:02:24 -0700382
383 if (mem_flags & __GFP_ZERO)
384 memset(retval, 0, pool->size);
385
Linus Torvalds1da177e2005-04-16 15:20:36 -0700386 return retval;
387}
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500388EXPORT_SYMBOL(dma_pool_alloc);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700389
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500390static struct dma_page *pool_find_page(struct dma_pool *pool, dma_addr_t dma)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700391{
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500392 struct dma_page *page;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700393
Linus Torvalds1da177e2005-04-16 15:20:36 -0700394 list_for_each_entry(page, &pool->page_list, page_list) {
395 if (dma < page->dma)
396 continue;
Robin Murphy676bd992015-10-01 15:37:19 -0700397 if ((dma - page->dma) < pool->allocation)
Rolf Eike Beer84bc2272011-01-13 15:47:24 -0800398 return page;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700399 }
Rolf Eike Beer84bc2272011-01-13 15:47:24 -0800400 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700401}
402
Linus Torvalds1da177e2005-04-16 15:20:36 -0700403/**
404 * dma_pool_free - put block back into dma pool
405 * @pool: the dma pool holding the block
406 * @vaddr: virtual address of block
407 * @dma: dma address of block
408 *
409 * Caller promises neither device nor driver will again touch this block
410 * unless it is first re-allocated.
411 */
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500412void dma_pool_free(struct dma_pool *pool, void *vaddr, dma_addr_t dma)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700413{
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500414 struct dma_page *page;
415 unsigned long flags;
Matthew Wilcoxa35a3452007-12-03 14:08:28 -0500416 unsigned int offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700417
Rolf Eike Beer84bc2272011-01-13 15:47:24 -0800418 spin_lock_irqsave(&pool->lock, flags);
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500419 page = pool_find_page(pool, dma);
420 if (!page) {
Rolf Eike Beer84bc2272011-01-13 15:47:24 -0800421 spin_unlock_irqrestore(&pool->lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700422 if (pool->dev)
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500423 dev_err(pool->dev,
424 "dma_pool_free %s, %p/%lx (bad dma)\n",
425 pool->name, vaddr, (unsigned long)dma);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700426 else
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500427 printk(KERN_ERR "dma_pool_free %s, %p/%lx (bad dma)\n",
428 pool->name, vaddr, (unsigned long)dma);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700429 return;
430 }
431
Matthew Wilcoxa35a3452007-12-03 14:08:28 -0500432 offset = vaddr - page->vaddr;
Andi Kleenb5ee5be2008-04-28 02:12:37 -0700433#ifdef DMAPOOL_DEBUG
Matthew Wilcoxa35a3452007-12-03 14:08:28 -0500434 if ((dma - page->dma) != offset) {
Rolf Eike Beer84bc2272011-01-13 15:47:24 -0800435 spin_unlock_irqrestore(&pool->lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700436 if (pool->dev)
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500437 dev_err(pool->dev,
438 "dma_pool_free %s, %p (bad vaddr)/%Lx\n",
439 pool->name, vaddr, (unsigned long long)dma);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700440 else
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500441 printk(KERN_ERR
442 "dma_pool_free %s, %p (bad vaddr)/%Lx\n",
443 pool->name, vaddr, (unsigned long long)dma);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700444 return;
445 }
Matthew Wilcoxa35a3452007-12-03 14:08:28 -0500446 {
447 unsigned int chain = page->offset;
448 while (chain < pool->allocation) {
449 if (chain != offset) {
450 chain = *(int *)(page->vaddr + chain);
451 continue;
452 }
Rolf Eike Beer84bc2272011-01-13 15:47:24 -0800453 spin_unlock_irqrestore(&pool->lock, flags);
Matthew Wilcoxa35a3452007-12-03 14:08:28 -0500454 if (pool->dev)
455 dev_err(pool->dev, "dma_pool_free %s, dma %Lx "
456 "already free\n", pool->name,
457 (unsigned long long)dma);
458 else
459 printk(KERN_ERR "dma_pool_free %s, dma %Lx "
460 "already free\n", pool->name,
461 (unsigned long long)dma);
462 return;
463 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700464 }
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500465 memset(vaddr, POOL_POISON_FREED, pool->size);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700466#endif
467
Linus Torvalds1da177e2005-04-16 15:20:36 -0700468 page->in_use--;
Matthew Wilcoxa35a3452007-12-03 14:08:28 -0500469 *(int *)vaddr = page->offset;
470 page->offset = offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700471 /*
472 * Resist a temptation to do
Matthew Wilcoxa35a3452007-12-03 14:08:28 -0500473 * if (!is_page_busy(page)) pool_free_page(pool, page);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700474 * Better have a few empty pages hang around.
475 */
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500476 spin_unlock_irqrestore(&pool->lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700477}
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500478EXPORT_SYMBOL(dma_pool_free);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700479
Tejun Heo9ac78492007-01-20 16:00:26 +0900480/*
481 * Managed DMA pool
482 */
483static void dmam_pool_release(struct device *dev, void *res)
484{
485 struct dma_pool *pool = *(struct dma_pool **)res;
486
487 dma_pool_destroy(pool);
488}
489
490static int dmam_pool_match(struct device *dev, void *res, void *match_data)
491{
492 return *(struct dma_pool **)res == match_data;
493}
494
495/**
496 * dmam_pool_create - Managed dma_pool_create()
497 * @name: name of pool, for diagnostics
498 * @dev: device that will be doing the DMA
499 * @size: size of the blocks in this pool.
500 * @align: alignment requirement for blocks; must be a power of two
501 * @allocation: returned blocks won't cross this boundary (or zero)
502 *
503 * Managed dma_pool_create(). DMA pool created with this function is
504 * automatically destroyed on driver detach.
505 */
506struct dma_pool *dmam_pool_create(const char *name, struct device *dev,
507 size_t size, size_t align, size_t allocation)
508{
509 struct dma_pool **ptr, *pool;
510
511 ptr = devres_alloc(dmam_pool_release, sizeof(*ptr), GFP_KERNEL);
512 if (!ptr)
513 return NULL;
514
515 pool = *ptr = dma_pool_create(name, dev, size, align, allocation);
516 if (pool)
517 devres_add(dev, ptr);
518 else
519 devres_free(ptr);
520
521 return pool;
522}
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500523EXPORT_SYMBOL(dmam_pool_create);
Tejun Heo9ac78492007-01-20 16:00:26 +0900524
525/**
526 * dmam_pool_destroy - Managed dma_pool_destroy()
527 * @pool: dma pool that will be destroyed
528 *
529 * Managed dma_pool_destroy().
530 */
531void dmam_pool_destroy(struct dma_pool *pool)
532{
533 struct device *dev = pool->dev;
534
Andy Shevchenko172cb4b2014-06-04 16:10:02 -0700535 WARN_ON(devres_release(dev, dmam_pool_release, dmam_pool_match, pool));
Tejun Heo9ac78492007-01-20 16:00:26 +0900536}
Matthew Wilcoxe87aa772007-12-03 12:04:31 -0500537EXPORT_SYMBOL(dmam_pool_destroy);