Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1 | /* |
| 2 | * arch/arm/common/dmabounce.c |
| 3 | * |
| 4 | * Special dma_{map/unmap/dma_sync}_* routines for systems that have |
| 5 | * limited DMA windows. These functions utilize bounce buffers to |
| 6 | * copy data to/from buffers located outside the DMA region. This |
| 7 | * only works for systems in which DMA memory is at the bottom of |
Erik Hovland | 3a2916a | 2006-03-22 21:02:11 +0000 | [diff] [blame] | 8 | * RAM, the remainder of memory is at the top and the DMA memory |
Simon Arlott | 6cbdc8c | 2007-05-11 20:40:30 +0100 | [diff] [blame] | 9 | * can be marked as ZONE_DMA. Anything beyond that such as discontiguous |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 10 | * DMA windows will require custom implementations that reserve memory |
| 11 | * areas at early bootup. |
| 12 | * |
| 13 | * Original version by Brad Parker (brad@heeltoe.com) |
| 14 | * Re-written by Christopher Hoover <ch@murgatroid.com> |
| 15 | * Made generic by Deepak Saxena <dsaxena@plexity.net> |
| 16 | * |
| 17 | * Copyright (C) 2002 Hewlett Packard Company. |
| 18 | * Copyright (C) 2004 MontaVista Software, Inc. |
| 19 | * |
| 20 | * This program is free software; you can redistribute it and/or |
| 21 | * modify it under the terms of the GNU General Public License |
| 22 | * version 2 as published by the Free Software Foundation. |
| 23 | */ |
| 24 | |
| 25 | #include <linux/module.h> |
| 26 | #include <linux/init.h> |
| 27 | #include <linux/slab.h> |
Nicolas Pitre | 58edb51 | 2008-09-09 15:54:13 -0400 | [diff] [blame] | 28 | #include <linux/page-flags.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 29 | #include <linux/device.h> |
| 30 | #include <linux/dma-mapping.h> |
| 31 | #include <linux/dmapool.h> |
| 32 | #include <linux/list.h> |
FUJITA Tomonori | 9f2326b | 2007-10-23 09:11:41 +0200 | [diff] [blame] | 33 | #include <linux/scatterlist.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 34 | |
Russell King | 14eb75b | 2005-06-20 16:56:08 +0100 | [diff] [blame] | 35 | #include <asm/cacheflush.h> |
| 36 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 37 | #undef STATS |
Russell King | cb7610d | 2005-10-30 21:12:08 +0000 | [diff] [blame] | 38 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 39 | #ifdef STATS |
| 40 | #define DO_STATS(X) do { X ; } while (0) |
| 41 | #else |
| 42 | #define DO_STATS(X) do { } while (0) |
| 43 | #endif |
| 44 | |
| 45 | /* ************************************************** */ |
| 46 | |
| 47 | struct safe_buffer { |
| 48 | struct list_head node; |
| 49 | |
| 50 | /* original request */ |
| 51 | void *ptr; |
| 52 | size_t size; |
| 53 | int direction; |
| 54 | |
| 55 | /* safe buffer info */ |
Russell King | cb7610d | 2005-10-30 21:12:08 +0000 | [diff] [blame] | 56 | struct dmabounce_pool *pool; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 57 | void *safe; |
| 58 | dma_addr_t safe_dma_addr; |
| 59 | }; |
| 60 | |
Russell King | cb7610d | 2005-10-30 21:12:08 +0000 | [diff] [blame] | 61 | struct dmabounce_pool { |
| 62 | unsigned long size; |
| 63 | struct dma_pool *pool; |
| 64 | #ifdef STATS |
| 65 | unsigned long allocs; |
| 66 | #endif |
| 67 | }; |
| 68 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 69 | struct dmabounce_device_info { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 70 | struct device *dev; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 71 | struct list_head safe_buffers; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 72 | #ifdef STATS |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 73 | unsigned long total_allocs; |
| 74 | unsigned long map_op_count; |
| 75 | unsigned long bounce_count; |
Russell King | 017cc02 | 2007-02-12 10:53:50 +0000 | [diff] [blame] | 76 | int attr_res; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 77 | #endif |
Russell King | cb7610d | 2005-10-30 21:12:08 +0000 | [diff] [blame] | 78 | struct dmabounce_pool small; |
| 79 | struct dmabounce_pool large; |
Kevin Hilman | 823588c1 | 2006-06-22 22:27:14 +0100 | [diff] [blame] | 80 | |
| 81 | rwlock_t lock; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 82 | }; |
| 83 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 84 | #ifdef STATS |
Russell King | 017cc02 | 2007-02-12 10:53:50 +0000 | [diff] [blame] | 85 | static ssize_t dmabounce_show(struct device *dev, struct device_attribute *attr, |
| 86 | char *buf) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 87 | { |
Russell King | 017cc02 | 2007-02-12 10:53:50 +0000 | [diff] [blame] | 88 | struct dmabounce_device_info *device_info = dev->archdata.dmabounce; |
| 89 | return sprintf(buf, "%lu %lu %lu %lu %lu %lu\n", |
| 90 | device_info->small.allocs, |
| 91 | device_info->large.allocs, |
Russell King | cb7610d | 2005-10-30 21:12:08 +0000 | [diff] [blame] | 92 | device_info->total_allocs - device_info->small.allocs - |
| 93 | device_info->large.allocs, |
Russell King | 017cc02 | 2007-02-12 10:53:50 +0000 | [diff] [blame] | 94 | device_info->total_allocs, |
| 95 | device_info->map_op_count, |
| 96 | device_info->bounce_count); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 97 | } |
Russell King | 017cc02 | 2007-02-12 10:53:50 +0000 | [diff] [blame] | 98 | |
| 99 | static DEVICE_ATTR(dmabounce_stats, 0400, dmabounce_show, NULL); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 100 | #endif |
| 101 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 102 | |
| 103 | /* allocate a 'safe' buffer and keep track of it */ |
| 104 | static inline struct safe_buffer * |
| 105 | alloc_safe_buffer(struct dmabounce_device_info *device_info, void *ptr, |
Russell King | cb7610d | 2005-10-30 21:12:08 +0000 | [diff] [blame] | 106 | size_t size, enum dma_data_direction dir) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 107 | { |
| 108 | struct safe_buffer *buf; |
Russell King | cb7610d | 2005-10-30 21:12:08 +0000 | [diff] [blame] | 109 | struct dmabounce_pool *pool; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 110 | struct device *dev = device_info->dev; |
Kevin Hilman | 823588c1 | 2006-06-22 22:27:14 +0100 | [diff] [blame] | 111 | unsigned long flags; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 112 | |
| 113 | dev_dbg(dev, "%s(ptr=%p, size=%d, dir=%d)\n", |
| 114 | __func__, ptr, size, dir); |
| 115 | |
Russell King | cb7610d | 2005-10-30 21:12:08 +0000 | [diff] [blame] | 116 | if (size <= device_info->small.size) { |
| 117 | pool = &device_info->small; |
| 118 | } else if (size <= device_info->large.size) { |
| 119 | pool = &device_info->large; |
| 120 | } else { |
| 121 | pool = NULL; |
| 122 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 123 | |
| 124 | buf = kmalloc(sizeof(struct safe_buffer), GFP_ATOMIC); |
| 125 | if (buf == NULL) { |
| 126 | dev_warn(dev, "%s: kmalloc failed\n", __func__); |
| 127 | return NULL; |
| 128 | } |
| 129 | |
Russell King | cb7610d | 2005-10-30 21:12:08 +0000 | [diff] [blame] | 130 | buf->ptr = ptr; |
| 131 | buf->size = size; |
| 132 | buf->direction = dir; |
| 133 | buf->pool = pool; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 134 | |
Russell King | cb7610d | 2005-10-30 21:12:08 +0000 | [diff] [blame] | 135 | if (pool) { |
| 136 | buf->safe = dma_pool_alloc(pool->pool, GFP_ATOMIC, |
| 137 | &buf->safe_dma_addr); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 138 | } else { |
Russell King | cb7610d | 2005-10-30 21:12:08 +0000 | [diff] [blame] | 139 | buf->safe = dma_alloc_coherent(dev, size, &buf->safe_dma_addr, |
| 140 | GFP_ATOMIC); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 141 | } |
| 142 | |
Russell King | cb7610d | 2005-10-30 21:12:08 +0000 | [diff] [blame] | 143 | if (buf->safe == NULL) { |
| 144 | dev_warn(dev, |
| 145 | "%s: could not alloc dma memory (size=%d)\n", |
| 146 | __func__, size); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 147 | kfree(buf); |
| 148 | return NULL; |
| 149 | } |
| 150 | |
| 151 | #ifdef STATS |
Russell King | cb7610d | 2005-10-30 21:12:08 +0000 | [diff] [blame] | 152 | if (pool) |
| 153 | pool->allocs++; |
| 154 | device_info->total_allocs++; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 155 | #endif |
| 156 | |
Kevin Hilman | 823588c1 | 2006-06-22 22:27:14 +0100 | [diff] [blame] | 157 | write_lock_irqsave(&device_info->lock, flags); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 158 | list_add(&buf->node, &device_info->safe_buffers); |
Kevin Hilman | 823588c1 | 2006-06-22 22:27:14 +0100 | [diff] [blame] | 159 | write_unlock_irqrestore(&device_info->lock, flags); |
| 160 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 161 | return buf; |
| 162 | } |
| 163 | |
| 164 | /* determine if a buffer is from our "safe" pool */ |
| 165 | static inline struct safe_buffer * |
| 166 | find_safe_buffer(struct dmabounce_device_info *device_info, dma_addr_t safe_dma_addr) |
| 167 | { |
Kevin Hilman | e2785f0 | 2006-08-18 15:32:14 +0100 | [diff] [blame] | 168 | struct safe_buffer *b, *rb = NULL; |
Kevin Hilman | 823588c1 | 2006-06-22 22:27:14 +0100 | [diff] [blame] | 169 | unsigned long flags; |
| 170 | |
| 171 | read_lock_irqsave(&device_info->lock, flags); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 172 | |
Russell King | b46a58f | 2005-06-22 21:25:58 +0100 | [diff] [blame] | 173 | list_for_each_entry(b, &device_info->safe_buffers, node) |
Kevin Hilman | e2785f0 | 2006-08-18 15:32:14 +0100 | [diff] [blame] | 174 | if (b->safe_dma_addr == safe_dma_addr) { |
| 175 | rb = b; |
Kevin Hilman | 823588c1 | 2006-06-22 22:27:14 +0100 | [diff] [blame] | 176 | break; |
Kevin Hilman | e2785f0 | 2006-08-18 15:32:14 +0100 | [diff] [blame] | 177 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 178 | |
Kevin Hilman | 823588c1 | 2006-06-22 22:27:14 +0100 | [diff] [blame] | 179 | read_unlock_irqrestore(&device_info->lock, flags); |
Kevin Hilman | e2785f0 | 2006-08-18 15:32:14 +0100 | [diff] [blame] | 180 | return rb; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 181 | } |
| 182 | |
| 183 | static inline void |
| 184 | free_safe_buffer(struct dmabounce_device_info *device_info, struct safe_buffer *buf) |
| 185 | { |
Kevin Hilman | 823588c1 | 2006-06-22 22:27:14 +0100 | [diff] [blame] | 186 | unsigned long flags; |
| 187 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 188 | dev_dbg(device_info->dev, "%s(buf=%p)\n", __func__, buf); |
| 189 | |
Kevin Hilman | 823588c1 | 2006-06-22 22:27:14 +0100 | [diff] [blame] | 190 | write_lock_irqsave(&device_info->lock, flags); |
| 191 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 192 | list_del(&buf->node); |
| 193 | |
Kevin Hilman | 823588c1 | 2006-06-22 22:27:14 +0100 | [diff] [blame] | 194 | write_unlock_irqrestore(&device_info->lock, flags); |
| 195 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 196 | if (buf->pool) |
Russell King | cb7610d | 2005-10-30 21:12:08 +0000 | [diff] [blame] | 197 | dma_pool_free(buf->pool->pool, buf->safe, buf->safe_dma_addr); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 198 | else |
| 199 | dma_free_coherent(device_info->dev, buf->size, buf->safe, |
| 200 | buf->safe_dma_addr); |
| 201 | |
| 202 | kfree(buf); |
| 203 | } |
| 204 | |
| 205 | /* ************************************************** */ |
| 206 | |
Russell King | 125ab12 | 2008-09-25 22:16:22 +0100 | [diff] [blame] | 207 | static struct safe_buffer *find_safe_buffer_dev(struct device *dev, |
| 208 | dma_addr_t dma_addr, const char *where) |
| 209 | { |
| 210 | if (!dev || !dev->archdata.dmabounce) |
| 211 | return NULL; |
| 212 | if (dma_mapping_error(dev, dma_addr)) { |
Russell King | e2f521e | 2011-07-03 23:53:13 +0100 | [diff] [blame] | 213 | dev_err(dev, "Trying to %s invalid mapping\n", where); |
Russell King | 125ab12 | 2008-09-25 22:16:22 +0100 | [diff] [blame] | 214 | return NULL; |
| 215 | } |
| 216 | return find_safe_buffer(dev->archdata.dmabounce, dma_addr); |
| 217 | } |
| 218 | |
Russell King | 23bc987 | 2011-07-03 22:28:32 +0100 | [diff] [blame] | 219 | static int needs_bounce(struct device *dev, dma_addr_t dma_addr, size_t size) |
| 220 | { |
| 221 | if (!dev || !dev->archdata.dmabounce) |
| 222 | return 0; |
| 223 | |
| 224 | if (dev->dma_mask) { |
| 225 | unsigned long limit, mask = *dev->dma_mask; |
| 226 | |
| 227 | limit = (mask + 1) & ~mask; |
| 228 | if (limit && size > limit) { |
| 229 | dev_err(dev, "DMA mapping too big (requested %#x " |
| 230 | "mask %#Lx)\n", size, *dev->dma_mask); |
| 231 | return -E2BIG; |
| 232 | } |
| 233 | |
| 234 | /* Figure out if we need to bounce from the DMA mask. */ |
| 235 | if ((dma_addr | (dma_addr + size - 1)) & ~mask) |
| 236 | return 1; |
| 237 | } |
| 238 | |
| 239 | return dma_needs_bounce(dev, dma_addr, size) ? 1 : 0; |
| 240 | } |
| 241 | |
Russell King | 3216a97 | 2008-09-25 22:23:31 +0100 | [diff] [blame] | 242 | static inline dma_addr_t map_single(struct device *dev, void *ptr, size_t size, |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 243 | enum dma_data_direction dir) |
| 244 | { |
Russell King | ab2c215 | 2007-02-12 10:28:24 +0000 | [diff] [blame] | 245 | struct dmabounce_device_info *device_info = dev->archdata.dmabounce; |
Russell King | dd3641f | 2011-07-03 22:39:43 +0100 | [diff] [blame] | 246 | struct safe_buffer *buf; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 247 | |
| 248 | if (device_info) |
| 249 | DO_STATS ( device_info->map_op_count++ ); |
| 250 | |
Russell King | dd3641f | 2011-07-03 22:39:43 +0100 | [diff] [blame] | 251 | buf = alloc_safe_buffer(device_info, ptr, size, dir); |
Russell King | dfa322f | 2011-07-03 23:54:34 +0100 | [diff] [blame^] | 252 | if (buf == NULL) { |
Russell King | dd3641f | 2011-07-03 22:39:43 +0100 | [diff] [blame] | 253 | dev_err(dev, "%s: unable to map unsafe buffer %p!\n", |
| 254 | __func__, ptr); |
Russell King | 23bc987 | 2011-07-03 22:28:32 +0100 | [diff] [blame] | 255 | return ~0; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 256 | } |
| 257 | |
Russell King | dd3641f | 2011-07-03 22:39:43 +0100 | [diff] [blame] | 258 | dev_dbg(dev, "%s: unsafe buffer %p (dma=%#x) mapped to %p (dma=%#x)\n", |
| 259 | __func__, buf->ptr, virt_to_dma(dev, buf->ptr), |
| 260 | buf->safe, buf->safe_dma_addr); |
| 261 | |
| 262 | if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL) { |
| 263 | dev_dbg(dev, "%s: copy unsafe %p to safe %p, size %d\n", |
| 264 | __func__, ptr, buf->safe, size); |
| 265 | memcpy(buf->safe, ptr, size); |
| 266 | } |
| 267 | |
| 268 | return buf->safe_dma_addr; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 269 | } |
| 270 | |
Russell King | dd3641f | 2011-07-03 22:39:43 +0100 | [diff] [blame] | 271 | static inline void unmap_single(struct device *dev, struct safe_buffer *buf, |
Russell King | 3216a97 | 2008-09-25 22:23:31 +0100 | [diff] [blame] | 272 | size_t size, enum dma_data_direction dir) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 273 | { |
Russell King | dd3641f | 2011-07-03 22:39:43 +0100 | [diff] [blame] | 274 | BUG_ON(buf->size != size); |
| 275 | BUG_ON(buf->direction != dir); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 276 | |
Russell King | dd3641f | 2011-07-03 22:39:43 +0100 | [diff] [blame] | 277 | dev_dbg(dev, "%s: unsafe buffer %p (dma=%#x) mapped to %p (dma=%#x)\n", |
| 278 | __func__, buf->ptr, virt_to_dma(dev, buf->ptr), |
| 279 | buf->safe, buf->safe_dma_addr); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 280 | |
Russell King | dd3641f | 2011-07-03 22:39:43 +0100 | [diff] [blame] | 281 | DO_STATS(dev->archdata.dmabounce->bounce_count++); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 282 | |
Russell King | dd3641f | 2011-07-03 22:39:43 +0100 | [diff] [blame] | 283 | if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL) { |
| 284 | void *ptr = buf->ptr; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 285 | |
Russell King | dd3641f | 2011-07-03 22:39:43 +0100 | [diff] [blame] | 286 | dev_dbg(dev, "%s: copy back safe %p to unsafe %p size %d\n", |
| 287 | __func__, buf->safe, ptr, size); |
| 288 | memcpy(ptr, buf->safe, size); |
Russell King | 5abc100 | 2005-06-20 12:31:14 +0100 | [diff] [blame] | 289 | |
Russell King | dd3641f | 2011-07-03 22:39:43 +0100 | [diff] [blame] | 290 | /* |
| 291 | * Since we may have written to a page cache page, |
| 292 | * we need to ensure that the data will be coherent |
| 293 | * with user mappings. |
| 294 | */ |
| 295 | __cpuc_flush_dcache_area(ptr, size); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 296 | } |
Russell King | dd3641f | 2011-07-03 22:39:43 +0100 | [diff] [blame] | 297 | free_safe_buffer(dev->archdata.dmabounce, buf); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 298 | } |
| 299 | |
| 300 | /* ************************************************** */ |
| 301 | |
| 302 | /* |
| 303 | * see if a buffer address is in an 'unsafe' range. if it is |
| 304 | * allocate a 'safe' buffer and copy the unsafe buffer into it. |
| 305 | * substitute the safe buffer for the unsafe one. |
| 306 | * (basically move the buffer from an unsafe area to a safe one) |
| 307 | */ |
Russell King | 24056f5 | 2011-01-03 11:29:28 +0000 | [diff] [blame] | 308 | dma_addr_t __dma_map_page(struct device *dev, struct page *page, |
Russell King | 3216a97 | 2008-09-25 22:23:31 +0100 | [diff] [blame] | 309 | unsigned long offset, size_t size, enum dma_data_direction dir) |
Russell King | 56f55f8 | 2008-09-25 20:59:12 +0100 | [diff] [blame] | 310 | { |
Russell King | dd3641f | 2011-07-03 22:39:43 +0100 | [diff] [blame] | 311 | dma_addr_t dma_addr; |
| 312 | int ret; |
| 313 | |
Russell King | 56f55f8 | 2008-09-25 20:59:12 +0100 | [diff] [blame] | 314 | dev_dbg(dev, "%s(page=%p,off=%#lx,size=%zx,dir=%x)\n", |
| 315 | __func__, page, offset, size, dir); |
| 316 | |
Russell King | dd3641f | 2011-07-03 22:39:43 +0100 | [diff] [blame] | 317 | dma_addr = pfn_to_dma(dev, page_to_pfn(page)) + offset; |
| 318 | |
| 319 | ret = needs_bounce(dev, dma_addr, size); |
| 320 | if (ret < 0) |
| 321 | return ~0; |
| 322 | |
| 323 | if (ret == 0) { |
| 324 | __dma_page_cpu_to_dev(page, offset, size, dir); |
| 325 | return dma_addr; |
| 326 | } |
| 327 | |
Nicolas Pitre | 58edb51 | 2008-09-09 15:54:13 -0400 | [diff] [blame] | 328 | if (PageHighMem(page)) { |
Russell King | dd3641f | 2011-07-03 22:39:43 +0100 | [diff] [blame] | 329 | dev_err(dev, "DMA buffer bouncing of HIGHMEM pages is not supported\n"); |
Nicolas Pitre | 58edb51 | 2008-09-09 15:54:13 -0400 | [diff] [blame] | 330 | return ~0; |
| 331 | } |
| 332 | |
Russell King | 56f55f8 | 2008-09-25 20:59:12 +0100 | [diff] [blame] | 333 | return map_single(dev, page_address(page) + offset, size, dir); |
| 334 | } |
Russell King | 24056f5 | 2011-01-03 11:29:28 +0000 | [diff] [blame] | 335 | EXPORT_SYMBOL(__dma_map_page); |
Russell King | 56f55f8 | 2008-09-25 20:59:12 +0100 | [diff] [blame] | 336 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 337 | /* |
| 338 | * see if a mapped address was really a "safe" buffer and if so, copy |
| 339 | * the data from the safe buffer back to the unsafe buffer and free up |
| 340 | * the safe buffer. (basically return things back to the way they |
| 341 | * should be) |
| 342 | */ |
Russell King | 24056f5 | 2011-01-03 11:29:28 +0000 | [diff] [blame] | 343 | void __dma_unmap_page(struct device *dev, dma_addr_t dma_addr, size_t size, |
Russell King | 3216a97 | 2008-09-25 22:23:31 +0100 | [diff] [blame] | 344 | enum dma_data_direction dir) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 345 | { |
Russell King | dd3641f | 2011-07-03 22:39:43 +0100 | [diff] [blame] | 346 | struct safe_buffer *buf; |
| 347 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 348 | dev_dbg(dev, "%s(ptr=%p,size=%d,dir=%x)\n", |
| 349 | __func__, (void *) dma_addr, size, dir); |
| 350 | |
Russell King | dd3641f | 2011-07-03 22:39:43 +0100 | [diff] [blame] | 351 | buf = find_safe_buffer_dev(dev, dma_addr, __func__); |
| 352 | if (!buf) { |
| 353 | __dma_page_dev_to_cpu(pfn_to_page(dma_to_pfn(dev, dma_addr)), |
| 354 | dma_addr & ~PAGE_MASK, size, dir); |
| 355 | return; |
| 356 | } |
| 357 | |
| 358 | unmap_single(dev, buf, size, dir); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 359 | } |
Russell King | 24056f5 | 2011-01-03 11:29:28 +0000 | [diff] [blame] | 360 | EXPORT_SYMBOL(__dma_unmap_page); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 361 | |
Russell King | 2638b4d | 2008-09-25 21:38:41 +0100 | [diff] [blame] | 362 | int dmabounce_sync_for_cpu(struct device *dev, dma_addr_t addr, |
| 363 | unsigned long off, size_t sz, enum dma_data_direction dir) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 364 | { |
Russell King | 125ab12 | 2008-09-25 22:16:22 +0100 | [diff] [blame] | 365 | struct safe_buffer *buf; |
| 366 | |
| 367 | dev_dbg(dev, "%s(dma=%#x,off=%#lx,sz=%zx,dir=%x)\n", |
Russell King | 2638b4d | 2008-09-25 21:38:41 +0100 | [diff] [blame] | 368 | __func__, addr, off, sz, dir); |
Russell King | 125ab12 | 2008-09-25 22:16:22 +0100 | [diff] [blame] | 369 | |
| 370 | buf = find_safe_buffer_dev(dev, addr, __func__); |
| 371 | if (!buf) |
| 372 | return 1; |
| 373 | |
Russell King | 0e18b5d | 2008-09-29 13:48:17 +0100 | [diff] [blame] | 374 | BUG_ON(buf->direction != dir); |
| 375 | |
Russell King | 125ab12 | 2008-09-25 22:16:22 +0100 | [diff] [blame] | 376 | dev_dbg(dev, "%s: unsafe buffer %p (dma=%#x) mapped to %p (dma=%#x)\n", |
| 377 | __func__, buf->ptr, virt_to_dma(dev, buf->ptr), |
| 378 | buf->safe, buf->safe_dma_addr); |
| 379 | |
| 380 | DO_STATS(dev->archdata.dmabounce->bounce_count++); |
| 381 | |
| 382 | if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL) { |
| 383 | dev_dbg(dev, "%s: copy back safe %p to unsafe %p size %d\n", |
| 384 | __func__, buf->safe + off, buf->ptr + off, sz); |
| 385 | memcpy(buf->ptr + off, buf->safe + off, sz); |
| 386 | } |
| 387 | return 0; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 388 | } |
Russell King | 2638b4d | 2008-09-25 21:38:41 +0100 | [diff] [blame] | 389 | EXPORT_SYMBOL(dmabounce_sync_for_cpu); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 390 | |
Russell King | 2638b4d | 2008-09-25 21:38:41 +0100 | [diff] [blame] | 391 | int dmabounce_sync_for_device(struct device *dev, dma_addr_t addr, |
| 392 | unsigned long off, size_t sz, enum dma_data_direction dir) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 393 | { |
Russell King | 125ab12 | 2008-09-25 22:16:22 +0100 | [diff] [blame] | 394 | struct safe_buffer *buf; |
| 395 | |
| 396 | dev_dbg(dev, "%s(dma=%#x,off=%#lx,sz=%zx,dir=%x)\n", |
Russell King | 2638b4d | 2008-09-25 21:38:41 +0100 | [diff] [blame] | 397 | __func__, addr, off, sz, dir); |
Russell King | 125ab12 | 2008-09-25 22:16:22 +0100 | [diff] [blame] | 398 | |
| 399 | buf = find_safe_buffer_dev(dev, addr, __func__); |
| 400 | if (!buf) |
| 401 | return 1; |
| 402 | |
Russell King | 0e18b5d | 2008-09-29 13:48:17 +0100 | [diff] [blame] | 403 | BUG_ON(buf->direction != dir); |
| 404 | |
Russell King | 125ab12 | 2008-09-25 22:16:22 +0100 | [diff] [blame] | 405 | dev_dbg(dev, "%s: unsafe buffer %p (dma=%#x) mapped to %p (dma=%#x)\n", |
| 406 | __func__, buf->ptr, virt_to_dma(dev, buf->ptr), |
| 407 | buf->safe, buf->safe_dma_addr); |
| 408 | |
| 409 | DO_STATS(dev->archdata.dmabounce->bounce_count++); |
| 410 | |
| 411 | if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL) { |
| 412 | dev_dbg(dev, "%s: copy out unsafe %p to safe %p, size %d\n", |
| 413 | __func__,buf->ptr + off, buf->safe + off, sz); |
| 414 | memcpy(buf->safe + off, buf->ptr + off, sz); |
| 415 | } |
| 416 | return 0; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 417 | } |
Russell King | 2638b4d | 2008-09-25 21:38:41 +0100 | [diff] [blame] | 418 | EXPORT_SYMBOL(dmabounce_sync_for_device); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 419 | |
Russell King | 3216a97 | 2008-09-25 22:23:31 +0100 | [diff] [blame] | 420 | static int dmabounce_init_pool(struct dmabounce_pool *pool, struct device *dev, |
| 421 | const char *name, unsigned long size) |
Russell King | cb7610d | 2005-10-30 21:12:08 +0000 | [diff] [blame] | 422 | { |
| 423 | pool->size = size; |
| 424 | DO_STATS(pool->allocs = 0); |
| 425 | pool->pool = dma_pool_create(name, dev, size, |
| 426 | 0 /* byte alignment */, |
| 427 | 0 /* no page-crossing issues */); |
| 428 | |
| 429 | return pool->pool ? 0 : -ENOMEM; |
| 430 | } |
| 431 | |
Russell King | 3216a97 | 2008-09-25 22:23:31 +0100 | [diff] [blame] | 432 | int dmabounce_register_dev(struct device *dev, unsigned long small_buffer_size, |
| 433 | unsigned long large_buffer_size) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 434 | { |
| 435 | struct dmabounce_device_info *device_info; |
Russell King | cb7610d | 2005-10-30 21:12:08 +0000 | [diff] [blame] | 436 | int ret; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 437 | |
| 438 | device_info = kmalloc(sizeof(struct dmabounce_device_info), GFP_ATOMIC); |
| 439 | if (!device_info) { |
Greg Kroah-Hartman | fc3a882 | 2008-05-02 06:02:41 +0200 | [diff] [blame] | 440 | dev_err(dev, |
| 441 | "Could not allocated dmabounce_device_info\n"); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 442 | return -ENOMEM; |
| 443 | } |
| 444 | |
Russell King | cb7610d | 2005-10-30 21:12:08 +0000 | [diff] [blame] | 445 | ret = dmabounce_init_pool(&device_info->small, dev, |
| 446 | "small_dmabounce_pool", small_buffer_size); |
| 447 | if (ret) { |
| 448 | dev_err(dev, |
| 449 | "dmabounce: could not allocate DMA pool for %ld byte objects\n", |
| 450 | small_buffer_size); |
| 451 | goto err_free; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 452 | } |
| 453 | |
| 454 | if (large_buffer_size) { |
Russell King | cb7610d | 2005-10-30 21:12:08 +0000 | [diff] [blame] | 455 | ret = dmabounce_init_pool(&device_info->large, dev, |
| 456 | "large_dmabounce_pool", |
| 457 | large_buffer_size); |
| 458 | if (ret) { |
| 459 | dev_err(dev, |
| 460 | "dmabounce: could not allocate DMA pool for %ld byte objects\n", |
| 461 | large_buffer_size); |
| 462 | goto err_destroy; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 463 | } |
| 464 | } |
| 465 | |
| 466 | device_info->dev = dev; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 467 | INIT_LIST_HEAD(&device_info->safe_buffers); |
Kevin Hilman | 823588c1 | 2006-06-22 22:27:14 +0100 | [diff] [blame] | 468 | rwlock_init(&device_info->lock); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 469 | |
| 470 | #ifdef STATS |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 471 | device_info->total_allocs = 0; |
| 472 | device_info->map_op_count = 0; |
| 473 | device_info->bounce_count = 0; |
Russell King | 017cc02 | 2007-02-12 10:53:50 +0000 | [diff] [blame] | 474 | device_info->attr_res = device_create_file(dev, &dev_attr_dmabounce_stats); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 475 | #endif |
| 476 | |
Russell King | ab2c215 | 2007-02-12 10:28:24 +0000 | [diff] [blame] | 477 | dev->archdata.dmabounce = device_info; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 478 | |
Greg Kroah-Hartman | fc3a882 | 2008-05-02 06:02:41 +0200 | [diff] [blame] | 479 | dev_info(dev, "dmabounce: registered device\n"); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 480 | |
| 481 | return 0; |
Russell King | cb7610d | 2005-10-30 21:12:08 +0000 | [diff] [blame] | 482 | |
| 483 | err_destroy: |
| 484 | dma_pool_destroy(device_info->small.pool); |
| 485 | err_free: |
| 486 | kfree(device_info); |
| 487 | return ret; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 488 | } |
Russell King | 3216a97 | 2008-09-25 22:23:31 +0100 | [diff] [blame] | 489 | EXPORT_SYMBOL(dmabounce_register_dev); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 490 | |
Russell King | 3216a97 | 2008-09-25 22:23:31 +0100 | [diff] [blame] | 491 | void dmabounce_unregister_dev(struct device *dev) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 492 | { |
Russell King | ab2c215 | 2007-02-12 10:28:24 +0000 | [diff] [blame] | 493 | struct dmabounce_device_info *device_info = dev->archdata.dmabounce; |
| 494 | |
| 495 | dev->archdata.dmabounce = NULL; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 496 | |
| 497 | if (!device_info) { |
Greg Kroah-Hartman | fc3a882 | 2008-05-02 06:02:41 +0200 | [diff] [blame] | 498 | dev_warn(dev, |
| 499 | "Never registered with dmabounce but attempting" |
| 500 | "to unregister!\n"); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 501 | return; |
| 502 | } |
| 503 | |
| 504 | if (!list_empty(&device_info->safe_buffers)) { |
Greg Kroah-Hartman | fc3a882 | 2008-05-02 06:02:41 +0200 | [diff] [blame] | 505 | dev_err(dev, |
| 506 | "Removing from dmabounce with pending buffers!\n"); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 507 | BUG(); |
| 508 | } |
| 509 | |
Russell King | cb7610d | 2005-10-30 21:12:08 +0000 | [diff] [blame] | 510 | if (device_info->small.pool) |
| 511 | dma_pool_destroy(device_info->small.pool); |
| 512 | if (device_info->large.pool) |
| 513 | dma_pool_destroy(device_info->large.pool); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 514 | |
| 515 | #ifdef STATS |
Russell King | 017cc02 | 2007-02-12 10:53:50 +0000 | [diff] [blame] | 516 | if (device_info->attr_res == 0) |
| 517 | device_remove_file(dev, &dev_attr_dmabounce_stats); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 518 | #endif |
| 519 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 520 | kfree(device_info); |
| 521 | |
Greg Kroah-Hartman | fc3a882 | 2008-05-02 06:02:41 +0200 | [diff] [blame] | 522 | dev_info(dev, "dmabounce: device unregistered\n"); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 523 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 524 | EXPORT_SYMBOL(dmabounce_unregister_dev); |
| 525 | |
| 526 | MODULE_AUTHOR("Christopher Hoover <ch@hpl.hp.com>, Deepak Saxena <dsaxena@plexity.net>"); |
| 527 | MODULE_DESCRIPTION("Special dma_{map/unmap/dma_sync}_* routines for systems with limited DMA windows"); |
| 528 | MODULE_LICENSE("GPL"); |