Chris Leech | c13c826 | 2006-05-23 17:18:44 -0700 | [diff] [blame] | 1 | /* |
| 2 | * Copyright(c) 2004 - 2006 Intel Corporation. All rights reserved. |
| 3 | * |
| 4 | * This program is free software; you can redistribute it and/or modify it |
| 5 | * under the terms of the GNU General Public License as published by the Free |
| 6 | * Software Foundation; either version 2 of the License, or (at your option) |
| 7 | * any later version. |
| 8 | * |
| 9 | * This program is distributed in the hope that it will be useful, but WITHOUT |
| 10 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
| 11 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for |
| 12 | * more details. |
| 13 | * |
| 14 | * You should have received a copy of the GNU General Public License along with |
| 15 | * this program; if not, write to the Free Software Foundation, Inc., 59 |
| 16 | * Temple Place - Suite 330, Boston, MA 02111-1307, USA. |
| 17 | * |
| 18 | * The full GNU General Public License is included in this distribution in the |
| 19 | * file called COPYING. |
| 20 | */ |
| 21 | |
| 22 | /* |
| 23 | * This code implements the DMA subsystem. It provides a HW-neutral interface |
| 24 | * for other kernel code to use asynchronous memory copy capabilities, |
| 25 | * if present, and allows different HW DMA drivers to register as providing |
| 26 | * this capability. |
| 27 | * |
| 28 | * Due to the fact we are accelerating what is already a relatively fast |
| 29 | * operation, the code goes to great lengths to avoid additional overhead, |
| 30 | * such as locking. |
| 31 | * |
| 32 | * LOCKING: |
| 33 | * |
| 34 | * The subsystem keeps two global lists, dma_device_list and dma_client_list. |
| 35 | * Both of these are protected by a mutex, dma_list_mutex. |
| 36 | * |
| 37 | * Each device has a channels list, which runs unlocked but is never modified |
| 38 | * once the device is registered, it's just setup by the driver. |
| 39 | * |
Dan Williams | d379b01 | 2007-07-09 11:56:42 -0700 | [diff] [blame] | 40 | * Each client is responsible for keeping track of the channels it uses. See |
| 41 | * the definition of dma_event_callback in dmaengine.h. |
Chris Leech | c13c826 | 2006-05-23 17:18:44 -0700 | [diff] [blame] | 42 | * |
| 43 | * Each device has a kref, which is initialized to 1 when the device is |
Tony Jones | 891f78e | 2007-09-25 02:03:03 +0200 | [diff] [blame] | 44 | * registered. A kref_get is done for each device registered. When the |
Sebastian Siewior | 8a5703f | 2008-04-21 22:38:45 +0000 | [diff] [blame] | 45 | * device is released, the corresponding kref_put is done in the release |
Chris Leech | c13c826 | 2006-05-23 17:18:44 -0700 | [diff] [blame] | 46 | * method. Every time one of the device's channels is allocated to a client, |
Sebastian Siewior | 8a5703f | 2008-04-21 22:38:45 +0000 | [diff] [blame] | 47 | * a kref_get occurs. When the channel is freed, the corresponding kref_put |
Chris Leech | c13c826 | 2006-05-23 17:18:44 -0700 | [diff] [blame] | 48 | * happens. The device's release function does a completion, so |
Tony Jones | 891f78e | 2007-09-25 02:03:03 +0200 | [diff] [blame] | 49 | * unregister_device does a remove event, device_unregister, a kref_put |
Chris Leech | c13c826 | 2006-05-23 17:18:44 -0700 | [diff] [blame] | 50 | * for the first reference, then waits on the completion for all other |
| 51 | * references to finish. |
| 52 | * |
| 53 | * Each channel has an open-coded implementation of Rusty Russell's "bigref," |
Dan Williams | d379b01 | 2007-07-09 11:56:42 -0700 | [diff] [blame] | 54 | * with a kref and a per_cpu local_t. A dma_chan_get is called when a client |
| 55 | * signals that it wants to use a channel, and dma_chan_put is called when |
Sebastian Siewior | 8a5703f | 2008-04-21 22:38:45 +0000 | [diff] [blame] | 56 | * a channel is removed or a client using it is unregistered. A client can |
Dan Williams | d379b01 | 2007-07-09 11:56:42 -0700 | [diff] [blame] | 57 | * take extra references per outstanding transaction, as is the case with |
| 58 | * the NET DMA client. The release function does a kref_put on the device. |
| 59 | * -ChrisL, DanW |
Chris Leech | c13c826 | 2006-05-23 17:18:44 -0700 | [diff] [blame] | 60 | */ |
| 61 | |
| 62 | #include <linux/init.h> |
| 63 | #include <linux/module.h> |
Dan Williams | 7405f74 | 2007-01-02 11:10:43 -0700 | [diff] [blame] | 64 | #include <linux/mm.h> |
Chris Leech | c13c826 | 2006-05-23 17:18:44 -0700 | [diff] [blame] | 65 | #include <linux/device.h> |
| 66 | #include <linux/dmaengine.h> |
| 67 | #include <linux/hardirq.h> |
| 68 | #include <linux/spinlock.h> |
| 69 | #include <linux/percpu.h> |
| 70 | #include <linux/rcupdate.h> |
| 71 | #include <linux/mutex.h> |
Dan Williams | 7405f74 | 2007-01-02 11:10:43 -0700 | [diff] [blame] | 72 | #include <linux/jiffies.h> |
Chris Leech | c13c826 | 2006-05-23 17:18:44 -0700 | [diff] [blame] | 73 | |
| 74 | static DEFINE_MUTEX(dma_list_mutex); |
| 75 | static LIST_HEAD(dma_device_list); |
| 76 | static LIST_HEAD(dma_client_list); |
| 77 | |
| 78 | /* --- sysfs implementation --- */ |
| 79 | |
Tony Jones | 891f78e | 2007-09-25 02:03:03 +0200 | [diff] [blame] | 80 | static ssize_t show_memcpy_count(struct device *dev, struct device_attribute *attr, char *buf) |
Chris Leech | c13c826 | 2006-05-23 17:18:44 -0700 | [diff] [blame] | 81 | { |
Tony Jones | 891f78e | 2007-09-25 02:03:03 +0200 | [diff] [blame] | 82 | struct dma_chan *chan = to_dma_chan(dev); |
Chris Leech | c13c826 | 2006-05-23 17:18:44 -0700 | [diff] [blame] | 83 | unsigned long count = 0; |
| 84 | int i; |
| 85 | |
Andrew Morton | 17f3ae0 | 2006-05-25 13:26:53 -0700 | [diff] [blame] | 86 | for_each_possible_cpu(i) |
Chris Leech | c13c826 | 2006-05-23 17:18:44 -0700 | [diff] [blame] | 87 | count += per_cpu_ptr(chan->local, i)->memcpy_count; |
| 88 | |
| 89 | return sprintf(buf, "%lu\n", count); |
| 90 | } |
| 91 | |
Tony Jones | 891f78e | 2007-09-25 02:03:03 +0200 | [diff] [blame] | 92 | static ssize_t show_bytes_transferred(struct device *dev, struct device_attribute *attr, |
| 93 | char *buf) |
Chris Leech | c13c826 | 2006-05-23 17:18:44 -0700 | [diff] [blame] | 94 | { |
Tony Jones | 891f78e | 2007-09-25 02:03:03 +0200 | [diff] [blame] | 95 | struct dma_chan *chan = to_dma_chan(dev); |
Chris Leech | c13c826 | 2006-05-23 17:18:44 -0700 | [diff] [blame] | 96 | unsigned long count = 0; |
| 97 | int i; |
| 98 | |
Andrew Morton | 17f3ae0 | 2006-05-25 13:26:53 -0700 | [diff] [blame] | 99 | for_each_possible_cpu(i) |
Chris Leech | c13c826 | 2006-05-23 17:18:44 -0700 | [diff] [blame] | 100 | count += per_cpu_ptr(chan->local, i)->bytes_transferred; |
| 101 | |
| 102 | return sprintf(buf, "%lu\n", count); |
| 103 | } |
| 104 | |
Tony Jones | 891f78e | 2007-09-25 02:03:03 +0200 | [diff] [blame] | 105 | static ssize_t show_in_use(struct device *dev, struct device_attribute *attr, char *buf) |
Chris Leech | c13c826 | 2006-05-23 17:18:44 -0700 | [diff] [blame] | 106 | { |
Tony Jones | 891f78e | 2007-09-25 02:03:03 +0200 | [diff] [blame] | 107 | struct dma_chan *chan = to_dma_chan(dev); |
Dan Williams | d379b01 | 2007-07-09 11:56:42 -0700 | [diff] [blame] | 108 | int in_use = 0; |
Chris Leech | c13c826 | 2006-05-23 17:18:44 -0700 | [diff] [blame] | 109 | |
Dan Williams | d379b01 | 2007-07-09 11:56:42 -0700 | [diff] [blame] | 110 | if (unlikely(chan->slow_ref) && |
| 111 | atomic_read(&chan->refcount.refcount) > 1) |
| 112 | in_use = 1; |
| 113 | else { |
| 114 | if (local_read(&(per_cpu_ptr(chan->local, |
| 115 | get_cpu())->refcount)) > 0) |
| 116 | in_use = 1; |
| 117 | put_cpu(); |
| 118 | } |
| 119 | |
| 120 | return sprintf(buf, "%d\n", in_use); |
Chris Leech | c13c826 | 2006-05-23 17:18:44 -0700 | [diff] [blame] | 121 | } |
| 122 | |
Tony Jones | 891f78e | 2007-09-25 02:03:03 +0200 | [diff] [blame] | 123 | static struct device_attribute dma_attrs[] = { |
Chris Leech | c13c826 | 2006-05-23 17:18:44 -0700 | [diff] [blame] | 124 | __ATTR(memcpy_count, S_IRUGO, show_memcpy_count, NULL), |
| 125 | __ATTR(bytes_transferred, S_IRUGO, show_bytes_transferred, NULL), |
| 126 | __ATTR(in_use, S_IRUGO, show_in_use, NULL), |
| 127 | __ATTR_NULL |
| 128 | }; |
| 129 | |
| 130 | static void dma_async_device_cleanup(struct kref *kref); |
| 131 | |
Tony Jones | 891f78e | 2007-09-25 02:03:03 +0200 | [diff] [blame] | 132 | static void dma_dev_release(struct device *dev) |
Chris Leech | c13c826 | 2006-05-23 17:18:44 -0700 | [diff] [blame] | 133 | { |
Tony Jones | 891f78e | 2007-09-25 02:03:03 +0200 | [diff] [blame] | 134 | struct dma_chan *chan = to_dma_chan(dev); |
Chris Leech | c13c826 | 2006-05-23 17:18:44 -0700 | [diff] [blame] | 135 | kref_put(&chan->device->refcount, dma_async_device_cleanup); |
| 136 | } |
| 137 | |
| 138 | static struct class dma_devclass = { |
Tony Jones | 891f78e | 2007-09-25 02:03:03 +0200 | [diff] [blame] | 139 | .name = "dma", |
| 140 | .dev_attrs = dma_attrs, |
| 141 | .dev_release = dma_dev_release, |
Chris Leech | c13c826 | 2006-05-23 17:18:44 -0700 | [diff] [blame] | 142 | }; |
| 143 | |
| 144 | /* --- client and device registration --- */ |
| 145 | |
Dan Williams | d379b01 | 2007-07-09 11:56:42 -0700 | [diff] [blame] | 146 | #define dma_chan_satisfies_mask(chan, mask) \ |
| 147 | __dma_chan_satisfies_mask((chan), &(mask)) |
| 148 | static int |
| 149 | __dma_chan_satisfies_mask(struct dma_chan *chan, dma_cap_mask_t *want) |
| 150 | { |
| 151 | dma_cap_mask_t has; |
| 152 | |
| 153 | bitmap_and(has.bits, want->bits, chan->device->cap_mask.bits, |
| 154 | DMA_TX_TYPE_END); |
| 155 | return bitmap_equal(want->bits, has.bits, DMA_TX_TYPE_END); |
| 156 | } |
| 157 | |
Chris Leech | c13c826 | 2006-05-23 17:18:44 -0700 | [diff] [blame] | 158 | /** |
Dan Williams | d379b01 | 2007-07-09 11:56:42 -0700 | [diff] [blame] | 159 | * dma_client_chan_alloc - try to allocate channels to a client |
Chris Leech | c13c826 | 2006-05-23 17:18:44 -0700 | [diff] [blame] | 160 | * @client: &dma_client |
| 161 | * |
| 162 | * Called with dma_list_mutex held. |
| 163 | */ |
Dan Williams | d379b01 | 2007-07-09 11:56:42 -0700 | [diff] [blame] | 164 | static void dma_client_chan_alloc(struct dma_client *client) |
Chris Leech | c13c826 | 2006-05-23 17:18:44 -0700 | [diff] [blame] | 165 | { |
| 166 | struct dma_device *device; |
| 167 | struct dma_chan *chan; |
Chris Leech | c13c826 | 2006-05-23 17:18:44 -0700 | [diff] [blame] | 168 | int desc; /* allocated descriptor count */ |
Dan Williams | d379b01 | 2007-07-09 11:56:42 -0700 | [diff] [blame] | 169 | enum dma_state_client ack; |
Chris Leech | c13c826 | 2006-05-23 17:18:44 -0700 | [diff] [blame] | 170 | |
Dan Williams | d379b01 | 2007-07-09 11:56:42 -0700 | [diff] [blame] | 171 | /* Find a channel */ |
| 172 | list_for_each_entry(device, &dma_device_list, global_node) |
Chris Leech | c13c826 | 2006-05-23 17:18:44 -0700 | [diff] [blame] | 173 | list_for_each_entry(chan, &device->channels, device_node) { |
Dan Williams | d379b01 | 2007-07-09 11:56:42 -0700 | [diff] [blame] | 174 | if (!dma_chan_satisfies_mask(chan, client->cap_mask)) |
Chris Leech | c13c826 | 2006-05-23 17:18:44 -0700 | [diff] [blame] | 175 | continue; |
| 176 | |
| 177 | desc = chan->device->device_alloc_chan_resources(chan); |
| 178 | if (desc >= 0) { |
Dan Williams | d379b01 | 2007-07-09 11:56:42 -0700 | [diff] [blame] | 179 | ack = client->event_callback(client, |
| 180 | chan, |
| 181 | DMA_RESOURCE_AVAILABLE); |
| 182 | |
| 183 | /* we are done once this client rejects |
| 184 | * an available resource |
| 185 | */ |
Dan Williams | 7cc5bf9 | 2008-07-08 11:58:21 -0700 | [diff] [blame^] | 186 | if (ack == DMA_ACK) { |
Dan Williams | d379b01 | 2007-07-09 11:56:42 -0700 | [diff] [blame] | 187 | dma_chan_get(chan); |
Dan Williams | 7cc5bf9 | 2008-07-08 11:58:21 -0700 | [diff] [blame^] | 188 | chan->client_count++; |
| 189 | } else if (ack == DMA_NAK) |
Dan Williams | d379b01 | 2007-07-09 11:56:42 -0700 | [diff] [blame] | 190 | return; |
Chris Leech | c13c826 | 2006-05-23 17:18:44 -0700 | [diff] [blame] | 191 | } |
| 192 | } |
Chris Leech | c13c826 | 2006-05-23 17:18:44 -0700 | [diff] [blame] | 193 | } |
| 194 | |
Dan Williams | 7405f74 | 2007-01-02 11:10:43 -0700 | [diff] [blame] | 195 | enum dma_status dma_sync_wait(struct dma_chan *chan, dma_cookie_t cookie) |
| 196 | { |
| 197 | enum dma_status status; |
| 198 | unsigned long dma_sync_wait_timeout = jiffies + msecs_to_jiffies(5000); |
| 199 | |
| 200 | dma_async_issue_pending(chan); |
| 201 | do { |
| 202 | status = dma_async_is_tx_complete(chan, cookie, NULL, NULL); |
| 203 | if (time_after_eq(jiffies, dma_sync_wait_timeout)) { |
| 204 | printk(KERN_ERR "dma_sync_wait_timeout!\n"); |
| 205 | return DMA_ERROR; |
| 206 | } |
| 207 | } while (status == DMA_IN_PROGRESS); |
| 208 | |
| 209 | return status; |
| 210 | } |
| 211 | EXPORT_SYMBOL(dma_sync_wait); |
| 212 | |
Chris Leech | c13c826 | 2006-05-23 17:18:44 -0700 | [diff] [blame] | 213 | /** |
Randy Dunlap | 6508871 | 2006-07-03 19:45:31 -0700 | [diff] [blame] | 214 | * dma_chan_cleanup - release a DMA channel's resources |
| 215 | * @kref: kernel reference structure that contains the DMA channel device |
Chris Leech | c13c826 | 2006-05-23 17:18:44 -0700 | [diff] [blame] | 216 | */ |
| 217 | void dma_chan_cleanup(struct kref *kref) |
| 218 | { |
| 219 | struct dma_chan *chan = container_of(kref, struct dma_chan, refcount); |
| 220 | chan->device->device_free_chan_resources(chan); |
Chris Leech | c13c826 | 2006-05-23 17:18:44 -0700 | [diff] [blame] | 221 | kref_put(&chan->device->refcount, dma_async_device_cleanup); |
| 222 | } |
David Brownell | 765e3d8 | 2007-03-16 13:38:05 -0800 | [diff] [blame] | 223 | EXPORT_SYMBOL(dma_chan_cleanup); |
Chris Leech | c13c826 | 2006-05-23 17:18:44 -0700 | [diff] [blame] | 224 | |
| 225 | static void dma_chan_free_rcu(struct rcu_head *rcu) |
| 226 | { |
| 227 | struct dma_chan *chan = container_of(rcu, struct dma_chan, rcu); |
| 228 | int bias = 0x7FFFFFFF; |
| 229 | int i; |
Andrew Morton | 17f3ae0 | 2006-05-25 13:26:53 -0700 | [diff] [blame] | 230 | for_each_possible_cpu(i) |
Chris Leech | c13c826 | 2006-05-23 17:18:44 -0700 | [diff] [blame] | 231 | bias -= local_read(&per_cpu_ptr(chan->local, i)->refcount); |
| 232 | atomic_sub(bias, &chan->refcount.refcount); |
| 233 | kref_put(&chan->refcount, dma_chan_cleanup); |
| 234 | } |
| 235 | |
Dan Williams | d379b01 | 2007-07-09 11:56:42 -0700 | [diff] [blame] | 236 | static void dma_chan_release(struct dma_chan *chan) |
Chris Leech | c13c826 | 2006-05-23 17:18:44 -0700 | [diff] [blame] | 237 | { |
| 238 | atomic_add(0x7FFFFFFF, &chan->refcount.refcount); |
| 239 | chan->slow_ref = 1; |
| 240 | call_rcu(&chan->rcu, dma_chan_free_rcu); |
| 241 | } |
| 242 | |
| 243 | /** |
Dan Williams | d379b01 | 2007-07-09 11:56:42 -0700 | [diff] [blame] | 244 | * dma_chans_notify_available - broadcast available channels to the clients |
Chris Leech | c13c826 | 2006-05-23 17:18:44 -0700 | [diff] [blame] | 245 | */ |
Dan Williams | d379b01 | 2007-07-09 11:56:42 -0700 | [diff] [blame] | 246 | static void dma_clients_notify_available(void) |
Chris Leech | c13c826 | 2006-05-23 17:18:44 -0700 | [diff] [blame] | 247 | { |
| 248 | struct dma_client *client; |
Dan Williams | d379b01 | 2007-07-09 11:56:42 -0700 | [diff] [blame] | 249 | |
| 250 | mutex_lock(&dma_list_mutex); |
| 251 | |
| 252 | list_for_each_entry(client, &dma_client_list, global_node) |
| 253 | dma_client_chan_alloc(client); |
| 254 | |
| 255 | mutex_unlock(&dma_list_mutex); |
| 256 | } |
| 257 | |
| 258 | /** |
| 259 | * dma_chans_notify_available - tell the clients that a channel is going away |
| 260 | * @chan: channel on its way out |
| 261 | */ |
| 262 | static void dma_clients_notify_removed(struct dma_chan *chan) |
| 263 | { |
| 264 | struct dma_client *client; |
| 265 | enum dma_state_client ack; |
Chris Leech | c13c826 | 2006-05-23 17:18:44 -0700 | [diff] [blame] | 266 | |
| 267 | mutex_lock(&dma_list_mutex); |
| 268 | |
| 269 | list_for_each_entry(client, &dma_client_list, global_node) { |
Dan Williams | d379b01 | 2007-07-09 11:56:42 -0700 | [diff] [blame] | 270 | ack = client->event_callback(client, chan, |
| 271 | DMA_RESOURCE_REMOVED); |
| 272 | |
| 273 | /* client was holding resources for this channel so |
| 274 | * free it |
| 275 | */ |
Dan Williams | 7cc5bf9 | 2008-07-08 11:58:21 -0700 | [diff] [blame^] | 276 | if (ack == DMA_ACK) { |
Dan Williams | d379b01 | 2007-07-09 11:56:42 -0700 | [diff] [blame] | 277 | dma_chan_put(chan); |
Dan Williams | 7cc5bf9 | 2008-07-08 11:58:21 -0700 | [diff] [blame^] | 278 | chan->client_count--; |
| 279 | } |
Chris Leech | c13c826 | 2006-05-23 17:18:44 -0700 | [diff] [blame] | 280 | } |
| 281 | |
| 282 | mutex_unlock(&dma_list_mutex); |
| 283 | } |
| 284 | |
| 285 | /** |
Dan Williams | d379b01 | 2007-07-09 11:56:42 -0700 | [diff] [blame] | 286 | * dma_async_client_register - register a &dma_client |
| 287 | * @client: ptr to a client structure with valid 'event_callback' and 'cap_mask' |
Chris Leech | c13c826 | 2006-05-23 17:18:44 -0700 | [diff] [blame] | 288 | */ |
Dan Williams | d379b01 | 2007-07-09 11:56:42 -0700 | [diff] [blame] | 289 | void dma_async_client_register(struct dma_client *client) |
Chris Leech | c13c826 | 2006-05-23 17:18:44 -0700 | [diff] [blame] | 290 | { |
Chris Leech | c13c826 | 2006-05-23 17:18:44 -0700 | [diff] [blame] | 291 | mutex_lock(&dma_list_mutex); |
| 292 | list_add_tail(&client->global_node, &dma_client_list); |
| 293 | mutex_unlock(&dma_list_mutex); |
Chris Leech | c13c826 | 2006-05-23 17:18:44 -0700 | [diff] [blame] | 294 | } |
David Brownell | 765e3d8 | 2007-03-16 13:38:05 -0800 | [diff] [blame] | 295 | EXPORT_SYMBOL(dma_async_client_register); |
Chris Leech | c13c826 | 2006-05-23 17:18:44 -0700 | [diff] [blame] | 296 | |
| 297 | /** |
| 298 | * dma_async_client_unregister - unregister a client and free the &dma_client |
Randy Dunlap | 6508871 | 2006-07-03 19:45:31 -0700 | [diff] [blame] | 299 | * @client: &dma_client to free |
Chris Leech | c13c826 | 2006-05-23 17:18:44 -0700 | [diff] [blame] | 300 | * |
| 301 | * Force frees any allocated DMA channels, frees the &dma_client memory |
| 302 | */ |
| 303 | void dma_async_client_unregister(struct dma_client *client) |
| 304 | { |
Dan Williams | d379b01 | 2007-07-09 11:56:42 -0700 | [diff] [blame] | 305 | struct dma_device *device; |
Chris Leech | c13c826 | 2006-05-23 17:18:44 -0700 | [diff] [blame] | 306 | struct dma_chan *chan; |
Dan Williams | d379b01 | 2007-07-09 11:56:42 -0700 | [diff] [blame] | 307 | enum dma_state_client ack; |
Chris Leech | c13c826 | 2006-05-23 17:18:44 -0700 | [diff] [blame] | 308 | |
| 309 | if (!client) |
| 310 | return; |
| 311 | |
Chris Leech | c13c826 | 2006-05-23 17:18:44 -0700 | [diff] [blame] | 312 | mutex_lock(&dma_list_mutex); |
Dan Williams | d379b01 | 2007-07-09 11:56:42 -0700 | [diff] [blame] | 313 | /* free all channels the client is holding */ |
| 314 | list_for_each_entry(device, &dma_device_list, global_node) |
| 315 | list_for_each_entry(chan, &device->channels, device_node) { |
| 316 | ack = client->event_callback(client, chan, |
| 317 | DMA_RESOURCE_REMOVED); |
| 318 | |
Dan Williams | 7cc5bf9 | 2008-07-08 11:58:21 -0700 | [diff] [blame^] | 319 | if (ack == DMA_ACK) { |
Dan Williams | d379b01 | 2007-07-09 11:56:42 -0700 | [diff] [blame] | 320 | dma_chan_put(chan); |
Dan Williams | 7cc5bf9 | 2008-07-08 11:58:21 -0700 | [diff] [blame^] | 321 | chan->client_count--; |
| 322 | } |
Dan Williams | d379b01 | 2007-07-09 11:56:42 -0700 | [diff] [blame] | 323 | } |
| 324 | |
Chris Leech | c13c826 | 2006-05-23 17:18:44 -0700 | [diff] [blame] | 325 | list_del(&client->global_node); |
| 326 | mutex_unlock(&dma_list_mutex); |
Chris Leech | c13c826 | 2006-05-23 17:18:44 -0700 | [diff] [blame] | 327 | } |
David Brownell | 765e3d8 | 2007-03-16 13:38:05 -0800 | [diff] [blame] | 328 | EXPORT_SYMBOL(dma_async_client_unregister); |
Chris Leech | c13c826 | 2006-05-23 17:18:44 -0700 | [diff] [blame] | 329 | |
| 330 | /** |
Dan Williams | d379b01 | 2007-07-09 11:56:42 -0700 | [diff] [blame] | 331 | * dma_async_client_chan_request - send all available channels to the |
| 332 | * client that satisfy the capability mask |
| 333 | * @client - requester |
Chris Leech | c13c826 | 2006-05-23 17:18:44 -0700 | [diff] [blame] | 334 | */ |
Dan Williams | d379b01 | 2007-07-09 11:56:42 -0700 | [diff] [blame] | 335 | void dma_async_client_chan_request(struct dma_client *client) |
Chris Leech | c13c826 | 2006-05-23 17:18:44 -0700 | [diff] [blame] | 336 | { |
Dan Williams | d379b01 | 2007-07-09 11:56:42 -0700 | [diff] [blame] | 337 | mutex_lock(&dma_list_mutex); |
| 338 | dma_client_chan_alloc(client); |
| 339 | mutex_unlock(&dma_list_mutex); |
Chris Leech | c13c826 | 2006-05-23 17:18:44 -0700 | [diff] [blame] | 340 | } |
David Brownell | 765e3d8 | 2007-03-16 13:38:05 -0800 | [diff] [blame] | 341 | EXPORT_SYMBOL(dma_async_client_chan_request); |
Chris Leech | c13c826 | 2006-05-23 17:18:44 -0700 | [diff] [blame] | 342 | |
| 343 | /** |
Randy Dunlap | 6508871 | 2006-07-03 19:45:31 -0700 | [diff] [blame] | 344 | * dma_async_device_register - registers DMA devices found |
Chris Leech | c13c826 | 2006-05-23 17:18:44 -0700 | [diff] [blame] | 345 | * @device: &dma_device |
| 346 | */ |
| 347 | int dma_async_device_register(struct dma_device *device) |
| 348 | { |
| 349 | static int id; |
Jeff Garzik | ff487fb | 2007-03-08 09:57:34 -0800 | [diff] [blame] | 350 | int chancnt = 0, rc; |
Chris Leech | c13c826 | 2006-05-23 17:18:44 -0700 | [diff] [blame] | 351 | struct dma_chan* chan; |
| 352 | |
| 353 | if (!device) |
| 354 | return -ENODEV; |
| 355 | |
Dan Williams | 7405f74 | 2007-01-02 11:10:43 -0700 | [diff] [blame] | 356 | /* validate device routines */ |
| 357 | BUG_ON(dma_has_cap(DMA_MEMCPY, device->cap_mask) && |
| 358 | !device->device_prep_dma_memcpy); |
| 359 | BUG_ON(dma_has_cap(DMA_XOR, device->cap_mask) && |
| 360 | !device->device_prep_dma_xor); |
| 361 | BUG_ON(dma_has_cap(DMA_ZERO_SUM, device->cap_mask) && |
| 362 | !device->device_prep_dma_zero_sum); |
| 363 | BUG_ON(dma_has_cap(DMA_MEMSET, device->cap_mask) && |
| 364 | !device->device_prep_dma_memset); |
Zhang Wei | 9b941c6 | 2008-03-13 17:45:28 -0700 | [diff] [blame] | 365 | BUG_ON(dma_has_cap(DMA_INTERRUPT, device->cap_mask) && |
Dan Williams | 7405f74 | 2007-01-02 11:10:43 -0700 | [diff] [blame] | 366 | !device->device_prep_dma_interrupt); |
| 367 | |
| 368 | BUG_ON(!device->device_alloc_chan_resources); |
| 369 | BUG_ON(!device->device_free_chan_resources); |
Dan Williams | 7405f74 | 2007-01-02 11:10:43 -0700 | [diff] [blame] | 370 | BUG_ON(!device->device_is_tx_complete); |
| 371 | BUG_ON(!device->device_issue_pending); |
| 372 | BUG_ON(!device->dev); |
| 373 | |
Chris Leech | c13c826 | 2006-05-23 17:18:44 -0700 | [diff] [blame] | 374 | init_completion(&device->done); |
| 375 | kref_init(&device->refcount); |
| 376 | device->dev_id = id++; |
| 377 | |
| 378 | /* represent channels in sysfs. Probably want devs too */ |
| 379 | list_for_each_entry(chan, &device->channels, device_node) { |
| 380 | chan->local = alloc_percpu(typeof(*chan->local)); |
| 381 | if (chan->local == NULL) |
| 382 | continue; |
| 383 | |
| 384 | chan->chan_id = chancnt++; |
Tony Jones | 891f78e | 2007-09-25 02:03:03 +0200 | [diff] [blame] | 385 | chan->dev.class = &dma_devclass; |
Haavard Skinnemoen | 1099dc7 | 2008-07-08 11:58:05 -0700 | [diff] [blame] | 386 | chan->dev.parent = device->dev; |
Tony Jones | 891f78e | 2007-09-25 02:03:03 +0200 | [diff] [blame] | 387 | snprintf(chan->dev.bus_id, BUS_ID_SIZE, "dma%dchan%d", |
Chris Leech | c13c826 | 2006-05-23 17:18:44 -0700 | [diff] [blame] | 388 | device->dev_id, chan->chan_id); |
| 389 | |
Tony Jones | 891f78e | 2007-09-25 02:03:03 +0200 | [diff] [blame] | 390 | rc = device_register(&chan->dev); |
Jeff Garzik | ff487fb | 2007-03-08 09:57:34 -0800 | [diff] [blame] | 391 | if (rc) { |
| 392 | chancnt--; |
| 393 | free_percpu(chan->local); |
| 394 | chan->local = NULL; |
| 395 | goto err_out; |
| 396 | } |
| 397 | |
Haavard Skinnemoen | 348badf | 2007-11-14 16:59:27 -0800 | [diff] [blame] | 398 | /* One for the channel, one of the class device */ |
| 399 | kref_get(&device->refcount); |
Chris Leech | c13c826 | 2006-05-23 17:18:44 -0700 | [diff] [blame] | 400 | kref_get(&device->refcount); |
Dan Williams | d379b01 | 2007-07-09 11:56:42 -0700 | [diff] [blame] | 401 | kref_init(&chan->refcount); |
Dan Williams | 7cc5bf9 | 2008-07-08 11:58:21 -0700 | [diff] [blame^] | 402 | chan->client_count = 0; |
Dan Williams | d379b01 | 2007-07-09 11:56:42 -0700 | [diff] [blame] | 403 | chan->slow_ref = 0; |
| 404 | INIT_RCU_HEAD(&chan->rcu); |
Chris Leech | c13c826 | 2006-05-23 17:18:44 -0700 | [diff] [blame] | 405 | } |
| 406 | |
| 407 | mutex_lock(&dma_list_mutex); |
| 408 | list_add_tail(&device->global_node, &dma_device_list); |
| 409 | mutex_unlock(&dma_list_mutex); |
| 410 | |
Dan Williams | d379b01 | 2007-07-09 11:56:42 -0700 | [diff] [blame] | 411 | dma_clients_notify_available(); |
Chris Leech | c13c826 | 2006-05-23 17:18:44 -0700 | [diff] [blame] | 412 | |
| 413 | return 0; |
Jeff Garzik | ff487fb | 2007-03-08 09:57:34 -0800 | [diff] [blame] | 414 | |
| 415 | err_out: |
| 416 | list_for_each_entry(chan, &device->channels, device_node) { |
| 417 | if (chan->local == NULL) |
| 418 | continue; |
| 419 | kref_put(&device->refcount, dma_async_device_cleanup); |
Tony Jones | 891f78e | 2007-09-25 02:03:03 +0200 | [diff] [blame] | 420 | device_unregister(&chan->dev); |
Jeff Garzik | ff487fb | 2007-03-08 09:57:34 -0800 | [diff] [blame] | 421 | chancnt--; |
| 422 | free_percpu(chan->local); |
| 423 | } |
| 424 | return rc; |
Chris Leech | c13c826 | 2006-05-23 17:18:44 -0700 | [diff] [blame] | 425 | } |
David Brownell | 765e3d8 | 2007-03-16 13:38:05 -0800 | [diff] [blame] | 426 | EXPORT_SYMBOL(dma_async_device_register); |
Chris Leech | c13c826 | 2006-05-23 17:18:44 -0700 | [diff] [blame] | 427 | |
| 428 | /** |
Randy Dunlap | 6508871 | 2006-07-03 19:45:31 -0700 | [diff] [blame] | 429 | * dma_async_device_cleanup - function called when all references are released |
| 430 | * @kref: kernel reference object |
Chris Leech | c13c826 | 2006-05-23 17:18:44 -0700 | [diff] [blame] | 431 | */ |
| 432 | static void dma_async_device_cleanup(struct kref *kref) |
| 433 | { |
| 434 | struct dma_device *device; |
| 435 | |
| 436 | device = container_of(kref, struct dma_device, refcount); |
| 437 | complete(&device->done); |
| 438 | } |
| 439 | |
Randy Dunlap | 6508871 | 2006-07-03 19:45:31 -0700 | [diff] [blame] | 440 | /** |
| 441 | * dma_async_device_unregister - unregisters DMA devices |
| 442 | * @device: &dma_device |
| 443 | */ |
| 444 | void dma_async_device_unregister(struct dma_device *device) |
Chris Leech | c13c826 | 2006-05-23 17:18:44 -0700 | [diff] [blame] | 445 | { |
| 446 | struct dma_chan *chan; |
Chris Leech | c13c826 | 2006-05-23 17:18:44 -0700 | [diff] [blame] | 447 | |
| 448 | mutex_lock(&dma_list_mutex); |
| 449 | list_del(&device->global_node); |
| 450 | mutex_unlock(&dma_list_mutex); |
| 451 | |
| 452 | list_for_each_entry(chan, &device->channels, device_node) { |
Dan Williams | d379b01 | 2007-07-09 11:56:42 -0700 | [diff] [blame] | 453 | dma_clients_notify_removed(chan); |
Tony Jones | 891f78e | 2007-09-25 02:03:03 +0200 | [diff] [blame] | 454 | device_unregister(&chan->dev); |
Dan Williams | d379b01 | 2007-07-09 11:56:42 -0700 | [diff] [blame] | 455 | dma_chan_release(chan); |
Chris Leech | c13c826 | 2006-05-23 17:18:44 -0700 | [diff] [blame] | 456 | } |
Chris Leech | c13c826 | 2006-05-23 17:18:44 -0700 | [diff] [blame] | 457 | |
| 458 | kref_put(&device->refcount, dma_async_device_cleanup); |
| 459 | wait_for_completion(&device->done); |
| 460 | } |
David Brownell | 765e3d8 | 2007-03-16 13:38:05 -0800 | [diff] [blame] | 461 | EXPORT_SYMBOL(dma_async_device_unregister); |
Chris Leech | c13c826 | 2006-05-23 17:18:44 -0700 | [diff] [blame] | 462 | |
Dan Williams | 7405f74 | 2007-01-02 11:10:43 -0700 | [diff] [blame] | 463 | /** |
| 464 | * dma_async_memcpy_buf_to_buf - offloaded copy between virtual addresses |
| 465 | * @chan: DMA channel to offload copy to |
| 466 | * @dest: destination address (virtual) |
| 467 | * @src: source address (virtual) |
| 468 | * @len: length |
| 469 | * |
| 470 | * Both @dest and @src must be mappable to a bus address according to the |
| 471 | * DMA mapping API rules for streaming mappings. |
| 472 | * Both @dest and @src must stay memory resident (kernel memory or locked |
| 473 | * user space pages). |
| 474 | */ |
| 475 | dma_cookie_t |
| 476 | dma_async_memcpy_buf_to_buf(struct dma_chan *chan, void *dest, |
| 477 | void *src, size_t len) |
| 478 | { |
| 479 | struct dma_device *dev = chan->device; |
| 480 | struct dma_async_tx_descriptor *tx; |
Dan Williams | 0036731 | 2008-02-02 19:49:57 -0700 | [diff] [blame] | 481 | dma_addr_t dma_dest, dma_src; |
Dan Williams | 7405f74 | 2007-01-02 11:10:43 -0700 | [diff] [blame] | 482 | dma_cookie_t cookie; |
| 483 | int cpu; |
| 484 | |
Dan Williams | 0036731 | 2008-02-02 19:49:57 -0700 | [diff] [blame] | 485 | dma_src = dma_map_single(dev->dev, src, len, DMA_TO_DEVICE); |
| 486 | dma_dest = dma_map_single(dev->dev, dest, len, DMA_FROM_DEVICE); |
Dan Williams | 636bdea | 2008-04-17 20:17:26 -0700 | [diff] [blame] | 487 | tx = dev->device_prep_dma_memcpy(chan, dma_dest, dma_src, len, |
| 488 | DMA_CTRL_ACK); |
Dan Williams | 0036731 | 2008-02-02 19:49:57 -0700 | [diff] [blame] | 489 | |
| 490 | if (!tx) { |
| 491 | dma_unmap_single(dev->dev, dma_src, len, DMA_TO_DEVICE); |
| 492 | dma_unmap_single(dev->dev, dma_dest, len, DMA_FROM_DEVICE); |
Dan Williams | 7405f74 | 2007-01-02 11:10:43 -0700 | [diff] [blame] | 493 | return -ENOMEM; |
Dan Williams | 0036731 | 2008-02-02 19:49:57 -0700 | [diff] [blame] | 494 | } |
Dan Williams | 7405f74 | 2007-01-02 11:10:43 -0700 | [diff] [blame] | 495 | |
Dan Williams | 7405f74 | 2007-01-02 11:10:43 -0700 | [diff] [blame] | 496 | tx->callback = NULL; |
Dan Williams | 7405f74 | 2007-01-02 11:10:43 -0700 | [diff] [blame] | 497 | cookie = tx->tx_submit(tx); |
| 498 | |
| 499 | cpu = get_cpu(); |
| 500 | per_cpu_ptr(chan->local, cpu)->bytes_transferred += len; |
| 501 | per_cpu_ptr(chan->local, cpu)->memcpy_count++; |
| 502 | put_cpu(); |
| 503 | |
| 504 | return cookie; |
| 505 | } |
| 506 | EXPORT_SYMBOL(dma_async_memcpy_buf_to_buf); |
| 507 | |
| 508 | /** |
| 509 | * dma_async_memcpy_buf_to_pg - offloaded copy from address to page |
| 510 | * @chan: DMA channel to offload copy to |
| 511 | * @page: destination page |
| 512 | * @offset: offset in page to copy to |
| 513 | * @kdata: source address (virtual) |
| 514 | * @len: length |
| 515 | * |
| 516 | * Both @page/@offset and @kdata must be mappable to a bus address according |
| 517 | * to the DMA mapping API rules for streaming mappings. |
| 518 | * Both @page/@offset and @kdata must stay memory resident (kernel memory or |
| 519 | * locked user space pages) |
| 520 | */ |
| 521 | dma_cookie_t |
| 522 | dma_async_memcpy_buf_to_pg(struct dma_chan *chan, struct page *page, |
| 523 | unsigned int offset, void *kdata, size_t len) |
| 524 | { |
| 525 | struct dma_device *dev = chan->device; |
| 526 | struct dma_async_tx_descriptor *tx; |
Dan Williams | 0036731 | 2008-02-02 19:49:57 -0700 | [diff] [blame] | 527 | dma_addr_t dma_dest, dma_src; |
Dan Williams | 7405f74 | 2007-01-02 11:10:43 -0700 | [diff] [blame] | 528 | dma_cookie_t cookie; |
| 529 | int cpu; |
| 530 | |
Dan Williams | 0036731 | 2008-02-02 19:49:57 -0700 | [diff] [blame] | 531 | dma_src = dma_map_single(dev->dev, kdata, len, DMA_TO_DEVICE); |
| 532 | dma_dest = dma_map_page(dev->dev, page, offset, len, DMA_FROM_DEVICE); |
Dan Williams | 636bdea | 2008-04-17 20:17:26 -0700 | [diff] [blame] | 533 | tx = dev->device_prep_dma_memcpy(chan, dma_dest, dma_src, len, |
| 534 | DMA_CTRL_ACK); |
Dan Williams | 0036731 | 2008-02-02 19:49:57 -0700 | [diff] [blame] | 535 | |
| 536 | if (!tx) { |
| 537 | dma_unmap_single(dev->dev, dma_src, len, DMA_TO_DEVICE); |
| 538 | dma_unmap_page(dev->dev, dma_dest, len, DMA_FROM_DEVICE); |
Dan Williams | 7405f74 | 2007-01-02 11:10:43 -0700 | [diff] [blame] | 539 | return -ENOMEM; |
Dan Williams | 0036731 | 2008-02-02 19:49:57 -0700 | [diff] [blame] | 540 | } |
Dan Williams | 7405f74 | 2007-01-02 11:10:43 -0700 | [diff] [blame] | 541 | |
Dan Williams | 7405f74 | 2007-01-02 11:10:43 -0700 | [diff] [blame] | 542 | tx->callback = NULL; |
Dan Williams | 7405f74 | 2007-01-02 11:10:43 -0700 | [diff] [blame] | 543 | cookie = tx->tx_submit(tx); |
| 544 | |
| 545 | cpu = get_cpu(); |
| 546 | per_cpu_ptr(chan->local, cpu)->bytes_transferred += len; |
| 547 | per_cpu_ptr(chan->local, cpu)->memcpy_count++; |
| 548 | put_cpu(); |
| 549 | |
| 550 | return cookie; |
| 551 | } |
| 552 | EXPORT_SYMBOL(dma_async_memcpy_buf_to_pg); |
| 553 | |
| 554 | /** |
| 555 | * dma_async_memcpy_pg_to_pg - offloaded copy from page to page |
| 556 | * @chan: DMA channel to offload copy to |
| 557 | * @dest_pg: destination page |
| 558 | * @dest_off: offset in page to copy to |
| 559 | * @src_pg: source page |
| 560 | * @src_off: offset in page to copy from |
| 561 | * @len: length |
| 562 | * |
| 563 | * Both @dest_page/@dest_off and @src_page/@src_off must be mappable to a bus |
| 564 | * address according to the DMA mapping API rules for streaming mappings. |
| 565 | * Both @dest_page/@dest_off and @src_page/@src_off must stay memory resident |
| 566 | * (kernel memory or locked user space pages). |
| 567 | */ |
| 568 | dma_cookie_t |
| 569 | dma_async_memcpy_pg_to_pg(struct dma_chan *chan, struct page *dest_pg, |
| 570 | unsigned int dest_off, struct page *src_pg, unsigned int src_off, |
| 571 | size_t len) |
| 572 | { |
| 573 | struct dma_device *dev = chan->device; |
| 574 | struct dma_async_tx_descriptor *tx; |
Dan Williams | 0036731 | 2008-02-02 19:49:57 -0700 | [diff] [blame] | 575 | dma_addr_t dma_dest, dma_src; |
Dan Williams | 7405f74 | 2007-01-02 11:10:43 -0700 | [diff] [blame] | 576 | dma_cookie_t cookie; |
| 577 | int cpu; |
| 578 | |
Dan Williams | 0036731 | 2008-02-02 19:49:57 -0700 | [diff] [blame] | 579 | dma_src = dma_map_page(dev->dev, src_pg, src_off, len, DMA_TO_DEVICE); |
| 580 | dma_dest = dma_map_page(dev->dev, dest_pg, dest_off, len, |
| 581 | DMA_FROM_DEVICE); |
Dan Williams | 636bdea | 2008-04-17 20:17:26 -0700 | [diff] [blame] | 582 | tx = dev->device_prep_dma_memcpy(chan, dma_dest, dma_src, len, |
| 583 | DMA_CTRL_ACK); |
Dan Williams | 0036731 | 2008-02-02 19:49:57 -0700 | [diff] [blame] | 584 | |
| 585 | if (!tx) { |
| 586 | dma_unmap_page(dev->dev, dma_src, len, DMA_TO_DEVICE); |
| 587 | dma_unmap_page(dev->dev, dma_dest, len, DMA_FROM_DEVICE); |
Dan Williams | 7405f74 | 2007-01-02 11:10:43 -0700 | [diff] [blame] | 588 | return -ENOMEM; |
Dan Williams | 0036731 | 2008-02-02 19:49:57 -0700 | [diff] [blame] | 589 | } |
Dan Williams | 7405f74 | 2007-01-02 11:10:43 -0700 | [diff] [blame] | 590 | |
Dan Williams | 7405f74 | 2007-01-02 11:10:43 -0700 | [diff] [blame] | 591 | tx->callback = NULL; |
Dan Williams | 7405f74 | 2007-01-02 11:10:43 -0700 | [diff] [blame] | 592 | cookie = tx->tx_submit(tx); |
| 593 | |
| 594 | cpu = get_cpu(); |
| 595 | per_cpu_ptr(chan->local, cpu)->bytes_transferred += len; |
| 596 | per_cpu_ptr(chan->local, cpu)->memcpy_count++; |
| 597 | put_cpu(); |
| 598 | |
| 599 | return cookie; |
| 600 | } |
| 601 | EXPORT_SYMBOL(dma_async_memcpy_pg_to_pg); |
| 602 | |
| 603 | void dma_async_tx_descriptor_init(struct dma_async_tx_descriptor *tx, |
| 604 | struct dma_chan *chan) |
| 605 | { |
| 606 | tx->chan = chan; |
| 607 | spin_lock_init(&tx->lock); |
Dan Williams | 7405f74 | 2007-01-02 11:10:43 -0700 | [diff] [blame] | 608 | } |
| 609 | EXPORT_SYMBOL(dma_async_tx_descriptor_init); |
| 610 | |
Chris Leech | c13c826 | 2006-05-23 17:18:44 -0700 | [diff] [blame] | 611 | static int __init dma_bus_init(void) |
| 612 | { |
| 613 | mutex_init(&dma_list_mutex); |
| 614 | return class_register(&dma_devclass); |
| 615 | } |
Chris Leech | c13c826 | 2006-05-23 17:18:44 -0700 | [diff] [blame] | 616 | subsys_initcall(dma_bus_init); |
| 617 | |