Dan Williams | 9bc89cd | 2007-01-02 11:10:44 -0700 | [diff] [blame] | 1 | /* |
| 2 | * core routines for the asynchronous memory transfer/transform api |
| 3 | * |
| 4 | * Copyright © 2006, Intel Corporation. |
| 5 | * |
| 6 | * Dan Williams <dan.j.williams@intel.com> |
| 7 | * |
| 8 | * with architecture considerations by: |
| 9 | * Neil Brown <neilb@suse.de> |
| 10 | * Jeff Garzik <jeff@garzik.org> |
| 11 | * |
| 12 | * This program is free software; you can redistribute it and/or modify it |
| 13 | * under the terms and conditions of the GNU General Public License, |
| 14 | * version 2, as published by the Free Software Foundation. |
| 15 | * |
| 16 | * This program is distributed in the hope it will be useful, but WITHOUT |
| 17 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
| 18 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for |
| 19 | * more details. |
| 20 | * |
| 21 | * You should have received a copy of the GNU General Public License along with |
| 22 | * this program; if not, write to the Free Software Foundation, Inc., |
| 23 | * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. |
| 24 | * |
| 25 | */ |
Franck Bui-Huu | 8252474 | 2008-05-12 21:21:05 +0200 | [diff] [blame] | 26 | #include <linux/rculist.h> |
Dan Williams | 9bc89cd | 2007-01-02 11:10:44 -0700 | [diff] [blame] | 27 | #include <linux/kernel.h> |
| 28 | #include <linux/async_tx.h> |
| 29 | |
| 30 | #ifdef CONFIG_DMA_ENGINE |
Dan Williams | bec0851 | 2009-01-06 11:38:14 -0700 | [diff] [blame] | 31 | static int __init async_tx_init(void) |
Dan Williams | 9bc89cd | 2007-01-02 11:10:44 -0700 | [diff] [blame] | 32 | { |
Dan Williams | 729b5d1 | 2009-03-25 09:13:25 -0700 | [diff] [blame] | 33 | async_dmaengine_get(); |
Dan Williams | 9bc89cd | 2007-01-02 11:10:44 -0700 | [diff] [blame] | 34 | |
| 35 | printk(KERN_INFO "async_tx: api initialized (async)\n"); |
| 36 | |
| 37 | return 0; |
Dan Williams | 9bc89cd | 2007-01-02 11:10:44 -0700 | [diff] [blame] | 38 | } |
| 39 | |
| 40 | static void __exit async_tx_exit(void) |
| 41 | { |
Dan Williams | 729b5d1 | 2009-03-25 09:13:25 -0700 | [diff] [blame] | 42 | async_dmaengine_put(); |
Dan Williams | 9bc89cd | 2007-01-02 11:10:44 -0700 | [diff] [blame] | 43 | } |
| 44 | |
Dan Williams | af1f951 | 2009-08-29 19:09:26 -0700 | [diff] [blame^] | 45 | module_init(async_tx_init); |
| 46 | module_exit(async_tx_exit); |
| 47 | |
Dan Williams | 9bc89cd | 2007-01-02 11:10:44 -0700 | [diff] [blame] | 48 | /** |
Dan Williams | 47437b2 | 2008-02-02 19:49:59 -0700 | [diff] [blame] | 49 | * __async_tx_find_channel - find a channel to carry out the operation or let |
Dan Williams | 9bc89cd | 2007-01-02 11:10:44 -0700 | [diff] [blame] | 50 | * the transaction execute synchronously |
Dan Williams | a08abd8 | 2009-06-03 11:43:59 -0700 | [diff] [blame] | 51 | * @submit: transaction dependency and submission modifiers |
Dan Williams | 9bc89cd | 2007-01-02 11:10:44 -0700 | [diff] [blame] | 52 | * @tx_type: transaction type |
| 53 | */ |
| 54 | struct dma_chan * |
Dan Williams | a08abd8 | 2009-06-03 11:43:59 -0700 | [diff] [blame] | 55 | __async_tx_find_channel(struct async_submit_ctl *submit, |
| 56 | enum dma_transaction_type tx_type) |
Dan Williams | 9bc89cd | 2007-01-02 11:10:44 -0700 | [diff] [blame] | 57 | { |
Dan Williams | a08abd8 | 2009-06-03 11:43:59 -0700 | [diff] [blame] | 58 | struct dma_async_tx_descriptor *depend_tx = submit->depend_tx; |
| 59 | |
Dan Williams | 9bc89cd | 2007-01-02 11:10:44 -0700 | [diff] [blame] | 60 | /* see if we can keep the chain on one channel */ |
| 61 | if (depend_tx && |
Dan Williams | bec0851 | 2009-01-06 11:38:14 -0700 | [diff] [blame] | 62 | dma_has_cap(tx_type, depend_tx->chan->device->cap_mask)) |
Dan Williams | 9bc89cd | 2007-01-02 11:10:44 -0700 | [diff] [blame] | 63 | return depend_tx->chan; |
Dan Williams | 729b5d1 | 2009-03-25 09:13:25 -0700 | [diff] [blame] | 64 | return async_dma_find_channel(tx_type); |
Dan Williams | 9bc89cd | 2007-01-02 11:10:44 -0700 | [diff] [blame] | 65 | } |
Dan Williams | 47437b2 | 2008-02-02 19:49:59 -0700 | [diff] [blame] | 66 | EXPORT_SYMBOL_GPL(__async_tx_find_channel); |
Dan Williams | 9bc89cd | 2007-01-02 11:10:44 -0700 | [diff] [blame] | 67 | #endif |
| 68 | |
Dan Williams | 19242d7 | 2008-04-17 20:17:25 -0700 | [diff] [blame] | 69 | |
| 70 | /** |
| 71 | * async_tx_channel_switch - queue an interrupt descriptor with a dependency |
| 72 | * pre-attached. |
| 73 | * @depend_tx: the operation that must finish before the new operation runs |
| 74 | * @tx: the new operation |
| 75 | */ |
| 76 | static void |
| 77 | async_tx_channel_switch(struct dma_async_tx_descriptor *depend_tx, |
| 78 | struct dma_async_tx_descriptor *tx) |
| 79 | { |
| 80 | struct dma_chan *chan; |
| 81 | struct dma_device *device; |
| 82 | struct dma_async_tx_descriptor *intr_tx = (void *) ~0; |
| 83 | |
| 84 | /* first check to see if we can still append to depend_tx */ |
| 85 | spin_lock_bh(&depend_tx->lock); |
| 86 | if (depend_tx->parent && depend_tx->chan == tx->chan) { |
| 87 | tx->parent = depend_tx; |
| 88 | depend_tx->next = tx; |
| 89 | intr_tx = NULL; |
| 90 | } |
| 91 | spin_unlock_bh(&depend_tx->lock); |
| 92 | |
| 93 | if (!intr_tx) |
| 94 | return; |
| 95 | |
| 96 | chan = depend_tx->chan; |
| 97 | device = chan->device; |
| 98 | |
| 99 | /* see if we can schedule an interrupt |
| 100 | * otherwise poll for completion |
| 101 | */ |
| 102 | if (dma_has_cap(DMA_INTERRUPT, device->cap_mask)) |
Dan Williams | 636bdea | 2008-04-17 20:17:26 -0700 | [diff] [blame] | 103 | intr_tx = device->device_prep_dma_interrupt(chan, 0); |
Dan Williams | 19242d7 | 2008-04-17 20:17:25 -0700 | [diff] [blame] | 104 | else |
| 105 | intr_tx = NULL; |
| 106 | |
| 107 | if (intr_tx) { |
| 108 | intr_tx->callback = NULL; |
| 109 | intr_tx->callback_param = NULL; |
| 110 | tx->parent = intr_tx; |
| 111 | /* safe to set ->next outside the lock since we know we are |
| 112 | * not submitted yet |
| 113 | */ |
| 114 | intr_tx->next = tx; |
| 115 | |
| 116 | /* check if we need to append */ |
| 117 | spin_lock_bh(&depend_tx->lock); |
| 118 | if (depend_tx->parent) { |
| 119 | intr_tx->parent = depend_tx; |
| 120 | depend_tx->next = intr_tx; |
| 121 | async_tx_ack(intr_tx); |
| 122 | intr_tx = NULL; |
| 123 | } |
| 124 | spin_unlock_bh(&depend_tx->lock); |
| 125 | |
| 126 | if (intr_tx) { |
| 127 | intr_tx->parent = NULL; |
| 128 | intr_tx->tx_submit(intr_tx); |
| 129 | async_tx_ack(intr_tx); |
| 130 | } |
| 131 | } else { |
| 132 | if (dma_wait_for_async_tx(depend_tx) == DMA_ERROR) |
| 133 | panic("%s: DMA_ERROR waiting for depend_tx\n", |
| 134 | __func__); |
| 135 | tx->tx_submit(tx); |
| 136 | } |
| 137 | } |
| 138 | |
| 139 | |
| 140 | /** |
Dan Williams | a08abd8 | 2009-06-03 11:43:59 -0700 | [diff] [blame] | 141 | * submit_disposition - flags for routing an incoming operation |
Dan Williams | 19242d7 | 2008-04-17 20:17:25 -0700 | [diff] [blame] | 142 | * @ASYNC_TX_SUBMITTED: we were able to append the new operation under the lock |
| 143 | * @ASYNC_TX_CHANNEL_SWITCH: when the lock is dropped schedule a channel switch |
| 144 | * @ASYNC_TX_DIRECT_SUBMIT: when the lock is dropped submit directly |
Dan Williams | a08abd8 | 2009-06-03 11:43:59 -0700 | [diff] [blame] | 145 | * |
| 146 | * while holding depend_tx->lock we must avoid submitting new operations |
| 147 | * to prevent a circular locking dependency with drivers that already |
| 148 | * hold a channel lock when calling async_tx_run_dependencies. |
Dan Williams | 19242d7 | 2008-04-17 20:17:25 -0700 | [diff] [blame] | 149 | */ |
| 150 | enum submit_disposition { |
| 151 | ASYNC_TX_SUBMITTED, |
| 152 | ASYNC_TX_CHANNEL_SWITCH, |
| 153 | ASYNC_TX_DIRECT_SUBMIT, |
| 154 | }; |
| 155 | |
Dan Williams | 9bc89cd | 2007-01-02 11:10:44 -0700 | [diff] [blame] | 156 | void |
| 157 | async_tx_submit(struct dma_chan *chan, struct dma_async_tx_descriptor *tx, |
Dan Williams | a08abd8 | 2009-06-03 11:43:59 -0700 | [diff] [blame] | 158 | struct async_submit_ctl *submit) |
Dan Williams | 9bc89cd | 2007-01-02 11:10:44 -0700 | [diff] [blame] | 159 | { |
Dan Williams | a08abd8 | 2009-06-03 11:43:59 -0700 | [diff] [blame] | 160 | struct dma_async_tx_descriptor *depend_tx = submit->depend_tx; |
| 161 | |
| 162 | tx->callback = submit->cb_fn; |
| 163 | tx->callback_param = submit->cb_param; |
Dan Williams | 9bc89cd | 2007-01-02 11:10:44 -0700 | [diff] [blame] | 164 | |
Dan Williams | 19242d7 | 2008-04-17 20:17:25 -0700 | [diff] [blame] | 165 | if (depend_tx) { |
| 166 | enum submit_disposition s; |
Dan Williams | 9bc89cd | 2007-01-02 11:10:44 -0700 | [diff] [blame] | 167 | |
Dan Williams | 19242d7 | 2008-04-17 20:17:25 -0700 | [diff] [blame] | 168 | /* sanity check the dependency chain: |
| 169 | * 1/ if ack is already set then we cannot be sure |
| 170 | * we are referring to the correct operation |
| 171 | * 2/ dependencies are 1:1 i.e. two transactions can |
| 172 | * not depend on the same parent |
| 173 | */ |
Dan Williams | 636bdea | 2008-04-17 20:17:26 -0700 | [diff] [blame] | 174 | BUG_ON(async_tx_test_ack(depend_tx) || depend_tx->next || |
| 175 | tx->parent); |
Dan Williams | 19242d7 | 2008-04-17 20:17:25 -0700 | [diff] [blame] | 176 | |
| 177 | /* the lock prevents async_tx_run_dependencies from missing |
| 178 | * the setting of ->next when ->parent != NULL |
| 179 | */ |
Dan Williams | 9bc89cd | 2007-01-02 11:10:44 -0700 | [diff] [blame] | 180 | spin_lock_bh(&depend_tx->lock); |
Dan Williams | 19242d7 | 2008-04-17 20:17:25 -0700 | [diff] [blame] | 181 | if (depend_tx->parent) { |
| 182 | /* we have a parent so we can not submit directly |
| 183 | * if we are staying on the same channel: append |
| 184 | * else: channel switch |
| 185 | */ |
| 186 | if (depend_tx->chan == chan) { |
| 187 | tx->parent = depend_tx; |
| 188 | depend_tx->next = tx; |
| 189 | s = ASYNC_TX_SUBMITTED; |
| 190 | } else |
| 191 | s = ASYNC_TX_CHANNEL_SWITCH; |
| 192 | } else { |
| 193 | /* we do not have a parent so we may be able to submit |
| 194 | * directly if we are staying on the same channel |
| 195 | */ |
| 196 | if (depend_tx->chan == chan) |
| 197 | s = ASYNC_TX_DIRECT_SUBMIT; |
| 198 | else |
| 199 | s = ASYNC_TX_CHANNEL_SWITCH; |
Dan Williams | 9bc89cd | 2007-01-02 11:10:44 -0700 | [diff] [blame] | 200 | } |
| 201 | spin_unlock_bh(&depend_tx->lock); |
| 202 | |
Dan Williams | 19242d7 | 2008-04-17 20:17:25 -0700 | [diff] [blame] | 203 | switch (s) { |
| 204 | case ASYNC_TX_SUBMITTED: |
| 205 | break; |
| 206 | case ASYNC_TX_CHANNEL_SWITCH: |
| 207 | async_tx_channel_switch(depend_tx, tx); |
| 208 | break; |
| 209 | case ASYNC_TX_DIRECT_SUBMIT: |
| 210 | tx->parent = NULL; |
| 211 | tx->tx_submit(tx); |
| 212 | break; |
| 213 | } |
Dan Williams | 9bc89cd | 2007-01-02 11:10:44 -0700 | [diff] [blame] | 214 | } else { |
| 215 | tx->parent = NULL; |
| 216 | tx->tx_submit(tx); |
| 217 | } |
| 218 | |
Dan Williams | a08abd8 | 2009-06-03 11:43:59 -0700 | [diff] [blame] | 219 | if (submit->flags & ASYNC_TX_ACK) |
Dan Williams | 9bc89cd | 2007-01-02 11:10:44 -0700 | [diff] [blame] | 220 | async_tx_ack(tx); |
| 221 | |
Dan Williams | 88ba2aa | 2009-04-09 16:16:18 -0700 | [diff] [blame] | 222 | if (depend_tx) |
Dan Williams | 9bc89cd | 2007-01-02 11:10:44 -0700 | [diff] [blame] | 223 | async_tx_ack(depend_tx); |
| 224 | } |
| 225 | EXPORT_SYMBOL_GPL(async_tx_submit); |
| 226 | |
| 227 | /** |
Dan Williams | a08abd8 | 2009-06-03 11:43:59 -0700 | [diff] [blame] | 228 | * async_trigger_callback - schedules the callback function to be run |
| 229 | * @submit: submission and completion parameters |
| 230 | * |
| 231 | * honored flags: ASYNC_TX_ACK |
| 232 | * |
| 233 | * The callback is run after any dependent operations have completed. |
Dan Williams | 9bc89cd | 2007-01-02 11:10:44 -0700 | [diff] [blame] | 234 | */ |
| 235 | struct dma_async_tx_descriptor * |
Dan Williams | a08abd8 | 2009-06-03 11:43:59 -0700 | [diff] [blame] | 236 | async_trigger_callback(struct async_submit_ctl *submit) |
Dan Williams | 9bc89cd | 2007-01-02 11:10:44 -0700 | [diff] [blame] | 237 | { |
| 238 | struct dma_chan *chan; |
| 239 | struct dma_device *device; |
| 240 | struct dma_async_tx_descriptor *tx; |
Dan Williams | a08abd8 | 2009-06-03 11:43:59 -0700 | [diff] [blame] | 241 | struct dma_async_tx_descriptor *depend_tx = submit->depend_tx; |
Dan Williams | 9bc89cd | 2007-01-02 11:10:44 -0700 | [diff] [blame] | 242 | |
| 243 | if (depend_tx) { |
| 244 | chan = depend_tx->chan; |
| 245 | device = chan->device; |
| 246 | |
| 247 | /* see if we can schedule an interrupt |
| 248 | * otherwise poll for completion |
| 249 | */ |
| 250 | if (device && !dma_has_cap(DMA_INTERRUPT, device->cap_mask)) |
| 251 | device = NULL; |
| 252 | |
Dan Williams | 636bdea | 2008-04-17 20:17:26 -0700 | [diff] [blame] | 253 | tx = device ? device->device_prep_dma_interrupt(chan, 0) : NULL; |
Dan Williams | 9bc89cd | 2007-01-02 11:10:44 -0700 | [diff] [blame] | 254 | } else |
| 255 | tx = NULL; |
| 256 | |
| 257 | if (tx) { |
Dan Williams | 3280ab3e | 2008-03-13 17:45:28 -0700 | [diff] [blame] | 258 | pr_debug("%s: (async)\n", __func__); |
Dan Williams | 9bc89cd | 2007-01-02 11:10:44 -0700 | [diff] [blame] | 259 | |
Dan Williams | a08abd8 | 2009-06-03 11:43:59 -0700 | [diff] [blame] | 260 | async_tx_submit(chan, tx, submit); |
Dan Williams | 9bc89cd | 2007-01-02 11:10:44 -0700 | [diff] [blame] | 261 | } else { |
Dan Williams | 3280ab3e | 2008-03-13 17:45:28 -0700 | [diff] [blame] | 262 | pr_debug("%s: (sync)\n", __func__); |
Dan Williams | 9bc89cd | 2007-01-02 11:10:44 -0700 | [diff] [blame] | 263 | |
| 264 | /* wait for any prerequisite operations */ |
Dan Williams | a08abd8 | 2009-06-03 11:43:59 -0700 | [diff] [blame] | 265 | async_tx_quiesce(&submit->depend_tx); |
Dan Williams | 9bc89cd | 2007-01-02 11:10:44 -0700 | [diff] [blame] | 266 | |
Dan Williams | a08abd8 | 2009-06-03 11:43:59 -0700 | [diff] [blame] | 267 | async_tx_sync_epilog(submit); |
Dan Williams | 9bc89cd | 2007-01-02 11:10:44 -0700 | [diff] [blame] | 268 | } |
| 269 | |
| 270 | return tx; |
| 271 | } |
| 272 | EXPORT_SYMBOL_GPL(async_trigger_callback); |
| 273 | |
Dan Williams | d2c52b7 | 2008-07-17 17:59:55 -0700 | [diff] [blame] | 274 | /** |
| 275 | * async_tx_quiesce - ensure tx is complete and freeable upon return |
| 276 | * @tx - transaction to quiesce |
| 277 | */ |
| 278 | void async_tx_quiesce(struct dma_async_tx_descriptor **tx) |
| 279 | { |
| 280 | if (*tx) { |
| 281 | /* if ack is already set then we cannot be sure |
| 282 | * we are referring to the correct operation |
| 283 | */ |
| 284 | BUG_ON(async_tx_test_ack(*tx)); |
| 285 | if (dma_wait_for_async_tx(*tx) == DMA_ERROR) |
| 286 | panic("DMA_ERROR waiting for transaction\n"); |
| 287 | async_tx_ack(*tx); |
| 288 | *tx = NULL; |
| 289 | } |
| 290 | } |
| 291 | EXPORT_SYMBOL_GPL(async_tx_quiesce); |
| 292 | |
Dan Williams | 9bc89cd | 2007-01-02 11:10:44 -0700 | [diff] [blame] | 293 | MODULE_AUTHOR("Intel Corporation"); |
| 294 | MODULE_DESCRIPTION("Asynchronous Bulk Memory Transactions API"); |
| 295 | MODULE_LICENSE("GPL"); |