Florian Meier | 96286b5 | 2014-01-06 20:18:24 +0100 | [diff] [blame] | 1 | /* |
| 2 | * BCM2835 DMA engine support |
| 3 | * |
| 4 | * This driver only supports cyclic DMA transfers |
| 5 | * as needed for the I2S module. |
| 6 | * |
| 7 | * Author: Florian Meier <florian.meier@koalo.de> |
| 8 | * Copyright 2013 |
| 9 | * |
| 10 | * Based on |
| 11 | * OMAP DMAengine support by Russell King |
| 12 | * |
| 13 | * BCM2708 DMA Driver |
| 14 | * Copyright (C) 2010 Broadcom |
| 15 | * |
| 16 | * Raspberry Pi PCM I2S ALSA Driver |
| 17 | * Copyright (c) by Phil Poole 2013 |
| 18 | * |
| 19 | * MARVELL MMP Peripheral DMA Driver |
| 20 | * Copyright 2012 Marvell International Ltd. |
| 21 | * |
| 22 | * This program is free software; you can redistribute it and/or modify |
| 23 | * it under the terms of the GNU General Public License as published by |
| 24 | * the Free Software Foundation; either version 2 of the License, or |
| 25 | * (at your option) any later version. |
| 26 | * |
| 27 | * This program is distributed in the hope that it will be useful, |
| 28 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 29 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 30 | * GNU General Public License for more details. |
| 31 | */ |
| 32 | #include <linux/dmaengine.h> |
| 33 | #include <linux/dma-mapping.h> |
Peter Ujfalusi | 27bc944 | 2015-11-16 13:09:03 +0200 | [diff] [blame] | 34 | #include <linux/dmapool.h> |
Florian Meier | 96286b5 | 2014-01-06 20:18:24 +0100 | [diff] [blame] | 35 | #include <linux/err.h> |
| 36 | #include <linux/init.h> |
| 37 | #include <linux/interrupt.h> |
| 38 | #include <linux/list.h> |
| 39 | #include <linux/module.h> |
| 40 | #include <linux/platform_device.h> |
| 41 | #include <linux/slab.h> |
| 42 | #include <linux/io.h> |
| 43 | #include <linux/spinlock.h> |
| 44 | #include <linux/of.h> |
| 45 | #include <linux/of_dma.h> |
| 46 | |
| 47 | #include "virt-dma.h" |
| 48 | |
| 49 | struct bcm2835_dmadev { |
| 50 | struct dma_device ddev; |
| 51 | spinlock_t lock; |
| 52 | void __iomem *base; |
| 53 | struct device_dma_parameters dma_parms; |
| 54 | }; |
| 55 | |
| 56 | struct bcm2835_dma_cb { |
| 57 | uint32_t info; |
| 58 | uint32_t src; |
| 59 | uint32_t dst; |
| 60 | uint32_t length; |
| 61 | uint32_t stride; |
| 62 | uint32_t next; |
| 63 | uint32_t pad[2]; |
| 64 | }; |
| 65 | |
Peter Ujfalusi | 27bc944 | 2015-11-16 13:09:03 +0200 | [diff] [blame] | 66 | struct bcm2835_cb_entry { |
| 67 | struct bcm2835_dma_cb *cb; |
| 68 | dma_addr_t paddr; |
| 69 | }; |
| 70 | |
Florian Meier | 96286b5 | 2014-01-06 20:18:24 +0100 | [diff] [blame] | 71 | struct bcm2835_chan { |
| 72 | struct virt_dma_chan vc; |
| 73 | struct list_head node; |
| 74 | |
| 75 | struct dma_slave_config cfg; |
Florian Meier | 96286b5 | 2014-01-06 20:18:24 +0100 | [diff] [blame] | 76 | unsigned int dreq; |
| 77 | |
| 78 | int ch; |
| 79 | struct bcm2835_desc *desc; |
Peter Ujfalusi | 27bc944 | 2015-11-16 13:09:03 +0200 | [diff] [blame] | 80 | struct dma_pool *cb_pool; |
Florian Meier | 96286b5 | 2014-01-06 20:18:24 +0100 | [diff] [blame] | 81 | |
| 82 | void __iomem *chan_base; |
| 83 | int irq_number; |
Martin Sperl | 4087412 | 2016-03-16 12:25:00 -0700 | [diff] [blame] | 84 | |
| 85 | bool is_lite_channel; |
Florian Meier | 96286b5 | 2014-01-06 20:18:24 +0100 | [diff] [blame] | 86 | }; |
| 87 | |
| 88 | struct bcm2835_desc { |
Peter Ujfalusi | 27bc944 | 2015-11-16 13:09:03 +0200 | [diff] [blame] | 89 | struct bcm2835_chan *c; |
Florian Meier | 96286b5 | 2014-01-06 20:18:24 +0100 | [diff] [blame] | 90 | struct virt_dma_desc vd; |
| 91 | enum dma_transfer_direction dir; |
| 92 | |
Florian Meier | 96286b5 | 2014-01-06 20:18:24 +0100 | [diff] [blame] | 93 | unsigned int frames; |
| 94 | size_t size; |
Martin Sperl | a4dcdd8 | 2016-03-16 12:24:58 -0700 | [diff] [blame] | 95 | |
| 96 | bool cyclic; |
Martin Sperl | 92153bb | 2016-03-16 12:24:59 -0700 | [diff] [blame] | 97 | |
| 98 | struct bcm2835_cb_entry cb_list[]; |
Florian Meier | 96286b5 | 2014-01-06 20:18:24 +0100 | [diff] [blame] | 99 | }; |
| 100 | |
| 101 | #define BCM2835_DMA_CS 0x00 |
| 102 | #define BCM2835_DMA_ADDR 0x04 |
Martin Sperl | e42685d | 2016-03-16 12:24:57 -0700 | [diff] [blame] | 103 | #define BCM2835_DMA_TI 0x08 |
Florian Meier | 96286b5 | 2014-01-06 20:18:24 +0100 | [diff] [blame] | 104 | #define BCM2835_DMA_SOURCE_AD 0x0c |
| 105 | #define BCM2835_DMA_DEST_AD 0x10 |
Martin Sperl | e42685d | 2016-03-16 12:24:57 -0700 | [diff] [blame] | 106 | #define BCM2835_DMA_LEN 0x14 |
| 107 | #define BCM2835_DMA_STRIDE 0x18 |
| 108 | #define BCM2835_DMA_NEXTCB 0x1c |
| 109 | #define BCM2835_DMA_DEBUG 0x20 |
Florian Meier | 96286b5 | 2014-01-06 20:18:24 +0100 | [diff] [blame] | 110 | |
| 111 | /* DMA CS Control and Status bits */ |
Martin Sperl | e42685d | 2016-03-16 12:24:57 -0700 | [diff] [blame] | 112 | #define BCM2835_DMA_ACTIVE BIT(0) /* activate the DMA */ |
| 113 | #define BCM2835_DMA_END BIT(1) /* current CB has ended */ |
| 114 | #define BCM2835_DMA_INT BIT(2) /* interrupt status */ |
| 115 | #define BCM2835_DMA_DREQ BIT(3) /* DREQ state */ |
Florian Meier | 96286b5 | 2014-01-06 20:18:24 +0100 | [diff] [blame] | 116 | #define BCM2835_DMA_ISPAUSED BIT(4) /* Pause requested or not active */ |
| 117 | #define BCM2835_DMA_ISHELD BIT(5) /* Is held by DREQ flow control */ |
Martin Sperl | e42685d | 2016-03-16 12:24:57 -0700 | [diff] [blame] | 118 | #define BCM2835_DMA_WAITING_FOR_WRITES BIT(6) /* waiting for last |
| 119 | * AXI-write to ack |
| 120 | */ |
| 121 | #define BCM2835_DMA_ERR BIT(8) |
| 122 | #define BCM2835_DMA_PRIORITY(x) ((x & 15) << 16) /* AXI priority */ |
| 123 | #define BCM2835_DMA_PANIC_PRIORITY(x) ((x & 15) << 20) /* panic priority */ |
| 124 | /* current value of TI.BCM2835_DMA_WAIT_RESP */ |
| 125 | #define BCM2835_DMA_WAIT_FOR_WRITES BIT(28) |
| 126 | #define BCM2835_DMA_DIS_DEBUG BIT(29) /* disable debug pause signal */ |
Florian Meier | 96286b5 | 2014-01-06 20:18:24 +0100 | [diff] [blame] | 127 | #define BCM2835_DMA_ABORT BIT(30) /* Stop current CB, go to next, WO */ |
| 128 | #define BCM2835_DMA_RESET BIT(31) /* WO, self clearing */ |
| 129 | |
Martin Sperl | e42685d | 2016-03-16 12:24:57 -0700 | [diff] [blame] | 130 | /* Transfer information bits - also bcm2835_cb.info field */ |
Florian Meier | 96286b5 | 2014-01-06 20:18:24 +0100 | [diff] [blame] | 131 | #define BCM2835_DMA_INT_EN BIT(0) |
Martin Sperl | e42685d | 2016-03-16 12:24:57 -0700 | [diff] [blame] | 132 | #define BCM2835_DMA_TDMODE BIT(1) /* 2D-Mode */ |
| 133 | #define BCM2835_DMA_WAIT_RESP BIT(3) /* wait for AXI-write to be acked */ |
Florian Meier | 96286b5 | 2014-01-06 20:18:24 +0100 | [diff] [blame] | 134 | #define BCM2835_DMA_D_INC BIT(4) |
Martin Sperl | e42685d | 2016-03-16 12:24:57 -0700 | [diff] [blame] | 135 | #define BCM2835_DMA_D_WIDTH BIT(5) /* 128bit writes if set */ |
| 136 | #define BCM2835_DMA_D_DREQ BIT(6) /* enable DREQ for destination */ |
| 137 | #define BCM2835_DMA_D_IGNORE BIT(7) /* ignore destination writes */ |
Florian Meier | 96286b5 | 2014-01-06 20:18:24 +0100 | [diff] [blame] | 138 | #define BCM2835_DMA_S_INC BIT(8) |
Martin Sperl | e42685d | 2016-03-16 12:24:57 -0700 | [diff] [blame] | 139 | #define BCM2835_DMA_S_WIDTH BIT(9) /* 128bit writes if set */ |
| 140 | #define BCM2835_DMA_S_DREQ BIT(10) /* enable SREQ for source */ |
| 141 | #define BCM2835_DMA_S_IGNORE BIT(11) /* ignore source reads - read 0 */ |
| 142 | #define BCM2835_DMA_BURST_LENGTH(x) ((x & 15) << 12) |
| 143 | #define BCM2835_DMA_PER_MAP(x) ((x & 31) << 16) /* REQ source */ |
| 144 | #define BCM2835_DMA_WAIT(x) ((x & 31) << 21) /* add DMA-wait cycles */ |
| 145 | #define BCM2835_DMA_NO_WIDE_BURSTS BIT(26) /* no 2 beat write bursts */ |
Florian Meier | 96286b5 | 2014-01-06 20:18:24 +0100 | [diff] [blame] | 146 | |
Martin Sperl | e42685d | 2016-03-16 12:24:57 -0700 | [diff] [blame] | 147 | /* debug register bits */ |
| 148 | #define BCM2835_DMA_DEBUG_LAST_NOT_SET_ERR BIT(0) |
| 149 | #define BCM2835_DMA_DEBUG_FIFO_ERR BIT(1) |
| 150 | #define BCM2835_DMA_DEBUG_READ_ERR BIT(2) |
| 151 | #define BCM2835_DMA_DEBUG_OUTSTANDING_WRITES_SHIFT 4 |
| 152 | #define BCM2835_DMA_DEBUG_OUTSTANDING_WRITES_BITS 4 |
| 153 | #define BCM2835_DMA_DEBUG_ID_SHIFT 16 |
| 154 | #define BCM2835_DMA_DEBUG_ID_BITS 9 |
| 155 | #define BCM2835_DMA_DEBUG_STATE_SHIFT 16 |
| 156 | #define BCM2835_DMA_DEBUG_STATE_BITS 9 |
| 157 | #define BCM2835_DMA_DEBUG_VERSION_SHIFT 25 |
| 158 | #define BCM2835_DMA_DEBUG_VERSION_BITS 3 |
| 159 | #define BCM2835_DMA_DEBUG_LITE BIT(28) |
| 160 | |
| 161 | /* shared registers for all dma channels */ |
| 162 | #define BCM2835_DMA_INT_STATUS 0xfe0 |
| 163 | #define BCM2835_DMA_ENABLE 0xff0 |
Florian Meier | 96286b5 | 2014-01-06 20:18:24 +0100 | [diff] [blame] | 164 | |
| 165 | #define BCM2835_DMA_DATA_TYPE_S8 1 |
| 166 | #define BCM2835_DMA_DATA_TYPE_S16 2 |
| 167 | #define BCM2835_DMA_DATA_TYPE_S32 4 |
| 168 | #define BCM2835_DMA_DATA_TYPE_S128 16 |
| 169 | |
Florian Meier | 96286b5 | 2014-01-06 20:18:24 +0100 | [diff] [blame] | 170 | /* Valid only for channels 0 - 14, 15 has its own base address */ |
| 171 | #define BCM2835_DMA_CHAN(n) ((n) << 8) /* Base address */ |
| 172 | #define BCM2835_DMA_CHANIO(base, n) ((base) + BCM2835_DMA_CHAN(n)) |
| 173 | |
Martin Sperl | 4087412 | 2016-03-16 12:25:00 -0700 | [diff] [blame] | 174 | /* the max dma length for different channels */ |
| 175 | #define MAX_DMA_LEN SZ_1G |
| 176 | #define MAX_LITE_DMA_LEN (SZ_64K - 4) |
| 177 | |
| 178 | static inline size_t bcm2835_dma_max_frame_length(struct bcm2835_chan *c) |
| 179 | { |
| 180 | /* lite and normal channels have different max frame length */ |
| 181 | return c->is_lite_channel ? MAX_LITE_DMA_LEN : MAX_DMA_LEN; |
| 182 | } |
| 183 | |
Martin Sperl | 92153bb | 2016-03-16 12:24:59 -0700 | [diff] [blame] | 184 | /* how many frames of max_len size do we need to transfer len bytes */ |
| 185 | static inline size_t bcm2835_dma_frames_for_length(size_t len, |
| 186 | size_t max_len) |
| 187 | { |
| 188 | return DIV_ROUND_UP(len, max_len); |
| 189 | } |
| 190 | |
Florian Meier | 96286b5 | 2014-01-06 20:18:24 +0100 | [diff] [blame] | 191 | static inline struct bcm2835_dmadev *to_bcm2835_dma_dev(struct dma_device *d) |
| 192 | { |
| 193 | return container_of(d, struct bcm2835_dmadev, ddev); |
| 194 | } |
| 195 | |
| 196 | static inline struct bcm2835_chan *to_bcm2835_dma_chan(struct dma_chan *c) |
| 197 | { |
| 198 | return container_of(c, struct bcm2835_chan, vc.chan); |
| 199 | } |
| 200 | |
| 201 | static inline struct bcm2835_desc *to_bcm2835_dma_desc( |
| 202 | struct dma_async_tx_descriptor *t) |
| 203 | { |
| 204 | return container_of(t, struct bcm2835_desc, vd.tx); |
| 205 | } |
| 206 | |
Martin Sperl | 92153bb | 2016-03-16 12:24:59 -0700 | [diff] [blame] | 207 | static void bcm2835_dma_free_cb_chain(struct bcm2835_desc *desc) |
Florian Meier | 96286b5 | 2014-01-06 20:18:24 +0100 | [diff] [blame] | 208 | { |
Martin Sperl | 92153bb | 2016-03-16 12:24:59 -0700 | [diff] [blame] | 209 | size_t i; |
Peter Ujfalusi | 27bc944 | 2015-11-16 13:09:03 +0200 | [diff] [blame] | 210 | |
| 211 | for (i = 0; i < desc->frames; i++) |
| 212 | dma_pool_free(desc->c->cb_pool, desc->cb_list[i].cb, |
| 213 | desc->cb_list[i].paddr); |
| 214 | |
Florian Meier | 96286b5 | 2014-01-06 20:18:24 +0100 | [diff] [blame] | 215 | kfree(desc); |
| 216 | } |
| 217 | |
Martin Sperl | 92153bb | 2016-03-16 12:24:59 -0700 | [diff] [blame] | 218 | static void bcm2835_dma_desc_free(struct virt_dma_desc *vd) |
| 219 | { |
| 220 | bcm2835_dma_free_cb_chain( |
| 221 | container_of(vd, struct bcm2835_desc, vd)); |
| 222 | } |
| 223 | |
| 224 | static void bcm2835_dma_create_cb_set_length( |
| 225 | struct bcm2835_chan *chan, |
| 226 | struct bcm2835_dma_cb *control_block, |
| 227 | size_t len, |
| 228 | size_t period_len, |
| 229 | size_t *total_len, |
| 230 | u32 finalextrainfo) |
| 231 | { |
Martin Sperl | 4087412 | 2016-03-16 12:25:00 -0700 | [diff] [blame] | 232 | size_t max_len = bcm2835_dma_max_frame_length(chan); |
| 233 | |
| 234 | /* set the length taking lite-channel limitations into account */ |
| 235 | control_block->length = min_t(u32, len, max_len); |
Martin Sperl | 92153bb | 2016-03-16 12:24:59 -0700 | [diff] [blame] | 236 | |
| 237 | /* finished if we have no period_length */ |
| 238 | if (!period_len) |
| 239 | return; |
| 240 | |
| 241 | /* |
| 242 | * period_len means: that we need to generate |
| 243 | * transfers that are terminating at every |
| 244 | * multiple of period_len - this is typically |
| 245 | * used to set the interrupt flag in info |
| 246 | * which is required during cyclic transfers |
| 247 | */ |
| 248 | |
| 249 | /* have we filled in period_length yet? */ |
| 250 | if (*total_len + control_block->length < period_len) |
| 251 | return; |
| 252 | |
| 253 | /* calculate the length that remains to reach period_length */ |
| 254 | control_block->length = period_len - *total_len; |
| 255 | |
| 256 | /* reset total_length for next period */ |
| 257 | *total_len = 0; |
| 258 | |
| 259 | /* add extrainfo bits in info */ |
| 260 | control_block->info |= finalextrainfo; |
| 261 | } |
| 262 | |
Martin Sperl | 388cc7a | 2016-03-16 12:25:01 -0700 | [diff] [blame] | 263 | static inline size_t bcm2835_dma_count_frames_for_sg( |
| 264 | struct bcm2835_chan *c, |
| 265 | struct scatterlist *sgl, |
| 266 | unsigned int sg_len) |
| 267 | { |
| 268 | size_t frames = 0; |
| 269 | struct scatterlist *sgent; |
| 270 | unsigned int i; |
| 271 | size_t plength = bcm2835_dma_max_frame_length(c); |
| 272 | |
| 273 | for_each_sg(sgl, sgent, sg_len, i) |
| 274 | frames += bcm2835_dma_frames_for_length( |
| 275 | sg_dma_len(sgent), plength); |
| 276 | |
| 277 | return frames; |
| 278 | } |
| 279 | |
Martin Sperl | 92153bb | 2016-03-16 12:24:59 -0700 | [diff] [blame] | 280 | /** |
| 281 | * bcm2835_dma_create_cb_chain - create a control block and fills data in |
| 282 | * |
| 283 | * @chan: the @dma_chan for which we run this |
| 284 | * @direction: the direction in which we transfer |
| 285 | * @cyclic: it is a cyclic transfer |
| 286 | * @info: the default info bits to apply per controlblock |
| 287 | * @frames: number of controlblocks to allocate |
| 288 | * @src: the src address to assign (if the S_INC bit is set |
| 289 | * in @info, then it gets incremented) |
| 290 | * @dst: the dst address to assign (if the D_INC bit is set |
| 291 | * in @info, then it gets incremented) |
| 292 | * @buf_len: the full buffer length (may also be 0) |
| 293 | * @period_len: the period length when to apply @finalextrainfo |
| 294 | * in addition to the last transfer |
| 295 | * this will also break some control-blocks early |
| 296 | * @finalextrainfo: additional bits in last controlblock |
| 297 | * (or when period_len is reached in case of cyclic) |
| 298 | * @gfp: the GFP flag to use for allocation |
| 299 | */ |
| 300 | static struct bcm2835_desc *bcm2835_dma_create_cb_chain( |
| 301 | struct dma_chan *chan, enum dma_transfer_direction direction, |
| 302 | bool cyclic, u32 info, u32 finalextrainfo, size_t frames, |
| 303 | dma_addr_t src, dma_addr_t dst, size_t buf_len, |
| 304 | size_t period_len, gfp_t gfp) |
| 305 | { |
| 306 | struct bcm2835_chan *c = to_bcm2835_dma_chan(chan); |
| 307 | size_t len = buf_len, total_len; |
| 308 | size_t frame; |
| 309 | struct bcm2835_desc *d; |
| 310 | struct bcm2835_cb_entry *cb_entry; |
| 311 | struct bcm2835_dma_cb *control_block; |
| 312 | |
Martin Sperl | d9f094a | 2016-03-16 12:25:02 -0700 | [diff] [blame^] | 313 | if (!frames) |
| 314 | return NULL; |
| 315 | |
Martin Sperl | 92153bb | 2016-03-16 12:24:59 -0700 | [diff] [blame] | 316 | /* allocate and setup the descriptor. */ |
| 317 | d = kzalloc(sizeof(*d) + frames * sizeof(struct bcm2835_cb_entry), |
| 318 | gfp); |
| 319 | if (!d) |
| 320 | return NULL; |
| 321 | |
| 322 | d->c = c; |
| 323 | d->dir = direction; |
| 324 | d->cyclic = cyclic; |
| 325 | |
| 326 | /* |
| 327 | * Iterate over all frames, create a control block |
| 328 | * for each frame and link them together. |
| 329 | */ |
| 330 | for (frame = 0, total_len = 0; frame < frames; d->frames++, frame++) { |
| 331 | cb_entry = &d->cb_list[frame]; |
| 332 | cb_entry->cb = dma_pool_alloc(c->cb_pool, gfp, |
| 333 | &cb_entry->paddr); |
| 334 | if (!cb_entry->cb) |
| 335 | goto error_cb; |
| 336 | |
| 337 | /* fill in the control block */ |
| 338 | control_block = cb_entry->cb; |
| 339 | control_block->info = info; |
| 340 | control_block->src = src; |
| 341 | control_block->dst = dst; |
| 342 | control_block->stride = 0; |
| 343 | control_block->next = 0; |
| 344 | /* set up length in control_block if requested */ |
| 345 | if (buf_len) { |
| 346 | /* calculate length honoring period_length */ |
| 347 | bcm2835_dma_create_cb_set_length( |
| 348 | c, control_block, |
| 349 | len, period_len, &total_len, |
| 350 | cyclic ? finalextrainfo : 0); |
| 351 | |
| 352 | /* calculate new remaining length */ |
| 353 | len -= control_block->length; |
| 354 | } |
| 355 | |
| 356 | /* link this the last controlblock */ |
| 357 | if (frame) |
| 358 | d->cb_list[frame - 1].cb->next = cb_entry->paddr; |
| 359 | |
| 360 | /* update src and dst and length */ |
| 361 | if (src && (info & BCM2835_DMA_S_INC)) |
| 362 | src += control_block->length; |
| 363 | if (dst && (info & BCM2835_DMA_D_INC)) |
| 364 | dst += control_block->length; |
| 365 | |
| 366 | /* Length of total transfer */ |
| 367 | d->size += control_block->length; |
| 368 | } |
| 369 | |
| 370 | /* the last frame requires extra flags */ |
| 371 | d->cb_list[d->frames - 1].cb->info |= finalextrainfo; |
| 372 | |
| 373 | /* detect a size missmatch */ |
| 374 | if (buf_len && (d->size != buf_len)) |
| 375 | goto error_cb; |
| 376 | |
| 377 | return d; |
| 378 | error_cb: |
| 379 | bcm2835_dma_free_cb_chain(d); |
| 380 | |
| 381 | return NULL; |
| 382 | } |
| 383 | |
Martin Sperl | 388cc7a | 2016-03-16 12:25:01 -0700 | [diff] [blame] | 384 | static void bcm2835_dma_fill_cb_chain_with_sg( |
| 385 | struct dma_chan *chan, |
| 386 | enum dma_transfer_direction direction, |
| 387 | struct bcm2835_cb_entry *cb, |
| 388 | struct scatterlist *sgl, |
| 389 | unsigned int sg_len) |
| 390 | { |
| 391 | struct bcm2835_chan *c = to_bcm2835_dma_chan(chan); |
| 392 | size_t max_len = bcm2835_dma_max_frame_length(c); |
| 393 | unsigned int i, len; |
| 394 | dma_addr_t addr; |
| 395 | struct scatterlist *sgent; |
| 396 | |
| 397 | for_each_sg(sgl, sgent, sg_len, i) { |
| 398 | for (addr = sg_dma_address(sgent), len = sg_dma_len(sgent); |
| 399 | len > 0; |
| 400 | addr += cb->cb->length, len -= cb->cb->length, cb++) { |
| 401 | if (direction == DMA_DEV_TO_MEM) |
| 402 | cb->cb->dst = addr; |
| 403 | else |
| 404 | cb->cb->src = addr; |
| 405 | cb->cb->length = min(len, max_len); |
| 406 | } |
| 407 | } |
| 408 | } |
| 409 | |
Florian Meier | 96286b5 | 2014-01-06 20:18:24 +0100 | [diff] [blame] | 410 | static int bcm2835_dma_abort(void __iomem *chan_base) |
| 411 | { |
| 412 | unsigned long cs; |
| 413 | long int timeout = 10000; |
| 414 | |
| 415 | cs = readl(chan_base + BCM2835_DMA_CS); |
| 416 | if (!(cs & BCM2835_DMA_ACTIVE)) |
| 417 | return 0; |
| 418 | |
| 419 | /* Write 0 to the active bit - Pause the DMA */ |
| 420 | writel(0, chan_base + BCM2835_DMA_CS); |
| 421 | |
| 422 | /* Wait for any current AXI transfer to complete */ |
| 423 | while ((cs & BCM2835_DMA_ISPAUSED) && --timeout) { |
| 424 | cpu_relax(); |
| 425 | cs = readl(chan_base + BCM2835_DMA_CS); |
| 426 | } |
| 427 | |
| 428 | /* We'll un-pause when we set of our next DMA */ |
| 429 | if (!timeout) |
| 430 | return -ETIMEDOUT; |
| 431 | |
| 432 | if (!(cs & BCM2835_DMA_ACTIVE)) |
| 433 | return 0; |
| 434 | |
| 435 | /* Terminate the control block chain */ |
| 436 | writel(0, chan_base + BCM2835_DMA_NEXTCB); |
| 437 | |
| 438 | /* Abort the whole DMA */ |
| 439 | writel(BCM2835_DMA_ABORT | BCM2835_DMA_ACTIVE, |
| 440 | chan_base + BCM2835_DMA_CS); |
| 441 | |
| 442 | return 0; |
| 443 | } |
| 444 | |
| 445 | static void bcm2835_dma_start_desc(struct bcm2835_chan *c) |
| 446 | { |
| 447 | struct virt_dma_desc *vd = vchan_next_desc(&c->vc); |
| 448 | struct bcm2835_desc *d; |
| 449 | |
| 450 | if (!vd) { |
| 451 | c->desc = NULL; |
| 452 | return; |
| 453 | } |
| 454 | |
| 455 | list_del(&vd->node); |
| 456 | |
| 457 | c->desc = d = to_bcm2835_dma_desc(&vd->tx); |
| 458 | |
Peter Ujfalusi | 27bc944 | 2015-11-16 13:09:03 +0200 | [diff] [blame] | 459 | writel(d->cb_list[0].paddr, c->chan_base + BCM2835_DMA_ADDR); |
Florian Meier | 96286b5 | 2014-01-06 20:18:24 +0100 | [diff] [blame] | 460 | writel(BCM2835_DMA_ACTIVE, c->chan_base + BCM2835_DMA_CS); |
| 461 | } |
| 462 | |
| 463 | static irqreturn_t bcm2835_dma_callback(int irq, void *data) |
| 464 | { |
| 465 | struct bcm2835_chan *c = data; |
| 466 | struct bcm2835_desc *d; |
| 467 | unsigned long flags; |
| 468 | |
| 469 | spin_lock_irqsave(&c->vc.lock, flags); |
| 470 | |
| 471 | /* Acknowledge interrupt */ |
| 472 | writel(BCM2835_DMA_INT, c->chan_base + BCM2835_DMA_CS); |
| 473 | |
| 474 | d = c->desc; |
| 475 | |
| 476 | if (d) { |
Martin Sperl | 388cc7a | 2016-03-16 12:25:01 -0700 | [diff] [blame] | 477 | if (d->cyclic) { |
| 478 | /* call the cyclic callback */ |
| 479 | vchan_cyclic_callback(&d->vd); |
Florian Meier | 96286b5 | 2014-01-06 20:18:24 +0100 | [diff] [blame] | 480 | |
Martin Sperl | 388cc7a | 2016-03-16 12:25:01 -0700 | [diff] [blame] | 481 | /* Keep the DMA engine running */ |
| 482 | writel(BCM2835_DMA_ACTIVE, |
| 483 | c->chan_base + BCM2835_DMA_CS); |
| 484 | } else { |
| 485 | vchan_cookie_complete(&c->desc->vd); |
| 486 | bcm2835_dma_start_desc(c); |
| 487 | } |
| 488 | } |
Florian Meier | 96286b5 | 2014-01-06 20:18:24 +0100 | [diff] [blame] | 489 | |
| 490 | spin_unlock_irqrestore(&c->vc.lock, flags); |
| 491 | |
| 492 | return IRQ_HANDLED; |
| 493 | } |
| 494 | |
| 495 | static int bcm2835_dma_alloc_chan_resources(struct dma_chan *chan) |
| 496 | { |
| 497 | struct bcm2835_chan *c = to_bcm2835_dma_chan(chan); |
Peter Ujfalusi | 27bc944 | 2015-11-16 13:09:03 +0200 | [diff] [blame] | 498 | struct device *dev = c->vc.chan.device->dev; |
Florian Meier | 96286b5 | 2014-01-06 20:18:24 +0100 | [diff] [blame] | 499 | |
Peter Ujfalusi | 27bc944 | 2015-11-16 13:09:03 +0200 | [diff] [blame] | 500 | dev_dbg(dev, "Allocating DMA channel %d\n", c->ch); |
| 501 | |
| 502 | c->cb_pool = dma_pool_create(dev_name(dev), dev, |
| 503 | sizeof(struct bcm2835_dma_cb), 0, 0); |
| 504 | if (!c->cb_pool) { |
| 505 | dev_err(dev, "unable to allocate descriptor pool\n"); |
| 506 | return -ENOMEM; |
| 507 | } |
Florian Meier | 96286b5 | 2014-01-06 20:18:24 +0100 | [diff] [blame] | 508 | |
| 509 | return request_irq(c->irq_number, |
| 510 | bcm2835_dma_callback, 0, "DMA IRQ", c); |
| 511 | } |
| 512 | |
| 513 | static void bcm2835_dma_free_chan_resources(struct dma_chan *chan) |
| 514 | { |
| 515 | struct bcm2835_chan *c = to_bcm2835_dma_chan(chan); |
| 516 | |
| 517 | vchan_free_chan_resources(&c->vc); |
| 518 | free_irq(c->irq_number, c); |
Peter Ujfalusi | 27bc944 | 2015-11-16 13:09:03 +0200 | [diff] [blame] | 519 | dma_pool_destroy(c->cb_pool); |
Florian Meier | 96286b5 | 2014-01-06 20:18:24 +0100 | [diff] [blame] | 520 | |
| 521 | dev_dbg(c->vc.chan.device->dev, "Freeing DMA channel %u\n", c->ch); |
| 522 | } |
| 523 | |
| 524 | static size_t bcm2835_dma_desc_size(struct bcm2835_desc *d) |
| 525 | { |
| 526 | return d->size; |
| 527 | } |
| 528 | |
| 529 | static size_t bcm2835_dma_desc_size_pos(struct bcm2835_desc *d, dma_addr_t addr) |
| 530 | { |
| 531 | unsigned int i; |
| 532 | size_t size; |
| 533 | |
| 534 | for (size = i = 0; i < d->frames; i++) { |
Peter Ujfalusi | 27bc944 | 2015-11-16 13:09:03 +0200 | [diff] [blame] | 535 | struct bcm2835_dma_cb *control_block = d->cb_list[i].cb; |
Florian Meier | 96286b5 | 2014-01-06 20:18:24 +0100 | [diff] [blame] | 536 | size_t this_size = control_block->length; |
| 537 | dma_addr_t dma; |
| 538 | |
| 539 | if (d->dir == DMA_DEV_TO_MEM) |
| 540 | dma = control_block->dst; |
| 541 | else |
| 542 | dma = control_block->src; |
| 543 | |
| 544 | if (size) |
| 545 | size += this_size; |
| 546 | else if (addr >= dma && addr < dma + this_size) |
| 547 | size += dma + this_size - addr; |
| 548 | } |
| 549 | |
| 550 | return size; |
| 551 | } |
| 552 | |
| 553 | static enum dma_status bcm2835_dma_tx_status(struct dma_chan *chan, |
| 554 | dma_cookie_t cookie, struct dma_tx_state *txstate) |
| 555 | { |
| 556 | struct bcm2835_chan *c = to_bcm2835_dma_chan(chan); |
| 557 | struct virt_dma_desc *vd; |
| 558 | enum dma_status ret; |
| 559 | unsigned long flags; |
| 560 | |
| 561 | ret = dma_cookie_status(chan, cookie, txstate); |
| 562 | if (ret == DMA_COMPLETE || !txstate) |
| 563 | return ret; |
| 564 | |
| 565 | spin_lock_irqsave(&c->vc.lock, flags); |
| 566 | vd = vchan_find_desc(&c->vc, cookie); |
| 567 | if (vd) { |
| 568 | txstate->residue = |
| 569 | bcm2835_dma_desc_size(to_bcm2835_dma_desc(&vd->tx)); |
| 570 | } else if (c->desc && c->desc->vd.tx.cookie == cookie) { |
| 571 | struct bcm2835_desc *d = c->desc; |
| 572 | dma_addr_t pos; |
| 573 | |
| 574 | if (d->dir == DMA_MEM_TO_DEV) |
| 575 | pos = readl(c->chan_base + BCM2835_DMA_SOURCE_AD); |
| 576 | else if (d->dir == DMA_DEV_TO_MEM) |
| 577 | pos = readl(c->chan_base + BCM2835_DMA_DEST_AD); |
| 578 | else |
| 579 | pos = 0; |
| 580 | |
| 581 | txstate->residue = bcm2835_dma_desc_size_pos(d, pos); |
| 582 | } else { |
| 583 | txstate->residue = 0; |
| 584 | } |
| 585 | |
| 586 | spin_unlock_irqrestore(&c->vc.lock, flags); |
| 587 | |
| 588 | return ret; |
| 589 | } |
| 590 | |
| 591 | static void bcm2835_dma_issue_pending(struct dma_chan *chan) |
| 592 | { |
| 593 | struct bcm2835_chan *c = to_bcm2835_dma_chan(chan); |
| 594 | unsigned long flags; |
| 595 | |
Florian Meier | 96286b5 | 2014-01-06 20:18:24 +0100 | [diff] [blame] | 596 | spin_lock_irqsave(&c->vc.lock, flags); |
| 597 | if (vchan_issue_pending(&c->vc) && !c->desc) |
| 598 | bcm2835_dma_start_desc(c); |
| 599 | |
| 600 | spin_unlock_irqrestore(&c->vc.lock, flags); |
| 601 | } |
| 602 | |
Martin Sperl | d9f094a | 2016-03-16 12:25:02 -0700 | [diff] [blame^] | 603 | struct dma_async_tx_descriptor *bcm2835_dma_prep_dma_memcpy( |
| 604 | struct dma_chan *chan, dma_addr_t dst, dma_addr_t src, |
| 605 | size_t len, unsigned long flags) |
| 606 | { |
| 607 | struct bcm2835_chan *c = to_bcm2835_dma_chan(chan); |
| 608 | struct bcm2835_desc *d; |
| 609 | u32 info = BCM2835_DMA_D_INC | BCM2835_DMA_S_INC; |
| 610 | u32 extra = BCM2835_DMA_INT_EN | BCM2835_DMA_WAIT_RESP; |
| 611 | size_t max_len = bcm2835_dma_max_frame_length(c); |
| 612 | size_t frames; |
| 613 | |
| 614 | /* if src, dst or len is not given return with an error */ |
| 615 | if (!src || !dst || !len) |
| 616 | return NULL; |
| 617 | |
| 618 | /* calculate number of frames */ |
| 619 | frames = bcm2835_dma_frames_for_length(len, max_len); |
| 620 | |
| 621 | /* allocate the CB chain - this also fills in the pointers */ |
| 622 | d = bcm2835_dma_create_cb_chain(chan, DMA_MEM_TO_MEM, false, |
| 623 | info, extra, frames, |
| 624 | src, dst, len, 0, GFP_KERNEL); |
| 625 | if (!d) |
| 626 | return NULL; |
| 627 | |
| 628 | return vchan_tx_prep(&c->vc, &d->vd, flags); |
| 629 | } |
| 630 | |
Martin Sperl | 388cc7a | 2016-03-16 12:25:01 -0700 | [diff] [blame] | 631 | static struct dma_async_tx_descriptor *bcm2835_dma_prep_slave_sg( |
| 632 | struct dma_chan *chan, |
| 633 | struct scatterlist *sgl, unsigned int sg_len, |
| 634 | enum dma_transfer_direction direction, |
| 635 | unsigned long flags, void *context) |
| 636 | { |
| 637 | struct bcm2835_chan *c = to_bcm2835_dma_chan(chan); |
| 638 | struct bcm2835_desc *d; |
| 639 | dma_addr_t src = 0, dst = 0; |
| 640 | u32 info = BCM2835_DMA_WAIT_RESP; |
| 641 | u32 extra = BCM2835_DMA_INT_EN; |
| 642 | size_t frames; |
| 643 | |
| 644 | if (!is_slave_direction(direction)) { |
| 645 | dev_err(chan->device->dev, |
| 646 | "%s: bad direction?\n", __func__); |
| 647 | return NULL; |
| 648 | } |
| 649 | |
| 650 | if (c->dreq != 0) |
| 651 | info |= BCM2835_DMA_PER_MAP(c->dreq); |
| 652 | |
| 653 | if (direction == DMA_DEV_TO_MEM) { |
| 654 | if (c->cfg.src_addr_width != DMA_SLAVE_BUSWIDTH_4_BYTES) |
| 655 | return NULL; |
| 656 | src = c->cfg.src_addr; |
| 657 | info |= BCM2835_DMA_S_DREQ | BCM2835_DMA_D_INC; |
| 658 | } else { |
| 659 | if (c->cfg.dst_addr_width != DMA_SLAVE_BUSWIDTH_4_BYTES) |
| 660 | return NULL; |
| 661 | dst = c->cfg.dst_addr; |
| 662 | info |= BCM2835_DMA_D_DREQ | BCM2835_DMA_S_INC; |
| 663 | } |
| 664 | |
| 665 | /* count frames in sg list */ |
| 666 | frames = bcm2835_dma_count_frames_for_sg(c, sgl, sg_len); |
| 667 | |
| 668 | /* allocate the CB chain */ |
| 669 | d = bcm2835_dma_create_cb_chain(chan, direction, false, |
| 670 | info, extra, |
| 671 | frames, src, dst, 0, 0, |
| 672 | GFP_KERNEL); |
| 673 | if (!d) |
| 674 | return NULL; |
| 675 | |
| 676 | /* fill in frames with scatterlist pointers */ |
| 677 | bcm2835_dma_fill_cb_chain_with_sg(chan, direction, d->cb_list, |
| 678 | sgl, sg_len); |
| 679 | |
| 680 | return vchan_tx_prep(&c->vc, &d->vd, flags); |
| 681 | } |
| 682 | |
Florian Meier | 96286b5 | 2014-01-06 20:18:24 +0100 | [diff] [blame] | 683 | static struct dma_async_tx_descriptor *bcm2835_dma_prep_dma_cyclic( |
| 684 | struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len, |
| 685 | size_t period_len, enum dma_transfer_direction direction, |
Laurent Pinchart | 31c1e5a | 2014-08-01 12:20:10 +0200 | [diff] [blame] | 686 | unsigned long flags) |
Florian Meier | 96286b5 | 2014-01-06 20:18:24 +0100 | [diff] [blame] | 687 | { |
| 688 | struct bcm2835_chan *c = to_bcm2835_dma_chan(chan); |
Florian Meier | 96286b5 | 2014-01-06 20:18:24 +0100 | [diff] [blame] | 689 | struct bcm2835_desc *d; |
Martin Sperl | 92153bb | 2016-03-16 12:24:59 -0700 | [diff] [blame] | 690 | dma_addr_t src, dst; |
| 691 | u32 info = BCM2835_DMA_WAIT_RESP; |
| 692 | u32 extra = BCM2835_DMA_INT_EN; |
Martin Sperl | 4087412 | 2016-03-16 12:25:00 -0700 | [diff] [blame] | 693 | size_t max_len = bcm2835_dma_max_frame_length(c); |
Martin Sperl | 92153bb | 2016-03-16 12:24:59 -0700 | [diff] [blame] | 694 | size_t frames; |
Florian Meier | 96286b5 | 2014-01-06 20:18:24 +0100 | [diff] [blame] | 695 | |
| 696 | /* Grab configuration */ |
| 697 | if (!is_slave_direction(direction)) { |
| 698 | dev_err(chan->device->dev, "%s: bad direction?\n", __func__); |
| 699 | return NULL; |
| 700 | } |
| 701 | |
Martin Sperl | 92153bb | 2016-03-16 12:24:59 -0700 | [diff] [blame] | 702 | if (!buf_len) { |
| 703 | dev_err(chan->device->dev, |
| 704 | "%s: bad buffer length (= 0)\n", __func__); |
Florian Meier | 96286b5 | 2014-01-06 20:18:24 +0100 | [diff] [blame] | 705 | return NULL; |
| 706 | } |
| 707 | |
Florian Meier | 96286b5 | 2014-01-06 20:18:24 +0100 | [diff] [blame] | 708 | /* |
Martin Sperl | 92153bb | 2016-03-16 12:24:59 -0700 | [diff] [blame] | 709 | * warn if buf_len is not a multiple of period_len - this may leed |
| 710 | * to unexpected latencies for interrupts and thus audiable clicks |
Florian Meier | 96286b5 | 2014-01-06 20:18:24 +0100 | [diff] [blame] | 711 | */ |
Martin Sperl | 92153bb | 2016-03-16 12:24:59 -0700 | [diff] [blame] | 712 | if (buf_len % period_len) |
| 713 | dev_warn_once(chan->device->dev, |
| 714 | "%s: buffer_length (%zd) is not a multiple of period_len (%zd)\n", |
| 715 | __func__, buf_len, period_len); |
Florian Meier | 96286b5 | 2014-01-06 20:18:24 +0100 | [diff] [blame] | 716 | |
Martin Sperl | 92153bb | 2016-03-16 12:24:59 -0700 | [diff] [blame] | 717 | /* Setup DREQ channel */ |
| 718 | if (c->dreq != 0) |
| 719 | info |= BCM2835_DMA_PER_MAP(c->dreq); |
Florian Meier | 96286b5 | 2014-01-06 20:18:24 +0100 | [diff] [blame] | 720 | |
Martin Sperl | 92153bb | 2016-03-16 12:24:59 -0700 | [diff] [blame] | 721 | if (direction == DMA_DEV_TO_MEM) { |
| 722 | if (c->cfg.src_addr_width != DMA_SLAVE_BUSWIDTH_4_BYTES) |
| 723 | return NULL; |
| 724 | src = c->cfg.src_addr; |
| 725 | dst = buf_addr; |
| 726 | info |= BCM2835_DMA_S_DREQ | BCM2835_DMA_D_INC; |
| 727 | } else { |
| 728 | if (c->cfg.dst_addr_width != DMA_SLAVE_BUSWIDTH_4_BYTES) |
| 729 | return NULL; |
| 730 | dst = c->cfg.dst_addr; |
| 731 | src = buf_addr; |
| 732 | info |= BCM2835_DMA_D_DREQ | BCM2835_DMA_S_INC; |
Florian Meier | 96286b5 | 2014-01-06 20:18:24 +0100 | [diff] [blame] | 733 | } |
| 734 | |
Martin Sperl | 92153bb | 2016-03-16 12:24:59 -0700 | [diff] [blame] | 735 | /* calculate number of frames */ |
Martin Sperl | 4087412 | 2016-03-16 12:25:00 -0700 | [diff] [blame] | 736 | frames = /* number of periods */ |
| 737 | DIV_ROUND_UP(buf_len, period_len) * |
| 738 | /* number of frames per period */ |
| 739 | bcm2835_dma_frames_for_length(period_len, max_len); |
Martin Sperl | 92153bb | 2016-03-16 12:24:59 -0700 | [diff] [blame] | 740 | |
| 741 | /* |
| 742 | * allocate the CB chain |
| 743 | * note that we need to use GFP_NOWAIT, as the ALSA i2s dmaengine |
| 744 | * implementation calls prep_dma_cyclic with interrupts disabled. |
| 745 | */ |
| 746 | d = bcm2835_dma_create_cb_chain(chan, direction, true, |
| 747 | info, extra, |
| 748 | frames, src, dst, buf_len, |
| 749 | period_len, GFP_NOWAIT); |
| 750 | if (!d) |
| 751 | return NULL; |
| 752 | |
| 753 | /* wrap around into a loop */ |
| 754 | d->cb_list[d->frames - 1].cb->next = d->cb_list[0].paddr; |
| 755 | |
Florian Meier | 96286b5 | 2014-01-06 20:18:24 +0100 | [diff] [blame] | 756 | return vchan_tx_prep(&c->vc, &d->vd, flags); |
| 757 | } |
| 758 | |
Maxime Ripard | 39159be | 2014-11-17 14:42:08 +0100 | [diff] [blame] | 759 | static int bcm2835_dma_slave_config(struct dma_chan *chan, |
| 760 | struct dma_slave_config *cfg) |
Florian Meier | 96286b5 | 2014-01-06 20:18:24 +0100 | [diff] [blame] | 761 | { |
Maxime Ripard | 39159be | 2014-11-17 14:42:08 +0100 | [diff] [blame] | 762 | struct bcm2835_chan *c = to_bcm2835_dma_chan(chan); |
| 763 | |
Florian Meier | 96286b5 | 2014-01-06 20:18:24 +0100 | [diff] [blame] | 764 | if ((cfg->direction == DMA_DEV_TO_MEM && |
| 765 | cfg->src_addr_width != DMA_SLAVE_BUSWIDTH_4_BYTES) || |
| 766 | (cfg->direction == DMA_MEM_TO_DEV && |
| 767 | cfg->dst_addr_width != DMA_SLAVE_BUSWIDTH_4_BYTES) || |
| 768 | !is_slave_direction(cfg->direction)) { |
| 769 | return -EINVAL; |
| 770 | } |
| 771 | |
| 772 | c->cfg = *cfg; |
| 773 | |
| 774 | return 0; |
| 775 | } |
| 776 | |
Maxime Ripard | 39159be | 2014-11-17 14:42:08 +0100 | [diff] [blame] | 777 | static int bcm2835_dma_terminate_all(struct dma_chan *chan) |
Florian Meier | 96286b5 | 2014-01-06 20:18:24 +0100 | [diff] [blame] | 778 | { |
Maxime Ripard | 39159be | 2014-11-17 14:42:08 +0100 | [diff] [blame] | 779 | struct bcm2835_chan *c = to_bcm2835_dma_chan(chan); |
Florian Meier | 96286b5 | 2014-01-06 20:18:24 +0100 | [diff] [blame] | 780 | struct bcm2835_dmadev *d = to_bcm2835_dma_dev(c->vc.chan.device); |
| 781 | unsigned long flags; |
| 782 | int timeout = 10000; |
| 783 | LIST_HEAD(head); |
| 784 | |
| 785 | spin_lock_irqsave(&c->vc.lock, flags); |
| 786 | |
| 787 | /* Prevent this channel being scheduled */ |
| 788 | spin_lock(&d->lock); |
| 789 | list_del_init(&c->node); |
| 790 | spin_unlock(&d->lock); |
| 791 | |
| 792 | /* |
| 793 | * Stop DMA activity: we assume the callback will not be called |
| 794 | * after bcm_dma_abort() returns (even if it does, it will see |
| 795 | * c->desc is NULL and exit.) |
| 796 | */ |
| 797 | if (c->desc) { |
Peter Ujfalusi | f931782 | 2015-03-27 13:35:53 +0200 | [diff] [blame] | 798 | bcm2835_dma_desc_free(&c->desc->vd); |
Florian Meier | 96286b5 | 2014-01-06 20:18:24 +0100 | [diff] [blame] | 799 | c->desc = NULL; |
| 800 | bcm2835_dma_abort(c->chan_base); |
| 801 | |
| 802 | /* Wait for stopping */ |
| 803 | while (--timeout) { |
| 804 | if (!(readl(c->chan_base + BCM2835_DMA_CS) & |
| 805 | BCM2835_DMA_ACTIVE)) |
| 806 | break; |
| 807 | |
| 808 | cpu_relax(); |
| 809 | } |
| 810 | |
| 811 | if (!timeout) |
| 812 | dev_err(d->ddev.dev, "DMA transfer could not be terminated\n"); |
| 813 | } |
| 814 | |
| 815 | vchan_get_all_descriptors(&c->vc, &head); |
| 816 | spin_unlock_irqrestore(&c->vc.lock, flags); |
| 817 | vchan_dma_desc_free_list(&c->vc, &head); |
| 818 | |
| 819 | return 0; |
| 820 | } |
| 821 | |
Florian Meier | 96286b5 | 2014-01-06 20:18:24 +0100 | [diff] [blame] | 822 | static int bcm2835_dma_chan_init(struct bcm2835_dmadev *d, int chan_id, int irq) |
| 823 | { |
| 824 | struct bcm2835_chan *c; |
| 825 | |
| 826 | c = devm_kzalloc(d->ddev.dev, sizeof(*c), GFP_KERNEL); |
| 827 | if (!c) |
| 828 | return -ENOMEM; |
| 829 | |
| 830 | c->vc.desc_free = bcm2835_dma_desc_free; |
| 831 | vchan_init(&c->vc, &d->ddev); |
| 832 | INIT_LIST_HEAD(&c->node); |
| 833 | |
Florian Meier | 96286b5 | 2014-01-06 20:18:24 +0100 | [diff] [blame] | 834 | c->chan_base = BCM2835_DMA_CHANIO(d->base, chan_id); |
| 835 | c->ch = chan_id; |
| 836 | c->irq_number = irq; |
| 837 | |
Martin Sperl | 4087412 | 2016-03-16 12:25:00 -0700 | [diff] [blame] | 838 | /* check in DEBUG register if this is a LITE channel */ |
| 839 | if (readl(c->chan_base + BCM2835_DMA_DEBUG) & |
| 840 | BCM2835_DMA_DEBUG_LITE) |
| 841 | c->is_lite_channel = true; |
| 842 | |
Florian Meier | 96286b5 | 2014-01-06 20:18:24 +0100 | [diff] [blame] | 843 | return 0; |
| 844 | } |
| 845 | |
| 846 | static void bcm2835_dma_free(struct bcm2835_dmadev *od) |
| 847 | { |
| 848 | struct bcm2835_chan *c, *next; |
| 849 | |
| 850 | list_for_each_entry_safe(c, next, &od->ddev.channels, |
| 851 | vc.chan.device_node) { |
| 852 | list_del(&c->vc.chan.device_node); |
| 853 | tasklet_kill(&c->vc.task); |
| 854 | } |
| 855 | } |
| 856 | |
| 857 | static const struct of_device_id bcm2835_dma_of_match[] = { |
| 858 | { .compatible = "brcm,bcm2835-dma", }, |
| 859 | {}, |
| 860 | }; |
| 861 | MODULE_DEVICE_TABLE(of, bcm2835_dma_of_match); |
| 862 | |
| 863 | static struct dma_chan *bcm2835_dma_xlate(struct of_phandle_args *spec, |
| 864 | struct of_dma *ofdma) |
| 865 | { |
| 866 | struct bcm2835_dmadev *d = ofdma->of_dma_data; |
| 867 | struct dma_chan *chan; |
| 868 | |
| 869 | chan = dma_get_any_slave_channel(&d->ddev); |
| 870 | if (!chan) |
| 871 | return NULL; |
| 872 | |
| 873 | /* Set DREQ from param */ |
| 874 | to_bcm2835_dma_chan(chan)->dreq = spec->args[0]; |
| 875 | |
| 876 | return chan; |
| 877 | } |
| 878 | |
Florian Meier | 96286b5 | 2014-01-06 20:18:24 +0100 | [diff] [blame] | 879 | static int bcm2835_dma_probe(struct platform_device *pdev) |
| 880 | { |
| 881 | struct bcm2835_dmadev *od; |
| 882 | struct resource *res; |
| 883 | void __iomem *base; |
| 884 | int rc; |
| 885 | int i; |
| 886 | int irq; |
| 887 | uint32_t chans_available; |
| 888 | |
| 889 | if (!pdev->dev.dma_mask) |
| 890 | pdev->dev.dma_mask = &pdev->dev.coherent_dma_mask; |
| 891 | |
| 892 | rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); |
| 893 | if (rc) |
| 894 | return rc; |
| 895 | |
| 896 | od = devm_kzalloc(&pdev->dev, sizeof(*od), GFP_KERNEL); |
| 897 | if (!od) |
| 898 | return -ENOMEM; |
| 899 | |
| 900 | pdev->dev.dma_parms = &od->dma_parms; |
| 901 | dma_set_max_seg_size(&pdev->dev, 0x3FFFFFFF); |
| 902 | |
| 903 | res = platform_get_resource(pdev, IORESOURCE_MEM, 0); |
| 904 | base = devm_ioremap_resource(&pdev->dev, res); |
| 905 | if (IS_ERR(base)) |
| 906 | return PTR_ERR(base); |
| 907 | |
| 908 | od->base = base; |
| 909 | |
| 910 | dma_cap_set(DMA_SLAVE, od->ddev.cap_mask); |
Florian Meier | 7f5ae35 | 2014-01-17 18:06:29 +0100 | [diff] [blame] | 911 | dma_cap_set(DMA_PRIVATE, od->ddev.cap_mask); |
Florian Meier | 96286b5 | 2014-01-06 20:18:24 +0100 | [diff] [blame] | 912 | dma_cap_set(DMA_CYCLIC, od->ddev.cap_mask); |
Martin Sperl | 388cc7a | 2016-03-16 12:25:01 -0700 | [diff] [blame] | 913 | dma_cap_set(DMA_SLAVE, od->ddev.cap_mask); |
Martin Sperl | d9f094a | 2016-03-16 12:25:02 -0700 | [diff] [blame^] | 914 | dma_cap_set(DMA_MEMCPY, od->ddev.cap_mask); |
Florian Meier | 96286b5 | 2014-01-06 20:18:24 +0100 | [diff] [blame] | 915 | od->ddev.device_alloc_chan_resources = bcm2835_dma_alloc_chan_resources; |
| 916 | od->ddev.device_free_chan_resources = bcm2835_dma_free_chan_resources; |
| 917 | od->ddev.device_tx_status = bcm2835_dma_tx_status; |
| 918 | od->ddev.device_issue_pending = bcm2835_dma_issue_pending; |
Florian Meier | 96286b5 | 2014-01-06 20:18:24 +0100 | [diff] [blame] | 919 | od->ddev.device_prep_dma_cyclic = bcm2835_dma_prep_dma_cyclic; |
Martin Sperl | 388cc7a | 2016-03-16 12:25:01 -0700 | [diff] [blame] | 920 | od->ddev.device_prep_slave_sg = bcm2835_dma_prep_slave_sg; |
Martin Sperl | d9f094a | 2016-03-16 12:25:02 -0700 | [diff] [blame^] | 921 | od->ddev.device_prep_dma_memcpy = bcm2835_dma_prep_dma_memcpy; |
Maxime Ripard | 39159be | 2014-11-17 14:42:08 +0100 | [diff] [blame] | 922 | od->ddev.device_config = bcm2835_dma_slave_config; |
| 923 | od->ddev.device_terminate_all = bcm2835_dma_terminate_all; |
Maxime Ripard | b574368 | 2014-11-17 14:42:45 +0100 | [diff] [blame] | 924 | od->ddev.src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); |
| 925 | od->ddev.dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); |
Martin Sperl | d9f094a | 2016-03-16 12:25:02 -0700 | [diff] [blame^] | 926 | od->ddev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV) | |
| 927 | BIT(DMA_MEM_TO_MEM); |
Martin Sperl | 0fa5867 | 2016-03-16 12:24:55 -0700 | [diff] [blame] | 928 | od->ddev.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; |
Florian Meier | 96286b5 | 2014-01-06 20:18:24 +0100 | [diff] [blame] | 929 | od->ddev.dev = &pdev->dev; |
| 930 | INIT_LIST_HEAD(&od->ddev.channels); |
| 931 | spin_lock_init(&od->lock); |
| 932 | |
| 933 | platform_set_drvdata(pdev, od); |
| 934 | |
| 935 | /* Request DMA channel mask from device tree */ |
| 936 | if (of_property_read_u32(pdev->dev.of_node, |
| 937 | "brcm,dma-channel-mask", |
| 938 | &chans_available)) { |
| 939 | dev_err(&pdev->dev, "Failed to get channel mask\n"); |
| 940 | rc = -EINVAL; |
| 941 | goto err_no_dma; |
| 942 | } |
| 943 | |
Florian Meier | 96286b5 | 2014-01-06 20:18:24 +0100 | [diff] [blame] | 944 | for (i = 0; i < pdev->num_resources; i++) { |
| 945 | irq = platform_get_irq(pdev, i); |
| 946 | if (irq < 0) |
| 947 | break; |
| 948 | |
| 949 | if (chans_available & (1 << i)) { |
| 950 | rc = bcm2835_dma_chan_init(od, i, irq); |
| 951 | if (rc) |
| 952 | goto err_no_dma; |
| 953 | } |
| 954 | } |
| 955 | |
| 956 | dev_dbg(&pdev->dev, "Initialized %i DMA channels\n", i); |
| 957 | |
| 958 | /* Device-tree DMA controller registration */ |
| 959 | rc = of_dma_controller_register(pdev->dev.of_node, |
| 960 | bcm2835_dma_xlate, od); |
| 961 | if (rc) { |
| 962 | dev_err(&pdev->dev, "Failed to register DMA controller\n"); |
| 963 | goto err_no_dma; |
| 964 | } |
| 965 | |
| 966 | rc = dma_async_device_register(&od->ddev); |
| 967 | if (rc) { |
| 968 | dev_err(&pdev->dev, |
| 969 | "Failed to register slave DMA engine device: %d\n", rc); |
| 970 | goto err_no_dma; |
| 971 | } |
| 972 | |
| 973 | dev_dbg(&pdev->dev, "Load BCM2835 DMA engine driver\n"); |
| 974 | |
| 975 | return 0; |
| 976 | |
| 977 | err_no_dma: |
| 978 | bcm2835_dma_free(od); |
| 979 | return rc; |
| 980 | } |
| 981 | |
| 982 | static int bcm2835_dma_remove(struct platform_device *pdev) |
| 983 | { |
| 984 | struct bcm2835_dmadev *od = platform_get_drvdata(pdev); |
| 985 | |
| 986 | dma_async_device_unregister(&od->ddev); |
| 987 | bcm2835_dma_free(od); |
| 988 | |
| 989 | return 0; |
| 990 | } |
| 991 | |
| 992 | static struct platform_driver bcm2835_dma_driver = { |
| 993 | .probe = bcm2835_dma_probe, |
| 994 | .remove = bcm2835_dma_remove, |
| 995 | .driver = { |
| 996 | .name = "bcm2835-dma", |
Florian Meier | 96286b5 | 2014-01-06 20:18:24 +0100 | [diff] [blame] | 997 | .of_match_table = of_match_ptr(bcm2835_dma_of_match), |
| 998 | }, |
| 999 | }; |
| 1000 | |
| 1001 | module_platform_driver(bcm2835_dma_driver); |
| 1002 | |
| 1003 | MODULE_ALIAS("platform:bcm2835-dma"); |
| 1004 | MODULE_DESCRIPTION("BCM2835 DMA engine driver"); |
| 1005 | MODULE_AUTHOR("Florian Meier <florian.meier@koalo.de>"); |
| 1006 | MODULE_LICENSE("GPL v2"); |