Lars-Peter Clausen | 7c169a4 | 2013-05-30 18:25:02 +0200 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (C) 2013, Lars-Peter Clausen <lars@metafoo.de> |
| 3 | * JZ4740 DMAC support |
| 4 | * |
| 5 | * This program is free software; you can redistribute it and/or modify it |
| 6 | * under the terms of the GNU General Public License as published by the |
| 7 | * Free Software Foundation; either version 2 of the License, or (at your |
| 8 | * option) any later version. |
| 9 | * |
Lars-Peter Clausen | 7c169a4 | 2013-05-30 18:25:02 +0200 | [diff] [blame] | 10 | */ |
| 11 | |
| 12 | #include <linux/dmaengine.h> |
| 13 | #include <linux/dma-mapping.h> |
| 14 | #include <linux/err.h> |
| 15 | #include <linux/init.h> |
| 16 | #include <linux/list.h> |
| 17 | #include <linux/module.h> |
| 18 | #include <linux/platform_device.h> |
| 19 | #include <linux/slab.h> |
| 20 | #include <linux/spinlock.h> |
Lars-Peter Clausen | 25ce6c3 | 2013-05-30 18:25:05 +0200 | [diff] [blame] | 21 | #include <linux/irq.h> |
| 22 | #include <linux/clk.h> |
Lars-Peter Clausen | 7c169a4 | 2013-05-30 18:25:02 +0200 | [diff] [blame] | 23 | |
| 24 | #include <asm/mach-jz4740/dma.h> |
| 25 | |
| 26 | #include "virt-dma.h" |
| 27 | |
| 28 | #define JZ_DMA_NR_CHANS 6 |
| 29 | |
Lars-Peter Clausen | 25ce6c3 | 2013-05-30 18:25:05 +0200 | [diff] [blame] | 30 | #define JZ_REG_DMA_SRC_ADDR(x) (0x00 + (x) * 0x20) |
| 31 | #define JZ_REG_DMA_DST_ADDR(x) (0x04 + (x) * 0x20) |
| 32 | #define JZ_REG_DMA_TRANSFER_COUNT(x) (0x08 + (x) * 0x20) |
| 33 | #define JZ_REG_DMA_REQ_TYPE(x) (0x0C + (x) * 0x20) |
| 34 | #define JZ_REG_DMA_STATUS_CTRL(x) (0x10 + (x) * 0x20) |
| 35 | #define JZ_REG_DMA_CMD(x) (0x14 + (x) * 0x20) |
| 36 | #define JZ_REG_DMA_DESC_ADDR(x) (0x18 + (x) * 0x20) |
| 37 | |
| 38 | #define JZ_REG_DMA_CTRL 0x300 |
| 39 | #define JZ_REG_DMA_IRQ 0x304 |
| 40 | #define JZ_REG_DMA_DOORBELL 0x308 |
| 41 | #define JZ_REG_DMA_DOORBELL_SET 0x30C |
| 42 | |
| 43 | #define JZ_DMA_STATUS_CTRL_NO_DESC BIT(31) |
| 44 | #define JZ_DMA_STATUS_CTRL_DESC_INV BIT(6) |
| 45 | #define JZ_DMA_STATUS_CTRL_ADDR_ERR BIT(4) |
| 46 | #define JZ_DMA_STATUS_CTRL_TRANSFER_DONE BIT(3) |
| 47 | #define JZ_DMA_STATUS_CTRL_HALT BIT(2) |
| 48 | #define JZ_DMA_STATUS_CTRL_COUNT_TERMINATE BIT(1) |
| 49 | #define JZ_DMA_STATUS_CTRL_ENABLE BIT(0) |
| 50 | |
| 51 | #define JZ_DMA_CMD_SRC_INC BIT(23) |
| 52 | #define JZ_DMA_CMD_DST_INC BIT(22) |
| 53 | #define JZ_DMA_CMD_RDIL_MASK (0xf << 16) |
| 54 | #define JZ_DMA_CMD_SRC_WIDTH_MASK (0x3 << 14) |
| 55 | #define JZ_DMA_CMD_DST_WIDTH_MASK (0x3 << 12) |
| 56 | #define JZ_DMA_CMD_INTERVAL_LENGTH_MASK (0x7 << 8) |
| 57 | #define JZ_DMA_CMD_BLOCK_MODE BIT(7) |
| 58 | #define JZ_DMA_CMD_DESC_VALID BIT(4) |
| 59 | #define JZ_DMA_CMD_DESC_VALID_MODE BIT(3) |
| 60 | #define JZ_DMA_CMD_VALID_IRQ_ENABLE BIT(2) |
| 61 | #define JZ_DMA_CMD_TRANSFER_IRQ_ENABLE BIT(1) |
| 62 | #define JZ_DMA_CMD_LINK_ENABLE BIT(0) |
| 63 | |
| 64 | #define JZ_DMA_CMD_FLAGS_OFFSET 22 |
| 65 | #define JZ_DMA_CMD_RDIL_OFFSET 16 |
| 66 | #define JZ_DMA_CMD_SRC_WIDTH_OFFSET 14 |
| 67 | #define JZ_DMA_CMD_DST_WIDTH_OFFSET 12 |
| 68 | #define JZ_DMA_CMD_TRANSFER_SIZE_OFFSET 8 |
| 69 | #define JZ_DMA_CMD_MODE_OFFSET 7 |
| 70 | |
| 71 | #define JZ_DMA_CTRL_PRIORITY_MASK (0x3 << 8) |
| 72 | #define JZ_DMA_CTRL_HALT BIT(3) |
| 73 | #define JZ_DMA_CTRL_ADDRESS_ERROR BIT(2) |
| 74 | #define JZ_DMA_CTRL_ENABLE BIT(0) |
| 75 | |
| 76 | enum jz4740_dma_width { |
| 77 | JZ4740_DMA_WIDTH_32BIT = 0, |
| 78 | JZ4740_DMA_WIDTH_8BIT = 1, |
| 79 | JZ4740_DMA_WIDTH_16BIT = 2, |
| 80 | }; |
| 81 | |
| 82 | enum jz4740_dma_transfer_size { |
| 83 | JZ4740_DMA_TRANSFER_SIZE_4BYTE = 0, |
| 84 | JZ4740_DMA_TRANSFER_SIZE_1BYTE = 1, |
| 85 | JZ4740_DMA_TRANSFER_SIZE_2BYTE = 2, |
| 86 | JZ4740_DMA_TRANSFER_SIZE_16BYTE = 3, |
| 87 | JZ4740_DMA_TRANSFER_SIZE_32BYTE = 4, |
| 88 | }; |
| 89 | |
| 90 | enum jz4740_dma_flags { |
| 91 | JZ4740_DMA_SRC_AUTOINC = 0x2, |
| 92 | JZ4740_DMA_DST_AUTOINC = 0x1, |
| 93 | }; |
| 94 | |
| 95 | enum jz4740_dma_mode { |
| 96 | JZ4740_DMA_MODE_SINGLE = 0, |
| 97 | JZ4740_DMA_MODE_BLOCK = 1, |
| 98 | }; |
| 99 | |
Lars-Peter Clausen | 7c169a4 | 2013-05-30 18:25:02 +0200 | [diff] [blame] | 100 | struct jz4740_dma_sg { |
| 101 | dma_addr_t addr; |
| 102 | unsigned int len; |
| 103 | }; |
| 104 | |
| 105 | struct jz4740_dma_desc { |
| 106 | struct virt_dma_desc vdesc; |
| 107 | |
| 108 | enum dma_transfer_direction direction; |
| 109 | bool cyclic; |
| 110 | |
| 111 | unsigned int num_sgs; |
| 112 | struct jz4740_dma_sg sg[]; |
| 113 | }; |
| 114 | |
| 115 | struct jz4740_dmaengine_chan { |
| 116 | struct virt_dma_chan vchan; |
Lars-Peter Clausen | 25ce6c3 | 2013-05-30 18:25:05 +0200 | [diff] [blame] | 117 | unsigned int id; |
Lars-Peter Clausen | 7c169a4 | 2013-05-30 18:25:02 +0200 | [diff] [blame] | 118 | |
| 119 | dma_addr_t fifo_addr; |
Lars-Peter Clausen | 25ce6c3 | 2013-05-30 18:25:05 +0200 | [diff] [blame] | 120 | unsigned int transfer_shift; |
Lars-Peter Clausen | 7c169a4 | 2013-05-30 18:25:02 +0200 | [diff] [blame] | 121 | |
| 122 | struct jz4740_dma_desc *desc; |
| 123 | unsigned int next_sg; |
| 124 | }; |
| 125 | |
| 126 | struct jz4740_dma_dev { |
| 127 | struct dma_device ddev; |
Lars-Peter Clausen | 25ce6c3 | 2013-05-30 18:25:05 +0200 | [diff] [blame] | 128 | void __iomem *base; |
| 129 | struct clk *clk; |
Lars-Peter Clausen | 7c169a4 | 2013-05-30 18:25:02 +0200 | [diff] [blame] | 130 | |
| 131 | struct jz4740_dmaengine_chan chan[JZ_DMA_NR_CHANS]; |
| 132 | }; |
| 133 | |
Lars-Peter Clausen | 25ce6c3 | 2013-05-30 18:25:05 +0200 | [diff] [blame] | 134 | static struct jz4740_dma_dev *jz4740_dma_chan_get_dev( |
| 135 | struct jz4740_dmaengine_chan *chan) |
| 136 | { |
| 137 | return container_of(chan->vchan.chan.device, struct jz4740_dma_dev, |
| 138 | ddev); |
| 139 | } |
| 140 | |
Lars-Peter Clausen | 7c169a4 | 2013-05-30 18:25:02 +0200 | [diff] [blame] | 141 | static struct jz4740_dmaengine_chan *to_jz4740_dma_chan(struct dma_chan *c) |
| 142 | { |
| 143 | return container_of(c, struct jz4740_dmaengine_chan, vchan.chan); |
| 144 | } |
| 145 | |
| 146 | static struct jz4740_dma_desc *to_jz4740_dma_desc(struct virt_dma_desc *vdesc) |
| 147 | { |
| 148 | return container_of(vdesc, struct jz4740_dma_desc, vdesc); |
| 149 | } |
| 150 | |
Lars-Peter Clausen | 25ce6c3 | 2013-05-30 18:25:05 +0200 | [diff] [blame] | 151 | static inline uint32_t jz4740_dma_read(struct jz4740_dma_dev *dmadev, |
| 152 | unsigned int reg) |
| 153 | { |
| 154 | return readl(dmadev->base + reg); |
| 155 | } |
| 156 | |
| 157 | static inline void jz4740_dma_write(struct jz4740_dma_dev *dmadev, |
| 158 | unsigned reg, uint32_t val) |
| 159 | { |
| 160 | writel(val, dmadev->base + reg); |
| 161 | } |
| 162 | |
| 163 | static inline void jz4740_dma_write_mask(struct jz4740_dma_dev *dmadev, |
| 164 | unsigned int reg, uint32_t val, uint32_t mask) |
| 165 | { |
| 166 | uint32_t tmp; |
| 167 | |
| 168 | tmp = jz4740_dma_read(dmadev, reg); |
| 169 | tmp &= ~mask; |
| 170 | tmp |= val; |
| 171 | jz4740_dma_write(dmadev, reg, tmp); |
| 172 | } |
| 173 | |
Lars-Peter Clausen | 7c169a4 | 2013-05-30 18:25:02 +0200 | [diff] [blame] | 174 | static struct jz4740_dma_desc *jz4740_dma_alloc_desc(unsigned int num_sgs) |
| 175 | { |
| 176 | return kzalloc(sizeof(struct jz4740_dma_desc) + |
| 177 | sizeof(struct jz4740_dma_sg) * num_sgs, GFP_ATOMIC); |
| 178 | } |
| 179 | |
| 180 | static enum jz4740_dma_width jz4740_dma_width(enum dma_slave_buswidth width) |
| 181 | { |
| 182 | switch (width) { |
| 183 | case DMA_SLAVE_BUSWIDTH_1_BYTE: |
| 184 | return JZ4740_DMA_WIDTH_8BIT; |
| 185 | case DMA_SLAVE_BUSWIDTH_2_BYTES: |
| 186 | return JZ4740_DMA_WIDTH_16BIT; |
| 187 | case DMA_SLAVE_BUSWIDTH_4_BYTES: |
| 188 | return JZ4740_DMA_WIDTH_32BIT; |
| 189 | default: |
| 190 | return JZ4740_DMA_WIDTH_32BIT; |
| 191 | } |
| 192 | } |
| 193 | |
| 194 | static enum jz4740_dma_transfer_size jz4740_dma_maxburst(u32 maxburst) |
| 195 | { |
| 196 | if (maxburst <= 1) |
| 197 | return JZ4740_DMA_TRANSFER_SIZE_1BYTE; |
| 198 | else if (maxburst <= 3) |
| 199 | return JZ4740_DMA_TRANSFER_SIZE_2BYTE; |
| 200 | else if (maxburst <= 15) |
| 201 | return JZ4740_DMA_TRANSFER_SIZE_4BYTE; |
| 202 | else if (maxburst <= 31) |
| 203 | return JZ4740_DMA_TRANSFER_SIZE_16BYTE; |
| 204 | |
| 205 | return JZ4740_DMA_TRANSFER_SIZE_32BYTE; |
| 206 | } |
| 207 | |
| 208 | static int jz4740_dma_slave_config(struct dma_chan *c, |
Maxime Ripard | 1d4c0b8 | 2014-11-17 14:42:11 +0100 | [diff] [blame] | 209 | struct dma_slave_config *config) |
Lars-Peter Clausen | 7c169a4 | 2013-05-30 18:25:02 +0200 | [diff] [blame] | 210 | { |
| 211 | struct jz4740_dmaengine_chan *chan = to_jz4740_dma_chan(c); |
Lars-Peter Clausen | 25ce6c3 | 2013-05-30 18:25:05 +0200 | [diff] [blame] | 212 | struct jz4740_dma_dev *dmadev = jz4740_dma_chan_get_dev(chan); |
| 213 | enum jz4740_dma_width src_width; |
| 214 | enum jz4740_dma_width dst_width; |
| 215 | enum jz4740_dma_transfer_size transfer_size; |
| 216 | enum jz4740_dma_flags flags; |
| 217 | uint32_t cmd; |
Lars-Peter Clausen | 7c169a4 | 2013-05-30 18:25:02 +0200 | [diff] [blame] | 218 | |
| 219 | switch (config->direction) { |
| 220 | case DMA_MEM_TO_DEV: |
Lars-Peter Clausen | 25ce6c3 | 2013-05-30 18:25:05 +0200 | [diff] [blame] | 221 | flags = JZ4740_DMA_SRC_AUTOINC; |
| 222 | transfer_size = jz4740_dma_maxburst(config->dst_maxburst); |
Lars-Peter Clausen | 7c169a4 | 2013-05-30 18:25:02 +0200 | [diff] [blame] | 223 | chan->fifo_addr = config->dst_addr; |
| 224 | break; |
| 225 | case DMA_DEV_TO_MEM: |
Lars-Peter Clausen | 25ce6c3 | 2013-05-30 18:25:05 +0200 | [diff] [blame] | 226 | flags = JZ4740_DMA_DST_AUTOINC; |
| 227 | transfer_size = jz4740_dma_maxburst(config->src_maxburst); |
Lars-Peter Clausen | 7c169a4 | 2013-05-30 18:25:02 +0200 | [diff] [blame] | 228 | chan->fifo_addr = config->src_addr; |
| 229 | break; |
| 230 | default: |
| 231 | return -EINVAL; |
| 232 | } |
| 233 | |
Lars-Peter Clausen | 25ce6c3 | 2013-05-30 18:25:05 +0200 | [diff] [blame] | 234 | src_width = jz4740_dma_width(config->src_addr_width); |
| 235 | dst_width = jz4740_dma_width(config->dst_addr_width); |
Lars-Peter Clausen | 7c169a4 | 2013-05-30 18:25:02 +0200 | [diff] [blame] | 236 | |
Lars-Peter Clausen | 25ce6c3 | 2013-05-30 18:25:05 +0200 | [diff] [blame] | 237 | switch (transfer_size) { |
| 238 | case JZ4740_DMA_TRANSFER_SIZE_2BYTE: |
| 239 | chan->transfer_shift = 1; |
| 240 | break; |
| 241 | case JZ4740_DMA_TRANSFER_SIZE_4BYTE: |
| 242 | chan->transfer_shift = 2; |
| 243 | break; |
| 244 | case JZ4740_DMA_TRANSFER_SIZE_16BYTE: |
| 245 | chan->transfer_shift = 4; |
| 246 | break; |
| 247 | case JZ4740_DMA_TRANSFER_SIZE_32BYTE: |
| 248 | chan->transfer_shift = 5; |
| 249 | break; |
| 250 | default: |
| 251 | chan->transfer_shift = 0; |
| 252 | break; |
| 253 | } |
Lars-Peter Clausen | 7c169a4 | 2013-05-30 18:25:02 +0200 | [diff] [blame] | 254 | |
Lars-Peter Clausen | 25ce6c3 | 2013-05-30 18:25:05 +0200 | [diff] [blame] | 255 | cmd = flags << JZ_DMA_CMD_FLAGS_OFFSET; |
| 256 | cmd |= src_width << JZ_DMA_CMD_SRC_WIDTH_OFFSET; |
| 257 | cmd |= dst_width << JZ_DMA_CMD_DST_WIDTH_OFFSET; |
| 258 | cmd |= transfer_size << JZ_DMA_CMD_TRANSFER_SIZE_OFFSET; |
| 259 | cmd |= JZ4740_DMA_MODE_SINGLE << JZ_DMA_CMD_MODE_OFFSET; |
| 260 | cmd |= JZ_DMA_CMD_TRANSFER_IRQ_ENABLE; |
| 261 | |
| 262 | jz4740_dma_write(dmadev, JZ_REG_DMA_CMD(chan->id), cmd); |
| 263 | jz4740_dma_write(dmadev, JZ_REG_DMA_STATUS_CTRL(chan->id), 0); |
| 264 | jz4740_dma_write(dmadev, JZ_REG_DMA_REQ_TYPE(chan->id), |
| 265 | config->slave_id); |
Lars-Peter Clausen | 7c169a4 | 2013-05-30 18:25:02 +0200 | [diff] [blame] | 266 | |
| 267 | return 0; |
| 268 | } |
| 269 | |
| 270 | static int jz4740_dma_terminate_all(struct dma_chan *c) |
| 271 | { |
| 272 | struct jz4740_dmaengine_chan *chan = to_jz4740_dma_chan(c); |
Lars-Peter Clausen | 25ce6c3 | 2013-05-30 18:25:05 +0200 | [diff] [blame] | 273 | struct jz4740_dma_dev *dmadev = jz4740_dma_chan_get_dev(chan); |
Lars-Peter Clausen | 7c169a4 | 2013-05-30 18:25:02 +0200 | [diff] [blame] | 274 | unsigned long flags; |
| 275 | LIST_HEAD(head); |
| 276 | |
| 277 | spin_lock_irqsave(&chan->vchan.lock, flags); |
Lars-Peter Clausen | 25ce6c3 | 2013-05-30 18:25:05 +0200 | [diff] [blame] | 278 | jz4740_dma_write_mask(dmadev, JZ_REG_DMA_STATUS_CTRL(chan->id), 0, |
| 279 | JZ_DMA_STATUS_CTRL_ENABLE); |
Lars-Peter Clausen | 7c169a4 | 2013-05-30 18:25:02 +0200 | [diff] [blame] | 280 | chan->desc = NULL; |
| 281 | vchan_get_all_descriptors(&chan->vchan, &head); |
| 282 | spin_unlock_irqrestore(&chan->vchan.lock, flags); |
| 283 | |
| 284 | vchan_dma_desc_free_list(&chan->vchan, &head); |
| 285 | |
| 286 | return 0; |
| 287 | } |
| 288 | |
Lars-Peter Clausen | 7c169a4 | 2013-05-30 18:25:02 +0200 | [diff] [blame] | 289 | static int jz4740_dma_start_transfer(struct jz4740_dmaengine_chan *chan) |
| 290 | { |
Lars-Peter Clausen | 25ce6c3 | 2013-05-30 18:25:05 +0200 | [diff] [blame] | 291 | struct jz4740_dma_dev *dmadev = jz4740_dma_chan_get_dev(chan); |
Lars-Peter Clausen | 7c169a4 | 2013-05-30 18:25:02 +0200 | [diff] [blame] | 292 | dma_addr_t src_addr, dst_addr; |
| 293 | struct virt_dma_desc *vdesc; |
| 294 | struct jz4740_dma_sg *sg; |
| 295 | |
Lars-Peter Clausen | 25ce6c3 | 2013-05-30 18:25:05 +0200 | [diff] [blame] | 296 | jz4740_dma_write_mask(dmadev, JZ_REG_DMA_STATUS_CTRL(chan->id), 0, |
| 297 | JZ_DMA_STATUS_CTRL_ENABLE); |
Lars-Peter Clausen | 7c169a4 | 2013-05-30 18:25:02 +0200 | [diff] [blame] | 298 | |
| 299 | if (!chan->desc) { |
| 300 | vdesc = vchan_next_desc(&chan->vchan); |
| 301 | if (!vdesc) |
| 302 | return 0; |
| 303 | chan->desc = to_jz4740_dma_desc(vdesc); |
| 304 | chan->next_sg = 0; |
| 305 | } |
| 306 | |
| 307 | if (chan->next_sg == chan->desc->num_sgs) |
| 308 | chan->next_sg = 0; |
| 309 | |
| 310 | sg = &chan->desc->sg[chan->next_sg]; |
| 311 | |
| 312 | if (chan->desc->direction == DMA_MEM_TO_DEV) { |
| 313 | src_addr = sg->addr; |
| 314 | dst_addr = chan->fifo_addr; |
| 315 | } else { |
| 316 | src_addr = chan->fifo_addr; |
| 317 | dst_addr = sg->addr; |
| 318 | } |
Lars-Peter Clausen | 25ce6c3 | 2013-05-30 18:25:05 +0200 | [diff] [blame] | 319 | jz4740_dma_write(dmadev, JZ_REG_DMA_SRC_ADDR(chan->id), src_addr); |
| 320 | jz4740_dma_write(dmadev, JZ_REG_DMA_DST_ADDR(chan->id), dst_addr); |
| 321 | jz4740_dma_write(dmadev, JZ_REG_DMA_TRANSFER_COUNT(chan->id), |
| 322 | sg->len >> chan->transfer_shift); |
Lars-Peter Clausen | 7c169a4 | 2013-05-30 18:25:02 +0200 | [diff] [blame] | 323 | |
| 324 | chan->next_sg++; |
| 325 | |
Lars-Peter Clausen | 25ce6c3 | 2013-05-30 18:25:05 +0200 | [diff] [blame] | 326 | jz4740_dma_write_mask(dmadev, JZ_REG_DMA_STATUS_CTRL(chan->id), |
| 327 | JZ_DMA_STATUS_CTRL_NO_DESC | JZ_DMA_STATUS_CTRL_ENABLE, |
| 328 | JZ_DMA_STATUS_CTRL_HALT | JZ_DMA_STATUS_CTRL_NO_DESC | |
| 329 | JZ_DMA_STATUS_CTRL_ENABLE); |
| 330 | |
| 331 | jz4740_dma_write_mask(dmadev, JZ_REG_DMA_CTRL, |
| 332 | JZ_DMA_CTRL_ENABLE, |
| 333 | JZ_DMA_CTRL_HALT | JZ_DMA_CTRL_ENABLE); |
Lars-Peter Clausen | 7c169a4 | 2013-05-30 18:25:02 +0200 | [diff] [blame] | 334 | |
| 335 | return 0; |
| 336 | } |
| 337 | |
Lars-Peter Clausen | 25ce6c3 | 2013-05-30 18:25:05 +0200 | [diff] [blame] | 338 | static void jz4740_dma_chan_irq(struct jz4740_dmaengine_chan *chan) |
Lars-Peter Clausen | 7c169a4 | 2013-05-30 18:25:02 +0200 | [diff] [blame] | 339 | { |
Lars-Peter Clausen | 7c169a4 | 2013-05-30 18:25:02 +0200 | [diff] [blame] | 340 | spin_lock(&chan->vchan.lock); |
| 341 | if (chan->desc) { |
Tapasweni Pathak | 26f7af37 | 2015-02-20 17:58:44 +0530 | [diff] [blame] | 342 | if (chan->desc->cyclic) { |
Lars-Peter Clausen | 7c169a4 | 2013-05-30 18:25:02 +0200 | [diff] [blame] | 343 | vchan_cyclic_callback(&chan->desc->vdesc); |
| 344 | } else { |
| 345 | if (chan->next_sg == chan->desc->num_sgs) { |
Lars-Peter Clausen | f498e06 | 2014-09-07 15:54:39 +0200 | [diff] [blame] | 346 | list_del(&chan->desc->vdesc.node); |
Lars-Peter Clausen | 7c169a4 | 2013-05-30 18:25:02 +0200 | [diff] [blame] | 347 | vchan_cookie_complete(&chan->desc->vdesc); |
Lars-Peter Clausen | f498e06 | 2014-09-07 15:54:39 +0200 | [diff] [blame] | 348 | chan->desc = NULL; |
Lars-Peter Clausen | 7c169a4 | 2013-05-30 18:25:02 +0200 | [diff] [blame] | 349 | } |
| 350 | } |
| 351 | } |
| 352 | jz4740_dma_start_transfer(chan); |
| 353 | spin_unlock(&chan->vchan.lock); |
| 354 | } |
| 355 | |
Lars-Peter Clausen | 25ce6c3 | 2013-05-30 18:25:05 +0200 | [diff] [blame] | 356 | static irqreturn_t jz4740_dma_irq(int irq, void *devid) |
| 357 | { |
| 358 | struct jz4740_dma_dev *dmadev = devid; |
| 359 | uint32_t irq_status; |
| 360 | unsigned int i; |
| 361 | |
| 362 | irq_status = readl(dmadev->base + JZ_REG_DMA_IRQ); |
| 363 | |
| 364 | for (i = 0; i < 6; ++i) { |
| 365 | if (irq_status & (1 << i)) { |
| 366 | jz4740_dma_write_mask(dmadev, |
| 367 | JZ_REG_DMA_STATUS_CTRL(i), 0, |
| 368 | JZ_DMA_STATUS_CTRL_ENABLE | |
| 369 | JZ_DMA_STATUS_CTRL_TRANSFER_DONE); |
| 370 | |
| 371 | jz4740_dma_chan_irq(&dmadev->chan[i]); |
| 372 | } |
| 373 | } |
| 374 | |
| 375 | return IRQ_HANDLED; |
| 376 | } |
| 377 | |
Lars-Peter Clausen | 7c169a4 | 2013-05-30 18:25:02 +0200 | [diff] [blame] | 378 | static void jz4740_dma_issue_pending(struct dma_chan *c) |
| 379 | { |
| 380 | struct jz4740_dmaengine_chan *chan = to_jz4740_dma_chan(c); |
| 381 | unsigned long flags; |
| 382 | |
| 383 | spin_lock_irqsave(&chan->vchan.lock, flags); |
| 384 | if (vchan_issue_pending(&chan->vchan) && !chan->desc) |
| 385 | jz4740_dma_start_transfer(chan); |
| 386 | spin_unlock_irqrestore(&chan->vchan.lock, flags); |
| 387 | } |
| 388 | |
| 389 | static struct dma_async_tx_descriptor *jz4740_dma_prep_slave_sg( |
| 390 | struct dma_chan *c, struct scatterlist *sgl, |
| 391 | unsigned int sg_len, enum dma_transfer_direction direction, |
| 392 | unsigned long flags, void *context) |
| 393 | { |
| 394 | struct jz4740_dmaengine_chan *chan = to_jz4740_dma_chan(c); |
| 395 | struct jz4740_dma_desc *desc; |
| 396 | struct scatterlist *sg; |
| 397 | unsigned int i; |
| 398 | |
| 399 | desc = jz4740_dma_alloc_desc(sg_len); |
| 400 | if (!desc) |
| 401 | return NULL; |
| 402 | |
| 403 | for_each_sg(sgl, sg, sg_len, i) { |
| 404 | desc->sg[i].addr = sg_dma_address(sg); |
| 405 | desc->sg[i].len = sg_dma_len(sg); |
| 406 | } |
| 407 | |
| 408 | desc->num_sgs = sg_len; |
| 409 | desc->direction = direction; |
| 410 | desc->cyclic = false; |
| 411 | |
| 412 | return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags); |
| 413 | } |
| 414 | |
| 415 | static struct dma_async_tx_descriptor *jz4740_dma_prep_dma_cyclic( |
| 416 | struct dma_chan *c, dma_addr_t buf_addr, size_t buf_len, |
| 417 | size_t period_len, enum dma_transfer_direction direction, |
Laurent Pinchart | 31c1e5a | 2014-08-01 12:20:10 +0200 | [diff] [blame] | 418 | unsigned long flags) |
Lars-Peter Clausen | 7c169a4 | 2013-05-30 18:25:02 +0200 | [diff] [blame] | 419 | { |
| 420 | struct jz4740_dmaengine_chan *chan = to_jz4740_dma_chan(c); |
| 421 | struct jz4740_dma_desc *desc; |
| 422 | unsigned int num_periods, i; |
| 423 | |
| 424 | if (buf_len % period_len) |
| 425 | return NULL; |
| 426 | |
| 427 | num_periods = buf_len / period_len; |
| 428 | |
| 429 | desc = jz4740_dma_alloc_desc(num_periods); |
| 430 | if (!desc) |
| 431 | return NULL; |
| 432 | |
| 433 | for (i = 0; i < num_periods; i++) { |
| 434 | desc->sg[i].addr = buf_addr; |
| 435 | desc->sg[i].len = period_len; |
| 436 | buf_addr += period_len; |
| 437 | } |
| 438 | |
| 439 | desc->num_sgs = num_periods; |
| 440 | desc->direction = direction; |
| 441 | desc->cyclic = true; |
| 442 | |
| 443 | return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags); |
| 444 | } |
| 445 | |
| 446 | static size_t jz4740_dma_desc_residue(struct jz4740_dmaengine_chan *chan, |
| 447 | struct jz4740_dma_desc *desc, unsigned int next_sg) |
| 448 | { |
Lars-Peter Clausen | 25ce6c3 | 2013-05-30 18:25:05 +0200 | [diff] [blame] | 449 | struct jz4740_dma_dev *dmadev = jz4740_dma_chan_get_dev(chan); |
| 450 | unsigned int residue, count; |
Lars-Peter Clausen | 7c169a4 | 2013-05-30 18:25:02 +0200 | [diff] [blame] | 451 | unsigned int i; |
| 452 | |
| 453 | residue = 0; |
| 454 | |
| 455 | for (i = next_sg; i < desc->num_sgs; i++) |
| 456 | residue += desc->sg[i].len; |
| 457 | |
Lars-Peter Clausen | 25ce6c3 | 2013-05-30 18:25:05 +0200 | [diff] [blame] | 458 | if (next_sg != 0) { |
| 459 | count = jz4740_dma_read(dmadev, |
| 460 | JZ_REG_DMA_TRANSFER_COUNT(chan->id)); |
| 461 | residue += count << chan->transfer_shift; |
| 462 | } |
Lars-Peter Clausen | 7c169a4 | 2013-05-30 18:25:02 +0200 | [diff] [blame] | 463 | |
| 464 | return residue; |
| 465 | } |
| 466 | |
| 467 | static enum dma_status jz4740_dma_tx_status(struct dma_chan *c, |
| 468 | dma_cookie_t cookie, struct dma_tx_state *state) |
| 469 | { |
| 470 | struct jz4740_dmaengine_chan *chan = to_jz4740_dma_chan(c); |
| 471 | struct virt_dma_desc *vdesc; |
| 472 | enum dma_status status; |
| 473 | unsigned long flags; |
| 474 | |
| 475 | status = dma_cookie_status(c, cookie, state); |
Vinod Koul | a605c48 | 2013-10-16 13:36:54 +0530 | [diff] [blame] | 476 | if (status == DMA_COMPLETE || !state) |
Lars-Peter Clausen | 7c169a4 | 2013-05-30 18:25:02 +0200 | [diff] [blame] | 477 | return status; |
| 478 | |
| 479 | spin_lock_irqsave(&chan->vchan.lock, flags); |
| 480 | vdesc = vchan_find_desc(&chan->vchan, cookie); |
| 481 | if (cookie == chan->desc->vdesc.tx.cookie) { |
| 482 | state->residue = jz4740_dma_desc_residue(chan, chan->desc, |
| 483 | chan->next_sg); |
| 484 | } else if (vdesc) { |
| 485 | state->residue = jz4740_dma_desc_residue(chan, |
| 486 | to_jz4740_dma_desc(vdesc), 0); |
| 487 | } else { |
| 488 | state->residue = 0; |
| 489 | } |
| 490 | spin_unlock_irqrestore(&chan->vchan.lock, flags); |
| 491 | |
| 492 | return status; |
| 493 | } |
| 494 | |
Lars-Peter Clausen | 7c169a4 | 2013-05-30 18:25:02 +0200 | [diff] [blame] | 495 | static void jz4740_dma_free_chan_resources(struct dma_chan *c) |
| 496 | { |
Lars-Peter Clausen | 25ce6c3 | 2013-05-30 18:25:05 +0200 | [diff] [blame] | 497 | vchan_free_chan_resources(to_virt_chan(c)); |
Lars-Peter Clausen | 7c169a4 | 2013-05-30 18:25:02 +0200 | [diff] [blame] | 498 | } |
| 499 | |
| 500 | static void jz4740_dma_desc_free(struct virt_dma_desc *vdesc) |
| 501 | { |
| 502 | kfree(container_of(vdesc, struct jz4740_dma_desc, vdesc)); |
| 503 | } |
| 504 | |
Lars-Peter Clausen | ca76683 | 2015-03-28 18:05:44 +0100 | [diff] [blame] | 505 | #define JZ4740_DMA_BUSWIDTHS (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \ |
| 506 | BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | BIT(DMA_SLAVE_BUSWIDTH_4_BYTES)) |
| 507 | |
Lars-Peter Clausen | 7c169a4 | 2013-05-30 18:25:02 +0200 | [diff] [blame] | 508 | static int jz4740_dma_probe(struct platform_device *pdev) |
| 509 | { |
| 510 | struct jz4740_dmaengine_chan *chan; |
| 511 | struct jz4740_dma_dev *dmadev; |
| 512 | struct dma_device *dd; |
| 513 | unsigned int i; |
Lars-Peter Clausen | 25ce6c3 | 2013-05-30 18:25:05 +0200 | [diff] [blame] | 514 | struct resource *res; |
Lars-Peter Clausen | 7c169a4 | 2013-05-30 18:25:02 +0200 | [diff] [blame] | 515 | int ret; |
Lars-Peter Clausen | 25ce6c3 | 2013-05-30 18:25:05 +0200 | [diff] [blame] | 516 | int irq; |
Lars-Peter Clausen | 7c169a4 | 2013-05-30 18:25:02 +0200 | [diff] [blame] | 517 | |
| 518 | dmadev = devm_kzalloc(&pdev->dev, sizeof(*dmadev), GFP_KERNEL); |
| 519 | if (!dmadev) |
| 520 | return -EINVAL; |
| 521 | |
| 522 | dd = &dmadev->ddev; |
| 523 | |
Lars-Peter Clausen | 25ce6c3 | 2013-05-30 18:25:05 +0200 | [diff] [blame] | 524 | res = platform_get_resource(pdev, IORESOURCE_MEM, 0); |
| 525 | dmadev->base = devm_ioremap_resource(&pdev->dev, res); |
| 526 | if (IS_ERR(dmadev->base)) |
| 527 | return PTR_ERR(dmadev->base); |
| 528 | |
| 529 | dmadev->clk = clk_get(&pdev->dev, "dma"); |
| 530 | if (IS_ERR(dmadev->clk)) |
| 531 | return PTR_ERR(dmadev->clk); |
| 532 | |
| 533 | clk_prepare_enable(dmadev->clk); |
| 534 | |
Lars-Peter Clausen | 7c169a4 | 2013-05-30 18:25:02 +0200 | [diff] [blame] | 535 | dma_cap_set(DMA_SLAVE, dd->cap_mask); |
| 536 | dma_cap_set(DMA_CYCLIC, dd->cap_mask); |
Lars-Peter Clausen | 7c169a4 | 2013-05-30 18:25:02 +0200 | [diff] [blame] | 537 | dd->device_free_chan_resources = jz4740_dma_free_chan_resources; |
| 538 | dd->device_tx_status = jz4740_dma_tx_status; |
| 539 | dd->device_issue_pending = jz4740_dma_issue_pending; |
| 540 | dd->device_prep_slave_sg = jz4740_dma_prep_slave_sg; |
| 541 | dd->device_prep_dma_cyclic = jz4740_dma_prep_dma_cyclic; |
Maxime Ripard | 1d4c0b8 | 2014-11-17 14:42:11 +0100 | [diff] [blame] | 542 | dd->device_config = jz4740_dma_slave_config; |
| 543 | dd->device_terminate_all = jz4740_dma_terminate_all; |
Lars-Peter Clausen | ca76683 | 2015-03-28 18:05:44 +0100 | [diff] [blame] | 544 | dd->src_addr_widths = JZ4740_DMA_BUSWIDTHS; |
| 545 | dd->dst_addr_widths = JZ4740_DMA_BUSWIDTHS; |
| 546 | dd->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); |
| 547 | dd->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; |
Lars-Peter Clausen | 7c169a4 | 2013-05-30 18:25:02 +0200 | [diff] [blame] | 548 | dd->dev = &pdev->dev; |
Lars-Peter Clausen | 7c169a4 | 2013-05-30 18:25:02 +0200 | [diff] [blame] | 549 | INIT_LIST_HEAD(&dd->channels); |
| 550 | |
Maxime Ripard | b2c100e | 2014-10-16 11:00:55 +0200 | [diff] [blame] | 551 | for (i = 0; i < JZ_DMA_NR_CHANS; i++) { |
Lars-Peter Clausen | 7c169a4 | 2013-05-30 18:25:02 +0200 | [diff] [blame] | 552 | chan = &dmadev->chan[i]; |
Lars-Peter Clausen | 25ce6c3 | 2013-05-30 18:25:05 +0200 | [diff] [blame] | 553 | chan->id = i; |
Lars-Peter Clausen | 7c169a4 | 2013-05-30 18:25:02 +0200 | [diff] [blame] | 554 | chan->vchan.desc_free = jz4740_dma_desc_free; |
| 555 | vchan_init(&chan->vchan, dd); |
| 556 | } |
| 557 | |
| 558 | ret = dma_async_device_register(dd); |
| 559 | if (ret) |
| 560 | return ret; |
| 561 | |
Lars-Peter Clausen | 25ce6c3 | 2013-05-30 18:25:05 +0200 | [diff] [blame] | 562 | irq = platform_get_irq(pdev, 0); |
| 563 | ret = request_irq(irq, jz4740_dma_irq, 0, dev_name(&pdev->dev), dmadev); |
| 564 | if (ret) |
| 565 | goto err_unregister; |
| 566 | |
Lars-Peter Clausen | 7c169a4 | 2013-05-30 18:25:02 +0200 | [diff] [blame] | 567 | platform_set_drvdata(pdev, dmadev); |
| 568 | |
| 569 | return 0; |
Lars-Peter Clausen | 25ce6c3 | 2013-05-30 18:25:05 +0200 | [diff] [blame] | 570 | |
| 571 | err_unregister: |
| 572 | dma_async_device_unregister(dd); |
| 573 | return ret; |
Lars-Peter Clausen | 7c169a4 | 2013-05-30 18:25:02 +0200 | [diff] [blame] | 574 | } |
| 575 | |
Vinod Koul | cec9cfa | 2016-07-02 14:48:02 +0530 | [diff] [blame] | 576 | static void jz4740_cleanup_vchan(struct dma_device *dmadev) |
| 577 | { |
| 578 | struct jz4740_dmaengine_chan *chan, *_chan; |
| 579 | |
| 580 | list_for_each_entry_safe(chan, _chan, |
| 581 | &dmadev->channels, vchan.chan.device_node) { |
| 582 | list_del(&chan->vchan.chan.device_node); |
| 583 | tasklet_kill(&chan->vchan.task); |
| 584 | } |
| 585 | } |
| 586 | |
| 587 | |
Lars-Peter Clausen | 7c169a4 | 2013-05-30 18:25:02 +0200 | [diff] [blame] | 588 | static int jz4740_dma_remove(struct platform_device *pdev) |
| 589 | { |
| 590 | struct jz4740_dma_dev *dmadev = platform_get_drvdata(pdev); |
Lars-Peter Clausen | 25ce6c3 | 2013-05-30 18:25:05 +0200 | [diff] [blame] | 591 | int irq = platform_get_irq(pdev, 0); |
Lars-Peter Clausen | 7c169a4 | 2013-05-30 18:25:02 +0200 | [diff] [blame] | 592 | |
Lars-Peter Clausen | 25ce6c3 | 2013-05-30 18:25:05 +0200 | [diff] [blame] | 593 | free_irq(irq, dmadev); |
Vinod Koul | cec9cfa | 2016-07-02 14:48:02 +0530 | [diff] [blame] | 594 | |
| 595 | jz4740_cleanup_vchan(&dmadev->ddev); |
Lars-Peter Clausen | 7c169a4 | 2013-05-30 18:25:02 +0200 | [diff] [blame] | 596 | dma_async_device_unregister(&dmadev->ddev); |
Lars-Peter Clausen | 25ce6c3 | 2013-05-30 18:25:05 +0200 | [diff] [blame] | 597 | clk_disable_unprepare(dmadev->clk); |
Lars-Peter Clausen | 7c169a4 | 2013-05-30 18:25:02 +0200 | [diff] [blame] | 598 | |
| 599 | return 0; |
| 600 | } |
| 601 | |
| 602 | static struct platform_driver jz4740_dma_driver = { |
| 603 | .probe = jz4740_dma_probe, |
| 604 | .remove = jz4740_dma_remove, |
| 605 | .driver = { |
| 606 | .name = "jz4740-dma", |
Lars-Peter Clausen | 7c169a4 | 2013-05-30 18:25:02 +0200 | [diff] [blame] | 607 | }, |
| 608 | }; |
| 609 | module_platform_driver(jz4740_dma_driver); |
| 610 | |
| 611 | MODULE_AUTHOR("Lars-Peter Clausen <lars@metafoo.de>"); |
| 612 | MODULE_DESCRIPTION("JZ4740 DMA driver"); |
Bjorn Helgaas | e2f9922 | 2014-07-15 17:24:38 -0600 | [diff] [blame] | 613 | MODULE_LICENSE("GPL v2"); |