Zhangfei Gao | 8e6152b | 2013-08-27 10:20:10 +0800 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (c) 2013 Linaro Ltd. |
| 3 | * Copyright (c) 2013 Hisilicon Limited. |
| 4 | * |
| 5 | * This program is free software; you can redistribute it and/or modify |
| 6 | * it under the terms of the GNU General Public License version 2 as |
| 7 | * published by the Free Software Foundation. |
| 8 | */ |
| 9 | #include <linux/sched.h> |
| 10 | #include <linux/device.h> |
| 11 | #include <linux/dmaengine.h> |
| 12 | #include <linux/init.h> |
| 13 | #include <linux/interrupt.h> |
| 14 | #include <linux/kernel.h> |
| 15 | #include <linux/module.h> |
| 16 | #include <linux/platform_device.h> |
| 17 | #include <linux/slab.h> |
| 18 | #include <linux/spinlock.h> |
| 19 | #include <linux/of_device.h> |
| 20 | #include <linux/of.h> |
| 21 | #include <linux/clk.h> |
| 22 | #include <linux/of_dma.h> |
| 23 | |
| 24 | #include "virt-dma.h" |
| 25 | |
| 26 | #define DRIVER_NAME "k3-dma" |
| 27 | #define DMA_ALIGN 3 |
| 28 | #define DMA_MAX_SIZE 0x1ffc |
| 29 | |
| 30 | #define INT_STAT 0x00 |
| 31 | #define INT_TC1 0x04 |
| 32 | #define INT_ERR1 0x0c |
| 33 | #define INT_ERR2 0x10 |
| 34 | #define INT_TC1_MASK 0x18 |
| 35 | #define INT_ERR1_MASK 0x20 |
| 36 | #define INT_ERR2_MASK 0x24 |
| 37 | #define INT_TC1_RAW 0x600 |
| 38 | #define INT_ERR1_RAW 0x608 |
| 39 | #define INT_ERR2_RAW 0x610 |
| 40 | #define CH_PRI 0x688 |
| 41 | #define CH_STAT 0x690 |
| 42 | #define CX_CUR_CNT 0x704 |
| 43 | #define CX_LLI 0x800 |
| 44 | #define CX_CNT 0x810 |
| 45 | #define CX_SRC 0x814 |
| 46 | #define CX_DST 0x818 |
| 47 | #define CX_CFG 0x81c |
| 48 | #define AXI_CFG 0x820 |
| 49 | #define AXI_CFG_DEFAULT 0x201201 |
| 50 | |
| 51 | #define CX_LLI_CHAIN_EN 0x2 |
| 52 | #define CX_CFG_EN 0x1 |
| 53 | #define CX_CFG_MEM2PER (0x1 << 2) |
| 54 | #define CX_CFG_PER2MEM (0x2 << 2) |
| 55 | #define CX_CFG_SRCINCR (0x1 << 31) |
| 56 | #define CX_CFG_DSTINCR (0x1 << 30) |
| 57 | |
| 58 | struct k3_desc_hw { |
| 59 | u32 lli; |
| 60 | u32 reserved[3]; |
| 61 | u32 count; |
| 62 | u32 saddr; |
| 63 | u32 daddr; |
| 64 | u32 config; |
| 65 | } __aligned(32); |
| 66 | |
| 67 | struct k3_dma_desc_sw { |
| 68 | struct virt_dma_desc vd; |
| 69 | dma_addr_t desc_hw_lli; |
| 70 | size_t desc_num; |
| 71 | size_t size; |
| 72 | struct k3_desc_hw desc_hw[0]; |
| 73 | }; |
| 74 | |
| 75 | struct k3_dma_phy; |
| 76 | |
| 77 | struct k3_dma_chan { |
| 78 | u32 ccfg; |
| 79 | struct virt_dma_chan vc; |
| 80 | struct k3_dma_phy *phy; |
| 81 | struct list_head node; |
| 82 | enum dma_transfer_direction dir; |
| 83 | dma_addr_t dev_addr; |
| 84 | enum dma_status status; |
| 85 | }; |
| 86 | |
| 87 | struct k3_dma_phy { |
| 88 | u32 idx; |
| 89 | void __iomem *base; |
| 90 | struct k3_dma_chan *vchan; |
| 91 | struct k3_dma_desc_sw *ds_run; |
| 92 | struct k3_dma_desc_sw *ds_done; |
| 93 | }; |
| 94 | |
| 95 | struct k3_dma_dev { |
| 96 | struct dma_device slave; |
| 97 | void __iomem *base; |
| 98 | struct tasklet_struct task; |
| 99 | spinlock_t lock; |
| 100 | struct list_head chan_pending; |
| 101 | struct k3_dma_phy *phy; |
| 102 | struct k3_dma_chan *chans; |
| 103 | struct clk *clk; |
| 104 | u32 dma_channels; |
| 105 | u32 dma_requests; |
| 106 | }; |
| 107 | |
| 108 | #define to_k3_dma(dmadev) container_of(dmadev, struct k3_dma_dev, slave) |
| 109 | |
| 110 | static struct k3_dma_chan *to_k3_chan(struct dma_chan *chan) |
| 111 | { |
| 112 | return container_of(chan, struct k3_dma_chan, vc.chan); |
| 113 | } |
| 114 | |
| 115 | static void k3_dma_pause_dma(struct k3_dma_phy *phy, bool on) |
| 116 | { |
| 117 | u32 val = 0; |
| 118 | |
| 119 | if (on) { |
| 120 | val = readl_relaxed(phy->base + CX_CFG); |
| 121 | val |= CX_CFG_EN; |
| 122 | writel_relaxed(val, phy->base + CX_CFG); |
| 123 | } else { |
| 124 | val = readl_relaxed(phy->base + CX_CFG); |
| 125 | val &= ~CX_CFG_EN; |
| 126 | writel_relaxed(val, phy->base + CX_CFG); |
| 127 | } |
| 128 | } |
| 129 | |
| 130 | static void k3_dma_terminate_chan(struct k3_dma_phy *phy, struct k3_dma_dev *d) |
| 131 | { |
| 132 | u32 val = 0; |
| 133 | |
| 134 | k3_dma_pause_dma(phy, false); |
| 135 | |
| 136 | val = 0x1 << phy->idx; |
| 137 | writel_relaxed(val, d->base + INT_TC1_RAW); |
| 138 | writel_relaxed(val, d->base + INT_ERR1_RAW); |
| 139 | writel_relaxed(val, d->base + INT_ERR2_RAW); |
| 140 | } |
| 141 | |
| 142 | static void k3_dma_set_desc(struct k3_dma_phy *phy, struct k3_desc_hw *hw) |
| 143 | { |
| 144 | writel_relaxed(hw->lli, phy->base + CX_LLI); |
| 145 | writel_relaxed(hw->count, phy->base + CX_CNT); |
| 146 | writel_relaxed(hw->saddr, phy->base + CX_SRC); |
| 147 | writel_relaxed(hw->daddr, phy->base + CX_DST); |
| 148 | writel_relaxed(AXI_CFG_DEFAULT, phy->base + AXI_CFG); |
| 149 | writel_relaxed(hw->config, phy->base + CX_CFG); |
| 150 | } |
| 151 | |
| 152 | static u32 k3_dma_get_curr_cnt(struct k3_dma_dev *d, struct k3_dma_phy *phy) |
| 153 | { |
| 154 | u32 cnt = 0; |
| 155 | |
| 156 | cnt = readl_relaxed(d->base + CX_CUR_CNT + phy->idx * 0x10); |
| 157 | cnt &= 0xffff; |
| 158 | return cnt; |
| 159 | } |
| 160 | |
| 161 | static u32 k3_dma_get_curr_lli(struct k3_dma_phy *phy) |
| 162 | { |
| 163 | return readl_relaxed(phy->base + CX_LLI); |
| 164 | } |
| 165 | |
| 166 | static u32 k3_dma_get_chan_stat(struct k3_dma_dev *d) |
| 167 | { |
| 168 | return readl_relaxed(d->base + CH_STAT); |
| 169 | } |
| 170 | |
| 171 | static void k3_dma_enable_dma(struct k3_dma_dev *d, bool on) |
| 172 | { |
| 173 | if (on) { |
| 174 | /* set same priority */ |
| 175 | writel_relaxed(0x0, d->base + CH_PRI); |
| 176 | |
| 177 | /* unmask irq */ |
| 178 | writel_relaxed(0xffff, d->base + INT_TC1_MASK); |
| 179 | writel_relaxed(0xffff, d->base + INT_ERR1_MASK); |
| 180 | writel_relaxed(0xffff, d->base + INT_ERR2_MASK); |
| 181 | } else { |
| 182 | /* mask irq */ |
| 183 | writel_relaxed(0x0, d->base + INT_TC1_MASK); |
| 184 | writel_relaxed(0x0, d->base + INT_ERR1_MASK); |
| 185 | writel_relaxed(0x0, d->base + INT_ERR2_MASK); |
| 186 | } |
| 187 | } |
| 188 | |
| 189 | static irqreturn_t k3_dma_int_handler(int irq, void *dev_id) |
| 190 | { |
| 191 | struct k3_dma_dev *d = (struct k3_dma_dev *)dev_id; |
| 192 | struct k3_dma_phy *p; |
| 193 | struct k3_dma_chan *c; |
| 194 | u32 stat = readl_relaxed(d->base + INT_STAT); |
| 195 | u32 tc1 = readl_relaxed(d->base + INT_TC1); |
| 196 | u32 err1 = readl_relaxed(d->base + INT_ERR1); |
| 197 | u32 err2 = readl_relaxed(d->base + INT_ERR2); |
| 198 | u32 i, irq_chan = 0; |
| 199 | |
| 200 | while (stat) { |
| 201 | i = __ffs(stat); |
| 202 | stat &= (stat - 1); |
| 203 | if (likely(tc1 & BIT(i))) { |
| 204 | p = &d->phy[i]; |
| 205 | c = p->vchan; |
| 206 | if (c) { |
| 207 | unsigned long flags; |
| 208 | |
| 209 | spin_lock_irqsave(&c->vc.lock, flags); |
| 210 | vchan_cookie_complete(&p->ds_run->vd); |
| 211 | p->ds_done = p->ds_run; |
| 212 | spin_unlock_irqrestore(&c->vc.lock, flags); |
| 213 | } |
| 214 | irq_chan |= BIT(i); |
| 215 | } |
| 216 | if (unlikely((err1 & BIT(i)) || (err2 & BIT(i)))) |
| 217 | dev_warn(d->slave.dev, "DMA ERR\n"); |
| 218 | } |
| 219 | |
| 220 | writel_relaxed(irq_chan, d->base + INT_TC1_RAW); |
| 221 | writel_relaxed(err1, d->base + INT_ERR1_RAW); |
| 222 | writel_relaxed(err2, d->base + INT_ERR2_RAW); |
| 223 | |
| 224 | if (irq_chan) { |
| 225 | tasklet_schedule(&d->task); |
| 226 | return IRQ_HANDLED; |
| 227 | } else |
| 228 | return IRQ_NONE; |
| 229 | } |
| 230 | |
| 231 | static int k3_dma_start_txd(struct k3_dma_chan *c) |
| 232 | { |
| 233 | struct k3_dma_dev *d = to_k3_dma(c->vc.chan.device); |
| 234 | struct virt_dma_desc *vd = vchan_next_desc(&c->vc); |
| 235 | |
| 236 | if (!c->phy) |
| 237 | return -EAGAIN; |
| 238 | |
| 239 | if (BIT(c->phy->idx) & k3_dma_get_chan_stat(d)) |
| 240 | return -EAGAIN; |
| 241 | |
| 242 | if (vd) { |
| 243 | struct k3_dma_desc_sw *ds = |
| 244 | container_of(vd, struct k3_dma_desc_sw, vd); |
| 245 | /* |
| 246 | * fetch and remove request from vc->desc_issued |
| 247 | * so vc->desc_issued only contains desc pending |
| 248 | */ |
| 249 | list_del(&ds->vd.node); |
| 250 | c->phy->ds_run = ds; |
| 251 | c->phy->ds_done = NULL; |
| 252 | /* start dma */ |
| 253 | k3_dma_set_desc(c->phy, &ds->desc_hw[0]); |
| 254 | return 0; |
| 255 | } |
| 256 | c->phy->ds_done = NULL; |
| 257 | c->phy->ds_run = NULL; |
| 258 | return -EAGAIN; |
| 259 | } |
| 260 | |
| 261 | static void k3_dma_tasklet(unsigned long arg) |
| 262 | { |
| 263 | struct k3_dma_dev *d = (struct k3_dma_dev *)arg; |
| 264 | struct k3_dma_phy *p; |
| 265 | struct k3_dma_chan *c, *cn; |
| 266 | unsigned pch, pch_alloc = 0; |
| 267 | |
| 268 | /* check new dma request of running channel in vc->desc_issued */ |
| 269 | list_for_each_entry_safe(c, cn, &d->slave.channels, vc.chan.device_node) { |
| 270 | spin_lock_irq(&c->vc.lock); |
| 271 | p = c->phy; |
| 272 | if (p && p->ds_done) { |
| 273 | if (k3_dma_start_txd(c)) { |
| 274 | /* No current txd associated with this channel */ |
| 275 | dev_dbg(d->slave.dev, "pchan %u: free\n", p->idx); |
| 276 | /* Mark this channel free */ |
| 277 | c->phy = NULL; |
| 278 | p->vchan = NULL; |
| 279 | } |
| 280 | } |
| 281 | spin_unlock_irq(&c->vc.lock); |
| 282 | } |
| 283 | |
| 284 | /* check new channel request in d->chan_pending */ |
| 285 | spin_lock_irq(&d->lock); |
| 286 | for (pch = 0; pch < d->dma_channels; pch++) { |
| 287 | p = &d->phy[pch]; |
| 288 | |
| 289 | if (p->vchan == NULL && !list_empty(&d->chan_pending)) { |
| 290 | c = list_first_entry(&d->chan_pending, |
| 291 | struct k3_dma_chan, node); |
| 292 | /* remove from d->chan_pending */ |
| 293 | list_del_init(&c->node); |
| 294 | pch_alloc |= 1 << pch; |
| 295 | /* Mark this channel allocated */ |
| 296 | p->vchan = c; |
| 297 | c->phy = p; |
| 298 | dev_dbg(d->slave.dev, "pchan %u: alloc vchan %p\n", pch, &c->vc); |
| 299 | } |
| 300 | } |
| 301 | spin_unlock_irq(&d->lock); |
| 302 | |
| 303 | for (pch = 0; pch < d->dma_channels; pch++) { |
| 304 | if (pch_alloc & (1 << pch)) { |
| 305 | p = &d->phy[pch]; |
| 306 | c = p->vchan; |
| 307 | if (c) { |
| 308 | spin_lock_irq(&c->vc.lock); |
| 309 | k3_dma_start_txd(c); |
| 310 | spin_unlock_irq(&c->vc.lock); |
| 311 | } |
| 312 | } |
| 313 | } |
| 314 | } |
| 315 | |
| 316 | static int k3_dma_alloc_chan_resources(struct dma_chan *chan) |
| 317 | { |
| 318 | return 0; |
| 319 | } |
| 320 | |
| 321 | static void k3_dma_free_chan_resources(struct dma_chan *chan) |
| 322 | { |
| 323 | struct k3_dma_chan *c = to_k3_chan(chan); |
| 324 | struct k3_dma_dev *d = to_k3_dma(chan->device); |
| 325 | unsigned long flags; |
| 326 | |
| 327 | spin_lock_irqsave(&d->lock, flags); |
| 328 | list_del_init(&c->node); |
| 329 | spin_unlock_irqrestore(&d->lock, flags); |
| 330 | |
| 331 | vchan_free_chan_resources(&c->vc); |
| 332 | c->ccfg = 0; |
| 333 | } |
| 334 | |
| 335 | static enum dma_status k3_dma_tx_status(struct dma_chan *chan, |
| 336 | dma_cookie_t cookie, struct dma_tx_state *state) |
| 337 | { |
| 338 | struct k3_dma_chan *c = to_k3_chan(chan); |
| 339 | struct k3_dma_dev *d = to_k3_dma(chan->device); |
| 340 | struct k3_dma_phy *p; |
| 341 | struct virt_dma_desc *vd; |
| 342 | unsigned long flags; |
| 343 | enum dma_status ret; |
| 344 | size_t bytes = 0; |
| 345 | |
| 346 | ret = dma_cookie_status(&c->vc.chan, cookie, state); |
Vinod Koul | bd2c348 | 2013-10-16 20:50:09 +0530 | [diff] [blame] | 347 | if (ret == DMA_COMPLETE) |
Zhangfei Gao | 8e6152b | 2013-08-27 10:20:10 +0800 | [diff] [blame] | 348 | return ret; |
| 349 | |
| 350 | spin_lock_irqsave(&c->vc.lock, flags); |
| 351 | p = c->phy; |
| 352 | ret = c->status; |
| 353 | |
| 354 | /* |
| 355 | * If the cookie is on our issue queue, then the residue is |
| 356 | * its total size. |
| 357 | */ |
| 358 | vd = vchan_find_desc(&c->vc, cookie); |
| 359 | if (vd) { |
| 360 | bytes = container_of(vd, struct k3_dma_desc_sw, vd)->size; |
| 361 | } else if ((!p) || (!p->ds_run)) { |
| 362 | bytes = 0; |
| 363 | } else { |
| 364 | struct k3_dma_desc_sw *ds = p->ds_run; |
| 365 | u32 clli = 0, index = 0; |
| 366 | |
| 367 | bytes = k3_dma_get_curr_cnt(d, p); |
| 368 | clli = k3_dma_get_curr_lli(p); |
| 369 | index = (clli - ds->desc_hw_lli) / sizeof(struct k3_desc_hw); |
| 370 | for (; index < ds->desc_num; index++) { |
| 371 | bytes += ds->desc_hw[index].count; |
| 372 | /* end of lli */ |
| 373 | if (!ds->desc_hw[index].lli) |
| 374 | break; |
| 375 | } |
| 376 | } |
| 377 | spin_unlock_irqrestore(&c->vc.lock, flags); |
| 378 | dma_set_residue(state, bytes); |
| 379 | return ret; |
| 380 | } |
| 381 | |
| 382 | static void k3_dma_issue_pending(struct dma_chan *chan) |
| 383 | { |
| 384 | struct k3_dma_chan *c = to_k3_chan(chan); |
| 385 | struct k3_dma_dev *d = to_k3_dma(chan->device); |
| 386 | unsigned long flags; |
| 387 | |
| 388 | spin_lock_irqsave(&c->vc.lock, flags); |
| 389 | /* add request to vc->desc_issued */ |
| 390 | if (vchan_issue_pending(&c->vc)) { |
| 391 | spin_lock(&d->lock); |
| 392 | if (!c->phy) { |
| 393 | if (list_empty(&c->node)) { |
| 394 | /* if new channel, add chan_pending */ |
| 395 | list_add_tail(&c->node, &d->chan_pending); |
| 396 | /* check in tasklet */ |
| 397 | tasklet_schedule(&d->task); |
| 398 | dev_dbg(d->slave.dev, "vchan %p: issued\n", &c->vc); |
| 399 | } |
| 400 | } |
| 401 | spin_unlock(&d->lock); |
| 402 | } else |
| 403 | dev_dbg(d->slave.dev, "vchan %p: nothing to issue\n", &c->vc); |
| 404 | spin_unlock_irqrestore(&c->vc.lock, flags); |
| 405 | } |
| 406 | |
| 407 | static void k3_dma_fill_desc(struct k3_dma_desc_sw *ds, dma_addr_t dst, |
| 408 | dma_addr_t src, size_t len, u32 num, u32 ccfg) |
| 409 | { |
| 410 | if ((num + 1) < ds->desc_num) |
| 411 | ds->desc_hw[num].lli = ds->desc_hw_lli + (num + 1) * |
| 412 | sizeof(struct k3_desc_hw); |
| 413 | ds->desc_hw[num].lli |= CX_LLI_CHAIN_EN; |
| 414 | ds->desc_hw[num].count = len; |
| 415 | ds->desc_hw[num].saddr = src; |
| 416 | ds->desc_hw[num].daddr = dst; |
| 417 | ds->desc_hw[num].config = ccfg; |
| 418 | } |
| 419 | |
| 420 | static struct dma_async_tx_descriptor *k3_dma_prep_memcpy( |
| 421 | struct dma_chan *chan, dma_addr_t dst, dma_addr_t src, |
| 422 | size_t len, unsigned long flags) |
| 423 | { |
| 424 | struct k3_dma_chan *c = to_k3_chan(chan); |
| 425 | struct k3_dma_desc_sw *ds; |
| 426 | size_t copy = 0; |
| 427 | int num = 0; |
| 428 | |
| 429 | if (!len) |
| 430 | return NULL; |
| 431 | |
| 432 | num = DIV_ROUND_UP(len, DMA_MAX_SIZE); |
| 433 | ds = kzalloc(sizeof(*ds) + num * sizeof(ds->desc_hw[0]), GFP_ATOMIC); |
| 434 | if (!ds) { |
| 435 | dev_dbg(chan->device->dev, "vchan %p: kzalloc fail\n", &c->vc); |
| 436 | return NULL; |
| 437 | } |
| 438 | ds->desc_hw_lli = __virt_to_phys((unsigned long)&ds->desc_hw[0]); |
| 439 | ds->size = len; |
| 440 | ds->desc_num = num; |
| 441 | num = 0; |
| 442 | |
| 443 | if (!c->ccfg) { |
| 444 | /* default is memtomem, without calling device_control */ |
| 445 | c->ccfg = CX_CFG_SRCINCR | CX_CFG_DSTINCR | CX_CFG_EN; |
| 446 | c->ccfg |= (0xf << 20) | (0xf << 24); /* burst = 16 */ |
| 447 | c->ccfg |= (0x3 << 12) | (0x3 << 16); /* width = 64 bit */ |
| 448 | } |
| 449 | |
| 450 | do { |
| 451 | copy = min_t(size_t, len, DMA_MAX_SIZE); |
| 452 | k3_dma_fill_desc(ds, dst, src, copy, num++, c->ccfg); |
| 453 | |
| 454 | if (c->dir == DMA_MEM_TO_DEV) { |
| 455 | src += copy; |
| 456 | } else if (c->dir == DMA_DEV_TO_MEM) { |
| 457 | dst += copy; |
| 458 | } else { |
| 459 | src += copy; |
| 460 | dst += copy; |
| 461 | } |
| 462 | len -= copy; |
| 463 | } while (len); |
| 464 | |
| 465 | ds->desc_hw[num-1].lli = 0; /* end of link */ |
| 466 | return vchan_tx_prep(&c->vc, &ds->vd, flags); |
| 467 | } |
| 468 | |
| 469 | static struct dma_async_tx_descriptor *k3_dma_prep_slave_sg( |
| 470 | struct dma_chan *chan, struct scatterlist *sgl, unsigned int sglen, |
| 471 | enum dma_transfer_direction dir, unsigned long flags, void *context) |
| 472 | { |
| 473 | struct k3_dma_chan *c = to_k3_chan(chan); |
| 474 | struct k3_dma_desc_sw *ds; |
| 475 | size_t len, avail, total = 0; |
| 476 | struct scatterlist *sg; |
| 477 | dma_addr_t addr, src = 0, dst = 0; |
| 478 | int num = sglen, i; |
| 479 | |
Zhangfei Gao | c61177c | 2014-01-14 11:37:43 +0800 | [diff] [blame] | 480 | if (sgl == NULL) |
Zhangfei Gao | 8e6152b | 2013-08-27 10:20:10 +0800 | [diff] [blame] | 481 | return NULL; |
| 482 | |
| 483 | for_each_sg(sgl, sg, sglen, i) { |
| 484 | avail = sg_dma_len(sg); |
| 485 | if (avail > DMA_MAX_SIZE) |
| 486 | num += DIV_ROUND_UP(avail, DMA_MAX_SIZE) - 1; |
| 487 | } |
| 488 | |
| 489 | ds = kzalloc(sizeof(*ds) + num * sizeof(ds->desc_hw[0]), GFP_ATOMIC); |
| 490 | if (!ds) { |
| 491 | dev_dbg(chan->device->dev, "vchan %p: kzalloc fail\n", &c->vc); |
| 492 | return NULL; |
| 493 | } |
| 494 | ds->desc_hw_lli = __virt_to_phys((unsigned long)&ds->desc_hw[0]); |
| 495 | ds->desc_num = num; |
| 496 | num = 0; |
| 497 | |
| 498 | for_each_sg(sgl, sg, sglen, i) { |
| 499 | addr = sg_dma_address(sg); |
| 500 | avail = sg_dma_len(sg); |
| 501 | total += avail; |
| 502 | |
| 503 | do { |
| 504 | len = min_t(size_t, avail, DMA_MAX_SIZE); |
| 505 | |
| 506 | if (dir == DMA_MEM_TO_DEV) { |
| 507 | src = addr; |
| 508 | dst = c->dev_addr; |
| 509 | } else if (dir == DMA_DEV_TO_MEM) { |
| 510 | src = c->dev_addr; |
| 511 | dst = addr; |
| 512 | } |
| 513 | |
| 514 | k3_dma_fill_desc(ds, dst, src, len, num++, c->ccfg); |
| 515 | |
| 516 | addr += len; |
| 517 | avail -= len; |
| 518 | } while (avail); |
| 519 | } |
| 520 | |
| 521 | ds->desc_hw[num-1].lli = 0; /* end of link */ |
| 522 | ds->size = total; |
| 523 | return vchan_tx_prep(&c->vc, &ds->vd, flags); |
| 524 | } |
| 525 | |
| 526 | static int k3_dma_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd, |
| 527 | unsigned long arg) |
| 528 | { |
| 529 | struct k3_dma_chan *c = to_k3_chan(chan); |
| 530 | struct k3_dma_dev *d = to_k3_dma(chan->device); |
| 531 | struct dma_slave_config *cfg = (void *)arg; |
| 532 | struct k3_dma_phy *p = c->phy; |
| 533 | unsigned long flags; |
| 534 | u32 maxburst = 0, val = 0; |
| 535 | enum dma_slave_buswidth width = DMA_SLAVE_BUSWIDTH_UNDEFINED; |
| 536 | LIST_HEAD(head); |
| 537 | |
| 538 | switch (cmd) { |
| 539 | case DMA_SLAVE_CONFIG: |
| 540 | if (cfg == NULL) |
| 541 | return -EINVAL; |
| 542 | c->dir = cfg->direction; |
| 543 | if (c->dir == DMA_DEV_TO_MEM) { |
| 544 | c->ccfg = CX_CFG_DSTINCR; |
| 545 | c->dev_addr = cfg->src_addr; |
| 546 | maxburst = cfg->src_maxburst; |
| 547 | width = cfg->src_addr_width; |
| 548 | } else if (c->dir == DMA_MEM_TO_DEV) { |
| 549 | c->ccfg = CX_CFG_SRCINCR; |
| 550 | c->dev_addr = cfg->dst_addr; |
| 551 | maxburst = cfg->dst_maxburst; |
| 552 | width = cfg->dst_addr_width; |
| 553 | } |
| 554 | switch (width) { |
| 555 | case DMA_SLAVE_BUSWIDTH_1_BYTE: |
| 556 | case DMA_SLAVE_BUSWIDTH_2_BYTES: |
| 557 | case DMA_SLAVE_BUSWIDTH_4_BYTES: |
| 558 | case DMA_SLAVE_BUSWIDTH_8_BYTES: |
| 559 | val = __ffs(width); |
| 560 | break; |
| 561 | default: |
| 562 | val = 3; |
| 563 | break; |
| 564 | } |
| 565 | c->ccfg |= (val << 12) | (val << 16); |
| 566 | |
| 567 | if ((maxburst == 0) || (maxburst > 16)) |
| 568 | val = 16; |
| 569 | else |
| 570 | val = maxburst - 1; |
| 571 | c->ccfg |= (val << 20) | (val << 24); |
| 572 | c->ccfg |= CX_CFG_MEM2PER | CX_CFG_EN; |
| 573 | |
| 574 | /* specific request line */ |
| 575 | c->ccfg |= c->vc.chan.chan_id << 4; |
| 576 | break; |
| 577 | |
| 578 | case DMA_TERMINATE_ALL: |
| 579 | dev_dbg(d->slave.dev, "vchan %p: terminate all\n", &c->vc); |
| 580 | |
| 581 | /* Prevent this channel being scheduled */ |
| 582 | spin_lock(&d->lock); |
| 583 | list_del_init(&c->node); |
| 584 | spin_unlock(&d->lock); |
| 585 | |
| 586 | /* Clear the tx descriptor lists */ |
| 587 | spin_lock_irqsave(&c->vc.lock, flags); |
| 588 | vchan_get_all_descriptors(&c->vc, &head); |
| 589 | if (p) { |
| 590 | /* vchan is assigned to a pchan - stop the channel */ |
| 591 | k3_dma_terminate_chan(p, d); |
| 592 | c->phy = NULL; |
| 593 | p->vchan = NULL; |
| 594 | p->ds_run = p->ds_done = NULL; |
| 595 | } |
| 596 | spin_unlock_irqrestore(&c->vc.lock, flags); |
| 597 | vchan_dma_desc_free_list(&c->vc, &head); |
| 598 | break; |
| 599 | |
| 600 | case DMA_PAUSE: |
| 601 | dev_dbg(d->slave.dev, "vchan %p: pause\n", &c->vc); |
| 602 | if (c->status == DMA_IN_PROGRESS) { |
| 603 | c->status = DMA_PAUSED; |
| 604 | if (p) { |
| 605 | k3_dma_pause_dma(p, false); |
| 606 | } else { |
| 607 | spin_lock(&d->lock); |
| 608 | list_del_init(&c->node); |
| 609 | spin_unlock(&d->lock); |
| 610 | } |
| 611 | } |
| 612 | break; |
| 613 | |
| 614 | case DMA_RESUME: |
| 615 | dev_dbg(d->slave.dev, "vchan %p: resume\n", &c->vc); |
| 616 | spin_lock_irqsave(&c->vc.lock, flags); |
| 617 | if (c->status == DMA_PAUSED) { |
| 618 | c->status = DMA_IN_PROGRESS; |
| 619 | if (p) { |
| 620 | k3_dma_pause_dma(p, true); |
| 621 | } else if (!list_empty(&c->vc.desc_issued)) { |
| 622 | spin_lock(&d->lock); |
| 623 | list_add_tail(&c->node, &d->chan_pending); |
| 624 | spin_unlock(&d->lock); |
| 625 | } |
| 626 | } |
| 627 | spin_unlock_irqrestore(&c->vc.lock, flags); |
| 628 | break; |
| 629 | default: |
| 630 | return -ENXIO; |
| 631 | } |
| 632 | return 0; |
| 633 | } |
| 634 | |
| 635 | static void k3_dma_free_desc(struct virt_dma_desc *vd) |
| 636 | { |
| 637 | struct k3_dma_desc_sw *ds = |
| 638 | container_of(vd, struct k3_dma_desc_sw, vd); |
| 639 | |
| 640 | kfree(ds); |
| 641 | } |
| 642 | |
| 643 | static struct of_device_id k3_pdma_dt_ids[] = { |
| 644 | { .compatible = "hisilicon,k3-dma-1.0", }, |
| 645 | {} |
| 646 | }; |
| 647 | MODULE_DEVICE_TABLE(of, k3_pdma_dt_ids); |
| 648 | |
| 649 | static struct dma_chan *k3_of_dma_simple_xlate(struct of_phandle_args *dma_spec, |
| 650 | struct of_dma *ofdma) |
| 651 | { |
| 652 | struct k3_dma_dev *d = ofdma->of_dma_data; |
| 653 | unsigned int request = dma_spec->args[0]; |
| 654 | |
| 655 | if (request > d->dma_requests) |
| 656 | return NULL; |
| 657 | |
| 658 | return dma_get_slave_channel(&(d->chans[request].vc.chan)); |
| 659 | } |
| 660 | |
| 661 | static int k3_dma_probe(struct platform_device *op) |
| 662 | { |
| 663 | struct k3_dma_dev *d; |
| 664 | const struct of_device_id *of_id; |
| 665 | struct resource *iores; |
| 666 | int i, ret, irq = 0; |
| 667 | |
| 668 | iores = platform_get_resource(op, IORESOURCE_MEM, 0); |
| 669 | if (!iores) |
| 670 | return -EINVAL; |
| 671 | |
| 672 | d = devm_kzalloc(&op->dev, sizeof(*d), GFP_KERNEL); |
| 673 | if (!d) |
| 674 | return -ENOMEM; |
| 675 | |
Jingoo Han | a576b7f | 2013-09-02 10:25:13 +0900 | [diff] [blame] | 676 | d->base = devm_ioremap_resource(&op->dev, iores); |
| 677 | if (IS_ERR(d->base)) |
| 678 | return PTR_ERR(d->base); |
Zhangfei Gao | 8e6152b | 2013-08-27 10:20:10 +0800 | [diff] [blame] | 679 | |
| 680 | of_id = of_match_device(k3_pdma_dt_ids, &op->dev); |
| 681 | if (of_id) { |
| 682 | of_property_read_u32((&op->dev)->of_node, |
| 683 | "dma-channels", &d->dma_channels); |
| 684 | of_property_read_u32((&op->dev)->of_node, |
| 685 | "dma-requests", &d->dma_requests); |
| 686 | } |
| 687 | |
| 688 | d->clk = devm_clk_get(&op->dev, NULL); |
| 689 | if (IS_ERR(d->clk)) { |
| 690 | dev_err(&op->dev, "no dma clk\n"); |
| 691 | return PTR_ERR(d->clk); |
| 692 | } |
| 693 | |
| 694 | irq = platform_get_irq(op, 0); |
| 695 | ret = devm_request_irq(&op->dev, irq, |
Michael Opdenacker | 174b537 | 2013-10-13 07:10:51 +0200 | [diff] [blame] | 696 | k3_dma_int_handler, 0, DRIVER_NAME, d); |
Zhangfei Gao | 8e6152b | 2013-08-27 10:20:10 +0800 | [diff] [blame] | 697 | if (ret) |
| 698 | return ret; |
| 699 | |
| 700 | /* init phy channel */ |
| 701 | d->phy = devm_kzalloc(&op->dev, |
| 702 | d->dma_channels * sizeof(struct k3_dma_phy), GFP_KERNEL); |
| 703 | if (d->phy == NULL) |
| 704 | return -ENOMEM; |
| 705 | |
| 706 | for (i = 0; i < d->dma_channels; i++) { |
| 707 | struct k3_dma_phy *p = &d->phy[i]; |
| 708 | |
| 709 | p->idx = i; |
| 710 | p->base = d->base + i * 0x40; |
| 711 | } |
| 712 | |
| 713 | INIT_LIST_HEAD(&d->slave.channels); |
| 714 | dma_cap_set(DMA_SLAVE, d->slave.cap_mask); |
| 715 | dma_cap_set(DMA_MEMCPY, d->slave.cap_mask); |
| 716 | d->slave.dev = &op->dev; |
| 717 | d->slave.device_alloc_chan_resources = k3_dma_alloc_chan_resources; |
| 718 | d->slave.device_free_chan_resources = k3_dma_free_chan_resources; |
| 719 | d->slave.device_tx_status = k3_dma_tx_status; |
| 720 | d->slave.device_prep_dma_memcpy = k3_dma_prep_memcpy; |
| 721 | d->slave.device_prep_slave_sg = k3_dma_prep_slave_sg; |
| 722 | d->slave.device_issue_pending = k3_dma_issue_pending; |
| 723 | d->slave.device_control = k3_dma_control; |
| 724 | d->slave.copy_align = DMA_ALIGN; |
| 725 | d->slave.chancnt = d->dma_requests; |
| 726 | |
| 727 | /* init virtual channel */ |
| 728 | d->chans = devm_kzalloc(&op->dev, |
| 729 | d->dma_requests * sizeof(struct k3_dma_chan), GFP_KERNEL); |
| 730 | if (d->chans == NULL) |
| 731 | return -ENOMEM; |
| 732 | |
| 733 | for (i = 0; i < d->dma_requests; i++) { |
| 734 | struct k3_dma_chan *c = &d->chans[i]; |
| 735 | |
| 736 | c->status = DMA_IN_PROGRESS; |
| 737 | INIT_LIST_HEAD(&c->node); |
| 738 | c->vc.desc_free = k3_dma_free_desc; |
| 739 | vchan_init(&c->vc, &d->slave); |
| 740 | } |
| 741 | |
| 742 | /* Enable clock before accessing registers */ |
| 743 | ret = clk_prepare_enable(d->clk); |
| 744 | if (ret < 0) { |
| 745 | dev_err(&op->dev, "clk_prepare_enable failed: %d\n", ret); |
| 746 | return ret; |
| 747 | } |
| 748 | |
| 749 | k3_dma_enable_dma(d, true); |
| 750 | |
| 751 | ret = dma_async_device_register(&d->slave); |
| 752 | if (ret) |
| 753 | return ret; |
| 754 | |
| 755 | ret = of_dma_controller_register((&op->dev)->of_node, |
| 756 | k3_of_dma_simple_xlate, d); |
| 757 | if (ret) |
| 758 | goto of_dma_register_fail; |
| 759 | |
| 760 | spin_lock_init(&d->lock); |
| 761 | INIT_LIST_HEAD(&d->chan_pending); |
| 762 | tasklet_init(&d->task, k3_dma_tasklet, (unsigned long)d); |
| 763 | platform_set_drvdata(op, d); |
| 764 | dev_info(&op->dev, "initialized\n"); |
| 765 | |
| 766 | return 0; |
| 767 | |
| 768 | of_dma_register_fail: |
| 769 | dma_async_device_unregister(&d->slave); |
| 770 | return ret; |
| 771 | } |
| 772 | |
| 773 | static int k3_dma_remove(struct platform_device *op) |
| 774 | { |
| 775 | struct k3_dma_chan *c, *cn; |
| 776 | struct k3_dma_dev *d = platform_get_drvdata(op); |
| 777 | |
| 778 | dma_async_device_unregister(&d->slave); |
| 779 | of_dma_controller_free((&op->dev)->of_node); |
| 780 | |
| 781 | list_for_each_entry_safe(c, cn, &d->slave.channels, vc.chan.device_node) { |
| 782 | list_del(&c->vc.chan.device_node); |
| 783 | tasklet_kill(&c->vc.task); |
| 784 | } |
| 785 | tasklet_kill(&d->task); |
| 786 | clk_disable_unprepare(d->clk); |
| 787 | return 0; |
| 788 | } |
| 789 | |
| 790 | static int k3_dma_suspend(struct device *dev) |
| 791 | { |
| 792 | struct k3_dma_dev *d = dev_get_drvdata(dev); |
| 793 | u32 stat = 0; |
| 794 | |
| 795 | stat = k3_dma_get_chan_stat(d); |
| 796 | if (stat) { |
| 797 | dev_warn(d->slave.dev, |
| 798 | "chan %d is running fail to suspend\n", stat); |
| 799 | return -1; |
| 800 | } |
| 801 | k3_dma_enable_dma(d, false); |
| 802 | clk_disable_unprepare(d->clk); |
| 803 | return 0; |
| 804 | } |
| 805 | |
| 806 | static int k3_dma_resume(struct device *dev) |
| 807 | { |
| 808 | struct k3_dma_dev *d = dev_get_drvdata(dev); |
| 809 | int ret = 0; |
| 810 | |
| 811 | ret = clk_prepare_enable(d->clk); |
| 812 | if (ret < 0) { |
| 813 | dev_err(d->slave.dev, "clk_prepare_enable failed: %d\n", ret); |
| 814 | return ret; |
| 815 | } |
| 816 | k3_dma_enable_dma(d, true); |
| 817 | return 0; |
| 818 | } |
| 819 | |
Zhangfei Gao | c61177c | 2014-01-14 11:37:43 +0800 | [diff] [blame] | 820 | static SIMPLE_DEV_PM_OPS(k3_dma_pmops, k3_dma_suspend, k3_dma_resume); |
Zhangfei Gao | 8e6152b | 2013-08-27 10:20:10 +0800 | [diff] [blame] | 821 | |
| 822 | static struct platform_driver k3_pdma_driver = { |
| 823 | .driver = { |
| 824 | .name = DRIVER_NAME, |
| 825 | .owner = THIS_MODULE, |
| 826 | .pm = &k3_dma_pmops, |
| 827 | .of_match_table = k3_pdma_dt_ids, |
| 828 | }, |
| 829 | .probe = k3_dma_probe, |
| 830 | .remove = k3_dma_remove, |
| 831 | }; |
| 832 | |
| 833 | module_platform_driver(k3_pdma_driver); |
| 834 | |
| 835 | MODULE_DESCRIPTION("Hisilicon k3 DMA Driver"); |
| 836 | MODULE_ALIAS("platform:k3dma"); |
| 837 | MODULE_LICENSE("GPL v2"); |