blob: ab434fbd8c35446863052fb7df19607c295b8228 [file] [log] [blame]
Felipe Balbi550a7372008-07-24 12:27:36 +03001/*
2 * Copyright (C) 2005-2006 by Texas Instruments
3 *
4 * This file implements a DMA interface using TI's CPPI DMA.
5 * For now it's DaVinci-only, but CPPI isn't specific to DaVinci or USB.
6 * The TUSB6020, using VLYNQ, has CPPI that looks much like DaVinci.
7 */
8
Sergei Shtylyov91e9c4fe2009-03-27 12:59:46 -07009#include <linux/platform_device.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090010#include <linux/slab.h>
Felipe Balbi550a7372008-07-24 12:27:36 +030011#include <linux/usb.h>
12
13#include "musb_core.h"
Hugo Villeneuve704a1482009-01-24 17:57:30 -080014#include "musb_debug.h"
Felipe Balbi550a7372008-07-24 12:27:36 +030015#include "cppi_dma.h"
16
17
18/* CPPI DMA status 7-mar-2006:
19 *
20 * - See musb_{host,gadget}.c for more info
21 *
22 * - Correct RX DMA generally forces the engine into irq-per-packet mode,
23 * which can easily saturate the CPU under non-mass-storage loads.
24 *
25 * NOTES 24-aug-2006 (2.6.18-rc4):
26 *
27 * - peripheral RXDMA wedged in a test with packets of length 512/512/1.
28 * evidently after the 1 byte packet was received and acked, the queue
29 * of BDs got garbaged so it wouldn't empty the fifo. (rxcsr 0x2003,
30 * and RX DMA0: 4 left, 80000000 8feff880, 8feff860 8feff860; 8f321401
31 * 004001ff 00000001 .. 8feff860) Host was just getting NAKed on tx
32 * of its next (512 byte) packet. IRQ issues?
33 *
34 * REVISIT: the "transfer DMA" glue between CPPI and USB fifos will
35 * evidently also directly update the RX and TX CSRs ... so audit all
36 * host and peripheral side DMA code to avoid CSR access after DMA has
37 * been started.
38 */
39
40/* REVISIT now we can avoid preallocating these descriptors; or
41 * more simply, switch to a global freelist not per-channel ones.
42 * Note: at full speed, 64 descriptors == 4K bulk data.
43 */
44#define NUM_TXCHAN_BD 64
45#define NUM_RXCHAN_BD 64
46
47static inline void cpu_drain_writebuffer(void)
48{
49 wmb();
50#ifdef CONFIG_CPU_ARM926T
51 /* REVISIT this "should not be needed",
52 * but lack of it sure seemed to hurt ...
53 */
54 asm("mcr p15, 0, r0, c7, c10, 4 @ drain write buffer\n");
55#endif
56}
57
58static inline struct cppi_descriptor *cppi_bd_alloc(struct cppi_channel *c)
59{
60 struct cppi_descriptor *bd = c->freelist;
61
62 if (bd)
63 c->freelist = bd->next;
64 return bd;
65}
66
67static inline void
68cppi_bd_free(struct cppi_channel *c, struct cppi_descriptor *bd)
69{
70 if (!bd)
71 return;
72 bd->next = c->freelist;
73 c->freelist = bd;
74}
75
76/*
77 * Start DMA controller
78 *
79 * Initialize the DMA controller as necessary.
80 */
81
82/* zero out entire rx state RAM entry for the channel */
83static void cppi_reset_rx(struct cppi_rx_stateram __iomem *rx)
84{
85 musb_writel(&rx->rx_skipbytes, 0, 0);
86 musb_writel(&rx->rx_head, 0, 0);
87 musb_writel(&rx->rx_sop, 0, 0);
88 musb_writel(&rx->rx_current, 0, 0);
89 musb_writel(&rx->rx_buf_current, 0, 0);
90 musb_writel(&rx->rx_len_len, 0, 0);
91 musb_writel(&rx->rx_cnt_cnt, 0, 0);
92}
93
94/* zero out entire tx state RAM entry for the channel */
95static void cppi_reset_tx(struct cppi_tx_stateram __iomem *tx, u32 ptr)
96{
97 musb_writel(&tx->tx_head, 0, 0);
98 musb_writel(&tx->tx_buf, 0, 0);
99 musb_writel(&tx->tx_current, 0, 0);
100 musb_writel(&tx->tx_buf_current, 0, 0);
101 musb_writel(&tx->tx_info, 0, 0);
102 musb_writel(&tx->tx_rem_len, 0, 0);
103 /* musb_writel(&tx->tx_dummy, 0, 0); */
104 musb_writel(&tx->tx_complete, 0, ptr);
105}
106
107static void __init cppi_pool_init(struct cppi *cppi, struct cppi_channel *c)
108{
109 int j;
110
111 /* initialize channel fields */
112 c->head = NULL;
113 c->tail = NULL;
114 c->last_processed = NULL;
115 c->channel.status = MUSB_DMA_STATUS_UNKNOWN;
116 c->controller = cppi;
117 c->is_rndis = 0;
118 c->freelist = NULL;
119
120 /* build the BD Free list for the channel */
121 for (j = 0; j < NUM_TXCHAN_BD + 1; j++) {
122 struct cppi_descriptor *bd;
123 dma_addr_t dma;
124
125 bd = dma_pool_alloc(cppi->pool, GFP_KERNEL, &dma);
126 bd->dma = dma;
127 cppi_bd_free(c, bd);
128 }
129}
130
131static int cppi_channel_abort(struct dma_channel *);
132
133static void cppi_pool_free(struct cppi_channel *c)
134{
135 struct cppi *cppi = c->controller;
136 struct cppi_descriptor *bd;
137
138 (void) cppi_channel_abort(&c->channel);
139 c->channel.status = MUSB_DMA_STATUS_UNKNOWN;
140 c->controller = NULL;
141
142 /* free all its bds */
143 bd = c->last_processed;
144 do {
145 if (bd)
146 dma_pool_free(cppi->pool, bd, bd->dma);
147 bd = cppi_bd_alloc(c);
148 } while (bd);
149 c->last_processed = NULL;
150}
151
152static int __init cppi_controller_start(struct dma_controller *c)
153{
154 struct cppi *controller;
155 void __iomem *tibase;
156 int i;
157
158 controller = container_of(c, struct cppi, controller);
159
160 /* do whatever is necessary to start controller */
161 for (i = 0; i < ARRAY_SIZE(controller->tx); i++) {
162 controller->tx[i].transmit = true;
163 controller->tx[i].index = i;
164 }
165 for (i = 0; i < ARRAY_SIZE(controller->rx); i++) {
166 controller->rx[i].transmit = false;
167 controller->rx[i].index = i;
168 }
169
170 /* setup BD list on a per channel basis */
171 for (i = 0; i < ARRAY_SIZE(controller->tx); i++)
172 cppi_pool_init(controller, controller->tx + i);
173 for (i = 0; i < ARRAY_SIZE(controller->rx); i++)
174 cppi_pool_init(controller, controller->rx + i);
175
176 tibase = controller->tibase;
177 INIT_LIST_HEAD(&controller->tx_complete);
178
179 /* initialise tx/rx channel head pointers to zero */
180 for (i = 0; i < ARRAY_SIZE(controller->tx); i++) {
181 struct cppi_channel *tx_ch = controller->tx + i;
182 struct cppi_tx_stateram __iomem *tx;
183
184 INIT_LIST_HEAD(&tx_ch->tx_complete);
185
186 tx = tibase + DAVINCI_TXCPPI_STATERAM_OFFSET(i);
187 tx_ch->state_ram = tx;
188 cppi_reset_tx(tx, 0);
189 }
190 for (i = 0; i < ARRAY_SIZE(controller->rx); i++) {
191 struct cppi_channel *rx_ch = controller->rx + i;
192 struct cppi_rx_stateram __iomem *rx;
193
194 INIT_LIST_HEAD(&rx_ch->tx_complete);
195
196 rx = tibase + DAVINCI_RXCPPI_STATERAM_OFFSET(i);
197 rx_ch->state_ram = rx;
198 cppi_reset_rx(rx);
199 }
200
201 /* enable individual cppi channels */
202 musb_writel(tibase, DAVINCI_TXCPPI_INTENAB_REG,
203 DAVINCI_DMA_ALL_CHANNELS_ENABLE);
204 musb_writel(tibase, DAVINCI_RXCPPI_INTENAB_REG,
205 DAVINCI_DMA_ALL_CHANNELS_ENABLE);
206
207 /* enable tx/rx CPPI control */
208 musb_writel(tibase, DAVINCI_TXCPPI_CTRL_REG, DAVINCI_DMA_CTRL_ENABLE);
209 musb_writel(tibase, DAVINCI_RXCPPI_CTRL_REG, DAVINCI_DMA_CTRL_ENABLE);
210
211 /* disable RNDIS mode, also host rx RNDIS autorequest */
212 musb_writel(tibase, DAVINCI_RNDIS_REG, 0);
213 musb_writel(tibase, DAVINCI_AUTOREQ_REG, 0);
214
215 return 0;
216}
217
218/*
219 * Stop DMA controller
220 *
221 * De-Init the DMA controller as necessary.
222 */
223
224static int cppi_controller_stop(struct dma_controller *c)
225{
226 struct cppi *controller;
227 void __iomem *tibase;
228 int i;
229
230 controller = container_of(c, struct cppi, controller);
231
232 tibase = controller->tibase;
233 /* DISABLE INDIVIDUAL CHANNEL Interrupts */
234 musb_writel(tibase, DAVINCI_TXCPPI_INTCLR_REG,
235 DAVINCI_DMA_ALL_CHANNELS_ENABLE);
236 musb_writel(tibase, DAVINCI_RXCPPI_INTCLR_REG,
237 DAVINCI_DMA_ALL_CHANNELS_ENABLE);
238
239 DBG(1, "Tearing down RX and TX Channels\n");
240 for (i = 0; i < ARRAY_SIZE(controller->tx); i++) {
241 /* FIXME restructure of txdma to use bds like rxdma */
242 controller->tx[i].last_processed = NULL;
243 cppi_pool_free(controller->tx + i);
244 }
245 for (i = 0; i < ARRAY_SIZE(controller->rx); i++)
246 cppi_pool_free(controller->rx + i);
247
248 /* in Tx Case proper teardown is supported. We resort to disabling
249 * Tx/Rx CPPI after cleanup of Tx channels. Before TX teardown is
250 * complete TX CPPI cannot be disabled.
251 */
252 /*disable tx/rx cppi */
253 musb_writel(tibase, DAVINCI_TXCPPI_CTRL_REG, DAVINCI_DMA_CTRL_DISABLE);
254 musb_writel(tibase, DAVINCI_RXCPPI_CTRL_REG, DAVINCI_DMA_CTRL_DISABLE);
255
256 return 0;
257}
258
259/* While dma channel is allocated, we only want the core irqs active
260 * for fault reports, otherwise we'd get irqs that we don't care about.
261 * Except for TX irqs, where dma done != fifo empty and reusable ...
262 *
263 * NOTE: docs don't say either way, but irq masking **enables** irqs.
264 *
265 * REVISIT same issue applies to pure PIO usage too, and non-cppi dma...
266 */
267static inline void core_rxirq_disable(void __iomem *tibase, unsigned epnum)
268{
269 musb_writel(tibase, DAVINCI_USB_INT_MASK_CLR_REG, 1 << (epnum + 8));
270}
271
272static inline void core_rxirq_enable(void __iomem *tibase, unsigned epnum)
273{
274 musb_writel(tibase, DAVINCI_USB_INT_MASK_SET_REG, 1 << (epnum + 8));
275}
276
277
278/*
279 * Allocate a CPPI Channel for DMA. With CPPI, channels are bound to
280 * each transfer direction of a non-control endpoint, so allocating
281 * (and deallocating) is mostly a way to notice bad housekeeping on
282 * the software side. We assume the irqs are always active.
283 */
284static struct dma_channel *
285cppi_channel_allocate(struct dma_controller *c,
286 struct musb_hw_ep *ep, u8 transmit)
287{
288 struct cppi *controller;
289 u8 index;
290 struct cppi_channel *cppi_ch;
291 void __iomem *tibase;
292
293 controller = container_of(c, struct cppi, controller);
294 tibase = controller->tibase;
295
296 /* ep0 doesn't use DMA; remember cppi indices are 0..N-1 */
297 index = ep->epnum - 1;
298
299 /* return the corresponding CPPI Channel Handle, and
300 * probably disable the non-CPPI irq until we need it.
301 */
302 if (transmit) {
303 if (index >= ARRAY_SIZE(controller->tx)) {
304 DBG(1, "no %cX%d CPPI channel\n", 'T', index);
305 return NULL;
306 }
307 cppi_ch = controller->tx + index;
308 } else {
309 if (index >= ARRAY_SIZE(controller->rx)) {
310 DBG(1, "no %cX%d CPPI channel\n", 'R', index);
311 return NULL;
312 }
313 cppi_ch = controller->rx + index;
314 core_rxirq_disable(tibase, ep->epnum);
315 }
316
317 /* REVISIT make this an error later once the same driver code works
318 * with the other DMA engine too
319 */
320 if (cppi_ch->hw_ep)
321 DBG(1, "re-allocating DMA%d %cX channel %p\n",
322 index, transmit ? 'T' : 'R', cppi_ch);
323 cppi_ch->hw_ep = ep;
324 cppi_ch->channel.status = MUSB_DMA_STATUS_FREE;
Ming Lei66af83d2010-09-20 10:32:06 +0300325 cppi_ch->channel.max_len = 0x7fffffff;
Felipe Balbi550a7372008-07-24 12:27:36 +0300326
327 DBG(4, "Allocate CPPI%d %cX\n", index, transmit ? 'T' : 'R');
328 return &cppi_ch->channel;
329}
330
331/* Release a CPPI Channel. */
332static void cppi_channel_release(struct dma_channel *channel)
333{
334 struct cppi_channel *c;
335 void __iomem *tibase;
336
337 /* REVISIT: for paranoia, check state and abort if needed... */
338
339 c = container_of(channel, struct cppi_channel, channel);
340 tibase = c->controller->tibase;
341 if (!c->hw_ep)
342 DBG(1, "releasing idle DMA channel %p\n", c);
343 else if (!c->transmit)
344 core_rxirq_enable(tibase, c->index + 1);
345
346 /* for now, leave its cppi IRQ enabled (we won't trigger it) */
347 c->hw_ep = NULL;
348 channel->status = MUSB_DMA_STATUS_UNKNOWN;
349}
350
351/* Context: controller irqlocked */
352static void
353cppi_dump_rx(int level, struct cppi_channel *c, const char *tag)
354{
355 void __iomem *base = c->controller->mregs;
356 struct cppi_rx_stateram __iomem *rx = c->state_ram;
357
358 musb_ep_select(base, c->index + 1);
359
360 DBG(level, "RX DMA%d%s: %d left, csr %04x, "
361 "%08x H%08x S%08x C%08x, "
362 "B%08x L%08x %08x .. %08x"
363 "\n",
364 c->index, tag,
365 musb_readl(c->controller->tibase,
366 DAVINCI_RXCPPI_BUFCNT0_REG + 4 * c->index),
367 musb_readw(c->hw_ep->regs, MUSB_RXCSR),
368
369 musb_readl(&rx->rx_skipbytes, 0),
370 musb_readl(&rx->rx_head, 0),
371 musb_readl(&rx->rx_sop, 0),
372 musb_readl(&rx->rx_current, 0),
373
374 musb_readl(&rx->rx_buf_current, 0),
375 musb_readl(&rx->rx_len_len, 0),
376 musb_readl(&rx->rx_cnt_cnt, 0),
377 musb_readl(&rx->rx_complete, 0)
378 );
379}
380
381/* Context: controller irqlocked */
382static void
383cppi_dump_tx(int level, struct cppi_channel *c, const char *tag)
384{
385 void __iomem *base = c->controller->mregs;
386 struct cppi_tx_stateram __iomem *tx = c->state_ram;
387
388 musb_ep_select(base, c->index + 1);
389
390 DBG(level, "TX DMA%d%s: csr %04x, "
391 "H%08x S%08x C%08x %08x, "
392 "F%08x L%08x .. %08x"
393 "\n",
394 c->index, tag,
395 musb_readw(c->hw_ep->regs, MUSB_TXCSR),
396
397 musb_readl(&tx->tx_head, 0),
398 musb_readl(&tx->tx_buf, 0),
399 musb_readl(&tx->tx_current, 0),
400 musb_readl(&tx->tx_buf_current, 0),
401
402 musb_readl(&tx->tx_info, 0),
403 musb_readl(&tx->tx_rem_len, 0),
404 /* dummy/unused word 6 */
405 musb_readl(&tx->tx_complete, 0)
406 );
407}
408
409/* Context: controller irqlocked */
410static inline void
411cppi_rndis_update(struct cppi_channel *c, int is_rx,
412 void __iomem *tibase, int is_rndis)
413{
414 /* we may need to change the rndis flag for this cppi channel */
415 if (c->is_rndis != is_rndis) {
416 u32 value = musb_readl(tibase, DAVINCI_RNDIS_REG);
417 u32 temp = 1 << (c->index);
418
419 if (is_rx)
420 temp <<= 16;
421 if (is_rndis)
422 value |= temp;
423 else
424 value &= ~temp;
425 musb_writel(tibase, DAVINCI_RNDIS_REG, value);
426 c->is_rndis = is_rndis;
427 }
428}
429
Hugo Villeneuve704a1482009-01-24 17:57:30 -0800430#ifdef CONFIG_USB_MUSB_DEBUG
Felipe Balbi550a7372008-07-24 12:27:36 +0300431static void cppi_dump_rxbd(const char *tag, struct cppi_descriptor *bd)
432{
433 pr_debug("RXBD/%s %08x: "
434 "nxt %08x buf %08x off.blen %08x opt.plen %08x\n",
435 tag, bd->dma,
436 bd->hw_next, bd->hw_bufp, bd->hw_off_len,
437 bd->hw_options);
438}
Hugo Villeneuve704a1482009-01-24 17:57:30 -0800439#endif
Felipe Balbi550a7372008-07-24 12:27:36 +0300440
441static void cppi_dump_rxq(int level, const char *tag, struct cppi_channel *rx)
442{
Hugo Villeneuve704a1482009-01-24 17:57:30 -0800443#ifdef CONFIG_USB_MUSB_DEBUG
Felipe Balbi550a7372008-07-24 12:27:36 +0300444 struct cppi_descriptor *bd;
445
446 if (!_dbg_level(level))
447 return;
448 cppi_dump_rx(level, rx, tag);
449 if (rx->last_processed)
450 cppi_dump_rxbd("last", rx->last_processed);
451 for (bd = rx->head; bd; bd = bd->next)
452 cppi_dump_rxbd("active", bd);
453#endif
454}
455
456
457/* NOTE: DaVinci autoreq is ignored except for host side "RNDIS" mode RX;
458 * so we won't ever use it (see "CPPI RX Woes" below).
459 */
460static inline int cppi_autoreq_update(struct cppi_channel *rx,
461 void __iomem *tibase, int onepacket, unsigned n_bds)
462{
463 u32 val;
464
465#ifdef RNDIS_RX_IS_USABLE
466 u32 tmp;
467 /* assert(is_host_active(musb)) */
468
469 /* start from "AutoReq never" */
470 tmp = musb_readl(tibase, DAVINCI_AUTOREQ_REG);
471 val = tmp & ~((0x3) << (rx->index * 2));
472
473 /* HCD arranged reqpkt for packet #1. we arrange int
474 * for all but the last one, maybe in two segments.
475 */
476 if (!onepacket) {
477#if 0
478 /* use two segments, autoreq "all" then the last "never" */
479 val |= ((0x3) << (rx->index * 2));
480 n_bds--;
481#else
482 /* one segment, autoreq "all-but-last" */
483 val |= ((0x1) << (rx->index * 2));
484#endif
485 }
486
487 if (val != tmp) {
488 int n = 100;
489
490 /* make sure that autoreq is updated before continuing */
491 musb_writel(tibase, DAVINCI_AUTOREQ_REG, val);
492 do {
493 tmp = musb_readl(tibase, DAVINCI_AUTOREQ_REG);
494 if (tmp == val)
495 break;
496 cpu_relax();
497 } while (n-- > 0);
498 }
499#endif
500
501 /* REQPKT is turned off after each segment */
502 if (n_bds && rx->channel.actual_len) {
503 void __iomem *regs = rx->hw_ep->regs;
504
505 val = musb_readw(regs, MUSB_RXCSR);
506 if (!(val & MUSB_RXCSR_H_REQPKT)) {
507 val |= MUSB_RXCSR_H_REQPKT | MUSB_RXCSR_H_WZC_BITS;
508 musb_writew(regs, MUSB_RXCSR, val);
509 /* flush writebufer */
510 val = musb_readw(regs, MUSB_RXCSR);
511 }
512 }
513 return n_bds;
514}
515
516
517/* Buffer enqueuing Logic:
518 *
519 * - RX builds new queues each time, to help handle routine "early
520 * termination" cases (faults, including errors and short reads)
521 * more correctly.
522 *
523 * - for now, TX reuses the same queue of BDs every time
524 *
525 * REVISIT long term, we want a normal dynamic model.
526 * ... the goal will be to append to the
527 * existing queue, processing completed "dma buffers" (segments) on the fly.
528 *
529 * Otherwise we force an IRQ latency between requests, which slows us a lot
530 * (especially in "transparent" dma). Unfortunately that model seems to be
531 * inherent in the DMA model from the Mentor code, except in the rare case
532 * of transfers big enough (~128+ KB) that we could append "middle" segments
533 * in the TX paths. (RX can't do this, see below.)
534 *
535 * That's true even in the CPPI- friendly iso case, where most urbs have
536 * several small segments provided in a group and where the "packet at a time"
537 * "transparent" DMA model is always correct, even on the RX side.
538 */
539
540/*
541 * CPPI TX:
542 * ========
543 * TX is a lot more reasonable than RX; it doesn't need to run in
544 * irq-per-packet mode very often. RNDIS mode seems to behave too
545 * (except how it handles the exactly-N-packets case). Building a
546 * txdma queue with multiple requests (urb or usb_request) looks
547 * like it would work ... but fault handling would need much testing.
548 *
549 * The main issue with TX mode RNDIS relates to transfer lengths that
550 * are an exact multiple of the packet length. It appears that there's
551 * a hiccup in that case (maybe the DMA completes before the ZLP gets
552 * written?) boiling down to not being able to rely on CPPI writing any
553 * terminating zero length packet before the next transfer is written.
554 * So that's punted to PIO; better yet, gadget drivers can avoid it.
555 *
556 * Plus, there's allegedly an undocumented constraint that rndis transfer
557 * length be a multiple of 64 bytes ... but the chip doesn't act that
558 * way, and we really don't _want_ that behavior anyway.
559 *
560 * On TX, "transparent" mode works ... although experiments have shown
561 * problems trying to use the SOP/EOP bits in different USB packets.
562 *
563 * REVISIT try to handle terminating zero length packets using CPPI
564 * instead of doing it by PIO after an IRQ. (Meanwhile, make Ethernet
565 * links avoid that issue by forcing them to avoid zlps.)
566 */
567static void
568cppi_next_tx_segment(struct musb *musb, struct cppi_channel *tx)
569{
570 unsigned maxpacket = tx->maxpacket;
571 dma_addr_t addr = tx->buf_dma + tx->offset;
572 size_t length = tx->buf_len - tx->offset;
573 struct cppi_descriptor *bd;
574 unsigned n_bds;
575 unsigned i;
576 struct cppi_tx_stateram __iomem *tx_ram = tx->state_ram;
577 int rndis;
578
579 /* TX can use the CPPI "rndis" mode, where we can probably fit this
580 * transfer in one BD and one IRQ. The only time we would NOT want
581 * to use it is when hardware constraints prevent it, or if we'd
582 * trigger the "send a ZLP?" confusion.
583 */
584 rndis = (maxpacket & 0x3f) == 0
Sergei Shtylyov6b6e9712009-03-26 18:29:19 -0700585 && length > maxpacket
Felipe Balbi550a7372008-07-24 12:27:36 +0300586 && length < 0xffff
587 && (length % maxpacket) != 0;
588
589 if (rndis) {
590 maxpacket = length;
591 n_bds = 1;
592 } else {
593 n_bds = length / maxpacket;
594 if (!length || (length % maxpacket))
595 n_bds++;
596 n_bds = min(n_bds, (unsigned) NUM_TXCHAN_BD);
597 length = min(n_bds * maxpacket, length);
598 }
599
Dan Carpenter2fbcf3f2011-03-20 14:16:17 +0300600 DBG(4, "TX DMA%d, pktSz %d %s bds %d dma 0x%llx len %u\n",
Felipe Balbi550a7372008-07-24 12:27:36 +0300601 tx->index,
602 maxpacket,
603 rndis ? "rndis" : "transparent",
604 n_bds,
Dan Carpenter2fbcf3f2011-03-20 14:16:17 +0300605 (unsigned long long)addr, length);
Felipe Balbi550a7372008-07-24 12:27:36 +0300606
607 cppi_rndis_update(tx, 0, musb->ctrl_base, rndis);
608
609 /* assuming here that channel_program is called during
610 * transfer initiation ... current code maintains state
611 * for one outstanding request only (no queues, not even
612 * the implicit ones of an iso urb).
613 */
614
615 bd = tx->freelist;
616 tx->head = bd;
617 tx->last_processed = NULL;
618
619 /* FIXME use BD pool like RX side does, and just queue
620 * the minimum number for this request.
621 */
622
623 /* Prepare queue of BDs first, then hand it to hardware.
624 * All BDs except maybe the last should be of full packet
625 * size; for RNDIS there _is_ only that last packet.
626 */
627 for (i = 0; i < n_bds; ) {
628 if (++i < n_bds && bd->next)
629 bd->hw_next = bd->next->dma;
630 else
631 bd->hw_next = 0;
632
633 bd->hw_bufp = tx->buf_dma + tx->offset;
634
635 /* FIXME set EOP only on the last packet,
636 * SOP only on the first ... avoid IRQs
637 */
638 if ((tx->offset + maxpacket) <= tx->buf_len) {
639 tx->offset += maxpacket;
640 bd->hw_off_len = maxpacket;
641 bd->hw_options = CPPI_SOP_SET | CPPI_EOP_SET
642 | CPPI_OWN_SET | maxpacket;
643 } else {
644 /* only this one may be a partial USB Packet */
645 u32 partial_len;
646
647 partial_len = tx->buf_len - tx->offset;
648 tx->offset = tx->buf_len;
649 bd->hw_off_len = partial_len;
650
651 bd->hw_options = CPPI_SOP_SET | CPPI_EOP_SET
652 | CPPI_OWN_SET | partial_len;
653 if (partial_len == 0)
654 bd->hw_options |= CPPI_ZERO_SET;
655 }
656
657 DBG(5, "TXBD %p: nxt %08x buf %08x len %04x opt %08x\n",
658 bd, bd->hw_next, bd->hw_bufp,
659 bd->hw_off_len, bd->hw_options);
660
661 /* update the last BD enqueued to the list */
662 tx->tail = bd;
663 bd = bd->next;
664 }
665
666 /* BDs live in DMA-coherent memory, but writes might be pending */
667 cpu_drain_writebuffer();
668
669 /* Write to the HeadPtr in state RAM to trigger */
670 musb_writel(&tx_ram->tx_head, 0, (u32)tx->freelist->dma);
671
672 cppi_dump_tx(5, tx, "/S");
673}
674
675/*
676 * CPPI RX Woes:
677 * =============
678 * Consider a 1KB bulk RX buffer in two scenarios: (a) it's fed two 300 byte
679 * packets back-to-back, and (b) it's fed two 512 byte packets back-to-back.
680 * (Full speed transfers have similar scenarios.)
681 *
682 * The correct behavior for Linux is that (a) fills the buffer with 300 bytes,
683 * and the next packet goes into a buffer that's queued later; while (b) fills
684 * the buffer with 1024 bytes. How to do that with CPPI?
685 *
686 * - RX queues in "rndis" mode -- one single BD -- handle (a) correctly, but
687 * (b) loses **BADLY** because nothing (!) happens when that second packet
688 * fills the buffer, much less when a third one arrives. (Which makes this
689 * not a "true" RNDIS mode. In the RNDIS protocol short-packet termination
690 * is optional, and it's fine if peripherals -- not hosts! -- pad messages
691 * out to end-of-buffer. Standard PCI host controller DMA descriptors
692 * implement that mode by default ... which is no accident.)
693 *
694 * - RX queues in "transparent" mode -- two BDs with 512 bytes each -- have
695 * converse problems: (b) is handled right, but (a) loses badly. CPPI RX
696 * ignores SOP/EOP markings and processes both of those BDs; so both packets
697 * are loaded into the buffer (with a 212 byte gap between them), and the next
698 * buffer queued will NOT get its 300 bytes of data. (It seems like SOP/EOP
699 * are intended as outputs for RX queues, not inputs...)
700 *
701 * - A variant of "transparent" mode -- one BD at a time -- is the only way to
702 * reliably make both cases work, with software handling both cases correctly
703 * and at the significant penalty of needing an IRQ per packet. (The lack of
704 * I/O overlap can be slightly ameliorated by enabling double buffering.)
705 *
706 * So how to get rid of IRQ-per-packet? The transparent multi-BD case could
707 * be used in special cases like mass storage, which sets URB_SHORT_NOT_OK
708 * (or maybe its peripheral side counterpart) to flag (a) scenarios as errors
709 * with guaranteed driver level fault recovery and scrubbing out what's left
710 * of that garbaged datastream.
711 *
712 * But there seems to be no way to identify the cases where CPPI RNDIS mode
713 * is appropriate -- which do NOT include RNDIS host drivers, but do include
714 * the CDC Ethernet driver! -- and the documentation is incomplete/wrong.
715 * So we can't _ever_ use RX RNDIS mode ... except by using a heuristic
716 * that applies best on the peripheral side (and which could fail rudely).
717 *
718 * Leaving only "transparent" mode; we avoid multi-bd modes in almost all
719 * cases other than mass storage class. Otherwise we're correct but slow,
720 * since CPPI penalizes our need for a "true RNDIS" default mode.
721 */
722
723
724/* Heuristic, intended to kick in for ethernet/rndis peripheral ONLY
725 *
726 * IFF
727 * (a) peripheral mode ... since rndis peripherals could pad their
728 * writes to hosts, causing i/o failure; or we'd have to cope with
729 * a largely unknowable variety of host side protocol variants
730 * (b) and short reads are NOT errors ... since full reads would
731 * cause those same i/o failures
732 * (c) and read length is
733 * - less than 64KB (max per cppi descriptor)
734 * - not a multiple of 4096 (g_zero default, full reads typical)
735 * - N (>1) packets long, ditto (full reads not EXPECTED)
736 * THEN
737 * try rx rndis mode
738 *
739 * Cost of heuristic failing: RXDMA wedges at the end of transfers that
740 * fill out the whole buffer. Buggy host side usb network drivers could
741 * trigger that, but "in the field" such bugs seem to be all but unknown.
742 *
743 * So this module parameter lets the heuristic be disabled. When using
744 * gadgetfs, the heuristic will probably need to be disabled.
745 */
746static int cppi_rx_rndis = 1;
747
748module_param(cppi_rx_rndis, bool, 0);
749MODULE_PARM_DESC(cppi_rx_rndis, "enable/disable RX RNDIS heuristic");
750
751
752/**
753 * cppi_next_rx_segment - dma read for the next chunk of a buffer
754 * @musb: the controller
755 * @rx: dma channel
756 * @onepacket: true unless caller treats short reads as errors, and
757 * performs fault recovery above usbcore.
758 * Context: controller irqlocked
759 *
760 * See above notes about why we can't use multi-BD RX queues except in
761 * rare cases (mass storage class), and can never use the hardware "rndis"
762 * mode (since it's not a "true" RNDIS mode) with complete safety..
763 *
764 * It's ESSENTIAL that callers specify "onepacket" mode unless they kick in
765 * code to recover from corrupted datastreams after each short transfer.
766 */
767static void
768cppi_next_rx_segment(struct musb *musb, struct cppi_channel *rx, int onepacket)
769{
770 unsigned maxpacket = rx->maxpacket;
771 dma_addr_t addr = rx->buf_dma + rx->offset;
772 size_t length = rx->buf_len - rx->offset;
773 struct cppi_descriptor *bd, *tail;
774 unsigned n_bds;
775 unsigned i;
776 void __iomem *tibase = musb->ctrl_base;
777 int is_rndis = 0;
778 struct cppi_rx_stateram __iomem *rx_ram = rx->state_ram;
779
780 if (onepacket) {
781 /* almost every USB driver, host or peripheral side */
782 n_bds = 1;
783
784 /* maybe apply the heuristic above */
785 if (cppi_rx_rndis
786 && is_peripheral_active(musb)
787 && length > maxpacket
788 && (length & ~0xffff) == 0
789 && (length & 0x0fff) != 0
790 && (length & (maxpacket - 1)) == 0) {
791 maxpacket = length;
792 is_rndis = 1;
793 }
794 } else {
795 /* virtually nothing except mass storage class */
796 if (length > 0xffff) {
797 n_bds = 0xffff / maxpacket;
798 length = n_bds * maxpacket;
799 } else {
800 n_bds = length / maxpacket;
801 if (length % maxpacket)
802 n_bds++;
803 }
804 if (n_bds == 1)
805 onepacket = 1;
806 else
807 n_bds = min(n_bds, (unsigned) NUM_RXCHAN_BD);
808 }
809
810 /* In host mode, autorequest logic can generate some IN tokens; it's
811 * tricky since we can't leave REQPKT set in RXCSR after the transfer
812 * finishes. So: multipacket transfers involve two or more segments.
813 * And always at least two IRQs ... RNDIS mode is not an option.
814 */
815 if (is_host_active(musb))
816 n_bds = cppi_autoreq_update(rx, tibase, onepacket, n_bds);
817
818 cppi_rndis_update(rx, 1, musb->ctrl_base, is_rndis);
819
820 length = min(n_bds * maxpacket, length);
821
822 DBG(4, "RX DMA%d seg, maxp %d %s bds %d (cnt %d) "
Dan Carpenter2fbcf3f2011-03-20 14:16:17 +0300823 "dma 0x%llx len %u %u/%u\n",
Felipe Balbi550a7372008-07-24 12:27:36 +0300824 rx->index, maxpacket,
825 onepacket
826 ? (is_rndis ? "rndis" : "onepacket")
827 : "multipacket",
828 n_bds,
829 musb_readl(tibase,
830 DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4))
831 & 0xffff,
Dan Carpenter2fbcf3f2011-03-20 14:16:17 +0300832 (unsigned long long)addr, length,
833 rx->channel.actual_len, rx->buf_len);
Felipe Balbi550a7372008-07-24 12:27:36 +0300834
835 /* only queue one segment at a time, since the hardware prevents
836 * correct queue shutdown after unexpected short packets
837 */
838 bd = cppi_bd_alloc(rx);
839 rx->head = bd;
840
841 /* Build BDs for all packets in this segment */
842 for (i = 0, tail = NULL; bd && i < n_bds; i++, tail = bd) {
843 u32 bd_len;
844
845 if (i) {
846 bd = cppi_bd_alloc(rx);
847 if (!bd)
848 break;
849 tail->next = bd;
850 tail->hw_next = bd->dma;
851 }
852 bd->hw_next = 0;
853
854 /* all but the last packet will be maxpacket size */
855 if (maxpacket < length)
856 bd_len = maxpacket;
857 else
858 bd_len = length;
859
860 bd->hw_bufp = addr;
861 addr += bd_len;
862 rx->offset += bd_len;
863
864 bd->hw_off_len = (0 /*offset*/ << 16) + bd_len;
865 bd->buflen = bd_len;
866
867 bd->hw_options = CPPI_OWN_SET | (i == 0 ? length : 0);
868 length -= bd_len;
869 }
870
871 /* we always expect at least one reusable BD! */
872 if (!tail) {
873 WARNING("rx dma%d -- no BDs? need %d\n", rx->index, n_bds);
874 return;
875 } else if (i < n_bds)
876 WARNING("rx dma%d -- only %d of %d BDs\n", rx->index, i, n_bds);
877
878 tail->next = NULL;
879 tail->hw_next = 0;
880
881 bd = rx->head;
882 rx->tail = tail;
883
884 /* short reads and other faults should terminate this entire
885 * dma segment. we want one "dma packet" per dma segment, not
886 * one per USB packet, terminating the whole queue at once...
887 * NOTE that current hardware seems to ignore SOP and EOP.
888 */
889 bd->hw_options |= CPPI_SOP_SET;
890 tail->hw_options |= CPPI_EOP_SET;
891
Hugo Villeneuve704a1482009-01-24 17:57:30 -0800892#ifdef CONFIG_USB_MUSB_DEBUG
893 if (_dbg_level(5)) {
Felipe Balbi550a7372008-07-24 12:27:36 +0300894 struct cppi_descriptor *d;
895
896 for (d = rx->head; d; d = d->next)
897 cppi_dump_rxbd("S", d);
898 }
Hugo Villeneuve704a1482009-01-24 17:57:30 -0800899#endif
Felipe Balbi550a7372008-07-24 12:27:36 +0300900
901 /* in case the preceding transfer left some state... */
902 tail = rx->last_processed;
903 if (tail) {
904 tail->next = bd;
905 tail->hw_next = bd->dma;
906 }
907
908 core_rxirq_enable(tibase, rx->index + 1);
909
910 /* BDs live in DMA-coherent memory, but writes might be pending */
911 cpu_drain_writebuffer();
912
913 /* REVISIT specs say to write this AFTER the BUFCNT register
914 * below ... but that loses badly.
915 */
916 musb_writel(&rx_ram->rx_head, 0, bd->dma);
917
918 /* bufferCount must be at least 3, and zeroes on completion
919 * unless it underflows below zero, or stops at two, or keeps
920 * growing ... grr.
921 */
922 i = musb_readl(tibase,
923 DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4))
924 & 0xffff;
925
926 if (!i)
927 musb_writel(tibase,
928 DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4),
929 n_bds + 2);
930 else if (n_bds > (i - 3))
931 musb_writel(tibase,
932 DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4),
933 n_bds - (i - 3));
934
935 i = musb_readl(tibase,
936 DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4))
937 & 0xffff;
938 if (i < (2 + n_bds)) {
939 DBG(2, "bufcnt%d underrun - %d (for %d)\n",
940 rx->index, i, n_bds);
941 musb_writel(tibase,
942 DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4),
943 n_bds + 2);
944 }
945
946 cppi_dump_rx(4, rx, "/S");
947}
948
949/**
950 * cppi_channel_program - program channel for data transfer
951 * @ch: the channel
952 * @maxpacket: max packet size
953 * @mode: For RX, 1 unless the usb protocol driver promised to treat
954 * all short reads as errors and kick in high level fault recovery.
955 * For TX, ignored because of RNDIS mode races/glitches.
956 * @dma_addr: dma address of buffer
957 * @len: length of buffer
958 * Context: controller irqlocked
959 */
960static int cppi_channel_program(struct dma_channel *ch,
961 u16 maxpacket, u8 mode,
962 dma_addr_t dma_addr, u32 len)
963{
964 struct cppi_channel *cppi_ch;
965 struct cppi *controller;
966 struct musb *musb;
967
968 cppi_ch = container_of(ch, struct cppi_channel, channel);
969 controller = cppi_ch->controller;
970 musb = controller->musb;
971
972 switch (ch->status) {
973 case MUSB_DMA_STATUS_BUS_ABORT:
974 case MUSB_DMA_STATUS_CORE_ABORT:
975 /* fault irq handler should have handled cleanup */
976 WARNING("%cX DMA%d not cleaned up after abort!\n",
977 cppi_ch->transmit ? 'T' : 'R',
978 cppi_ch->index);
979 /* WARN_ON(1); */
980 break;
981 case MUSB_DMA_STATUS_BUSY:
982 WARNING("program active channel? %cX DMA%d\n",
983 cppi_ch->transmit ? 'T' : 'R',
984 cppi_ch->index);
985 /* WARN_ON(1); */
986 break;
987 case MUSB_DMA_STATUS_UNKNOWN:
988 DBG(1, "%cX DMA%d not allocated!\n",
989 cppi_ch->transmit ? 'T' : 'R',
990 cppi_ch->index);
991 /* FALLTHROUGH */
992 case MUSB_DMA_STATUS_FREE:
993 break;
994 }
995
996 ch->status = MUSB_DMA_STATUS_BUSY;
997
998 /* set transfer parameters, then queue up its first segment */
999 cppi_ch->buf_dma = dma_addr;
1000 cppi_ch->offset = 0;
1001 cppi_ch->maxpacket = maxpacket;
1002 cppi_ch->buf_len = len;
Swaminathan S191b7762009-01-24 17:57:37 -08001003 cppi_ch->channel.actual_len = 0;
Felipe Balbi550a7372008-07-24 12:27:36 +03001004
1005 /* TX channel? or RX? */
1006 if (cppi_ch->transmit)
1007 cppi_next_tx_segment(musb, cppi_ch);
1008 else
1009 cppi_next_rx_segment(musb, cppi_ch, mode);
1010
1011 return true;
1012}
1013
1014static bool cppi_rx_scan(struct cppi *cppi, unsigned ch)
1015{
1016 struct cppi_channel *rx = &cppi->rx[ch];
1017 struct cppi_rx_stateram __iomem *state = rx->state_ram;
1018 struct cppi_descriptor *bd;
1019 struct cppi_descriptor *last = rx->last_processed;
1020 bool completed = false;
1021 bool acked = false;
1022 int i;
1023 dma_addr_t safe2ack;
1024 void __iomem *regs = rx->hw_ep->regs;
1025
1026 cppi_dump_rx(6, rx, "/K");
1027
1028 bd = last ? last->next : rx->head;
1029 if (!bd)
1030 return false;
1031
1032 /* run through all completed BDs */
1033 for (i = 0, safe2ack = musb_readl(&state->rx_complete, 0);
1034 (safe2ack || completed) && bd && i < NUM_RXCHAN_BD;
1035 i++, bd = bd->next) {
1036 u16 len;
1037
1038 /* catch latest BD writes from CPPI */
1039 rmb();
1040 if (!completed && (bd->hw_options & CPPI_OWN_SET))
1041 break;
1042
Dan Carpenter2fbcf3f2011-03-20 14:16:17 +03001043 DBG(5, "C/RXBD %llx: nxt %08x buf %08x "
Felipe Balbi550a7372008-07-24 12:27:36 +03001044 "off.len %08x opt.len %08x (%d)\n",
Dan Carpenter2fbcf3f2011-03-20 14:16:17 +03001045 (unsigned long long)bd->dma, bd->hw_next, bd->hw_bufp,
Felipe Balbi550a7372008-07-24 12:27:36 +03001046 bd->hw_off_len, bd->hw_options,
1047 rx->channel.actual_len);
1048
1049 /* actual packet received length */
1050 if ((bd->hw_options & CPPI_SOP_SET) && !completed)
1051 len = bd->hw_off_len & CPPI_RECV_PKTLEN_MASK;
1052 else
1053 len = 0;
1054
1055 if (bd->hw_options & CPPI_EOQ_MASK)
1056 completed = true;
1057
1058 if (!completed && len < bd->buflen) {
1059 /* NOTE: when we get a short packet, RXCSR_H_REQPKT
1060 * must have been cleared, and no more DMA packets may
1061 * active be in the queue... TI docs didn't say, but
1062 * CPPI ignores those BDs even though OWN is still set.
1063 */
1064 completed = true;
1065 DBG(3, "rx short %d/%d (%d)\n",
1066 len, bd->buflen,
1067 rx->channel.actual_len);
1068 }
1069
1070 /* If we got here, we expect to ack at least one BD; meanwhile
1071 * CPPI may completing other BDs while we scan this list...
1072 *
1073 * RACE: we can notice OWN cleared before CPPI raises the
1074 * matching irq by writing that BD as the completion pointer.
1075 * In such cases, stop scanning and wait for the irq, avoiding
1076 * lost acks and states where BD ownership is unclear.
1077 */
1078 if (bd->dma == safe2ack) {
1079 musb_writel(&state->rx_complete, 0, safe2ack);
1080 safe2ack = musb_readl(&state->rx_complete, 0);
1081 acked = true;
1082 if (bd->dma == safe2ack)
1083 safe2ack = 0;
1084 }
1085
1086 rx->channel.actual_len += len;
1087
1088 cppi_bd_free(rx, last);
1089 last = bd;
1090
1091 /* stop scanning on end-of-segment */
1092 if (bd->hw_next == 0)
1093 completed = true;
1094 }
1095 rx->last_processed = last;
1096
1097 /* dma abort, lost ack, or ... */
1098 if (!acked && last) {
1099 int csr;
1100
1101 if (safe2ack == 0 || safe2ack == rx->last_processed->dma)
1102 musb_writel(&state->rx_complete, 0, safe2ack);
1103 if (safe2ack == 0) {
1104 cppi_bd_free(rx, last);
1105 rx->last_processed = NULL;
1106
1107 /* if we land here on the host side, H_REQPKT will
1108 * be clear and we need to restart the queue...
1109 */
1110 WARN_ON(rx->head);
1111 }
1112 musb_ep_select(cppi->mregs, rx->index + 1);
1113 csr = musb_readw(regs, MUSB_RXCSR);
1114 if (csr & MUSB_RXCSR_DMAENAB) {
Dan Carpenter2fbcf3f2011-03-20 14:16:17 +03001115 DBG(4, "list%d %p/%p, last %llx%s, csr %04x\n",
Felipe Balbi550a7372008-07-24 12:27:36 +03001116 rx->index,
1117 rx->head, rx->tail,
1118 rx->last_processed
Dan Carpenter2fbcf3f2011-03-20 14:16:17 +03001119 ? (unsigned long long)
1120 rx->last_processed->dma
Felipe Balbi550a7372008-07-24 12:27:36 +03001121 : 0,
1122 completed ? ", completed" : "",
1123 csr);
1124 cppi_dump_rxq(4, "/what?", rx);
1125 }
1126 }
1127 if (!completed) {
1128 int csr;
1129
1130 rx->head = bd;
1131
1132 /* REVISIT seems like "autoreq all but EOP" doesn't...
1133 * setting it here "should" be racey, but seems to work
1134 */
1135 csr = musb_readw(rx->hw_ep->regs, MUSB_RXCSR);
1136 if (is_host_active(cppi->musb)
1137 && bd
1138 && !(csr & MUSB_RXCSR_H_REQPKT)) {
1139 csr |= MUSB_RXCSR_H_REQPKT;
1140 musb_writew(regs, MUSB_RXCSR,
1141 MUSB_RXCSR_H_WZC_BITS | csr);
1142 csr = musb_readw(rx->hw_ep->regs, MUSB_RXCSR);
1143 }
1144 } else {
1145 rx->head = NULL;
1146 rx->tail = NULL;
1147 }
1148
1149 cppi_dump_rx(6, rx, completed ? "/completed" : "/cleaned");
1150 return completed;
1151}
1152
Sergei Shtylyov91e9c4fe2009-03-27 12:59:46 -07001153irqreturn_t cppi_interrupt(int irq, void *dev_id)
Felipe Balbi550a7372008-07-24 12:27:36 +03001154{
Sergei Shtylyov91e9c4fe2009-03-27 12:59:46 -07001155 struct musb *musb = dev_id;
Felipe Balbi550a7372008-07-24 12:27:36 +03001156 struct cppi *cppi;
Sergei Shtylyov91e9c4fe2009-03-27 12:59:46 -07001157 void __iomem *tibase;
Felipe Balbi550a7372008-07-24 12:27:36 +03001158 struct musb_hw_ep *hw_ep = NULL;
Sergei Shtylyov91e9c4fe2009-03-27 12:59:46 -07001159 u32 rx, tx;
1160 int i, index;
Jon Povey2f8d5cd2010-09-24 13:44:10 +03001161 unsigned long uninitialized_var(flags);
Felipe Balbi550a7372008-07-24 12:27:36 +03001162
1163 cppi = container_of(musb->dma_controller, struct cppi, controller);
Swaminathan S93aa3da2009-12-15 13:30:00 +02001164 if (cppi->irq)
1165 spin_lock_irqsave(&musb->lock, flags);
Felipe Balbi550a7372008-07-24 12:27:36 +03001166
1167 tibase = musb->ctrl_base;
1168
Sergei Shtylyov91e9c4fe2009-03-27 12:59:46 -07001169 tx = musb_readl(tibase, DAVINCI_TXCPPI_MASKED_REG);
1170 rx = musb_readl(tibase, DAVINCI_RXCPPI_MASKED_REG);
1171
Dan Carpenterec63bf62011-03-20 14:14:36 +03001172 if (!tx && !rx) {
1173 if (cppi->irq)
1174 spin_unlock_irqrestore(&musb->lock, flags);
Sergei Shtylyov91e9c4fe2009-03-27 12:59:46 -07001175 return IRQ_NONE;
Dan Carpenterec63bf62011-03-20 14:14:36 +03001176 }
Sergei Shtylyov91e9c4fe2009-03-27 12:59:46 -07001177
1178 DBG(4, "CPPI IRQ Tx%x Rx%x\n", tx, rx);
1179
Felipe Balbi550a7372008-07-24 12:27:36 +03001180 /* process TX channels */
1181 for (index = 0; tx; tx = tx >> 1, index++) {
1182 struct cppi_channel *tx_ch;
1183 struct cppi_tx_stateram __iomem *tx_ram;
1184 bool completed = false;
1185 struct cppi_descriptor *bd;
1186
1187 if (!(tx & 1))
1188 continue;
1189
1190 tx_ch = cppi->tx + index;
1191 tx_ram = tx_ch->state_ram;
1192
1193 /* FIXME need a cppi_tx_scan() routine, which
1194 * can also be called from abort code
1195 */
1196
1197 cppi_dump_tx(5, tx_ch, "/E");
1198
1199 bd = tx_ch->head;
1200
Swaminathan S56596922009-12-28 13:40:38 +02001201 /*
1202 * If Head is null then this could mean that a abort interrupt
1203 * that needs to be acknowledged.
1204 */
Felipe Balbi550a7372008-07-24 12:27:36 +03001205 if (NULL == bd) {
1206 DBG(1, "null BD\n");
Dan Carpenter2e10f5e2011-03-20 14:18:26 +03001207 musb_writel(&tx_ram->tx_complete, 0, 0);
Felipe Balbi550a7372008-07-24 12:27:36 +03001208 continue;
1209 }
1210
1211 /* run through all completed BDs */
1212 for (i = 0; !completed && bd && i < NUM_TXCHAN_BD;
1213 i++, bd = bd->next) {
1214 u16 len;
1215
1216 /* catch latest BD writes from CPPI */
1217 rmb();
1218 if (bd->hw_options & CPPI_OWN_SET)
1219 break;
1220
1221 DBG(5, "C/TXBD %p n %x b %x off %x opt %x\n",
1222 bd, bd->hw_next, bd->hw_bufp,
1223 bd->hw_off_len, bd->hw_options);
1224
1225 len = bd->hw_off_len & CPPI_BUFFER_LEN_MASK;
1226 tx_ch->channel.actual_len += len;
1227
1228 tx_ch->last_processed = bd;
1229
1230 /* write completion register to acknowledge
1231 * processing of completed BDs, and possibly
1232 * release the IRQ; EOQ might not be set ...
1233 *
1234 * REVISIT use the same ack strategy as rx
1235 *
1236 * REVISIT have observed bit 18 set; huh??
1237 */
1238 /* if ((bd->hw_options & CPPI_EOQ_MASK)) */
1239 musb_writel(&tx_ram->tx_complete, 0, bd->dma);
1240
1241 /* stop scanning on end-of-segment */
1242 if (bd->hw_next == 0)
1243 completed = true;
1244 }
1245
1246 /* on end of segment, maybe go to next one */
1247 if (completed) {
1248 /* cppi_dump_tx(4, tx_ch, "/complete"); */
1249
1250 /* transfer more, or report completion */
1251 if (tx_ch->offset >= tx_ch->buf_len) {
1252 tx_ch->head = NULL;
1253 tx_ch->tail = NULL;
1254 tx_ch->channel.status = MUSB_DMA_STATUS_FREE;
1255
1256 hw_ep = tx_ch->hw_ep;
1257
Sergei Shtylyovc7bbc052009-03-26 18:26:40 -07001258 musb_dma_completion(musb, index + 1, 1);
Felipe Balbi550a7372008-07-24 12:27:36 +03001259
1260 } else {
1261 /* Bigger transfer than we could fit in
1262 * that first batch of descriptors...
1263 */
1264 cppi_next_tx_segment(musb, tx_ch);
1265 }
1266 } else
1267 tx_ch->head = bd;
1268 }
1269
1270 /* Start processing the RX block */
1271 for (index = 0; rx; rx = rx >> 1, index++) {
1272
1273 if (rx & 1) {
1274 struct cppi_channel *rx_ch;
1275
1276 rx_ch = cppi->rx + index;
1277
1278 /* let incomplete dma segments finish */
1279 if (!cppi_rx_scan(cppi, index))
1280 continue;
1281
1282 /* start another dma segment if needed */
1283 if (rx_ch->channel.actual_len != rx_ch->buf_len
1284 && rx_ch->channel.actual_len
1285 == rx_ch->offset) {
1286 cppi_next_rx_segment(musb, rx_ch, 1);
1287 continue;
1288 }
1289
1290 /* all segments completed! */
1291 rx_ch->channel.status = MUSB_DMA_STATUS_FREE;
1292
1293 hw_ep = rx_ch->hw_ep;
1294
1295 core_rxirq_disable(tibase, index + 1);
1296 musb_dma_completion(musb, index + 1, 0);
1297 }
1298 }
1299
1300 /* write to CPPI EOI register to re-enable interrupts */
1301 musb_writel(tibase, DAVINCI_CPPI_EOI_REG, 0);
Sergei Shtylyov91e9c4fe2009-03-27 12:59:46 -07001302
Swaminathan S93aa3da2009-12-15 13:30:00 +02001303 if (cppi->irq)
1304 spin_unlock_irqrestore(&musb->lock, flags);
1305
Sergei Shtylyov91e9c4fe2009-03-27 12:59:46 -07001306 return IRQ_HANDLED;
Felipe Balbi550a7372008-07-24 12:27:36 +03001307}
1308
1309/* Instantiate a software object representing a DMA controller. */
1310struct dma_controller *__init
1311dma_controller_create(struct musb *musb, void __iomem *mregs)
1312{
1313 struct cppi *controller;
Sergei Shtylyov91e9c4fe2009-03-27 12:59:46 -07001314 struct device *dev = musb->controller;
1315 struct platform_device *pdev = to_platform_device(dev);
Hema Kalliguddifcf173e2010-09-29 11:26:39 -05001316 int irq = platform_get_irq_byname(pdev, "dma");
Felipe Balbi550a7372008-07-24 12:27:36 +03001317
1318 controller = kzalloc(sizeof *controller, GFP_KERNEL);
1319 if (!controller)
1320 return NULL;
1321
1322 controller->mregs = mregs;
1323 controller->tibase = mregs - DAVINCI_BASE_OFFSET;
1324
1325 controller->musb = musb;
1326 controller->controller.start = cppi_controller_start;
1327 controller->controller.stop = cppi_controller_stop;
1328 controller->controller.channel_alloc = cppi_channel_allocate;
1329 controller->controller.channel_release = cppi_channel_release;
1330 controller->controller.channel_program = cppi_channel_program;
1331 controller->controller.channel_abort = cppi_channel_abort;
1332
1333 /* NOTE: allocating from on-chip SRAM would give the least
1334 * contention for memory access, if that ever matters here.
1335 */
1336
1337 /* setup BufferPool */
1338 controller->pool = dma_pool_create("cppi",
1339 controller->musb->controller,
1340 sizeof(struct cppi_descriptor),
1341 CPPI_DESCRIPTOR_ALIGN, 0);
1342 if (!controller->pool) {
1343 kfree(controller);
1344 return NULL;
1345 }
1346
Sergei Shtylyov91e9c4fe2009-03-27 12:59:46 -07001347 if (irq > 0) {
1348 if (request_irq(irq, cppi_interrupt, 0, "cppi-dma", musb)) {
1349 dev_err(dev, "request_irq %d failed!\n", irq);
1350 dma_controller_destroy(&controller->controller);
1351 return NULL;
1352 }
1353 controller->irq = irq;
1354 }
1355
Felipe Balbi550a7372008-07-24 12:27:36 +03001356 return &controller->controller;
1357}
1358
1359/*
1360 * Destroy a previously-instantiated DMA controller.
1361 */
1362void dma_controller_destroy(struct dma_controller *c)
1363{
1364 struct cppi *cppi;
1365
1366 cppi = container_of(c, struct cppi, controller);
1367
Sergei Shtylyov91e9c4fe2009-03-27 12:59:46 -07001368 if (cppi->irq)
1369 free_irq(cppi->irq, cppi->musb);
1370
Felipe Balbi550a7372008-07-24 12:27:36 +03001371 /* assert: caller stopped the controller first */
1372 dma_pool_destroy(cppi->pool);
1373
1374 kfree(cppi);
1375}
1376
1377/*
1378 * Context: controller irqlocked, endpoint selected
1379 */
1380static int cppi_channel_abort(struct dma_channel *channel)
1381{
1382 struct cppi_channel *cppi_ch;
1383 struct cppi *controller;
1384 void __iomem *mbase;
1385 void __iomem *tibase;
1386 void __iomem *regs;
1387 u32 value;
1388 struct cppi_descriptor *queue;
1389
1390 cppi_ch = container_of(channel, struct cppi_channel, channel);
1391
1392 controller = cppi_ch->controller;
1393
1394 switch (channel->status) {
1395 case MUSB_DMA_STATUS_BUS_ABORT:
1396 case MUSB_DMA_STATUS_CORE_ABORT:
1397 /* from RX or TX fault irq handler */
1398 case MUSB_DMA_STATUS_BUSY:
1399 /* the hardware needs shutting down */
1400 regs = cppi_ch->hw_ep->regs;
1401 break;
1402 case MUSB_DMA_STATUS_UNKNOWN:
1403 case MUSB_DMA_STATUS_FREE:
1404 return 0;
1405 default:
1406 return -EINVAL;
1407 }
1408
1409 if (!cppi_ch->transmit && cppi_ch->head)
1410 cppi_dump_rxq(3, "/abort", cppi_ch);
1411
1412 mbase = controller->mregs;
1413 tibase = controller->tibase;
1414
1415 queue = cppi_ch->head;
1416 cppi_ch->head = NULL;
1417 cppi_ch->tail = NULL;
1418
1419 /* REVISIT should rely on caller having done this,
1420 * and caller should rely on us not changing it.
1421 * peripheral code is safe ... check host too.
1422 */
1423 musb_ep_select(mbase, cppi_ch->index + 1);
1424
1425 if (cppi_ch->transmit) {
1426 struct cppi_tx_stateram __iomem *tx_ram;
Felipe Balbi550a7372008-07-24 12:27:36 +03001427 /* REVISIT put timeouts on these controller handshakes */
1428
1429 cppi_dump_tx(6, cppi_ch, " (teardown)");
1430
1431 /* teardown DMA engine then usb core */
1432 do {
1433 value = musb_readl(tibase, DAVINCI_TXCPPI_TEAR_REG);
1434 } while (!(value & CPPI_TEAR_READY));
1435 musb_writel(tibase, DAVINCI_TXCPPI_TEAR_REG, cppi_ch->index);
1436
1437 tx_ram = cppi_ch->state_ram;
1438 do {
1439 value = musb_readl(&tx_ram->tx_complete, 0);
1440 } while (0xFFFFFFFC != value);
Felipe Balbi550a7372008-07-24 12:27:36 +03001441
1442 /* FIXME clean up the transfer state ... here?
1443 * the completion routine should get called with
1444 * an appropriate status code.
1445 */
1446
1447 value = musb_readw(regs, MUSB_TXCSR);
1448 value &= ~MUSB_TXCSR_DMAENAB;
1449 value |= MUSB_TXCSR_FLUSHFIFO;
1450 musb_writew(regs, MUSB_TXCSR, value);
1451 musb_writew(regs, MUSB_TXCSR, value);
1452
Swaminathan S56596922009-12-28 13:40:38 +02001453 /*
Felipe Balbi550a7372008-07-24 12:27:36 +03001454 * 1. Write to completion Ptr value 0x1(bit 0 set)
1455 * (write back mode)
Swaminathan S56596922009-12-28 13:40:38 +02001456 * 2. Wait for abort interrupt and then put the channel in
1457 * compare mode by writing 1 to the tx_complete register.
Felipe Balbi550a7372008-07-24 12:27:36 +03001458 */
1459 cppi_reset_tx(tx_ram, 1);
Dan Carpenteraca7f352011-03-20 14:15:24 +03001460 cppi_ch->head = NULL;
Swaminathan S56596922009-12-28 13:40:38 +02001461 musb_writel(&tx_ram->tx_complete, 0, 1);
Felipe Balbi550a7372008-07-24 12:27:36 +03001462 cppi_dump_tx(5, cppi_ch, " (done teardown)");
1463
1464 /* REVISIT tx side _should_ clean up the same way
1465 * as the RX side ... this does no cleanup at all!
1466 */
1467
1468 } else /* RX */ {
1469 u16 csr;
1470
1471 /* NOTE: docs don't guarantee any of this works ... we
1472 * expect that if the usb core stops telling the cppi core
1473 * to pull more data from it, then it'll be safe to flush
1474 * current RX DMA state iff any pending fifo transfer is done.
1475 */
1476
1477 core_rxirq_disable(tibase, cppi_ch->index + 1);
1478
1479 /* for host, ensure ReqPkt is never set again */
1480 if (is_host_active(cppi_ch->controller->musb)) {
1481 value = musb_readl(tibase, DAVINCI_AUTOREQ_REG);
1482 value &= ~((0x3) << (cppi_ch->index * 2));
1483 musb_writel(tibase, DAVINCI_AUTOREQ_REG, value);
1484 }
1485
1486 csr = musb_readw(regs, MUSB_RXCSR);
1487
1488 /* for host, clear (just) ReqPkt at end of current packet(s) */
1489 if (is_host_active(cppi_ch->controller->musb)) {
1490 csr |= MUSB_RXCSR_H_WZC_BITS;
1491 csr &= ~MUSB_RXCSR_H_REQPKT;
1492 } else
1493 csr |= MUSB_RXCSR_P_WZC_BITS;
1494
1495 /* clear dma enable */
1496 csr &= ~(MUSB_RXCSR_DMAENAB);
1497 musb_writew(regs, MUSB_RXCSR, csr);
1498 csr = musb_readw(regs, MUSB_RXCSR);
1499
1500 /* Quiesce: wait for current dma to finish (if not cleanup).
1501 * We can't use bit zero of stateram->rx_sop, since that
1502 * refers to an entire "DMA packet" not just emptying the
1503 * current fifo. Most segments need multiple usb packets.
1504 */
1505 if (channel->status == MUSB_DMA_STATUS_BUSY)
1506 udelay(50);
1507
1508 /* scan the current list, reporting any data that was
1509 * transferred and acking any IRQ
1510 */
1511 cppi_rx_scan(controller, cppi_ch->index);
1512
1513 /* clobber the existing state once it's idle
1514 *
1515 * NOTE: arguably, we should also wait for all the other
1516 * RX channels to quiesce (how??) and then temporarily
1517 * disable RXCPPI_CTRL_REG ... but it seems that we can
1518 * rely on the controller restarting from state ram, with
1519 * only RXCPPI_BUFCNT state being bogus. BUFCNT will
1520 * correct itself after the next DMA transfer though.
1521 *
1522 * REVISIT does using rndis mode change that?
1523 */
1524 cppi_reset_rx(cppi_ch->state_ram);
1525
1526 /* next DMA request _should_ load cppi head ptr */
1527
1528 /* ... we don't "free" that list, only mutate it in place. */
1529 cppi_dump_rx(5, cppi_ch, " (done abort)");
1530
1531 /* clean up previously pending bds */
1532 cppi_bd_free(cppi_ch, cppi_ch->last_processed);
1533 cppi_ch->last_processed = NULL;
1534
1535 while (queue) {
1536 struct cppi_descriptor *tmp = queue->next;
1537
1538 cppi_bd_free(cppi_ch, queue);
1539 queue = tmp;
1540 }
1541 }
1542
1543 channel->status = MUSB_DMA_STATUS_FREE;
1544 cppi_ch->buf_dma = 0;
1545 cppi_ch->offset = 0;
1546 cppi_ch->buf_len = 0;
1547 cppi_ch->maxpacket = 0;
1548 return 0;
1549}
1550
1551/* TBD Queries:
1552 *
1553 * Power Management ... probably turn off cppi during suspend, restart;
1554 * check state ram? Clocking is presumably shared with usb core.
1555 */