| /* |
| * Copyright (C) 2005-2006 by Texas Instruments |
| * |
| * This file implements a DMA interface using TI's CPPI DMA. |
| * For now it's DaVinci-only, but CPPI isn't specific to DaVinci or USB. |
| * The TUSB6020, using VLYNQ, has CPPI that looks much like DaVinci. |
| */ |
| |
| #include <linux/usb.h> |
| |
| #include "musb_core.h" |
| #include "musb_debug.h" |
| #include "cppi_dma.h" |
| |
| |
| /* CPPI DMA status 7-mar-2006: |
| * |
| * - See musb_{host,gadget}.c for more info |
| * |
| * - Correct RX DMA generally forces the engine into irq-per-packet mode, |
| * which can easily saturate the CPU under non-mass-storage loads. |
| * |
| * NOTES 24-aug-2006 (2.6.18-rc4): |
| * |
| * - peripheral RXDMA wedged in a test with packets of length 512/512/1. |
| * evidently after the 1 byte packet was received and acked, the queue |
| * of BDs got garbaged so it wouldn't empty the fifo. (rxcsr 0x2003, |
| * and RX DMA0: 4 left, 80000000 8feff880, 8feff860 8feff860; 8f321401 |
| * 004001ff 00000001 .. 8feff860) Host was just getting NAKed on tx |
| * of its next (512 byte) packet. IRQ issues? |
| * |
| * REVISIT: the "transfer DMA" glue between CPPI and USB fifos will |
| * evidently also directly update the RX and TX CSRs ... so audit all |
| * host and peripheral side DMA code to avoid CSR access after DMA has |
| * been started. |
| */ |
| |
| /* REVISIT now we can avoid preallocating these descriptors; or |
| * more simply, switch to a global freelist not per-channel ones. |
| * Note: at full speed, 64 descriptors == 4K bulk data. |
| */ |
| #define NUM_TXCHAN_BD 64 |
| #define NUM_RXCHAN_BD 64 |
| |
| static inline void cpu_drain_writebuffer(void) |
| { |
| wmb(); |
| #ifdef CONFIG_CPU_ARM926T |
| /* REVISIT this "should not be needed", |
| * but lack of it sure seemed to hurt ... |
| */ |
| asm("mcr p15, 0, r0, c7, c10, 4 @ drain write buffer\n"); |
| #endif |
| } |
| |
| static inline struct cppi_descriptor *cppi_bd_alloc(struct cppi_channel *c) |
| { |
| struct cppi_descriptor *bd = c->freelist; |
| |
| if (bd) |
| c->freelist = bd->next; |
| return bd; |
| } |
| |
| static inline void |
| cppi_bd_free(struct cppi_channel *c, struct cppi_descriptor *bd) |
| { |
| if (!bd) |
| return; |
| bd->next = c->freelist; |
| c->freelist = bd; |
| } |
| |
| /* |
| * Start DMA controller |
| * |
| * Initialize the DMA controller as necessary. |
| */ |
| |
| /* zero out entire rx state RAM entry for the channel */ |
| static void cppi_reset_rx(struct cppi_rx_stateram __iomem *rx) |
| { |
| musb_writel(&rx->rx_skipbytes, 0, 0); |
| musb_writel(&rx->rx_head, 0, 0); |
| musb_writel(&rx->rx_sop, 0, 0); |
| musb_writel(&rx->rx_current, 0, 0); |
| musb_writel(&rx->rx_buf_current, 0, 0); |
| musb_writel(&rx->rx_len_len, 0, 0); |
| musb_writel(&rx->rx_cnt_cnt, 0, 0); |
| } |
| |
| /* zero out entire tx state RAM entry for the channel */ |
| static void cppi_reset_tx(struct cppi_tx_stateram __iomem *tx, u32 ptr) |
| { |
| musb_writel(&tx->tx_head, 0, 0); |
| musb_writel(&tx->tx_buf, 0, 0); |
| musb_writel(&tx->tx_current, 0, 0); |
| musb_writel(&tx->tx_buf_current, 0, 0); |
| musb_writel(&tx->tx_info, 0, 0); |
| musb_writel(&tx->tx_rem_len, 0, 0); |
| /* musb_writel(&tx->tx_dummy, 0, 0); */ |
| musb_writel(&tx->tx_complete, 0, ptr); |
| } |
| |
| static void __init cppi_pool_init(struct cppi *cppi, struct cppi_channel *c) |
| { |
| int j; |
| |
| /* initialize channel fields */ |
| c->head = NULL; |
| c->tail = NULL; |
| c->last_processed = NULL; |
| c->channel.status = MUSB_DMA_STATUS_UNKNOWN; |
| c->controller = cppi; |
| c->is_rndis = 0; |
| c->freelist = NULL; |
| |
| /* build the BD Free list for the channel */ |
| for (j = 0; j < NUM_TXCHAN_BD + 1; j++) { |
| struct cppi_descriptor *bd; |
| dma_addr_t dma; |
| |
| bd = dma_pool_alloc(cppi->pool, GFP_KERNEL, &dma); |
| bd->dma = dma; |
| cppi_bd_free(c, bd); |
| } |
| } |
| |
| static int cppi_channel_abort(struct dma_channel *); |
| |
| static void cppi_pool_free(struct cppi_channel *c) |
| { |
| struct cppi *cppi = c->controller; |
| struct cppi_descriptor *bd; |
| |
| (void) cppi_channel_abort(&c->channel); |
| c->channel.status = MUSB_DMA_STATUS_UNKNOWN; |
| c->controller = NULL; |
| |
| /* free all its bds */ |
| bd = c->last_processed; |
| do { |
| if (bd) |
| dma_pool_free(cppi->pool, bd, bd->dma); |
| bd = cppi_bd_alloc(c); |
| } while (bd); |
| c->last_processed = NULL; |
| } |
| |
| static int __init cppi_controller_start(struct dma_controller *c) |
| { |
| struct cppi *controller; |
| void __iomem *tibase; |
| int i; |
| |
| controller = container_of(c, struct cppi, controller); |
| |
| /* do whatever is necessary to start controller */ |
| for (i = 0; i < ARRAY_SIZE(controller->tx); i++) { |
| controller->tx[i].transmit = true; |
| controller->tx[i].index = i; |
| } |
| for (i = 0; i < ARRAY_SIZE(controller->rx); i++) { |
| controller->rx[i].transmit = false; |
| controller->rx[i].index = i; |
| } |
| |
| /* setup BD list on a per channel basis */ |
| for (i = 0; i < ARRAY_SIZE(controller->tx); i++) |
| cppi_pool_init(controller, controller->tx + i); |
| for (i = 0; i < ARRAY_SIZE(controller->rx); i++) |
| cppi_pool_init(controller, controller->rx + i); |
| |
| tibase = controller->tibase; |
| INIT_LIST_HEAD(&controller->tx_complete); |
| |
| /* initialise tx/rx channel head pointers to zero */ |
| for (i = 0; i < ARRAY_SIZE(controller->tx); i++) { |
| struct cppi_channel *tx_ch = controller->tx + i; |
| struct cppi_tx_stateram __iomem *tx; |
| |
| INIT_LIST_HEAD(&tx_ch->tx_complete); |
| |
| tx = tibase + DAVINCI_TXCPPI_STATERAM_OFFSET(i); |
| tx_ch->state_ram = tx; |
| cppi_reset_tx(tx, 0); |
| } |
| for (i = 0; i < ARRAY_SIZE(controller->rx); i++) { |
| struct cppi_channel *rx_ch = controller->rx + i; |
| struct cppi_rx_stateram __iomem *rx; |
| |
| INIT_LIST_HEAD(&rx_ch->tx_complete); |
| |
| rx = tibase + DAVINCI_RXCPPI_STATERAM_OFFSET(i); |
| rx_ch->state_ram = rx; |
| cppi_reset_rx(rx); |
| } |
| |
| /* enable individual cppi channels */ |
| musb_writel(tibase, DAVINCI_TXCPPI_INTENAB_REG, |
| DAVINCI_DMA_ALL_CHANNELS_ENABLE); |
| musb_writel(tibase, DAVINCI_RXCPPI_INTENAB_REG, |
| DAVINCI_DMA_ALL_CHANNELS_ENABLE); |
| |
| /* enable tx/rx CPPI control */ |
| musb_writel(tibase, DAVINCI_TXCPPI_CTRL_REG, DAVINCI_DMA_CTRL_ENABLE); |
| musb_writel(tibase, DAVINCI_RXCPPI_CTRL_REG, DAVINCI_DMA_CTRL_ENABLE); |
| |
| /* disable RNDIS mode, also host rx RNDIS autorequest */ |
| musb_writel(tibase, DAVINCI_RNDIS_REG, 0); |
| musb_writel(tibase, DAVINCI_AUTOREQ_REG, 0); |
| |
| return 0; |
| } |
| |
| /* |
| * Stop DMA controller |
| * |
| * De-Init the DMA controller as necessary. |
| */ |
| |
| static int cppi_controller_stop(struct dma_controller *c) |
| { |
| struct cppi *controller; |
| void __iomem *tibase; |
| int i; |
| |
| controller = container_of(c, struct cppi, controller); |
| |
| tibase = controller->tibase; |
| /* DISABLE INDIVIDUAL CHANNEL Interrupts */ |
| musb_writel(tibase, DAVINCI_TXCPPI_INTCLR_REG, |
| DAVINCI_DMA_ALL_CHANNELS_ENABLE); |
| musb_writel(tibase, DAVINCI_RXCPPI_INTCLR_REG, |
| DAVINCI_DMA_ALL_CHANNELS_ENABLE); |
| |
| DBG(1, "Tearing down RX and TX Channels\n"); |
| for (i = 0; i < ARRAY_SIZE(controller->tx); i++) { |
| /* FIXME restructure of txdma to use bds like rxdma */ |
| controller->tx[i].last_processed = NULL; |
| cppi_pool_free(controller->tx + i); |
| } |
| for (i = 0; i < ARRAY_SIZE(controller->rx); i++) |
| cppi_pool_free(controller->rx + i); |
| |
| /* in Tx Case proper teardown is supported. We resort to disabling |
| * Tx/Rx CPPI after cleanup of Tx channels. Before TX teardown is |
| * complete TX CPPI cannot be disabled. |
| */ |
| /*disable tx/rx cppi */ |
| musb_writel(tibase, DAVINCI_TXCPPI_CTRL_REG, DAVINCI_DMA_CTRL_DISABLE); |
| musb_writel(tibase, DAVINCI_RXCPPI_CTRL_REG, DAVINCI_DMA_CTRL_DISABLE); |
| |
| return 0; |
| } |
| |
| /* While dma channel is allocated, we only want the core irqs active |
| * for fault reports, otherwise we'd get irqs that we don't care about. |
| * Except for TX irqs, where dma done != fifo empty and reusable ... |
| * |
| * NOTE: docs don't say either way, but irq masking **enables** irqs. |
| * |
| * REVISIT same issue applies to pure PIO usage too, and non-cppi dma... |
| */ |
| static inline void core_rxirq_disable(void __iomem *tibase, unsigned epnum) |
| { |
| musb_writel(tibase, DAVINCI_USB_INT_MASK_CLR_REG, 1 << (epnum + 8)); |
| } |
| |
| static inline void core_rxirq_enable(void __iomem *tibase, unsigned epnum) |
| { |
| musb_writel(tibase, DAVINCI_USB_INT_MASK_SET_REG, 1 << (epnum + 8)); |
| } |
| |
| |
| /* |
| * Allocate a CPPI Channel for DMA. With CPPI, channels are bound to |
| * each transfer direction of a non-control endpoint, so allocating |
| * (and deallocating) is mostly a way to notice bad housekeeping on |
| * the software side. We assume the irqs are always active. |
| */ |
| static struct dma_channel * |
| cppi_channel_allocate(struct dma_controller *c, |
| struct musb_hw_ep *ep, u8 transmit) |
| { |
| struct cppi *controller; |
| u8 index; |
| struct cppi_channel *cppi_ch; |
| void __iomem *tibase; |
| |
| controller = container_of(c, struct cppi, controller); |
| tibase = controller->tibase; |
| |
| /* ep0 doesn't use DMA; remember cppi indices are 0..N-1 */ |
| index = ep->epnum - 1; |
| |
| /* return the corresponding CPPI Channel Handle, and |
| * probably disable the non-CPPI irq until we need it. |
| */ |
| if (transmit) { |
| if (index >= ARRAY_SIZE(controller->tx)) { |
| DBG(1, "no %cX%d CPPI channel\n", 'T', index); |
| return NULL; |
| } |
| cppi_ch = controller->tx + index; |
| } else { |
| if (index >= ARRAY_SIZE(controller->rx)) { |
| DBG(1, "no %cX%d CPPI channel\n", 'R', index); |
| return NULL; |
| } |
| cppi_ch = controller->rx + index; |
| core_rxirq_disable(tibase, ep->epnum); |
| } |
| |
| /* REVISIT make this an error later once the same driver code works |
| * with the other DMA engine too |
| */ |
| if (cppi_ch->hw_ep) |
| DBG(1, "re-allocating DMA%d %cX channel %p\n", |
| index, transmit ? 'T' : 'R', cppi_ch); |
| cppi_ch->hw_ep = ep; |
| cppi_ch->channel.status = MUSB_DMA_STATUS_FREE; |
| |
| DBG(4, "Allocate CPPI%d %cX\n", index, transmit ? 'T' : 'R'); |
| return &cppi_ch->channel; |
| } |
| |
| /* Release a CPPI Channel. */ |
| static void cppi_channel_release(struct dma_channel *channel) |
| { |
| struct cppi_channel *c; |
| void __iomem *tibase; |
| |
| /* REVISIT: for paranoia, check state and abort if needed... */ |
| |
| c = container_of(channel, struct cppi_channel, channel); |
| tibase = c->controller->tibase; |
| if (!c->hw_ep) |
| DBG(1, "releasing idle DMA channel %p\n", c); |
| else if (!c->transmit) |
| core_rxirq_enable(tibase, c->index + 1); |
| |
| /* for now, leave its cppi IRQ enabled (we won't trigger it) */ |
| c->hw_ep = NULL; |
| channel->status = MUSB_DMA_STATUS_UNKNOWN; |
| } |
| |
| /* Context: controller irqlocked */ |
| static void |
| cppi_dump_rx(int level, struct cppi_channel *c, const char *tag) |
| { |
| void __iomem *base = c->controller->mregs; |
| struct cppi_rx_stateram __iomem *rx = c->state_ram; |
| |
| musb_ep_select(base, c->index + 1); |
| |
| DBG(level, "RX DMA%d%s: %d left, csr %04x, " |
| "%08x H%08x S%08x C%08x, " |
| "B%08x L%08x %08x .. %08x" |
| "\n", |
| c->index, tag, |
| musb_readl(c->controller->tibase, |
| DAVINCI_RXCPPI_BUFCNT0_REG + 4 * c->index), |
| musb_readw(c->hw_ep->regs, MUSB_RXCSR), |
| |
| musb_readl(&rx->rx_skipbytes, 0), |
| musb_readl(&rx->rx_head, 0), |
| musb_readl(&rx->rx_sop, 0), |
| musb_readl(&rx->rx_current, 0), |
| |
| musb_readl(&rx->rx_buf_current, 0), |
| musb_readl(&rx->rx_len_len, 0), |
| musb_readl(&rx->rx_cnt_cnt, 0), |
| musb_readl(&rx->rx_complete, 0) |
| ); |
| } |
| |
| /* Context: controller irqlocked */ |
| static void |
| cppi_dump_tx(int level, struct cppi_channel *c, const char *tag) |
| { |
| void __iomem *base = c->controller->mregs; |
| struct cppi_tx_stateram __iomem *tx = c->state_ram; |
| |
| musb_ep_select(base, c->index + 1); |
| |
| DBG(level, "TX DMA%d%s: csr %04x, " |
| "H%08x S%08x C%08x %08x, " |
| "F%08x L%08x .. %08x" |
| "\n", |
| c->index, tag, |
| musb_readw(c->hw_ep->regs, MUSB_TXCSR), |
| |
| musb_readl(&tx->tx_head, 0), |
| musb_readl(&tx->tx_buf, 0), |
| musb_readl(&tx->tx_current, 0), |
| musb_readl(&tx->tx_buf_current, 0), |
| |
| musb_readl(&tx->tx_info, 0), |
| musb_readl(&tx->tx_rem_len, 0), |
| /* dummy/unused word 6 */ |
| musb_readl(&tx->tx_complete, 0) |
| ); |
| } |
| |
| /* Context: controller irqlocked */ |
| static inline void |
| cppi_rndis_update(struct cppi_channel *c, int is_rx, |
| void __iomem *tibase, int is_rndis) |
| { |
| /* we may need to change the rndis flag for this cppi channel */ |
| if (c->is_rndis != is_rndis) { |
| u32 value = musb_readl(tibase, DAVINCI_RNDIS_REG); |
| u32 temp = 1 << (c->index); |
| |
| if (is_rx) |
| temp <<= 16; |
| if (is_rndis) |
| value |= temp; |
| else |
| value &= ~temp; |
| musb_writel(tibase, DAVINCI_RNDIS_REG, value); |
| c->is_rndis = is_rndis; |
| } |
| } |
| |
| #ifdef CONFIG_USB_MUSB_DEBUG |
| static void cppi_dump_rxbd(const char *tag, struct cppi_descriptor *bd) |
| { |
| pr_debug("RXBD/%s %08x: " |
| "nxt %08x buf %08x off.blen %08x opt.plen %08x\n", |
| tag, bd->dma, |
| bd->hw_next, bd->hw_bufp, bd->hw_off_len, |
| bd->hw_options); |
| } |
| #endif |
| |
| static void cppi_dump_rxq(int level, const char *tag, struct cppi_channel *rx) |
| { |
| #ifdef CONFIG_USB_MUSB_DEBUG |
| struct cppi_descriptor *bd; |
| |
| if (!_dbg_level(level)) |
| return; |
| cppi_dump_rx(level, rx, tag); |
| if (rx->last_processed) |
| cppi_dump_rxbd("last", rx->last_processed); |
| for (bd = rx->head; bd; bd = bd->next) |
| cppi_dump_rxbd("active", bd); |
| #endif |
| } |
| |
| |
| /* NOTE: DaVinci autoreq is ignored except for host side "RNDIS" mode RX; |
| * so we won't ever use it (see "CPPI RX Woes" below). |
| */ |
| static inline int cppi_autoreq_update(struct cppi_channel *rx, |
| void __iomem *tibase, int onepacket, unsigned n_bds) |
| { |
| u32 val; |
| |
| #ifdef RNDIS_RX_IS_USABLE |
| u32 tmp; |
| /* assert(is_host_active(musb)) */ |
| |
| /* start from "AutoReq never" */ |
| tmp = musb_readl(tibase, DAVINCI_AUTOREQ_REG); |
| val = tmp & ~((0x3) << (rx->index * 2)); |
| |
| /* HCD arranged reqpkt for packet #1. we arrange int |
| * for all but the last one, maybe in two segments. |
| */ |
| if (!onepacket) { |
| #if 0 |
| /* use two segments, autoreq "all" then the last "never" */ |
| val |= ((0x3) << (rx->index * 2)); |
| n_bds--; |
| #else |
| /* one segment, autoreq "all-but-last" */ |
| val |= ((0x1) << (rx->index * 2)); |
| #endif |
| } |
| |
| if (val != tmp) { |
| int n = 100; |
| |
| /* make sure that autoreq is updated before continuing */ |
| musb_writel(tibase, DAVINCI_AUTOREQ_REG, val); |
| do { |
| tmp = musb_readl(tibase, DAVINCI_AUTOREQ_REG); |
| if (tmp == val) |
| break; |
| cpu_relax(); |
| } while (n-- > 0); |
| } |
| #endif |
| |
| /* REQPKT is turned off after each segment */ |
| if (n_bds && rx->channel.actual_len) { |
| void __iomem *regs = rx->hw_ep->regs; |
| |
| val = musb_readw(regs, MUSB_RXCSR); |
| if (!(val & MUSB_RXCSR_H_REQPKT)) { |
| val |= MUSB_RXCSR_H_REQPKT | MUSB_RXCSR_H_WZC_BITS; |
| musb_writew(regs, MUSB_RXCSR, val); |
| /* flush writebufer */ |
| val = musb_readw(regs, MUSB_RXCSR); |
| } |
| } |
| return n_bds; |
| } |
| |
| |
| /* Buffer enqueuing Logic: |
| * |
| * - RX builds new queues each time, to help handle routine "early |
| * termination" cases (faults, including errors and short reads) |
| * more correctly. |
| * |
| * - for now, TX reuses the same queue of BDs every time |
| * |
| * REVISIT long term, we want a normal dynamic model. |
| * ... the goal will be to append to the |
| * existing queue, processing completed "dma buffers" (segments) on the fly. |
| * |
| * Otherwise we force an IRQ latency between requests, which slows us a lot |
| * (especially in "transparent" dma). Unfortunately that model seems to be |
| * inherent in the DMA model from the Mentor code, except in the rare case |
| * of transfers big enough (~128+ KB) that we could append "middle" segments |
| * in the TX paths. (RX can't do this, see below.) |
| * |
| * That's true even in the CPPI- friendly iso case, where most urbs have |
| * several small segments provided in a group and where the "packet at a time" |
| * "transparent" DMA model is always correct, even on the RX side. |
| */ |
| |
| /* |
| * CPPI TX: |
| * ======== |
| * TX is a lot more reasonable than RX; it doesn't need to run in |
| * irq-per-packet mode very often. RNDIS mode seems to behave too |
| * (except how it handles the exactly-N-packets case). Building a |
| * txdma queue with multiple requests (urb or usb_request) looks |
| * like it would work ... but fault handling would need much testing. |
| * |
| * The main issue with TX mode RNDIS relates to transfer lengths that |
| * are an exact multiple of the packet length. It appears that there's |
| * a hiccup in that case (maybe the DMA completes before the ZLP gets |
| * written?) boiling down to not being able to rely on CPPI writing any |
| * terminating zero length packet before the next transfer is written. |
| * So that's punted to PIO; better yet, gadget drivers can avoid it. |
| * |
| * Plus, there's allegedly an undocumented constraint that rndis transfer |
| * length be a multiple of 64 bytes ... but the chip doesn't act that |
| * way, and we really don't _want_ that behavior anyway. |
| * |
| * On TX, "transparent" mode works ... although experiments have shown |
| * problems trying to use the SOP/EOP bits in different USB packets. |
| * |
| * REVISIT try to handle terminating zero length packets using CPPI |
| * instead of doing it by PIO after an IRQ. (Meanwhile, make Ethernet |
| * links avoid that issue by forcing them to avoid zlps.) |
| */ |
| static void |
| cppi_next_tx_segment(struct musb *musb, struct cppi_channel *tx) |
| { |
| unsigned maxpacket = tx->maxpacket; |
| dma_addr_t addr = tx->buf_dma + tx->offset; |
| size_t length = tx->buf_len - tx->offset; |
| struct cppi_descriptor *bd; |
| unsigned n_bds; |
| unsigned i; |
| struct cppi_tx_stateram __iomem *tx_ram = tx->state_ram; |
| int rndis; |
| |
| /* TX can use the CPPI "rndis" mode, where we can probably fit this |
| * transfer in one BD and one IRQ. The only time we would NOT want |
| * to use it is when hardware constraints prevent it, or if we'd |
| * trigger the "send a ZLP?" confusion. |
| */ |
| rndis = (maxpacket & 0x3f) == 0 |
| && length < 0xffff |
| && (length % maxpacket) != 0; |
| |
| if (rndis) { |
| maxpacket = length; |
| n_bds = 1; |
| } else { |
| n_bds = length / maxpacket; |
| if (!length || (length % maxpacket)) |
| n_bds++; |
| n_bds = min(n_bds, (unsigned) NUM_TXCHAN_BD); |
| length = min(n_bds * maxpacket, length); |
| } |
| |
| DBG(4, "TX DMA%d, pktSz %d %s bds %d dma 0x%x len %u\n", |
| tx->index, |
| maxpacket, |
| rndis ? "rndis" : "transparent", |
| n_bds, |
| addr, length); |
| |
| cppi_rndis_update(tx, 0, musb->ctrl_base, rndis); |
| |
| /* assuming here that channel_program is called during |
| * transfer initiation ... current code maintains state |
| * for one outstanding request only (no queues, not even |
| * the implicit ones of an iso urb). |
| */ |
| |
| bd = tx->freelist; |
| tx->head = bd; |
| tx->last_processed = NULL; |
| |
| /* FIXME use BD pool like RX side does, and just queue |
| * the minimum number for this request. |
| */ |
| |
| /* Prepare queue of BDs first, then hand it to hardware. |
| * All BDs except maybe the last should be of full packet |
| * size; for RNDIS there _is_ only that last packet. |
| */ |
| for (i = 0; i < n_bds; ) { |
| if (++i < n_bds && bd->next) |
| bd->hw_next = bd->next->dma; |
| else |
| bd->hw_next = 0; |
| |
| bd->hw_bufp = tx->buf_dma + tx->offset; |
| |
| /* FIXME set EOP only on the last packet, |
| * SOP only on the first ... avoid IRQs |
| */ |
| if ((tx->offset + maxpacket) <= tx->buf_len) { |
| tx->offset += maxpacket; |
| bd->hw_off_len = maxpacket; |
| bd->hw_options = CPPI_SOP_SET | CPPI_EOP_SET |
| | CPPI_OWN_SET | maxpacket; |
| } else { |
| /* only this one may be a partial USB Packet */ |
| u32 partial_len; |
| |
| partial_len = tx->buf_len - tx->offset; |
| tx->offset = tx->buf_len; |
| bd->hw_off_len = partial_len; |
| |
| bd->hw_options = CPPI_SOP_SET | CPPI_EOP_SET |
| | CPPI_OWN_SET | partial_len; |
| if (partial_len == 0) |
| bd->hw_options |= CPPI_ZERO_SET; |
| } |
| |
| DBG(5, "TXBD %p: nxt %08x buf %08x len %04x opt %08x\n", |
| bd, bd->hw_next, bd->hw_bufp, |
| bd->hw_off_len, bd->hw_options); |
| |
| /* update the last BD enqueued to the list */ |
| tx->tail = bd; |
| bd = bd->next; |
| } |
| |
| /* BDs live in DMA-coherent memory, but writes might be pending */ |
| cpu_drain_writebuffer(); |
| |
| /* Write to the HeadPtr in state RAM to trigger */ |
| musb_writel(&tx_ram->tx_head, 0, (u32)tx->freelist->dma); |
| |
| cppi_dump_tx(5, tx, "/S"); |
| } |
| |
| /* |
| * CPPI RX Woes: |
| * ============= |
| * Consider a 1KB bulk RX buffer in two scenarios: (a) it's fed two 300 byte |
| * packets back-to-back, and (b) it's fed two 512 byte packets back-to-back. |
| * (Full speed transfers have similar scenarios.) |
| * |
| * The correct behavior for Linux is that (a) fills the buffer with 300 bytes, |
| * and the next packet goes into a buffer that's queued later; while (b) fills |
| * the buffer with 1024 bytes. How to do that with CPPI? |
| * |
| * - RX queues in "rndis" mode -- one single BD -- handle (a) correctly, but |
| * (b) loses **BADLY** because nothing (!) happens when that second packet |
| * fills the buffer, much less when a third one arrives. (Which makes this |
| * not a "true" RNDIS mode. In the RNDIS protocol short-packet termination |
| * is optional, and it's fine if peripherals -- not hosts! -- pad messages |
| * out to end-of-buffer. Standard PCI host controller DMA descriptors |
| * implement that mode by default ... which is no accident.) |
| * |
| * - RX queues in "transparent" mode -- two BDs with 512 bytes each -- have |
| * converse problems: (b) is handled right, but (a) loses badly. CPPI RX |
| * ignores SOP/EOP markings and processes both of those BDs; so both packets |
| * are loaded into the buffer (with a 212 byte gap between them), and the next |
| * buffer queued will NOT get its 300 bytes of data. (It seems like SOP/EOP |
| * are intended as outputs for RX queues, not inputs...) |
| * |
| * - A variant of "transparent" mode -- one BD at a time -- is the only way to |
| * reliably make both cases work, with software handling both cases correctly |
| * and at the significant penalty of needing an IRQ per packet. (The lack of |
| * I/O overlap can be slightly ameliorated by enabling double buffering.) |
| * |
| * So how to get rid of IRQ-per-packet? The transparent multi-BD case could |
| * be used in special cases like mass storage, which sets URB_SHORT_NOT_OK |
| * (or maybe its peripheral side counterpart) to flag (a) scenarios as errors |
| * with guaranteed driver level fault recovery and scrubbing out what's left |
| * of that garbaged datastream. |
| * |
| * But there seems to be no way to identify the cases where CPPI RNDIS mode |
| * is appropriate -- which do NOT include RNDIS host drivers, but do include |
| * the CDC Ethernet driver! -- and the documentation is incomplete/wrong. |
| * So we can't _ever_ use RX RNDIS mode ... except by using a heuristic |
| * that applies best on the peripheral side (and which could fail rudely). |
| * |
| * Leaving only "transparent" mode; we avoid multi-bd modes in almost all |
| * cases other than mass storage class. Otherwise we're correct but slow, |
| * since CPPI penalizes our need for a "true RNDIS" default mode. |
| */ |
| |
| |
| /* Heuristic, intended to kick in for ethernet/rndis peripheral ONLY |
| * |
| * IFF |
| * (a) peripheral mode ... since rndis peripherals could pad their |
| * writes to hosts, causing i/o failure; or we'd have to cope with |
| * a largely unknowable variety of host side protocol variants |
| * (b) and short reads are NOT errors ... since full reads would |
| * cause those same i/o failures |
| * (c) and read length is |
| * - less than 64KB (max per cppi descriptor) |
| * - not a multiple of 4096 (g_zero default, full reads typical) |
| * - N (>1) packets long, ditto (full reads not EXPECTED) |
| * THEN |
| * try rx rndis mode |
| * |
| * Cost of heuristic failing: RXDMA wedges at the end of transfers that |
| * fill out the whole buffer. Buggy host side usb network drivers could |
| * trigger that, but "in the field" such bugs seem to be all but unknown. |
| * |
| * So this module parameter lets the heuristic be disabled. When using |
| * gadgetfs, the heuristic will probably need to be disabled. |
| */ |
| static int cppi_rx_rndis = 1; |
| |
| module_param(cppi_rx_rndis, bool, 0); |
| MODULE_PARM_DESC(cppi_rx_rndis, "enable/disable RX RNDIS heuristic"); |
| |
| |
| /** |
| * cppi_next_rx_segment - dma read for the next chunk of a buffer |
| * @musb: the controller |
| * @rx: dma channel |
| * @onepacket: true unless caller treats short reads as errors, and |
| * performs fault recovery above usbcore. |
| * Context: controller irqlocked |
| * |
| * See above notes about why we can't use multi-BD RX queues except in |
| * rare cases (mass storage class), and can never use the hardware "rndis" |
| * mode (since it's not a "true" RNDIS mode) with complete safety.. |
| * |
| * It's ESSENTIAL that callers specify "onepacket" mode unless they kick in |
| * code to recover from corrupted datastreams after each short transfer. |
| */ |
| static void |
| cppi_next_rx_segment(struct musb *musb, struct cppi_channel *rx, int onepacket) |
| { |
| unsigned maxpacket = rx->maxpacket; |
| dma_addr_t addr = rx->buf_dma + rx->offset; |
| size_t length = rx->buf_len - rx->offset; |
| struct cppi_descriptor *bd, *tail; |
| unsigned n_bds; |
| unsigned i; |
| void __iomem *tibase = musb->ctrl_base; |
| int is_rndis = 0; |
| struct cppi_rx_stateram __iomem *rx_ram = rx->state_ram; |
| |
| if (onepacket) { |
| /* almost every USB driver, host or peripheral side */ |
| n_bds = 1; |
| |
| /* maybe apply the heuristic above */ |
| if (cppi_rx_rndis |
| && is_peripheral_active(musb) |
| && length > maxpacket |
| && (length & ~0xffff) == 0 |
| && (length & 0x0fff) != 0 |
| && (length & (maxpacket - 1)) == 0) { |
| maxpacket = length; |
| is_rndis = 1; |
| } |
| } else { |
| /* virtually nothing except mass storage class */ |
| if (length > 0xffff) { |
| n_bds = 0xffff / maxpacket; |
| length = n_bds * maxpacket; |
| } else { |
| n_bds = length / maxpacket; |
| if (length % maxpacket) |
| n_bds++; |
| } |
| if (n_bds == 1) |
| onepacket = 1; |
| else |
| n_bds = min(n_bds, (unsigned) NUM_RXCHAN_BD); |
| } |
| |
| /* In host mode, autorequest logic can generate some IN tokens; it's |
| * tricky since we can't leave REQPKT set in RXCSR after the transfer |
| * finishes. So: multipacket transfers involve two or more segments. |
| * And always at least two IRQs ... RNDIS mode is not an option. |
| */ |
| if (is_host_active(musb)) |
| n_bds = cppi_autoreq_update(rx, tibase, onepacket, n_bds); |
| |
| cppi_rndis_update(rx, 1, musb->ctrl_base, is_rndis); |
| |
| length = min(n_bds * maxpacket, length); |
| |
| DBG(4, "RX DMA%d seg, maxp %d %s bds %d (cnt %d) " |
| "dma 0x%x len %u %u/%u\n", |
| rx->index, maxpacket, |
| onepacket |
| ? (is_rndis ? "rndis" : "onepacket") |
| : "multipacket", |
| n_bds, |
| musb_readl(tibase, |
| DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4)) |
| & 0xffff, |
| addr, length, rx->channel.actual_len, rx->buf_len); |
| |
| /* only queue one segment at a time, since the hardware prevents |
| * correct queue shutdown after unexpected short packets |
| */ |
| bd = cppi_bd_alloc(rx); |
| rx->head = bd; |
| |
| /* Build BDs for all packets in this segment */ |
| for (i = 0, tail = NULL; bd && i < n_bds; i++, tail = bd) { |
| u32 bd_len; |
| |
| if (i) { |
| bd = cppi_bd_alloc(rx); |
| if (!bd) |
| break; |
| tail->next = bd; |
| tail->hw_next = bd->dma; |
| } |
| bd->hw_next = 0; |
| |
| /* all but the last packet will be maxpacket size */ |
| if (maxpacket < length) |
| bd_len = maxpacket; |
| else |
| bd_len = length; |
| |
| bd->hw_bufp = addr; |
| addr += bd_len; |
| rx->offset += bd_len; |
| |
| bd->hw_off_len = (0 /*offset*/ << 16) + bd_len; |
| bd->buflen = bd_len; |
| |
| bd->hw_options = CPPI_OWN_SET | (i == 0 ? length : 0); |
| length -= bd_len; |
| } |
| |
| /* we always expect at least one reusable BD! */ |
| if (!tail) { |
| WARNING("rx dma%d -- no BDs? need %d\n", rx->index, n_bds); |
| return; |
| } else if (i < n_bds) |
| WARNING("rx dma%d -- only %d of %d BDs\n", rx->index, i, n_bds); |
| |
| tail->next = NULL; |
| tail->hw_next = 0; |
| |
| bd = rx->head; |
| rx->tail = tail; |
| |
| /* short reads and other faults should terminate this entire |
| * dma segment. we want one "dma packet" per dma segment, not |
| * one per USB packet, terminating the whole queue at once... |
| * NOTE that current hardware seems to ignore SOP and EOP. |
| */ |
| bd->hw_options |= CPPI_SOP_SET; |
| tail->hw_options |= CPPI_EOP_SET; |
| |
| #ifdef CONFIG_USB_MUSB_DEBUG |
| if (_dbg_level(5)) { |
| struct cppi_descriptor *d; |
| |
| for (d = rx->head; d; d = d->next) |
| cppi_dump_rxbd("S", d); |
| } |
| #endif |
| |
| /* in case the preceding transfer left some state... */ |
| tail = rx->last_processed; |
| if (tail) { |
| tail->next = bd; |
| tail->hw_next = bd->dma; |
| } |
| |
| core_rxirq_enable(tibase, rx->index + 1); |
| |
| /* BDs live in DMA-coherent memory, but writes might be pending */ |
| cpu_drain_writebuffer(); |
| |
| /* REVISIT specs say to write this AFTER the BUFCNT register |
| * below ... but that loses badly. |
| */ |
| musb_writel(&rx_ram->rx_head, 0, bd->dma); |
| |
| /* bufferCount must be at least 3, and zeroes on completion |
| * unless it underflows below zero, or stops at two, or keeps |
| * growing ... grr. |
| */ |
| i = musb_readl(tibase, |
| DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4)) |
| & 0xffff; |
| |
| if (!i) |
| musb_writel(tibase, |
| DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4), |
| n_bds + 2); |
| else if (n_bds > (i - 3)) |
| musb_writel(tibase, |
| DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4), |
| n_bds - (i - 3)); |
| |
| i = musb_readl(tibase, |
| DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4)) |
| & 0xffff; |
| if (i < (2 + n_bds)) { |
| DBG(2, "bufcnt%d underrun - %d (for %d)\n", |
| rx->index, i, n_bds); |
| musb_writel(tibase, |
| DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4), |
| n_bds + 2); |
| } |
| |
| cppi_dump_rx(4, rx, "/S"); |
| } |
| |
| /** |
| * cppi_channel_program - program channel for data transfer |
| * @ch: the channel |
| * @maxpacket: max packet size |
| * @mode: For RX, 1 unless the usb protocol driver promised to treat |
| * all short reads as errors and kick in high level fault recovery. |
| * For TX, ignored because of RNDIS mode races/glitches. |
| * @dma_addr: dma address of buffer |
| * @len: length of buffer |
| * Context: controller irqlocked |
| */ |
| static int cppi_channel_program(struct dma_channel *ch, |
| u16 maxpacket, u8 mode, |
| dma_addr_t dma_addr, u32 len) |
| { |
| struct cppi_channel *cppi_ch; |
| struct cppi *controller; |
| struct musb *musb; |
| |
| cppi_ch = container_of(ch, struct cppi_channel, channel); |
| controller = cppi_ch->controller; |
| musb = controller->musb; |
| |
| switch (ch->status) { |
| case MUSB_DMA_STATUS_BUS_ABORT: |
| case MUSB_DMA_STATUS_CORE_ABORT: |
| /* fault irq handler should have handled cleanup */ |
| WARNING("%cX DMA%d not cleaned up after abort!\n", |
| cppi_ch->transmit ? 'T' : 'R', |
| cppi_ch->index); |
| /* WARN_ON(1); */ |
| break; |
| case MUSB_DMA_STATUS_BUSY: |
| WARNING("program active channel? %cX DMA%d\n", |
| cppi_ch->transmit ? 'T' : 'R', |
| cppi_ch->index); |
| /* WARN_ON(1); */ |
| break; |
| case MUSB_DMA_STATUS_UNKNOWN: |
| DBG(1, "%cX DMA%d not allocated!\n", |
| cppi_ch->transmit ? 'T' : 'R', |
| cppi_ch->index); |
| /* FALLTHROUGH */ |
| case MUSB_DMA_STATUS_FREE: |
| break; |
| } |
| |
| ch->status = MUSB_DMA_STATUS_BUSY; |
| |
| /* set transfer parameters, then queue up its first segment */ |
| cppi_ch->buf_dma = dma_addr; |
| cppi_ch->offset = 0; |
| cppi_ch->maxpacket = maxpacket; |
| cppi_ch->buf_len = len; |
| |
| /* TX channel? or RX? */ |
| if (cppi_ch->transmit) |
| cppi_next_tx_segment(musb, cppi_ch); |
| else |
| cppi_next_rx_segment(musb, cppi_ch, mode); |
| |
| return true; |
| } |
| |
| static bool cppi_rx_scan(struct cppi *cppi, unsigned ch) |
| { |
| struct cppi_channel *rx = &cppi->rx[ch]; |
| struct cppi_rx_stateram __iomem *state = rx->state_ram; |
| struct cppi_descriptor *bd; |
| struct cppi_descriptor *last = rx->last_processed; |
| bool completed = false; |
| bool acked = false; |
| int i; |
| dma_addr_t safe2ack; |
| void __iomem *regs = rx->hw_ep->regs; |
| |
| cppi_dump_rx(6, rx, "/K"); |
| |
| bd = last ? last->next : rx->head; |
| if (!bd) |
| return false; |
| |
| /* run through all completed BDs */ |
| for (i = 0, safe2ack = musb_readl(&state->rx_complete, 0); |
| (safe2ack || completed) && bd && i < NUM_RXCHAN_BD; |
| i++, bd = bd->next) { |
| u16 len; |
| |
| /* catch latest BD writes from CPPI */ |
| rmb(); |
| if (!completed && (bd->hw_options & CPPI_OWN_SET)) |
| break; |
| |
| DBG(5, "C/RXBD %08x: nxt %08x buf %08x " |
| "off.len %08x opt.len %08x (%d)\n", |
| bd->dma, bd->hw_next, bd->hw_bufp, |
| bd->hw_off_len, bd->hw_options, |
| rx->channel.actual_len); |
| |
| /* actual packet received length */ |
| if ((bd->hw_options & CPPI_SOP_SET) && !completed) |
| len = bd->hw_off_len & CPPI_RECV_PKTLEN_MASK; |
| else |
| len = 0; |
| |
| if (bd->hw_options & CPPI_EOQ_MASK) |
| completed = true; |
| |
| if (!completed && len < bd->buflen) { |
| /* NOTE: when we get a short packet, RXCSR_H_REQPKT |
| * must have been cleared, and no more DMA packets may |
| * active be in the queue... TI docs didn't say, but |
| * CPPI ignores those BDs even though OWN is still set. |
| */ |
| completed = true; |
| DBG(3, "rx short %d/%d (%d)\n", |
| len, bd->buflen, |
| rx->channel.actual_len); |
| } |
| |
| /* If we got here, we expect to ack at least one BD; meanwhile |
| * CPPI may completing other BDs while we scan this list... |
| * |
| * RACE: we can notice OWN cleared before CPPI raises the |
| * matching irq by writing that BD as the completion pointer. |
| * In such cases, stop scanning and wait for the irq, avoiding |
| * lost acks and states where BD ownership is unclear. |
| */ |
| if (bd->dma == safe2ack) { |
| musb_writel(&state->rx_complete, 0, safe2ack); |
| safe2ack = musb_readl(&state->rx_complete, 0); |
| acked = true; |
| if (bd->dma == safe2ack) |
| safe2ack = 0; |
| } |
| |
| rx->channel.actual_len += len; |
| |
| cppi_bd_free(rx, last); |
| last = bd; |
| |
| /* stop scanning on end-of-segment */ |
| if (bd->hw_next == 0) |
| completed = true; |
| } |
| rx->last_processed = last; |
| |
| /* dma abort, lost ack, or ... */ |
| if (!acked && last) { |
| int csr; |
| |
| if (safe2ack == 0 || safe2ack == rx->last_processed->dma) |
| musb_writel(&state->rx_complete, 0, safe2ack); |
| if (safe2ack == 0) { |
| cppi_bd_free(rx, last); |
| rx->last_processed = NULL; |
| |
| /* if we land here on the host side, H_REQPKT will |
| * be clear and we need to restart the queue... |
| */ |
| WARN_ON(rx->head); |
| } |
| musb_ep_select(cppi->mregs, rx->index + 1); |
| csr = musb_readw(regs, MUSB_RXCSR); |
| if (csr & MUSB_RXCSR_DMAENAB) { |
| DBG(4, "list%d %p/%p, last %08x%s, csr %04x\n", |
| rx->index, |
| rx->head, rx->tail, |
| rx->last_processed |
| ? rx->last_processed->dma |
| : 0, |
| completed ? ", completed" : "", |
| csr); |
| cppi_dump_rxq(4, "/what?", rx); |
| } |
| } |
| if (!completed) { |
| int csr; |
| |
| rx->head = bd; |
| |
| /* REVISIT seems like "autoreq all but EOP" doesn't... |
| * setting it here "should" be racey, but seems to work |
| */ |
| csr = musb_readw(rx->hw_ep->regs, MUSB_RXCSR); |
| if (is_host_active(cppi->musb) |
| && bd |
| && !(csr & MUSB_RXCSR_H_REQPKT)) { |
| csr |= MUSB_RXCSR_H_REQPKT; |
| musb_writew(regs, MUSB_RXCSR, |
| MUSB_RXCSR_H_WZC_BITS | csr); |
| csr = musb_readw(rx->hw_ep->regs, MUSB_RXCSR); |
| } |
| } else { |
| rx->head = NULL; |
| rx->tail = NULL; |
| } |
| |
| cppi_dump_rx(6, rx, completed ? "/completed" : "/cleaned"); |
| return completed; |
| } |
| |
| void cppi_completion(struct musb *musb, u32 rx, u32 tx) |
| { |
| void __iomem *tibase; |
| int i, index; |
| struct cppi *cppi; |
| struct musb_hw_ep *hw_ep = NULL; |
| |
| cppi = container_of(musb->dma_controller, struct cppi, controller); |
| |
| tibase = musb->ctrl_base; |
| |
| /* process TX channels */ |
| for (index = 0; tx; tx = tx >> 1, index++) { |
| struct cppi_channel *tx_ch; |
| struct cppi_tx_stateram __iomem *tx_ram; |
| bool completed = false; |
| struct cppi_descriptor *bd; |
| |
| if (!(tx & 1)) |
| continue; |
| |
| tx_ch = cppi->tx + index; |
| tx_ram = tx_ch->state_ram; |
| |
| /* FIXME need a cppi_tx_scan() routine, which |
| * can also be called from abort code |
| */ |
| |
| cppi_dump_tx(5, tx_ch, "/E"); |
| |
| bd = tx_ch->head; |
| |
| if (NULL == bd) { |
| DBG(1, "null BD\n"); |
| continue; |
| } |
| |
| /* run through all completed BDs */ |
| for (i = 0; !completed && bd && i < NUM_TXCHAN_BD; |
| i++, bd = bd->next) { |
| u16 len; |
| |
| /* catch latest BD writes from CPPI */ |
| rmb(); |
| if (bd->hw_options & CPPI_OWN_SET) |
| break; |
| |
| DBG(5, "C/TXBD %p n %x b %x off %x opt %x\n", |
| bd, bd->hw_next, bd->hw_bufp, |
| bd->hw_off_len, bd->hw_options); |
| |
| len = bd->hw_off_len & CPPI_BUFFER_LEN_MASK; |
| tx_ch->channel.actual_len += len; |
| |
| tx_ch->last_processed = bd; |
| |
| /* write completion register to acknowledge |
| * processing of completed BDs, and possibly |
| * release the IRQ; EOQ might not be set ... |
| * |
| * REVISIT use the same ack strategy as rx |
| * |
| * REVISIT have observed bit 18 set; huh?? |
| */ |
| /* if ((bd->hw_options & CPPI_EOQ_MASK)) */ |
| musb_writel(&tx_ram->tx_complete, 0, bd->dma); |
| |
| /* stop scanning on end-of-segment */ |
| if (bd->hw_next == 0) |
| completed = true; |
| } |
| |
| /* on end of segment, maybe go to next one */ |
| if (completed) { |
| /* cppi_dump_tx(4, tx_ch, "/complete"); */ |
| |
| /* transfer more, or report completion */ |
| if (tx_ch->offset >= tx_ch->buf_len) { |
| tx_ch->head = NULL; |
| tx_ch->tail = NULL; |
| tx_ch->channel.status = MUSB_DMA_STATUS_FREE; |
| |
| hw_ep = tx_ch->hw_ep; |
| |
| /* Peripheral role never repurposes the |
| * endpoint, so immediate completion is |
| * safe. Host role waits for the fifo |
| * to empty (TXPKTRDY irq) before going |
| * to the next queued bulk transfer. |
| */ |
| if (is_host_active(cppi->musb)) { |
| #if 0 |
| /* WORKAROUND because we may |
| * not always get TXKPTRDY ... |
| */ |
| int csr; |
| |
| csr = musb_readw(hw_ep->regs, |
| MUSB_TXCSR); |
| if (csr & MUSB_TXCSR_TXPKTRDY) |
| #endif |
| completed = false; |
| } |
| if (completed) |
| musb_dma_completion(musb, index + 1, 1); |
| |
| } else { |
| /* Bigger transfer than we could fit in |
| * that first batch of descriptors... |
| */ |
| cppi_next_tx_segment(musb, tx_ch); |
| } |
| } else |
| tx_ch->head = bd; |
| } |
| |
| /* Start processing the RX block */ |
| for (index = 0; rx; rx = rx >> 1, index++) { |
| |
| if (rx & 1) { |
| struct cppi_channel *rx_ch; |
| |
| rx_ch = cppi->rx + index; |
| |
| /* let incomplete dma segments finish */ |
| if (!cppi_rx_scan(cppi, index)) |
| continue; |
| |
| /* start another dma segment if needed */ |
| if (rx_ch->channel.actual_len != rx_ch->buf_len |
| && rx_ch->channel.actual_len |
| == rx_ch->offset) { |
| cppi_next_rx_segment(musb, rx_ch, 1); |
| continue; |
| } |
| |
| /* all segments completed! */ |
| rx_ch->channel.status = MUSB_DMA_STATUS_FREE; |
| |
| hw_ep = rx_ch->hw_ep; |
| |
| core_rxirq_disable(tibase, index + 1); |
| musb_dma_completion(musb, index + 1, 0); |
| } |
| } |
| |
| /* write to CPPI EOI register to re-enable interrupts */ |
| musb_writel(tibase, DAVINCI_CPPI_EOI_REG, 0); |
| } |
| |
| /* Instantiate a software object representing a DMA controller. */ |
| struct dma_controller *__init |
| dma_controller_create(struct musb *musb, void __iomem *mregs) |
| { |
| struct cppi *controller; |
| |
| controller = kzalloc(sizeof *controller, GFP_KERNEL); |
| if (!controller) |
| return NULL; |
| |
| controller->mregs = mregs; |
| controller->tibase = mregs - DAVINCI_BASE_OFFSET; |
| |
| controller->musb = musb; |
| controller->controller.start = cppi_controller_start; |
| controller->controller.stop = cppi_controller_stop; |
| controller->controller.channel_alloc = cppi_channel_allocate; |
| controller->controller.channel_release = cppi_channel_release; |
| controller->controller.channel_program = cppi_channel_program; |
| controller->controller.channel_abort = cppi_channel_abort; |
| |
| /* NOTE: allocating from on-chip SRAM would give the least |
| * contention for memory access, if that ever matters here. |
| */ |
| |
| /* setup BufferPool */ |
| controller->pool = dma_pool_create("cppi", |
| controller->musb->controller, |
| sizeof(struct cppi_descriptor), |
| CPPI_DESCRIPTOR_ALIGN, 0); |
| if (!controller->pool) { |
| kfree(controller); |
| return NULL; |
| } |
| |
| return &controller->controller; |
| } |
| |
| /* |
| * Destroy a previously-instantiated DMA controller. |
| */ |
| void dma_controller_destroy(struct dma_controller *c) |
| { |
| struct cppi *cppi; |
| |
| cppi = container_of(c, struct cppi, controller); |
| |
| /* assert: caller stopped the controller first */ |
| dma_pool_destroy(cppi->pool); |
| |
| kfree(cppi); |
| } |
| |
| /* |
| * Context: controller irqlocked, endpoint selected |
| */ |
| static int cppi_channel_abort(struct dma_channel *channel) |
| { |
| struct cppi_channel *cppi_ch; |
| struct cppi *controller; |
| void __iomem *mbase; |
| void __iomem *tibase; |
| void __iomem *regs; |
| u32 value; |
| struct cppi_descriptor *queue; |
| |
| cppi_ch = container_of(channel, struct cppi_channel, channel); |
| |
| controller = cppi_ch->controller; |
| |
| switch (channel->status) { |
| case MUSB_DMA_STATUS_BUS_ABORT: |
| case MUSB_DMA_STATUS_CORE_ABORT: |
| /* from RX or TX fault irq handler */ |
| case MUSB_DMA_STATUS_BUSY: |
| /* the hardware needs shutting down */ |
| regs = cppi_ch->hw_ep->regs; |
| break; |
| case MUSB_DMA_STATUS_UNKNOWN: |
| case MUSB_DMA_STATUS_FREE: |
| return 0; |
| default: |
| return -EINVAL; |
| } |
| |
| if (!cppi_ch->transmit && cppi_ch->head) |
| cppi_dump_rxq(3, "/abort", cppi_ch); |
| |
| mbase = controller->mregs; |
| tibase = controller->tibase; |
| |
| queue = cppi_ch->head; |
| cppi_ch->head = NULL; |
| cppi_ch->tail = NULL; |
| |
| /* REVISIT should rely on caller having done this, |
| * and caller should rely on us not changing it. |
| * peripheral code is safe ... check host too. |
| */ |
| musb_ep_select(mbase, cppi_ch->index + 1); |
| |
| if (cppi_ch->transmit) { |
| struct cppi_tx_stateram __iomem *tx_ram; |
| int enabled; |
| |
| /* mask interrupts raised to signal teardown complete. */ |
| enabled = musb_readl(tibase, DAVINCI_TXCPPI_INTENAB_REG) |
| & (1 << cppi_ch->index); |
| if (enabled) |
| musb_writel(tibase, DAVINCI_TXCPPI_INTCLR_REG, |
| (1 << cppi_ch->index)); |
| |
| /* REVISIT put timeouts on these controller handshakes */ |
| |
| cppi_dump_tx(6, cppi_ch, " (teardown)"); |
| |
| /* teardown DMA engine then usb core */ |
| do { |
| value = musb_readl(tibase, DAVINCI_TXCPPI_TEAR_REG); |
| } while (!(value & CPPI_TEAR_READY)); |
| musb_writel(tibase, DAVINCI_TXCPPI_TEAR_REG, cppi_ch->index); |
| |
| tx_ram = cppi_ch->state_ram; |
| do { |
| value = musb_readl(&tx_ram->tx_complete, 0); |
| } while (0xFFFFFFFC != value); |
| musb_writel(&tx_ram->tx_complete, 0, 0xFFFFFFFC); |
| |
| /* FIXME clean up the transfer state ... here? |
| * the completion routine should get called with |
| * an appropriate status code. |
| */ |
| |
| value = musb_readw(regs, MUSB_TXCSR); |
| value &= ~MUSB_TXCSR_DMAENAB; |
| value |= MUSB_TXCSR_FLUSHFIFO; |
| musb_writew(regs, MUSB_TXCSR, value); |
| musb_writew(regs, MUSB_TXCSR, value); |
| |
| /* re-enable interrupt */ |
| if (enabled) |
| musb_writel(tibase, DAVINCI_TXCPPI_INTENAB_REG, |
| (1 << cppi_ch->index)); |
| |
| /* While we scrub the TX state RAM, ensure that we clean |
| * up any interrupt that's currently asserted: |
| * 1. Write to completion Ptr value 0x1(bit 0 set) |
| * (write back mode) |
| * 2. Write to completion Ptr value 0x0(bit 0 cleared) |
| * (compare mode) |
| * Value written is compared(for bits 31:2) and when |
| * equal, interrupt is deasserted. |
| */ |
| cppi_reset_tx(tx_ram, 1); |
| musb_writel(&tx_ram->tx_complete, 0, 0); |
| |
| cppi_dump_tx(5, cppi_ch, " (done teardown)"); |
| |
| /* REVISIT tx side _should_ clean up the same way |
| * as the RX side ... this does no cleanup at all! |
| */ |
| |
| } else /* RX */ { |
| u16 csr; |
| |
| /* NOTE: docs don't guarantee any of this works ... we |
| * expect that if the usb core stops telling the cppi core |
| * to pull more data from it, then it'll be safe to flush |
| * current RX DMA state iff any pending fifo transfer is done. |
| */ |
| |
| core_rxirq_disable(tibase, cppi_ch->index + 1); |
| |
| /* for host, ensure ReqPkt is never set again */ |
| if (is_host_active(cppi_ch->controller->musb)) { |
| value = musb_readl(tibase, DAVINCI_AUTOREQ_REG); |
| value &= ~((0x3) << (cppi_ch->index * 2)); |
| musb_writel(tibase, DAVINCI_AUTOREQ_REG, value); |
| } |
| |
| csr = musb_readw(regs, MUSB_RXCSR); |
| |
| /* for host, clear (just) ReqPkt at end of current packet(s) */ |
| if (is_host_active(cppi_ch->controller->musb)) { |
| csr |= MUSB_RXCSR_H_WZC_BITS; |
| csr &= ~MUSB_RXCSR_H_REQPKT; |
| } else |
| csr |= MUSB_RXCSR_P_WZC_BITS; |
| |
| /* clear dma enable */ |
| csr &= ~(MUSB_RXCSR_DMAENAB); |
| musb_writew(regs, MUSB_RXCSR, csr); |
| csr = musb_readw(regs, MUSB_RXCSR); |
| |
| /* Quiesce: wait for current dma to finish (if not cleanup). |
| * We can't use bit zero of stateram->rx_sop, since that |
| * refers to an entire "DMA packet" not just emptying the |
| * current fifo. Most segments need multiple usb packets. |
| */ |
| if (channel->status == MUSB_DMA_STATUS_BUSY) |
| udelay(50); |
| |
| /* scan the current list, reporting any data that was |
| * transferred and acking any IRQ |
| */ |
| cppi_rx_scan(controller, cppi_ch->index); |
| |
| /* clobber the existing state once it's idle |
| * |
| * NOTE: arguably, we should also wait for all the other |
| * RX channels to quiesce (how??) and then temporarily |
| * disable RXCPPI_CTRL_REG ... but it seems that we can |
| * rely on the controller restarting from state ram, with |
| * only RXCPPI_BUFCNT state being bogus. BUFCNT will |
| * correct itself after the next DMA transfer though. |
| * |
| * REVISIT does using rndis mode change that? |
| */ |
| cppi_reset_rx(cppi_ch->state_ram); |
| |
| /* next DMA request _should_ load cppi head ptr */ |
| |
| /* ... we don't "free" that list, only mutate it in place. */ |
| cppi_dump_rx(5, cppi_ch, " (done abort)"); |
| |
| /* clean up previously pending bds */ |
| cppi_bd_free(cppi_ch, cppi_ch->last_processed); |
| cppi_ch->last_processed = NULL; |
| |
| while (queue) { |
| struct cppi_descriptor *tmp = queue->next; |
| |
| cppi_bd_free(cppi_ch, queue); |
| queue = tmp; |
| } |
| } |
| |
| channel->status = MUSB_DMA_STATUS_FREE; |
| cppi_ch->buf_dma = 0; |
| cppi_ch->offset = 0; |
| cppi_ch->buf_len = 0; |
| cppi_ch->maxpacket = 0; |
| return 0; |
| } |
| |
| /* TBD Queries: |
| * |
| * Power Management ... probably turn off cppi during suspend, restart; |
| * check state ram? Clocking is presumably shared with usb core. |
| */ |