ioat1: move descriptor allocation from submit to prep
The async_tx api assumes that after a successful ->prep a subsequent
->submit will not fail due to a lack of resources.
This also fixes a bug in the allocation failure case. Previously the
descriptors allocated prior to the allocation failure would not be
returned to the free list.
Signed-off-by: Maciej Sosnowski <maciej.sosnowski@intel.com>
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
diff --git a/drivers/dma/ioat/dma.c b/drivers/dma/ioat/dma.c
index 4840d48..c4333be 100644
--- a/drivers/dma/ioat/dma.c
+++ b/drivers/dma/ioat/dma.c
@@ -420,95 +420,29 @@
static dma_cookie_t ioat1_tx_submit(struct dma_async_tx_descriptor *tx)
{
struct ioat_dma_chan *ioat_chan = to_ioat_chan(tx->chan);
- struct ioat_desc_sw *first = tx_to_ioat_desc(tx);
- struct ioat_desc_sw *prev, *new;
- struct ioat_dma_descriptor *hw;
+ struct ioat_desc_sw *desc = tx_to_ioat_desc(tx);
+ struct ioat_desc_sw *first;
+ struct ioat_desc_sw *chain_tail;
dma_cookie_t cookie;
- LIST_HEAD(new_chain);
- u32 copy;
- size_t len;
- dma_addr_t src, dst;
- unsigned long orig_flags;
- unsigned int desc_count = 0;
-
- /* src and dest and len are stored in the initial descriptor */
- len = first->len;
- src = first->src;
- dst = first->dst;
- orig_flags = first->txd.flags;
- new = first;
spin_lock_bh(&ioat_chan->desc_lock);
- prev = to_ioat_desc(ioat_chan->used_desc.prev);
- prefetch(prev->hw);
- do {
- copy = min_t(size_t, len, ioat_chan->xfercap);
-
- async_tx_ack(&new->txd);
-
- hw = new->hw;
- hw->size = copy;
- hw->ctl = 0;
- hw->src_addr = src;
- hw->dst_addr = dst;
- hw->next = 0;
-
- /* chain together the physical address list for the HW */
- wmb();
- prev->hw->next = (u64) new->txd.phys;
-
- len -= copy;
- dst += copy;
- src += copy;
-
- list_add_tail(&new->node, &new_chain);
- desc_count++;
- prev = new;
- } while (len && (new = ioat1_dma_get_next_descriptor(ioat_chan)));
-
- if (!new) {
- dev_err(to_dev(ioat_chan), "tx submit failed\n");
- spin_unlock_bh(&ioat_chan->desc_lock);
- return -ENOMEM;
- }
-
- hw->ctl_f.compl_write = 1;
- if (first->txd.callback) {
- hw->ctl_f.int_en = 1;
- if (first != new) {
- /* move callback into to last desc */
- new->txd.callback = first->txd.callback;
- new->txd.callback_param
- = first->txd.callback_param;
- first->txd.callback = NULL;
- first->txd.callback_param = NULL;
- }
- }
-
- new->tx_cnt = desc_count;
- new->txd.flags = orig_flags; /* client is in control of this ack */
-
- /* store the original values for use in later cleanup */
- if (new != first) {
- new->src = first->src;
- new->dst = first->dst;
- new->len = first->len;
- }
-
/* cookie incr and addition to used_list must be atomic */
cookie = ioat_chan->common.cookie;
cookie++;
if (cookie < 0)
cookie = 1;
- ioat_chan->common.cookie = new->txd.cookie = cookie;
+ ioat_chan->common.cookie = tx->cookie = cookie;
/* write address into NextDescriptor field of last desc in chain */
- to_ioat_desc(ioat_chan->used_desc.prev)->hw->next =
- first->txd.phys;
- list_splice_tail(&new_chain, &ioat_chan->used_desc);
+ first = to_ioat_desc(tx->tx_list.next);
+ chain_tail = to_ioat_desc(ioat_chan->used_desc.prev);
+ /* make descriptor updates globally visible before chaining */
+ wmb();
+ chain_tail->hw->next = first->txd.phys;
+ list_splice_tail_init(&tx->tx_list, &ioat_chan->used_desc);
- ioat_chan->dmacount += desc_count;
- ioat_chan->pending += desc_count;
+ ioat_chan->dmacount += desc->tx_cnt;
+ ioat_chan->pending += desc->tx_cnt;
if (ioat_chan->pending >= ioat_pending_level)
__ioat1_dma_memcpy_issue_pending(ioat_chan);
spin_unlock_bh(&ioat_chan->desc_lock);
@@ -937,24 +871,66 @@
dma_addr_t dma_src, size_t len, unsigned long flags)
{
struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
- struct ioat_desc_sw *new;
+ struct ioat_desc_sw *desc;
+ size_t copy;
+ LIST_HEAD(chain);
+ dma_addr_t src = dma_src;
+ dma_addr_t dest = dma_dest;
+ size_t total_len = len;
+ struct ioat_dma_descriptor *hw = NULL;
+ int tx_cnt = 0;
spin_lock_bh(&ioat_chan->desc_lock);
- new = ioat_dma_get_next_descriptor(ioat_chan);
- spin_unlock_bh(&ioat_chan->desc_lock);
+ desc = ioat_dma_get_next_descriptor(ioat_chan);
+ do {
+ if (!desc)
+ break;
- if (new) {
- new->len = len;
- new->dst = dma_dest;
- new->src = dma_src;
- new->txd.flags = flags;
- return &new->txd;
- } else {
+ tx_cnt++;
+ copy = min_t(size_t, len, ioat_chan->xfercap);
+
+ hw = desc->hw;
+ hw->size = copy;
+ hw->ctl = 0;
+ hw->src_addr = src;
+ hw->dst_addr = dest;
+
+ list_add_tail(&desc->node, &chain);
+
+ len -= copy;
+ dest += copy;
+ src += copy;
+ if (len) {
+ struct ioat_desc_sw *next;
+
+ async_tx_ack(&desc->txd);
+ next = ioat_dma_get_next_descriptor(ioat_chan);
+ hw->next = next ? next->txd.phys : 0;
+ desc = next;
+ } else
+ hw->next = 0;
+ } while (len);
+
+ if (!desc) {
dev_err(to_dev(ioat_chan),
"chan%d - get_next_desc failed: %d descs waiting, %d total desc\n",
chan_num(ioat_chan), ioat_chan->dmacount, ioat_chan->desccount);
+ list_splice(&chain, &ioat_chan->free_desc);
+ spin_unlock_bh(&ioat_chan->desc_lock);
return NULL;
}
+ spin_unlock_bh(&ioat_chan->desc_lock);
+
+ desc->txd.flags = flags;
+ desc->tx_cnt = tx_cnt;
+ desc->src = dma_src;
+ desc->dst = dma_dest;
+ desc->len = total_len;
+ list_splice(&chain, &desc->txd.tx_list);
+ hw->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT);
+ hw->ctl_f.compl_write = 1;
+
+ return &desc->txd;
}
static struct dma_async_tx_descriptor *