blob: 13980190821762aa47a31729b688e5b36001ef77 [file] [log] [blame]
Ben Hutchings8ceee662008-04-27 12:55:59 +01001/****************************************************************************
2 * Driver for Solarflare Solarstorm network controllers and boards
3 * Copyright 2005-2006 Fen Systems Ltd.
Ben Hutchings0a6f40c2011-02-25 00:01:34 +00004 * Copyright 2005-2010 Solarflare Communications Inc.
Ben Hutchings8ceee662008-04-27 12:55:59 +01005 *
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 as published
8 * by the Free Software Foundation, incorporated herein by reference.
9 */
10
11#include <linux/pci.h>
12#include <linux/tcp.h>
13#include <linux/ip.h>
14#include <linux/in.h>
Ben Hutchings738a8f42009-11-29 15:16:05 +000015#include <linux/ipv6.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090016#include <linux/slab.h>
Ben Hutchings738a8f42009-11-29 15:16:05 +000017#include <net/ipv6.h>
Ben Hutchings8ceee662008-04-27 12:55:59 +010018#include <linux/if_ether.h>
19#include <linux/highmem.h>
20#include "net_driver.h"
Ben Hutchings8ceee662008-04-27 12:55:59 +010021#include "efx.h"
Ben Hutchings744093c2009-11-29 15:12:08 +000022#include "nic.h"
Ben Hutchings8ceee662008-04-27 12:55:59 +010023#include "workarounds.h"
24
25/*
26 * TX descriptor ring full threshold
27 *
28 * The tx_queue descriptor ring fill-level must fall below this value
29 * before we restart the netif queue
30 */
Steve Hodgsonecc910f2010-09-10 06:42:22 +000031#define EFX_TXQ_THRESHOLD(_efx) ((_efx)->txq_entries / 2u)
Ben Hutchings8ceee662008-04-27 12:55:59 +010032
Ben Hutchings4d566062008-09-01 12:47:12 +010033static void efx_dequeue_buffer(struct efx_tx_queue *tx_queue,
34 struct efx_tx_buffer *buffer)
Ben Hutchings8ceee662008-04-27 12:55:59 +010035{
36 if (buffer->unmap_len) {
37 struct pci_dev *pci_dev = tx_queue->efx->pci_dev;
Ben Hutchingscc12dac2008-09-01 12:46:43 +010038 dma_addr_t unmap_addr = (buffer->dma_addr + buffer->len -
39 buffer->unmap_len);
Ben Hutchings8ceee662008-04-27 12:55:59 +010040 if (buffer->unmap_single)
Ben Hutchingscc12dac2008-09-01 12:46:43 +010041 pci_unmap_single(pci_dev, unmap_addr, buffer->unmap_len,
42 PCI_DMA_TODEVICE);
Ben Hutchings8ceee662008-04-27 12:55:59 +010043 else
Ben Hutchingscc12dac2008-09-01 12:46:43 +010044 pci_unmap_page(pci_dev, unmap_addr, buffer->unmap_len,
45 PCI_DMA_TODEVICE);
Ben Hutchings8ceee662008-04-27 12:55:59 +010046 buffer->unmap_len = 0;
Ben Hutchingsdc8cfa52008-09-01 12:46:50 +010047 buffer->unmap_single = false;
Ben Hutchings8ceee662008-04-27 12:55:59 +010048 }
49
50 if (buffer->skb) {
51 dev_kfree_skb_any((struct sk_buff *) buffer->skb);
52 buffer->skb = NULL;
Ben Hutchings62776d02010-06-23 11:30:07 +000053 netif_vdbg(tx_queue->efx, tx_done, tx_queue->efx->net_dev,
54 "TX queue %d transmission id %x complete\n",
55 tx_queue->queue, tx_queue->read_count);
Ben Hutchings8ceee662008-04-27 12:55:59 +010056 }
57}
58
Ben Hutchingsb9b39b62008-05-07 12:51:12 +010059/**
60 * struct efx_tso_header - a DMA mapped buffer for packet headers
61 * @next: Linked list of free ones.
62 * The list is protected by the TX queue lock.
63 * @dma_unmap_len: Length to unmap for an oversize buffer, or 0.
64 * @dma_addr: The DMA address of the header below.
65 *
66 * This controls the memory used for a TSO header. Use TSOH_DATA()
67 * to find the packet header data. Use TSOH_SIZE() to calculate the
68 * total size required for a given packet header length. TSO headers
69 * in the free list are exactly %TSOH_STD_SIZE bytes in size.
70 */
71struct efx_tso_header {
72 union {
73 struct efx_tso_header *next;
74 size_t unmap_len;
75 };
76 dma_addr_t dma_addr;
77};
78
79static int efx_enqueue_skb_tso(struct efx_tx_queue *tx_queue,
Ben Hutchings740847d2008-09-01 12:48:23 +010080 struct sk_buff *skb);
Ben Hutchingsb9b39b62008-05-07 12:51:12 +010081static void efx_fini_tso(struct efx_tx_queue *tx_queue);
82static void efx_tsoh_heap_free(struct efx_tx_queue *tx_queue,
83 struct efx_tso_header *tsoh);
84
Ben Hutchings4d566062008-09-01 12:47:12 +010085static void efx_tsoh_free(struct efx_tx_queue *tx_queue,
86 struct efx_tx_buffer *buffer)
Ben Hutchingsb9b39b62008-05-07 12:51:12 +010087{
88 if (buffer->tsoh) {
89 if (likely(!buffer->tsoh->unmap_len)) {
90 buffer->tsoh->next = tx_queue->tso_headers_free;
91 tx_queue->tso_headers_free = buffer->tsoh;
92 } else {
93 efx_tsoh_heap_free(tx_queue, buffer->tsoh);
94 }
95 buffer->tsoh = NULL;
96 }
97}
98
Ben Hutchings8ceee662008-04-27 12:55:59 +010099
Ben Hutchings63f19882009-10-23 08:31:20 +0000100static inline unsigned
101efx_max_tx_len(struct efx_nic *efx, dma_addr_t dma_addr)
102{
103 /* Depending on the NIC revision, we can use descriptor
104 * lengths up to 8K or 8K-1. However, since PCI Express
105 * devices must split read requests at 4K boundaries, there is
106 * little benefit from using descriptors that cross those
107 * boundaries and we keep things simple by not doing so.
108 */
109 unsigned len = (~dma_addr & 0xfff) + 1;
110
111 /* Work around hardware bug for unaligned buffers. */
112 if (EFX_WORKAROUND_5391(efx) && (dma_addr & 0xf))
113 len = min_t(unsigned, len, 512 - (dma_addr & 0xf));
114
115 return len;
116}
117
Ben Hutchings8ceee662008-04-27 12:55:59 +0100118/*
119 * Add a socket buffer to a TX queue
120 *
121 * This maps all fragments of a socket buffer for DMA and adds them to
122 * the TX queue. The queue's insert pointer will be incremented by
123 * the number of fragments in the socket buffer.
124 *
125 * If any DMA mapping fails, any mapped fragments will be unmapped,
126 * the queue's insert pointer will be restored to its original value.
127 *
Ben Hutchings497f5ba2009-11-23 16:07:05 +0000128 * This function is split out from efx_hard_start_xmit to allow the
129 * loopback test to direct packets via specific TX queues.
130 *
Ben Hutchings8ceee662008-04-27 12:55:59 +0100131 * Returns NETDEV_TX_OK or NETDEV_TX_BUSY
132 * You must hold netif_tx_lock() to call this function.
133 */
Ben Hutchings497f5ba2009-11-23 16:07:05 +0000134netdev_tx_t efx_enqueue_skb(struct efx_tx_queue *tx_queue, struct sk_buff *skb)
Ben Hutchings8ceee662008-04-27 12:55:59 +0100135{
136 struct efx_nic *efx = tx_queue->efx;
137 struct pci_dev *pci_dev = efx->pci_dev;
138 struct efx_tx_buffer *buffer;
139 skb_frag_t *fragment;
140 struct page *page;
141 int page_offset;
Ben Hutchings63f19882009-10-23 08:31:20 +0000142 unsigned int len, unmap_len = 0, fill_level, insert_ptr;
Ben Hutchings8ceee662008-04-27 12:55:59 +0100143 dma_addr_t dma_addr, unmap_addr = 0;
144 unsigned int dma_len;
Ben Hutchingsdc8cfa52008-09-01 12:46:50 +0100145 bool unmap_single;
Ben Hutchings8ceee662008-04-27 12:55:59 +0100146 int q_space, i = 0;
Stephen Hemminger613573252009-08-31 19:50:58 +0000147 netdev_tx_t rc = NETDEV_TX_OK;
Ben Hutchings8ceee662008-04-27 12:55:59 +0100148
149 EFX_BUG_ON_PARANOID(tx_queue->write_count != tx_queue->insert_count);
150
Ben Hutchings9bc183d2009-11-23 16:06:47 +0000151 if (skb_shinfo(skb)->gso_size)
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100152 return efx_enqueue_skb_tso(tx_queue, skb);
153
Ben Hutchings8ceee662008-04-27 12:55:59 +0100154 /* Get size of the initial fragment */
155 len = skb_headlen(skb);
156
Ben Hutchingsbb145a92009-03-20 13:25:39 +0000157 /* Pad if necessary */
158 if (EFX_WORKAROUND_15592(efx) && skb->len <= 32) {
159 EFX_BUG_ON_PARANOID(skb->data_len);
160 len = 32 + 1;
161 if (skb_pad(skb, len - skb->len))
162 return NETDEV_TX_OK;
163 }
164
Ben Hutchings8ceee662008-04-27 12:55:59 +0100165 fill_level = tx_queue->insert_count - tx_queue->old_read_count;
Steve Hodgsonecc910f2010-09-10 06:42:22 +0000166 q_space = efx->txq_entries - 1 - fill_level;
Ben Hutchings8ceee662008-04-27 12:55:59 +0100167
168 /* Map for DMA. Use pci_map_single rather than pci_map_page
169 * since this is more efficient on machines with sparse
170 * memory.
171 */
Ben Hutchingsdc8cfa52008-09-01 12:46:50 +0100172 unmap_single = true;
Ben Hutchings8ceee662008-04-27 12:55:59 +0100173 dma_addr = pci_map_single(pci_dev, skb->data, len, PCI_DMA_TODEVICE);
174
175 /* Process all fragments */
176 while (1) {
FUJITA Tomonori8d8bb392008-07-25 19:44:49 -0700177 if (unlikely(pci_dma_mapping_error(pci_dev, dma_addr)))
Ben Hutchings8ceee662008-04-27 12:55:59 +0100178 goto pci_err;
179
180 /* Store fields for marking in the per-fragment final
181 * descriptor */
182 unmap_len = len;
183 unmap_addr = dma_addr;
184
185 /* Add to TX queue, splitting across DMA boundaries */
186 do {
187 if (unlikely(q_space-- <= 0)) {
188 /* It might be that completions have
189 * happened since the xmit path last
190 * checked. Update the xmit path's
191 * copy of read_count.
192 */
Ben Hutchingsc04bfc62010-12-10 01:24:16 +0000193 netif_tx_stop_queue(tx_queue->core_txq);
Ben Hutchings8ceee662008-04-27 12:55:59 +0100194 /* This memory barrier protects the
Ben Hutchingsc04bfc62010-12-10 01:24:16 +0000195 * change of queue state from the access
Ben Hutchings8ceee662008-04-27 12:55:59 +0100196 * of read_count. */
197 smp_mb();
198 tx_queue->old_read_count =
Ben Hutchings51c56f42010-11-10 18:46:40 +0000199 ACCESS_ONCE(tx_queue->read_count);
Ben Hutchings8ceee662008-04-27 12:55:59 +0100200 fill_level = (tx_queue->insert_count
201 - tx_queue->old_read_count);
Steve Hodgsonecc910f2010-09-10 06:42:22 +0000202 q_space = efx->txq_entries - 1 - fill_level;
Ben Hutchingsc04bfc62010-12-10 01:24:16 +0000203 if (unlikely(q_space-- <= 0)) {
204 rc = NETDEV_TX_BUSY;
205 goto unwind;
206 }
Ben Hutchings8ceee662008-04-27 12:55:59 +0100207 smp_mb();
Ben Hutchingsc04bfc62010-12-10 01:24:16 +0000208 netif_tx_start_queue(tx_queue->core_txq);
Ben Hutchings8ceee662008-04-27 12:55:59 +0100209 }
210
Steve Hodgsonecc910f2010-09-10 06:42:22 +0000211 insert_ptr = tx_queue->insert_count & tx_queue->ptr_mask;
Ben Hutchings8ceee662008-04-27 12:55:59 +0100212 buffer = &tx_queue->buffer[insert_ptr];
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100213 efx_tsoh_free(tx_queue, buffer);
214 EFX_BUG_ON_PARANOID(buffer->tsoh);
Ben Hutchings8ceee662008-04-27 12:55:59 +0100215 EFX_BUG_ON_PARANOID(buffer->skb);
216 EFX_BUG_ON_PARANOID(buffer->len);
Ben Hutchingsdc8cfa52008-09-01 12:46:50 +0100217 EFX_BUG_ON_PARANOID(!buffer->continuation);
Ben Hutchings8ceee662008-04-27 12:55:59 +0100218 EFX_BUG_ON_PARANOID(buffer->unmap_len);
219
Ben Hutchings63f19882009-10-23 08:31:20 +0000220 dma_len = efx_max_tx_len(efx, dma_addr);
221 if (likely(dma_len >= len))
Ben Hutchings8ceee662008-04-27 12:55:59 +0100222 dma_len = len;
223
Ben Hutchings8ceee662008-04-27 12:55:59 +0100224 /* Fill out per descriptor fields */
225 buffer->len = dma_len;
226 buffer->dma_addr = dma_addr;
227 len -= dma_len;
228 dma_addr += dma_len;
229 ++tx_queue->insert_count;
230 } while (len);
231
232 /* Transfer ownership of the unmapping to the final buffer */
Ben Hutchings8ceee662008-04-27 12:55:59 +0100233 buffer->unmap_single = unmap_single;
234 buffer->unmap_len = unmap_len;
235 unmap_len = 0;
236
237 /* Get address and size of next fragment */
238 if (i >= skb_shinfo(skb)->nr_frags)
239 break;
240 fragment = &skb_shinfo(skb)->frags[i];
241 len = fragment->size;
242 page = fragment->page;
243 page_offset = fragment->page_offset;
244 i++;
245 /* Map for DMA */
Ben Hutchingsdc8cfa52008-09-01 12:46:50 +0100246 unmap_single = false;
Ben Hutchings8ceee662008-04-27 12:55:59 +0100247 dma_addr = pci_map_page(pci_dev, page, page_offset, len,
248 PCI_DMA_TODEVICE);
249 }
250
251 /* Transfer ownership of the skb to the final buffer */
252 buffer->skb = skb;
Ben Hutchingsdc8cfa52008-09-01 12:46:50 +0100253 buffer->continuation = false;
Ben Hutchings8ceee662008-04-27 12:55:59 +0100254
255 /* Pass off to hardware */
Ben Hutchings152b6a62009-11-29 03:43:56 +0000256 efx_nic_push_buffers(tx_queue);
Ben Hutchings8ceee662008-04-27 12:55:59 +0100257
258 return NETDEV_TX_OK;
259
260 pci_err:
Ben Hutchings62776d02010-06-23 11:30:07 +0000261 netif_err(efx, tx_err, efx->net_dev,
262 " TX queue %d could not map skb with %d bytes %d "
263 "fragments for DMA\n", tx_queue->queue, skb->len,
264 skb_shinfo(skb)->nr_frags + 1);
Ben Hutchings8ceee662008-04-27 12:55:59 +0100265
266 /* Mark the packet as transmitted, and free the SKB ourselves */
Ben Hutchings9bc183d2009-11-23 16:06:47 +0000267 dev_kfree_skb_any(skb);
Ben Hutchings8ceee662008-04-27 12:55:59 +0100268
269 unwind:
270 /* Work backwards until we hit the original insert pointer value */
271 while (tx_queue->insert_count != tx_queue->write_count) {
272 --tx_queue->insert_count;
Steve Hodgsonecc910f2010-09-10 06:42:22 +0000273 insert_ptr = tx_queue->insert_count & tx_queue->ptr_mask;
Ben Hutchings8ceee662008-04-27 12:55:59 +0100274 buffer = &tx_queue->buffer[insert_ptr];
275 efx_dequeue_buffer(tx_queue, buffer);
276 buffer->len = 0;
277 }
278
279 /* Free the fragment we were mid-way through pushing */
Ben Hutchingsecbd95c2008-09-01 12:46:40 +0100280 if (unmap_len) {
281 if (unmap_single)
282 pci_unmap_single(pci_dev, unmap_addr, unmap_len,
283 PCI_DMA_TODEVICE);
284 else
285 pci_unmap_page(pci_dev, unmap_addr, unmap_len,
286 PCI_DMA_TODEVICE);
287 }
Ben Hutchings8ceee662008-04-27 12:55:59 +0100288
289 return rc;
290}
291
292/* Remove packets from the TX queue
293 *
294 * This removes packets from the TX queue, up to and including the
295 * specified index.
296 */
Ben Hutchings4d566062008-09-01 12:47:12 +0100297static void efx_dequeue_buffers(struct efx_tx_queue *tx_queue,
298 unsigned int index)
Ben Hutchings8ceee662008-04-27 12:55:59 +0100299{
300 struct efx_nic *efx = tx_queue->efx;
301 unsigned int stop_index, read_ptr;
Ben Hutchings8ceee662008-04-27 12:55:59 +0100302
Steve Hodgsonecc910f2010-09-10 06:42:22 +0000303 stop_index = (index + 1) & tx_queue->ptr_mask;
304 read_ptr = tx_queue->read_count & tx_queue->ptr_mask;
Ben Hutchings8ceee662008-04-27 12:55:59 +0100305
306 while (read_ptr != stop_index) {
307 struct efx_tx_buffer *buffer = &tx_queue->buffer[read_ptr];
308 if (unlikely(buffer->len == 0)) {
Ben Hutchings62776d02010-06-23 11:30:07 +0000309 netif_err(efx, tx_err, efx->net_dev,
310 "TX queue %d spurious TX completion id %x\n",
311 tx_queue->queue, read_ptr);
Ben Hutchings8ceee662008-04-27 12:55:59 +0100312 efx_schedule_reset(efx, RESET_TYPE_TX_SKIP);
313 return;
314 }
315
316 efx_dequeue_buffer(tx_queue, buffer);
Ben Hutchingsdc8cfa52008-09-01 12:46:50 +0100317 buffer->continuation = true;
Ben Hutchings8ceee662008-04-27 12:55:59 +0100318 buffer->len = 0;
319
320 ++tx_queue->read_count;
Steve Hodgsonecc910f2010-09-10 06:42:22 +0000321 read_ptr = tx_queue->read_count & tx_queue->ptr_mask;
Ben Hutchings8ceee662008-04-27 12:55:59 +0100322 }
323}
324
Ben Hutchings8ceee662008-04-27 12:55:59 +0100325/* Initiate a packet transmission. We use one channel per CPU
326 * (sharing when we have more CPUs than channels). On Falcon, the TX
327 * completion events will be directed back to the CPU that transmitted
328 * the packet, which should be cache-efficient.
329 *
330 * Context: non-blocking.
331 * Note that returning anything other than NETDEV_TX_OK will cause the
332 * OS to free the skb.
333 */
Stephen Hemminger613573252009-08-31 19:50:58 +0000334netdev_tx_t efx_hard_start_xmit(struct sk_buff *skb,
335 struct net_device *net_dev)
Ben Hutchings8ceee662008-04-27 12:55:59 +0100336{
Ben Hutchings767e4682008-09-01 12:43:14 +0100337 struct efx_nic *efx = netdev_priv(net_dev);
Ben Hutchings60ac1062008-09-01 12:44:59 +0100338 struct efx_tx_queue *tx_queue;
Ben Hutchings94b274b2011-01-10 21:18:20 +0000339 unsigned index, type;
Ben Hutchings60ac1062008-09-01 12:44:59 +0100340
Ben Hutchingsa7ef5932009-03-04 09:52:37 +0000341 if (unlikely(efx->port_inhibited))
342 return NETDEV_TX_BUSY;
343
Ben Hutchings94b274b2011-01-10 21:18:20 +0000344 index = skb_get_queue_mapping(skb);
345 type = skb->ip_summed == CHECKSUM_PARTIAL ? EFX_TXQ_TYPE_OFFLOAD : 0;
346 if (index >= efx->n_tx_channels) {
347 index -= efx->n_tx_channels;
348 type |= EFX_TXQ_TYPE_HIGHPRI;
349 }
350 tx_queue = efx_get_tx_queue(efx, index, type);
Ben Hutchings60ac1062008-09-01 12:44:59 +0100351
Ben Hutchings497f5ba2009-11-23 16:07:05 +0000352 return efx_enqueue_skb(tx_queue, skb);
Ben Hutchings8ceee662008-04-27 12:55:59 +0100353}
354
Ben Hutchings60031fc2011-01-12 18:39:40 +0000355void efx_init_tx_queue_core_txq(struct efx_tx_queue *tx_queue)
356{
Ben Hutchings94b274b2011-01-10 21:18:20 +0000357 struct efx_nic *efx = tx_queue->efx;
358
Ben Hutchings60031fc2011-01-12 18:39:40 +0000359 /* Must be inverse of queue lookup in efx_hard_start_xmit() */
Ben Hutchings94b274b2011-01-10 21:18:20 +0000360 tx_queue->core_txq =
361 netdev_get_tx_queue(efx->net_dev,
362 tx_queue->queue / EFX_TXQ_TYPES +
363 ((tx_queue->queue & EFX_TXQ_TYPE_HIGHPRI) ?
364 efx->n_tx_channels : 0));
365}
366
367int efx_setup_tc(struct net_device *net_dev, u8 num_tc)
368{
369 struct efx_nic *efx = netdev_priv(net_dev);
370 struct efx_channel *channel;
371 struct efx_tx_queue *tx_queue;
372 unsigned tc;
373 int rc;
374
375 if (efx_nic_rev(efx) < EFX_REV_FALCON_B0 || num_tc > EFX_MAX_TX_TC)
376 return -EINVAL;
377
378 if (num_tc == net_dev->num_tc)
379 return 0;
380
381 for (tc = 0; tc < num_tc; tc++) {
382 net_dev->tc_to_txq[tc].offset = tc * efx->n_tx_channels;
383 net_dev->tc_to_txq[tc].count = efx->n_tx_channels;
384 }
385
386 if (num_tc > net_dev->num_tc) {
387 /* Initialise high-priority queues as necessary */
388 efx_for_each_channel(channel, efx) {
389 efx_for_each_possible_channel_tx_queue(tx_queue,
390 channel) {
391 if (!(tx_queue->queue & EFX_TXQ_TYPE_HIGHPRI))
392 continue;
393 if (!tx_queue->buffer) {
394 rc = efx_probe_tx_queue(tx_queue);
395 if (rc)
396 return rc;
397 }
398 if (!tx_queue->initialised)
399 efx_init_tx_queue(tx_queue);
400 efx_init_tx_queue_core_txq(tx_queue);
401 }
402 }
403 } else {
404 /* Reduce number of classes before number of queues */
405 net_dev->num_tc = num_tc;
406 }
407
408 rc = netif_set_real_num_tx_queues(net_dev,
409 max_t(int, num_tc, 1) *
410 efx->n_tx_channels);
411 if (rc)
412 return rc;
413
414 /* Do not destroy high-priority queues when they become
415 * unused. We would have to flush them first, and it is
416 * fairly difficult to flush a subset of TX queues. Leave
417 * it to efx_fini_channels().
418 */
419
420 net_dev->num_tc = num_tc;
421 return 0;
Ben Hutchings60031fc2011-01-12 18:39:40 +0000422}
423
Ben Hutchings8ceee662008-04-27 12:55:59 +0100424void efx_xmit_done(struct efx_tx_queue *tx_queue, unsigned int index)
425{
426 unsigned fill_level;
427 struct efx_nic *efx = tx_queue->efx;
428
Steve Hodgsonecc910f2010-09-10 06:42:22 +0000429 EFX_BUG_ON_PARANOID(index > tx_queue->ptr_mask);
Ben Hutchings8ceee662008-04-27 12:55:59 +0100430
431 efx_dequeue_buffers(tx_queue, index);
432
433 /* See if we need to restart the netif queue. This barrier
Ben Hutchingsc04bfc62010-12-10 01:24:16 +0000434 * separates the update of read_count from the test of the
435 * queue state. */
Ben Hutchings8ceee662008-04-27 12:55:59 +0100436 smp_mb();
Ben Hutchingsc04bfc62010-12-10 01:24:16 +0000437 if (unlikely(netif_tx_queue_stopped(tx_queue->core_txq)) &&
438 likely(efx->port_enabled)) {
Ben Hutchings8ceee662008-04-27 12:55:59 +0100439 fill_level = tx_queue->insert_count - tx_queue->read_count;
Steve Hodgsonecc910f2010-09-10 06:42:22 +0000440 if (fill_level < EFX_TXQ_THRESHOLD(efx)) {
Ben Hutchings55668612008-05-16 21:16:10 +0100441 EFX_BUG_ON_PARANOID(!efx_dev_registered(efx));
Ben Hutchingsc04bfc62010-12-10 01:24:16 +0000442 netif_tx_wake_queue(tx_queue->core_txq);
Ben Hutchings8ceee662008-04-27 12:55:59 +0100443 }
444 }
Ben Hutchingscd385572010-11-15 23:53:11 +0000445
446 /* Check whether the hardware queue is now empty */
447 if ((int)(tx_queue->read_count - tx_queue->old_write_count) >= 0) {
448 tx_queue->old_write_count = ACCESS_ONCE(tx_queue->write_count);
449 if (tx_queue->read_count == tx_queue->old_write_count) {
450 smp_mb();
451 tx_queue->empty_read_count =
452 tx_queue->read_count | EFX_EMPTY_COUNT_VALID;
453 }
454 }
Ben Hutchings8ceee662008-04-27 12:55:59 +0100455}
456
457int efx_probe_tx_queue(struct efx_tx_queue *tx_queue)
458{
459 struct efx_nic *efx = tx_queue->efx;
Steve Hodgsonecc910f2010-09-10 06:42:22 +0000460 unsigned int entries;
Ben Hutchings8ceee662008-04-27 12:55:59 +0100461 int i, rc;
462
Steve Hodgsonecc910f2010-09-10 06:42:22 +0000463 /* Create the smallest power-of-two aligned ring */
464 entries = max(roundup_pow_of_two(efx->txq_entries), EFX_MIN_DMAQ_SIZE);
465 EFX_BUG_ON_PARANOID(entries > EFX_MAX_DMAQ_SIZE);
466 tx_queue->ptr_mask = entries - 1;
467
468 netif_dbg(efx, probe, efx->net_dev,
469 "creating TX queue %d size %#x mask %#x\n",
470 tx_queue->queue, efx->txq_entries, tx_queue->ptr_mask);
Ben Hutchings8ceee662008-04-27 12:55:59 +0100471
472 /* Allocate software ring */
Steve Hodgsonecc910f2010-09-10 06:42:22 +0000473 tx_queue->buffer = kzalloc(entries * sizeof(*tx_queue->buffer),
474 GFP_KERNEL);
Ben Hutchings60ac1062008-09-01 12:44:59 +0100475 if (!tx_queue->buffer)
476 return -ENOMEM;
Steve Hodgsonecc910f2010-09-10 06:42:22 +0000477 for (i = 0; i <= tx_queue->ptr_mask; ++i)
Ben Hutchingsdc8cfa52008-09-01 12:46:50 +0100478 tx_queue->buffer[i].continuation = true;
Ben Hutchings8ceee662008-04-27 12:55:59 +0100479
480 /* Allocate hardware ring */
Ben Hutchings152b6a62009-11-29 03:43:56 +0000481 rc = efx_nic_probe_tx(tx_queue);
Ben Hutchings8ceee662008-04-27 12:55:59 +0100482 if (rc)
Ben Hutchings60ac1062008-09-01 12:44:59 +0100483 goto fail;
Ben Hutchings8ceee662008-04-27 12:55:59 +0100484
485 return 0;
486
Ben Hutchings60ac1062008-09-01 12:44:59 +0100487 fail:
Ben Hutchings8ceee662008-04-27 12:55:59 +0100488 kfree(tx_queue->buffer);
489 tx_queue->buffer = NULL;
Ben Hutchings8ceee662008-04-27 12:55:59 +0100490 return rc;
491}
492
Ben Hutchingsbc3c90a2008-09-01 12:48:46 +0100493void efx_init_tx_queue(struct efx_tx_queue *tx_queue)
Ben Hutchings8ceee662008-04-27 12:55:59 +0100494{
Ben Hutchings62776d02010-06-23 11:30:07 +0000495 netif_dbg(tx_queue->efx, drv, tx_queue->efx->net_dev,
496 "initialising TX queue %d\n", tx_queue->queue);
Ben Hutchings8ceee662008-04-27 12:55:59 +0100497
498 tx_queue->insert_count = 0;
499 tx_queue->write_count = 0;
Ben Hutchingscd385572010-11-15 23:53:11 +0000500 tx_queue->old_write_count = 0;
Ben Hutchings8ceee662008-04-27 12:55:59 +0100501 tx_queue->read_count = 0;
502 tx_queue->old_read_count = 0;
Ben Hutchingscd385572010-11-15 23:53:11 +0000503 tx_queue->empty_read_count = 0 | EFX_EMPTY_COUNT_VALID;
Ben Hutchings8ceee662008-04-27 12:55:59 +0100504
505 /* Set up TX descriptor ring */
Ben Hutchings152b6a62009-11-29 03:43:56 +0000506 efx_nic_init_tx(tx_queue);
Ben Hutchings94b274b2011-01-10 21:18:20 +0000507
508 tx_queue->initialised = true;
Ben Hutchings8ceee662008-04-27 12:55:59 +0100509}
510
511void efx_release_tx_buffers(struct efx_tx_queue *tx_queue)
512{
513 struct efx_tx_buffer *buffer;
514
515 if (!tx_queue->buffer)
516 return;
517
518 /* Free any buffers left in the ring */
519 while (tx_queue->read_count != tx_queue->write_count) {
Steve Hodgsonecc910f2010-09-10 06:42:22 +0000520 buffer = &tx_queue->buffer[tx_queue->read_count & tx_queue->ptr_mask];
Ben Hutchings8ceee662008-04-27 12:55:59 +0100521 efx_dequeue_buffer(tx_queue, buffer);
Ben Hutchingsdc8cfa52008-09-01 12:46:50 +0100522 buffer->continuation = true;
Ben Hutchings8ceee662008-04-27 12:55:59 +0100523 buffer->len = 0;
524
525 ++tx_queue->read_count;
526 }
527}
528
529void efx_fini_tx_queue(struct efx_tx_queue *tx_queue)
530{
Ben Hutchings94b274b2011-01-10 21:18:20 +0000531 if (!tx_queue->initialised)
532 return;
533
Ben Hutchings62776d02010-06-23 11:30:07 +0000534 netif_dbg(tx_queue->efx, drv, tx_queue->efx->net_dev,
535 "shutting down TX queue %d\n", tx_queue->queue);
Ben Hutchings8ceee662008-04-27 12:55:59 +0100536
Ben Hutchings94b274b2011-01-10 21:18:20 +0000537 tx_queue->initialised = false;
538
Ben Hutchings8ceee662008-04-27 12:55:59 +0100539 /* Flush TX queue, remove descriptor ring */
Ben Hutchings152b6a62009-11-29 03:43:56 +0000540 efx_nic_fini_tx(tx_queue);
Ben Hutchings8ceee662008-04-27 12:55:59 +0100541
542 efx_release_tx_buffers(tx_queue);
543
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100544 /* Free up TSO header cache */
545 efx_fini_tso(tx_queue);
Ben Hutchings8ceee662008-04-27 12:55:59 +0100546}
547
548void efx_remove_tx_queue(struct efx_tx_queue *tx_queue)
549{
Ben Hutchings94b274b2011-01-10 21:18:20 +0000550 if (!tx_queue->buffer)
551 return;
552
Ben Hutchings62776d02010-06-23 11:30:07 +0000553 netif_dbg(tx_queue->efx, drv, tx_queue->efx->net_dev,
554 "destroying TX queue %d\n", tx_queue->queue);
Ben Hutchings152b6a62009-11-29 03:43:56 +0000555 efx_nic_remove_tx(tx_queue);
Ben Hutchings8ceee662008-04-27 12:55:59 +0100556
557 kfree(tx_queue->buffer);
558 tx_queue->buffer = NULL;
Ben Hutchings8ceee662008-04-27 12:55:59 +0100559}
560
561
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100562/* Efx TCP segmentation acceleration.
563 *
564 * Why? Because by doing it here in the driver we can go significantly
565 * faster than the GSO.
566 *
567 * Requires TX checksum offload support.
568 */
569
570/* Number of bytes inserted at the start of a TSO header buffer,
571 * similar to NET_IP_ALIGN.
572 */
Ben Hutchings13e9ab12008-09-01 12:50:28 +0100573#ifdef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100574#define TSOH_OFFSET 0
575#else
576#define TSOH_OFFSET NET_IP_ALIGN
577#endif
578
579#define TSOH_BUFFER(tsoh) ((u8 *)(tsoh + 1) + TSOH_OFFSET)
580
581/* Total size of struct efx_tso_header, buffer and padding */
582#define TSOH_SIZE(hdr_len) \
583 (sizeof(struct efx_tso_header) + TSOH_OFFSET + hdr_len)
584
585/* Size of blocks on free list. Larger blocks must be allocated from
586 * the heap.
587 */
588#define TSOH_STD_SIZE 128
589
590#define PTR_DIFF(p1, p2) ((u8 *)(p1) - (u8 *)(p2))
591#define ETH_HDR_LEN(skb) (skb_network_header(skb) - (skb)->data)
592#define SKB_TCP_OFF(skb) PTR_DIFF(tcp_hdr(skb), (skb)->data)
593#define SKB_IPV4_OFF(skb) PTR_DIFF(ip_hdr(skb), (skb)->data)
Ben Hutchings738a8f42009-11-29 15:16:05 +0000594#define SKB_IPV6_OFF(skb) PTR_DIFF(ipv6_hdr(skb), (skb)->data)
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100595
596/**
597 * struct tso_state - TSO state for an SKB
Ben Hutchings23d9e602008-09-01 12:47:02 +0100598 * @out_len: Remaining length in current segment
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100599 * @seqnum: Current sequence number
Ben Hutchings23d9e602008-09-01 12:47:02 +0100600 * @ipv4_id: Current IPv4 ID, host endian
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100601 * @packet_space: Remaining space in current packet
Ben Hutchings23d9e602008-09-01 12:47:02 +0100602 * @dma_addr: DMA address of current position
603 * @in_len: Remaining length in current SKB fragment
604 * @unmap_len: Length of SKB fragment
605 * @unmap_addr: DMA address of SKB fragment
606 * @unmap_single: DMA single vs page mapping flag
Ben Hutchings738a8f42009-11-29 15:16:05 +0000607 * @protocol: Network protocol (after any VLAN header)
Ben Hutchings23d9e602008-09-01 12:47:02 +0100608 * @header_len: Number of bytes of header
609 * @full_packet_size: Number of bytes to put in each outgoing segment
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100610 *
611 * The state used during segmentation. It is put into this data structure
612 * just to make it easy to pass into inline functions.
613 */
614struct tso_state {
Ben Hutchings23d9e602008-09-01 12:47:02 +0100615 /* Output position */
616 unsigned out_len;
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100617 unsigned seqnum;
Ben Hutchings23d9e602008-09-01 12:47:02 +0100618 unsigned ipv4_id;
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100619 unsigned packet_space;
620
Ben Hutchings23d9e602008-09-01 12:47:02 +0100621 /* Input position */
622 dma_addr_t dma_addr;
623 unsigned in_len;
624 unsigned unmap_len;
625 dma_addr_t unmap_addr;
626 bool unmap_single;
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100627
Ben Hutchings738a8f42009-11-29 15:16:05 +0000628 __be16 protocol;
Ben Hutchings23d9e602008-09-01 12:47:02 +0100629 unsigned header_len;
630 int full_packet_size;
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100631};
632
633
634/*
635 * Verify that our various assumptions about sk_buffs and the conditions
Ben Hutchings738a8f42009-11-29 15:16:05 +0000636 * under which TSO will be attempted hold true. Return the protocol number.
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100637 */
Ben Hutchings738a8f42009-11-29 15:16:05 +0000638static __be16 efx_tso_check_protocol(struct sk_buff *skb)
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100639{
Ben Hutchings740847d2008-09-01 12:48:23 +0100640 __be16 protocol = skb->protocol;
641
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100642 EFX_BUG_ON_PARANOID(((struct ethhdr *)skb->data)->h_proto !=
Ben Hutchings740847d2008-09-01 12:48:23 +0100643 protocol);
644 if (protocol == htons(ETH_P_8021Q)) {
645 /* Find the encapsulated protocol; reset network header
646 * and transport header based on that. */
647 struct vlan_ethhdr *veh = (struct vlan_ethhdr *)skb->data;
648 protocol = veh->h_vlan_encapsulated_proto;
649 skb_set_network_header(skb, sizeof(*veh));
650 if (protocol == htons(ETH_P_IP))
651 skb_set_transport_header(skb, sizeof(*veh) +
652 4 * ip_hdr(skb)->ihl);
Ben Hutchings738a8f42009-11-29 15:16:05 +0000653 else if (protocol == htons(ETH_P_IPV6))
654 skb_set_transport_header(skb, sizeof(*veh) +
655 sizeof(struct ipv6hdr));
Ben Hutchings740847d2008-09-01 12:48:23 +0100656 }
657
Ben Hutchings738a8f42009-11-29 15:16:05 +0000658 if (protocol == htons(ETH_P_IP)) {
659 EFX_BUG_ON_PARANOID(ip_hdr(skb)->protocol != IPPROTO_TCP);
660 } else {
661 EFX_BUG_ON_PARANOID(protocol != htons(ETH_P_IPV6));
662 EFX_BUG_ON_PARANOID(ipv6_hdr(skb)->nexthdr != NEXTHDR_TCP);
663 }
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100664 EFX_BUG_ON_PARANOID((PTR_DIFF(tcp_hdr(skb), skb->data)
665 + (tcp_hdr(skb)->doff << 2u)) >
666 skb_headlen(skb));
Ben Hutchings738a8f42009-11-29 15:16:05 +0000667
668 return protocol;
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100669}
670
671
672/*
673 * Allocate a page worth of efx_tso_header structures, and string them
674 * into the tx_queue->tso_headers_free linked list. Return 0 or -ENOMEM.
675 */
676static int efx_tsoh_block_alloc(struct efx_tx_queue *tx_queue)
677{
678
679 struct pci_dev *pci_dev = tx_queue->efx->pci_dev;
680 struct efx_tso_header *tsoh;
681 dma_addr_t dma_addr;
682 u8 *base_kva, *kva;
683
684 base_kva = pci_alloc_consistent(pci_dev, PAGE_SIZE, &dma_addr);
685 if (base_kva == NULL) {
Ben Hutchings62776d02010-06-23 11:30:07 +0000686 netif_err(tx_queue->efx, tx_err, tx_queue->efx->net_dev,
687 "Unable to allocate page for TSO headers\n");
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100688 return -ENOMEM;
689 }
690
691 /* pci_alloc_consistent() allocates pages. */
692 EFX_BUG_ON_PARANOID(dma_addr & (PAGE_SIZE - 1u));
693
694 for (kva = base_kva; kva < base_kva + PAGE_SIZE; kva += TSOH_STD_SIZE) {
695 tsoh = (struct efx_tso_header *)kva;
696 tsoh->dma_addr = dma_addr + (TSOH_BUFFER(tsoh) - base_kva);
697 tsoh->next = tx_queue->tso_headers_free;
698 tx_queue->tso_headers_free = tsoh;
699 }
700
701 return 0;
702}
703
704
705/* Free up a TSO header, and all others in the same page. */
706static void efx_tsoh_block_free(struct efx_tx_queue *tx_queue,
707 struct efx_tso_header *tsoh,
708 struct pci_dev *pci_dev)
709{
710 struct efx_tso_header **p;
711 unsigned long base_kva;
712 dma_addr_t base_dma;
713
714 base_kva = (unsigned long)tsoh & PAGE_MASK;
715 base_dma = tsoh->dma_addr & PAGE_MASK;
716
717 p = &tx_queue->tso_headers_free;
Ben Hutchingsb3475642008-05-16 21:15:49 +0100718 while (*p != NULL) {
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100719 if (((unsigned long)*p & PAGE_MASK) == base_kva)
720 *p = (*p)->next;
721 else
722 p = &(*p)->next;
Ben Hutchingsb3475642008-05-16 21:15:49 +0100723 }
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100724
725 pci_free_consistent(pci_dev, PAGE_SIZE, (void *)base_kva, base_dma);
726}
727
728static struct efx_tso_header *
729efx_tsoh_heap_alloc(struct efx_tx_queue *tx_queue, size_t header_len)
730{
731 struct efx_tso_header *tsoh;
732
733 tsoh = kmalloc(TSOH_SIZE(header_len), GFP_ATOMIC | GFP_DMA);
734 if (unlikely(!tsoh))
735 return NULL;
736
737 tsoh->dma_addr = pci_map_single(tx_queue->efx->pci_dev,
738 TSOH_BUFFER(tsoh), header_len,
739 PCI_DMA_TODEVICE);
FUJITA Tomonori8d8bb392008-07-25 19:44:49 -0700740 if (unlikely(pci_dma_mapping_error(tx_queue->efx->pci_dev,
741 tsoh->dma_addr))) {
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100742 kfree(tsoh);
743 return NULL;
744 }
745
746 tsoh->unmap_len = header_len;
747 return tsoh;
748}
749
750static void
751efx_tsoh_heap_free(struct efx_tx_queue *tx_queue, struct efx_tso_header *tsoh)
752{
753 pci_unmap_single(tx_queue->efx->pci_dev,
754 tsoh->dma_addr, tsoh->unmap_len,
755 PCI_DMA_TODEVICE);
756 kfree(tsoh);
757}
758
759/**
760 * efx_tx_queue_insert - push descriptors onto the TX queue
761 * @tx_queue: Efx TX queue
762 * @dma_addr: DMA address of fragment
763 * @len: Length of fragment
Ben Hutchingsecbd95c2008-09-01 12:46:40 +0100764 * @final_buffer: The final buffer inserted into the queue
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100765 *
766 * Push descriptors onto the TX queue. Return 0 on success or 1 if
767 * @tx_queue full.
768 */
769static int efx_tx_queue_insert(struct efx_tx_queue *tx_queue,
770 dma_addr_t dma_addr, unsigned len,
Ben Hutchingsecbd95c2008-09-01 12:46:40 +0100771 struct efx_tx_buffer **final_buffer)
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100772{
773 struct efx_tx_buffer *buffer;
774 struct efx_nic *efx = tx_queue->efx;
Ben Hutchings63f19882009-10-23 08:31:20 +0000775 unsigned dma_len, fill_level, insert_ptr;
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100776 int q_space;
777
778 EFX_BUG_ON_PARANOID(len <= 0);
779
780 fill_level = tx_queue->insert_count - tx_queue->old_read_count;
781 /* -1 as there is no way to represent all descriptors used */
Steve Hodgsonecc910f2010-09-10 06:42:22 +0000782 q_space = efx->txq_entries - 1 - fill_level;
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100783
784 while (1) {
785 if (unlikely(q_space-- <= 0)) {
786 /* It might be that completions have happened
787 * since the xmit path last checked. Update
788 * the xmit path's copy of read_count.
789 */
Ben Hutchingsc04bfc62010-12-10 01:24:16 +0000790 netif_tx_stop_queue(tx_queue->core_txq);
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100791 /* This memory barrier protects the change of
Ben Hutchingsc04bfc62010-12-10 01:24:16 +0000792 * queue state from the access of read_count. */
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100793 smp_mb();
794 tx_queue->old_read_count =
Ben Hutchings51c56f42010-11-10 18:46:40 +0000795 ACCESS_ONCE(tx_queue->read_count);
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100796 fill_level = (tx_queue->insert_count
797 - tx_queue->old_read_count);
Steve Hodgsonecc910f2010-09-10 06:42:22 +0000798 q_space = efx->txq_entries - 1 - fill_level;
Ben Hutchingsecbd95c2008-09-01 12:46:40 +0100799 if (unlikely(q_space-- <= 0)) {
800 *final_buffer = NULL;
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100801 return 1;
Ben Hutchingsecbd95c2008-09-01 12:46:40 +0100802 }
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100803 smp_mb();
Ben Hutchingsc04bfc62010-12-10 01:24:16 +0000804 netif_tx_start_queue(tx_queue->core_txq);
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100805 }
806
Steve Hodgsonecc910f2010-09-10 06:42:22 +0000807 insert_ptr = tx_queue->insert_count & tx_queue->ptr_mask;
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100808 buffer = &tx_queue->buffer[insert_ptr];
809 ++tx_queue->insert_count;
810
811 EFX_BUG_ON_PARANOID(tx_queue->insert_count -
Steve Hodgsonecc910f2010-09-10 06:42:22 +0000812 tx_queue->read_count >=
813 efx->txq_entries);
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100814
815 efx_tsoh_free(tx_queue, buffer);
816 EFX_BUG_ON_PARANOID(buffer->len);
817 EFX_BUG_ON_PARANOID(buffer->unmap_len);
818 EFX_BUG_ON_PARANOID(buffer->skb);
Ben Hutchingsdc8cfa52008-09-01 12:46:50 +0100819 EFX_BUG_ON_PARANOID(!buffer->continuation);
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100820 EFX_BUG_ON_PARANOID(buffer->tsoh);
821
822 buffer->dma_addr = dma_addr;
823
Ben Hutchings63f19882009-10-23 08:31:20 +0000824 dma_len = efx_max_tx_len(efx, dma_addr);
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100825
826 /* If there is enough space to send then do so */
827 if (dma_len >= len)
828 break;
829
830 buffer->len = dma_len; /* Don't set the other members */
831 dma_addr += dma_len;
832 len -= dma_len;
833 }
834
835 EFX_BUG_ON_PARANOID(!len);
836 buffer->len = len;
Ben Hutchingsecbd95c2008-09-01 12:46:40 +0100837 *final_buffer = buffer;
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100838 return 0;
839}
840
841
842/*
843 * Put a TSO header into the TX queue.
844 *
845 * This is special-cased because we know that it is small enough to fit in
846 * a single fragment, and we know it doesn't cross a page boundary. It
847 * also allows us to not worry about end-of-packet etc.
848 */
Ben Hutchings4d566062008-09-01 12:47:12 +0100849static void efx_tso_put_header(struct efx_tx_queue *tx_queue,
850 struct efx_tso_header *tsoh, unsigned len)
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100851{
852 struct efx_tx_buffer *buffer;
853
Steve Hodgsonecc910f2010-09-10 06:42:22 +0000854 buffer = &tx_queue->buffer[tx_queue->insert_count & tx_queue->ptr_mask];
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100855 efx_tsoh_free(tx_queue, buffer);
856 EFX_BUG_ON_PARANOID(buffer->len);
857 EFX_BUG_ON_PARANOID(buffer->unmap_len);
858 EFX_BUG_ON_PARANOID(buffer->skb);
Ben Hutchingsdc8cfa52008-09-01 12:46:50 +0100859 EFX_BUG_ON_PARANOID(!buffer->continuation);
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100860 EFX_BUG_ON_PARANOID(buffer->tsoh);
861 buffer->len = len;
862 buffer->dma_addr = tsoh->dma_addr;
863 buffer->tsoh = tsoh;
864
865 ++tx_queue->insert_count;
866}
867
868
869/* Remove descriptors put into a tx_queue. */
870static void efx_enqueue_unwind(struct efx_tx_queue *tx_queue)
871{
872 struct efx_tx_buffer *buffer;
Ben Hutchingscc12dac2008-09-01 12:46:43 +0100873 dma_addr_t unmap_addr;
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100874
875 /* Work backwards until we hit the original insert pointer value */
876 while (tx_queue->insert_count != tx_queue->write_count) {
877 --tx_queue->insert_count;
878 buffer = &tx_queue->buffer[tx_queue->insert_count &
Steve Hodgsonecc910f2010-09-10 06:42:22 +0000879 tx_queue->ptr_mask];
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100880 efx_tsoh_free(tx_queue, buffer);
881 EFX_BUG_ON_PARANOID(buffer->skb);
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100882 if (buffer->unmap_len) {
Ben Hutchingscc12dac2008-09-01 12:46:43 +0100883 unmap_addr = (buffer->dma_addr + buffer->len -
884 buffer->unmap_len);
Ben Hutchingsecbd95c2008-09-01 12:46:40 +0100885 if (buffer->unmap_single)
886 pci_unmap_single(tx_queue->efx->pci_dev,
Ben Hutchingscc12dac2008-09-01 12:46:43 +0100887 unmap_addr, buffer->unmap_len,
Ben Hutchingsecbd95c2008-09-01 12:46:40 +0100888 PCI_DMA_TODEVICE);
889 else
890 pci_unmap_page(tx_queue->efx->pci_dev,
Ben Hutchingscc12dac2008-09-01 12:46:43 +0100891 unmap_addr, buffer->unmap_len,
Ben Hutchingsecbd95c2008-09-01 12:46:40 +0100892 PCI_DMA_TODEVICE);
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100893 buffer->unmap_len = 0;
894 }
Neil Turtona7ebd272009-12-23 13:47:13 +0000895 buffer->len = 0;
896 buffer->continuation = true;
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100897 }
898}
899
900
901/* Parse the SKB header and initialise state. */
Ben Hutchings4d566062008-09-01 12:47:12 +0100902static void tso_start(struct tso_state *st, const struct sk_buff *skb)
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100903{
904 /* All ethernet/IP/TCP headers combined size is TCP header size
905 * plus offset of TCP header relative to start of packet.
906 */
Ben Hutchings23d9e602008-09-01 12:47:02 +0100907 st->header_len = ((tcp_hdr(skb)->doff << 2u)
908 + PTR_DIFF(tcp_hdr(skb), skb->data));
909 st->full_packet_size = st->header_len + skb_shinfo(skb)->gso_size;
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100910
Ben Hutchings738a8f42009-11-29 15:16:05 +0000911 if (st->protocol == htons(ETH_P_IP))
912 st->ipv4_id = ntohs(ip_hdr(skb)->id);
913 else
914 st->ipv4_id = 0;
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100915 st->seqnum = ntohl(tcp_hdr(skb)->seq);
916
917 EFX_BUG_ON_PARANOID(tcp_hdr(skb)->urg);
918 EFX_BUG_ON_PARANOID(tcp_hdr(skb)->syn);
919 EFX_BUG_ON_PARANOID(tcp_hdr(skb)->rst);
920
Ben Hutchings23d9e602008-09-01 12:47:02 +0100921 st->packet_space = st->full_packet_size;
922 st->out_len = skb->len - st->header_len;
923 st->unmap_len = 0;
924 st->unmap_single = false;
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100925}
926
Ben Hutchings4d566062008-09-01 12:47:12 +0100927static int tso_get_fragment(struct tso_state *st, struct efx_nic *efx,
928 skb_frag_t *frag)
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100929{
Ben Hutchings23d9e602008-09-01 12:47:02 +0100930 st->unmap_addr = pci_map_page(efx->pci_dev, frag->page,
931 frag->page_offset, frag->size,
932 PCI_DMA_TODEVICE);
933 if (likely(!pci_dma_mapping_error(efx->pci_dev, st->unmap_addr))) {
934 st->unmap_single = false;
935 st->unmap_len = frag->size;
936 st->in_len = frag->size;
937 st->dma_addr = st->unmap_addr;
Ben Hutchingsecbd95c2008-09-01 12:46:40 +0100938 return 0;
939 }
940 return -ENOMEM;
941}
942
Ben Hutchings4d566062008-09-01 12:47:12 +0100943static int tso_get_head_fragment(struct tso_state *st, struct efx_nic *efx,
944 const struct sk_buff *skb)
Ben Hutchingsecbd95c2008-09-01 12:46:40 +0100945{
Ben Hutchings23d9e602008-09-01 12:47:02 +0100946 int hl = st->header_len;
Ben Hutchingsecbd95c2008-09-01 12:46:40 +0100947 int len = skb_headlen(skb) - hl;
948
Ben Hutchings23d9e602008-09-01 12:47:02 +0100949 st->unmap_addr = pci_map_single(efx->pci_dev, skb->data + hl,
950 len, PCI_DMA_TODEVICE);
951 if (likely(!pci_dma_mapping_error(efx->pci_dev, st->unmap_addr))) {
952 st->unmap_single = true;
953 st->unmap_len = len;
954 st->in_len = len;
955 st->dma_addr = st->unmap_addr;
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100956 return 0;
957 }
958 return -ENOMEM;
959}
960
961
962/**
963 * tso_fill_packet_with_fragment - form descriptors for the current fragment
964 * @tx_queue: Efx TX queue
965 * @skb: Socket buffer
966 * @st: TSO state
967 *
968 * Form descriptors for the current fragment, until we reach the end
969 * of fragment or end-of-packet. Return 0 on success, 1 if not enough
970 * space in @tx_queue.
971 */
Ben Hutchings4d566062008-09-01 12:47:12 +0100972static int tso_fill_packet_with_fragment(struct efx_tx_queue *tx_queue,
973 const struct sk_buff *skb,
974 struct tso_state *st)
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100975{
Ben Hutchingsecbd95c2008-09-01 12:46:40 +0100976 struct efx_tx_buffer *buffer;
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100977 int n, end_of_packet, rc;
978
Ben Hutchings23d9e602008-09-01 12:47:02 +0100979 if (st->in_len == 0)
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100980 return 0;
981 if (st->packet_space == 0)
982 return 0;
983
Ben Hutchings23d9e602008-09-01 12:47:02 +0100984 EFX_BUG_ON_PARANOID(st->in_len <= 0);
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100985 EFX_BUG_ON_PARANOID(st->packet_space <= 0);
986
Ben Hutchings23d9e602008-09-01 12:47:02 +0100987 n = min(st->in_len, st->packet_space);
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100988
989 st->packet_space -= n;
Ben Hutchings23d9e602008-09-01 12:47:02 +0100990 st->out_len -= n;
991 st->in_len -= n;
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100992
Ben Hutchings23d9e602008-09-01 12:47:02 +0100993 rc = efx_tx_queue_insert(tx_queue, st->dma_addr, n, &buffer);
Ben Hutchingsecbd95c2008-09-01 12:46:40 +0100994 if (likely(rc == 0)) {
Ben Hutchings23d9e602008-09-01 12:47:02 +0100995 if (st->out_len == 0)
Ben Hutchingsecbd95c2008-09-01 12:46:40 +0100996 /* Transfer ownership of the skb */
997 buffer->skb = skb;
998
Ben Hutchings23d9e602008-09-01 12:47:02 +0100999 end_of_packet = st->out_len == 0 || st->packet_space == 0;
Ben Hutchingsecbd95c2008-09-01 12:46:40 +01001000 buffer->continuation = !end_of_packet;
1001
Ben Hutchings23d9e602008-09-01 12:47:02 +01001002 if (st->in_len == 0) {
Ben Hutchingsecbd95c2008-09-01 12:46:40 +01001003 /* Transfer ownership of the pci mapping */
Ben Hutchings23d9e602008-09-01 12:47:02 +01001004 buffer->unmap_len = st->unmap_len;
1005 buffer->unmap_single = st->unmap_single;
1006 st->unmap_len = 0;
Ben Hutchingsecbd95c2008-09-01 12:46:40 +01001007 }
1008 }
Ben Hutchingsb9b39b62008-05-07 12:51:12 +01001009
Ben Hutchings23d9e602008-09-01 12:47:02 +01001010 st->dma_addr += n;
Ben Hutchingsb9b39b62008-05-07 12:51:12 +01001011 return rc;
1012}
1013
1014
1015/**
1016 * tso_start_new_packet - generate a new header and prepare for the new packet
1017 * @tx_queue: Efx TX queue
1018 * @skb: Socket buffer
1019 * @st: TSO state
1020 *
1021 * Generate a new header and prepare for the new packet. Return 0 on
1022 * success, or -1 if failed to alloc header.
1023 */
Ben Hutchings4d566062008-09-01 12:47:12 +01001024static int tso_start_new_packet(struct efx_tx_queue *tx_queue,
1025 const struct sk_buff *skb,
1026 struct tso_state *st)
Ben Hutchingsb9b39b62008-05-07 12:51:12 +01001027{
1028 struct efx_tso_header *tsoh;
Ben Hutchingsb9b39b62008-05-07 12:51:12 +01001029 struct tcphdr *tsoh_th;
1030 unsigned ip_length;
1031 u8 *header;
1032
1033 /* Allocate a DMA-mapped header buffer. */
Ben Hutchings23d9e602008-09-01 12:47:02 +01001034 if (likely(TSOH_SIZE(st->header_len) <= TSOH_STD_SIZE)) {
Ben Hutchingsb3475642008-05-16 21:15:49 +01001035 if (tx_queue->tso_headers_free == NULL) {
Ben Hutchingsb9b39b62008-05-07 12:51:12 +01001036 if (efx_tsoh_block_alloc(tx_queue))
1037 return -1;
Ben Hutchingsb3475642008-05-16 21:15:49 +01001038 }
Ben Hutchingsb9b39b62008-05-07 12:51:12 +01001039 EFX_BUG_ON_PARANOID(!tx_queue->tso_headers_free);
1040 tsoh = tx_queue->tso_headers_free;
1041 tx_queue->tso_headers_free = tsoh->next;
1042 tsoh->unmap_len = 0;
1043 } else {
1044 tx_queue->tso_long_headers++;
Ben Hutchings23d9e602008-09-01 12:47:02 +01001045 tsoh = efx_tsoh_heap_alloc(tx_queue, st->header_len);
Ben Hutchingsb9b39b62008-05-07 12:51:12 +01001046 if (unlikely(!tsoh))
1047 return -1;
1048 }
1049
1050 header = TSOH_BUFFER(tsoh);
1051 tsoh_th = (struct tcphdr *)(header + SKB_TCP_OFF(skb));
Ben Hutchingsb9b39b62008-05-07 12:51:12 +01001052
1053 /* Copy and update the headers. */
Ben Hutchings23d9e602008-09-01 12:47:02 +01001054 memcpy(header, skb->data, st->header_len);
Ben Hutchingsb9b39b62008-05-07 12:51:12 +01001055
1056 tsoh_th->seq = htonl(st->seqnum);
1057 st->seqnum += skb_shinfo(skb)->gso_size;
Ben Hutchings23d9e602008-09-01 12:47:02 +01001058 if (st->out_len > skb_shinfo(skb)->gso_size) {
Ben Hutchingsb9b39b62008-05-07 12:51:12 +01001059 /* This packet will not finish the TSO burst. */
Ben Hutchings23d9e602008-09-01 12:47:02 +01001060 ip_length = st->full_packet_size - ETH_HDR_LEN(skb);
Ben Hutchingsb9b39b62008-05-07 12:51:12 +01001061 tsoh_th->fin = 0;
1062 tsoh_th->psh = 0;
1063 } else {
1064 /* This packet will be the last in the TSO burst. */
Ben Hutchings23d9e602008-09-01 12:47:02 +01001065 ip_length = st->header_len - ETH_HDR_LEN(skb) + st->out_len;
Ben Hutchingsb9b39b62008-05-07 12:51:12 +01001066 tsoh_th->fin = tcp_hdr(skb)->fin;
1067 tsoh_th->psh = tcp_hdr(skb)->psh;
1068 }
Ben Hutchingsb9b39b62008-05-07 12:51:12 +01001069
Ben Hutchings738a8f42009-11-29 15:16:05 +00001070 if (st->protocol == htons(ETH_P_IP)) {
1071 struct iphdr *tsoh_iph =
1072 (struct iphdr *)(header + SKB_IPV4_OFF(skb));
1073
1074 tsoh_iph->tot_len = htons(ip_length);
1075
1076 /* Linux leaves suitable gaps in the IP ID space for us to fill. */
1077 tsoh_iph->id = htons(st->ipv4_id);
1078 st->ipv4_id++;
1079 } else {
1080 struct ipv6hdr *tsoh_iph =
1081 (struct ipv6hdr *)(header + SKB_IPV6_OFF(skb));
1082
1083 tsoh_iph->payload_len = htons(ip_length - sizeof(*tsoh_iph));
1084 }
Ben Hutchingsb9b39b62008-05-07 12:51:12 +01001085
1086 st->packet_space = skb_shinfo(skb)->gso_size;
1087 ++tx_queue->tso_packets;
1088
1089 /* Form a descriptor for this header. */
Ben Hutchings23d9e602008-09-01 12:47:02 +01001090 efx_tso_put_header(tx_queue, tsoh, st->header_len);
Ben Hutchingsb9b39b62008-05-07 12:51:12 +01001091
1092 return 0;
1093}
1094
1095
1096/**
1097 * efx_enqueue_skb_tso - segment and transmit a TSO socket buffer
1098 * @tx_queue: Efx TX queue
1099 * @skb: Socket buffer
1100 *
1101 * Context: You must hold netif_tx_lock() to call this function.
1102 *
1103 * Add socket buffer @skb to @tx_queue, doing TSO or return != 0 if
1104 * @skb was not enqueued. In all cases @skb is consumed. Return
1105 * %NETDEV_TX_OK or %NETDEV_TX_BUSY.
1106 */
1107static int efx_enqueue_skb_tso(struct efx_tx_queue *tx_queue,
Ben Hutchings740847d2008-09-01 12:48:23 +01001108 struct sk_buff *skb)
Ben Hutchingsb9b39b62008-05-07 12:51:12 +01001109{
Ben Hutchingsecbd95c2008-09-01 12:46:40 +01001110 struct efx_nic *efx = tx_queue->efx;
Ben Hutchingsb9b39b62008-05-07 12:51:12 +01001111 int frag_i, rc, rc2 = NETDEV_TX_OK;
1112 struct tso_state state;
Ben Hutchingsb9b39b62008-05-07 12:51:12 +01001113
Ben Hutchings738a8f42009-11-29 15:16:05 +00001114 /* Find the packet protocol and sanity-check it */
1115 state.protocol = efx_tso_check_protocol(skb);
Ben Hutchingsb9b39b62008-05-07 12:51:12 +01001116
1117 EFX_BUG_ON_PARANOID(tx_queue->write_count != tx_queue->insert_count);
1118
1119 tso_start(&state, skb);
1120
1121 /* Assume that skb header area contains exactly the headers, and
1122 * all payload is in the frag list.
1123 */
Ben Hutchings23d9e602008-09-01 12:47:02 +01001124 if (skb_headlen(skb) == state.header_len) {
Ben Hutchingsb9b39b62008-05-07 12:51:12 +01001125 /* Grab the first payload fragment. */
1126 EFX_BUG_ON_PARANOID(skb_shinfo(skb)->nr_frags < 1);
1127 frag_i = 0;
Ben Hutchingsecbd95c2008-09-01 12:46:40 +01001128 rc = tso_get_fragment(&state, efx,
1129 skb_shinfo(skb)->frags + frag_i);
Ben Hutchingsb9b39b62008-05-07 12:51:12 +01001130 if (rc)
1131 goto mem_err;
1132 } else {
Ben Hutchingsecbd95c2008-09-01 12:46:40 +01001133 rc = tso_get_head_fragment(&state, efx, skb);
Ben Hutchingsb9b39b62008-05-07 12:51:12 +01001134 if (rc)
1135 goto mem_err;
1136 frag_i = -1;
1137 }
1138
1139 if (tso_start_new_packet(tx_queue, skb, &state) < 0)
1140 goto mem_err;
1141
1142 while (1) {
1143 rc = tso_fill_packet_with_fragment(tx_queue, skb, &state);
Ben Hutchingsc04bfc62010-12-10 01:24:16 +00001144 if (unlikely(rc)) {
1145 rc2 = NETDEV_TX_BUSY;
1146 goto unwind;
1147 }
Ben Hutchingsb9b39b62008-05-07 12:51:12 +01001148
1149 /* Move onto the next fragment? */
Ben Hutchings23d9e602008-09-01 12:47:02 +01001150 if (state.in_len == 0) {
Ben Hutchingsb9b39b62008-05-07 12:51:12 +01001151 if (++frag_i >= skb_shinfo(skb)->nr_frags)
1152 /* End of payload reached. */
1153 break;
Ben Hutchingsecbd95c2008-09-01 12:46:40 +01001154 rc = tso_get_fragment(&state, efx,
1155 skb_shinfo(skb)->frags + frag_i);
Ben Hutchingsb9b39b62008-05-07 12:51:12 +01001156 if (rc)
1157 goto mem_err;
1158 }
1159
1160 /* Start at new packet? */
1161 if (state.packet_space == 0 &&
1162 tso_start_new_packet(tx_queue, skb, &state) < 0)
1163 goto mem_err;
1164 }
1165
1166 /* Pass off to hardware */
Ben Hutchings152b6a62009-11-29 03:43:56 +00001167 efx_nic_push_buffers(tx_queue);
Ben Hutchingsb9b39b62008-05-07 12:51:12 +01001168
1169 tx_queue->tso_bursts++;
1170 return NETDEV_TX_OK;
1171
1172 mem_err:
Ben Hutchings62776d02010-06-23 11:30:07 +00001173 netif_err(efx, tx_err, efx->net_dev,
1174 "Out of memory for TSO headers, or PCI mapping error\n");
Ben Hutchings9bc183d2009-11-23 16:06:47 +00001175 dev_kfree_skb_any(skb);
Ben Hutchingsb9b39b62008-05-07 12:51:12 +01001176
1177 unwind:
Ben Hutchings5988b632008-09-01 12:46:36 +01001178 /* Free the DMA mapping we were in the process of writing out */
Ben Hutchings23d9e602008-09-01 12:47:02 +01001179 if (state.unmap_len) {
1180 if (state.unmap_single)
1181 pci_unmap_single(efx->pci_dev, state.unmap_addr,
1182 state.unmap_len, PCI_DMA_TODEVICE);
Ben Hutchingsecbd95c2008-09-01 12:46:40 +01001183 else
Ben Hutchings23d9e602008-09-01 12:47:02 +01001184 pci_unmap_page(efx->pci_dev, state.unmap_addr,
1185 state.unmap_len, PCI_DMA_TODEVICE);
Ben Hutchingsecbd95c2008-09-01 12:46:40 +01001186 }
Ben Hutchings5988b632008-09-01 12:46:36 +01001187
Ben Hutchingsb9b39b62008-05-07 12:51:12 +01001188 efx_enqueue_unwind(tx_queue);
1189 return rc2;
1190}
1191
1192
1193/*
1194 * Free up all TSO datastructures associated with tx_queue. This
1195 * routine should be called only once the tx_queue is both empty and
1196 * will no longer be used.
1197 */
1198static void efx_fini_tso(struct efx_tx_queue *tx_queue)
1199{
1200 unsigned i;
1201
Ben Hutchingsb3475642008-05-16 21:15:49 +01001202 if (tx_queue->buffer) {
Steve Hodgsonecc910f2010-09-10 06:42:22 +00001203 for (i = 0; i <= tx_queue->ptr_mask; ++i)
Ben Hutchingsb9b39b62008-05-07 12:51:12 +01001204 efx_tsoh_free(tx_queue, &tx_queue->buffer[i]);
Ben Hutchingsb3475642008-05-16 21:15:49 +01001205 }
Ben Hutchingsb9b39b62008-05-07 12:51:12 +01001206
1207 while (tx_queue->tso_headers_free != NULL)
1208 efx_tsoh_block_free(tx_queue, tx_queue->tso_headers_free,
1209 tx_queue->efx->pci_dev);
1210}