blob: e669f94e821bde87e23d842fe25005e9568a93a8 [file] [log] [blame]
Ben Hutchings8ceee662008-04-27 12:55:59 +01001/****************************************************************************
2 * Driver for Solarflare Solarstorm network controllers and boards
3 * Copyright 2005-2006 Fen Systems Ltd.
Ben Hutchings906bb262009-11-29 15:16:19 +00004 * Copyright 2005-2009 Solarflare Communications Inc.
Ben Hutchings8ceee662008-04-27 12:55:59 +01005 *
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 as published
8 * by the Free Software Foundation, incorporated herein by reference.
9 */
10
11#include <linux/pci.h>
12#include <linux/tcp.h>
13#include <linux/ip.h>
14#include <linux/in.h>
Ben Hutchings738a8f42009-11-29 15:16:05 +000015#include <linux/ipv6.h>
16#include <net/ipv6.h>
Ben Hutchings8ceee662008-04-27 12:55:59 +010017#include <linux/if_ether.h>
18#include <linux/highmem.h>
19#include "net_driver.h"
Ben Hutchings8ceee662008-04-27 12:55:59 +010020#include "efx.h"
Ben Hutchings744093c2009-11-29 15:12:08 +000021#include "nic.h"
Ben Hutchings8ceee662008-04-27 12:55:59 +010022#include "workarounds.h"
23
24/*
25 * TX descriptor ring full threshold
26 *
27 * The tx_queue descriptor ring fill-level must fall below this value
28 * before we restart the netif queue
29 */
Ben Hutchings3ffeabd2009-10-23 08:30:58 +000030#define EFX_TXQ_THRESHOLD (EFX_TXQ_MASK / 2u)
Ben Hutchings8ceee662008-04-27 12:55:59 +010031
32/* We want to be able to nest calls to netif_stop_queue(), since each
33 * channel can have an individual stop on the queue.
34 */
35void efx_stop_queue(struct efx_nic *efx)
36{
37 spin_lock_bh(&efx->netif_stop_lock);
38 EFX_TRACE(efx, "stop TX queue\n");
39
40 atomic_inc(&efx->netif_stop_count);
41 netif_stop_queue(efx->net_dev);
42
43 spin_unlock_bh(&efx->netif_stop_lock);
44}
45
46/* Wake netif's TX queue
47 * We want to be able to nest calls to netif_stop_queue(), since each
48 * channel can have an individual stop on the queue.
49 */
Ben Hutchings4d566062008-09-01 12:47:12 +010050void efx_wake_queue(struct efx_nic *efx)
Ben Hutchings8ceee662008-04-27 12:55:59 +010051{
52 local_bh_disable();
53 if (atomic_dec_and_lock(&efx->netif_stop_count,
54 &efx->netif_stop_lock)) {
55 EFX_TRACE(efx, "waking TX queue\n");
56 netif_wake_queue(efx->net_dev);
57 spin_unlock(&efx->netif_stop_lock);
58 }
59 local_bh_enable();
60}
61
Ben Hutchings4d566062008-09-01 12:47:12 +010062static void efx_dequeue_buffer(struct efx_tx_queue *tx_queue,
63 struct efx_tx_buffer *buffer)
Ben Hutchings8ceee662008-04-27 12:55:59 +010064{
65 if (buffer->unmap_len) {
66 struct pci_dev *pci_dev = tx_queue->efx->pci_dev;
Ben Hutchingscc12dac2008-09-01 12:46:43 +010067 dma_addr_t unmap_addr = (buffer->dma_addr + buffer->len -
68 buffer->unmap_len);
Ben Hutchings8ceee662008-04-27 12:55:59 +010069 if (buffer->unmap_single)
Ben Hutchingscc12dac2008-09-01 12:46:43 +010070 pci_unmap_single(pci_dev, unmap_addr, buffer->unmap_len,
71 PCI_DMA_TODEVICE);
Ben Hutchings8ceee662008-04-27 12:55:59 +010072 else
Ben Hutchingscc12dac2008-09-01 12:46:43 +010073 pci_unmap_page(pci_dev, unmap_addr, buffer->unmap_len,
74 PCI_DMA_TODEVICE);
Ben Hutchings8ceee662008-04-27 12:55:59 +010075 buffer->unmap_len = 0;
Ben Hutchingsdc8cfa52008-09-01 12:46:50 +010076 buffer->unmap_single = false;
Ben Hutchings8ceee662008-04-27 12:55:59 +010077 }
78
79 if (buffer->skb) {
80 dev_kfree_skb_any((struct sk_buff *) buffer->skb);
81 buffer->skb = NULL;
82 EFX_TRACE(tx_queue->efx, "TX queue %d transmission id %x "
83 "complete\n", tx_queue->queue, read_ptr);
84 }
85}
86
Ben Hutchingsb9b39b62008-05-07 12:51:12 +010087/**
88 * struct efx_tso_header - a DMA mapped buffer for packet headers
89 * @next: Linked list of free ones.
90 * The list is protected by the TX queue lock.
91 * @dma_unmap_len: Length to unmap for an oversize buffer, or 0.
92 * @dma_addr: The DMA address of the header below.
93 *
94 * This controls the memory used for a TSO header. Use TSOH_DATA()
95 * to find the packet header data. Use TSOH_SIZE() to calculate the
96 * total size required for a given packet header length. TSO headers
97 * in the free list are exactly %TSOH_STD_SIZE bytes in size.
98 */
99struct efx_tso_header {
100 union {
101 struct efx_tso_header *next;
102 size_t unmap_len;
103 };
104 dma_addr_t dma_addr;
105};
106
107static int efx_enqueue_skb_tso(struct efx_tx_queue *tx_queue,
Ben Hutchings740847d2008-09-01 12:48:23 +0100108 struct sk_buff *skb);
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100109static void efx_fini_tso(struct efx_tx_queue *tx_queue);
110static void efx_tsoh_heap_free(struct efx_tx_queue *tx_queue,
111 struct efx_tso_header *tsoh);
112
Ben Hutchings4d566062008-09-01 12:47:12 +0100113static void efx_tsoh_free(struct efx_tx_queue *tx_queue,
114 struct efx_tx_buffer *buffer)
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100115{
116 if (buffer->tsoh) {
117 if (likely(!buffer->tsoh->unmap_len)) {
118 buffer->tsoh->next = tx_queue->tso_headers_free;
119 tx_queue->tso_headers_free = buffer->tsoh;
120 } else {
121 efx_tsoh_heap_free(tx_queue, buffer->tsoh);
122 }
123 buffer->tsoh = NULL;
124 }
125}
126
Ben Hutchings8ceee662008-04-27 12:55:59 +0100127
Ben Hutchings63f19882009-10-23 08:31:20 +0000128static inline unsigned
129efx_max_tx_len(struct efx_nic *efx, dma_addr_t dma_addr)
130{
131 /* Depending on the NIC revision, we can use descriptor
132 * lengths up to 8K or 8K-1. However, since PCI Express
133 * devices must split read requests at 4K boundaries, there is
134 * little benefit from using descriptors that cross those
135 * boundaries and we keep things simple by not doing so.
136 */
137 unsigned len = (~dma_addr & 0xfff) + 1;
138
139 /* Work around hardware bug for unaligned buffers. */
140 if (EFX_WORKAROUND_5391(efx) && (dma_addr & 0xf))
141 len = min_t(unsigned, len, 512 - (dma_addr & 0xf));
142
143 return len;
144}
145
Ben Hutchings8ceee662008-04-27 12:55:59 +0100146/*
147 * Add a socket buffer to a TX queue
148 *
149 * This maps all fragments of a socket buffer for DMA and adds them to
150 * the TX queue. The queue's insert pointer will be incremented by
151 * the number of fragments in the socket buffer.
152 *
153 * If any DMA mapping fails, any mapped fragments will be unmapped,
154 * the queue's insert pointer will be restored to its original value.
155 *
Ben Hutchings497f5ba2009-11-23 16:07:05 +0000156 * This function is split out from efx_hard_start_xmit to allow the
157 * loopback test to direct packets via specific TX queues.
158 *
Ben Hutchings8ceee662008-04-27 12:55:59 +0100159 * Returns NETDEV_TX_OK or NETDEV_TX_BUSY
160 * You must hold netif_tx_lock() to call this function.
161 */
Ben Hutchings497f5ba2009-11-23 16:07:05 +0000162netdev_tx_t efx_enqueue_skb(struct efx_tx_queue *tx_queue, struct sk_buff *skb)
Ben Hutchings8ceee662008-04-27 12:55:59 +0100163{
164 struct efx_nic *efx = tx_queue->efx;
165 struct pci_dev *pci_dev = efx->pci_dev;
166 struct efx_tx_buffer *buffer;
167 skb_frag_t *fragment;
168 struct page *page;
169 int page_offset;
Ben Hutchings63f19882009-10-23 08:31:20 +0000170 unsigned int len, unmap_len = 0, fill_level, insert_ptr;
Ben Hutchings8ceee662008-04-27 12:55:59 +0100171 dma_addr_t dma_addr, unmap_addr = 0;
172 unsigned int dma_len;
Ben Hutchingsdc8cfa52008-09-01 12:46:50 +0100173 bool unmap_single;
Ben Hutchings8ceee662008-04-27 12:55:59 +0100174 int q_space, i = 0;
Stephen Hemminger613573252009-08-31 19:50:58 +0000175 netdev_tx_t rc = NETDEV_TX_OK;
Ben Hutchings8ceee662008-04-27 12:55:59 +0100176
177 EFX_BUG_ON_PARANOID(tx_queue->write_count != tx_queue->insert_count);
178
Ben Hutchings9bc183d2009-11-23 16:06:47 +0000179 if (skb_shinfo(skb)->gso_size)
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100180 return efx_enqueue_skb_tso(tx_queue, skb);
181
Ben Hutchings8ceee662008-04-27 12:55:59 +0100182 /* Get size of the initial fragment */
183 len = skb_headlen(skb);
184
Ben Hutchingsbb145a92009-03-20 13:25:39 +0000185 /* Pad if necessary */
186 if (EFX_WORKAROUND_15592(efx) && skb->len <= 32) {
187 EFX_BUG_ON_PARANOID(skb->data_len);
188 len = 32 + 1;
189 if (skb_pad(skb, len - skb->len))
190 return NETDEV_TX_OK;
191 }
192
Ben Hutchings8ceee662008-04-27 12:55:59 +0100193 fill_level = tx_queue->insert_count - tx_queue->old_read_count;
Ben Hutchings3ffeabd2009-10-23 08:30:58 +0000194 q_space = EFX_TXQ_MASK - 1 - fill_level;
Ben Hutchings8ceee662008-04-27 12:55:59 +0100195
196 /* Map for DMA. Use pci_map_single rather than pci_map_page
197 * since this is more efficient on machines with sparse
198 * memory.
199 */
Ben Hutchingsdc8cfa52008-09-01 12:46:50 +0100200 unmap_single = true;
Ben Hutchings8ceee662008-04-27 12:55:59 +0100201 dma_addr = pci_map_single(pci_dev, skb->data, len, PCI_DMA_TODEVICE);
202
203 /* Process all fragments */
204 while (1) {
FUJITA Tomonori8d8bb392008-07-25 19:44:49 -0700205 if (unlikely(pci_dma_mapping_error(pci_dev, dma_addr)))
Ben Hutchings8ceee662008-04-27 12:55:59 +0100206 goto pci_err;
207
208 /* Store fields for marking in the per-fragment final
209 * descriptor */
210 unmap_len = len;
211 unmap_addr = dma_addr;
212
213 /* Add to TX queue, splitting across DMA boundaries */
214 do {
215 if (unlikely(q_space-- <= 0)) {
216 /* It might be that completions have
217 * happened since the xmit path last
218 * checked. Update the xmit path's
219 * copy of read_count.
220 */
221 ++tx_queue->stopped;
222 /* This memory barrier protects the
223 * change of stopped from the access
224 * of read_count. */
225 smp_mb();
226 tx_queue->old_read_count =
227 *(volatile unsigned *)
228 &tx_queue->read_count;
229 fill_level = (tx_queue->insert_count
230 - tx_queue->old_read_count);
Ben Hutchings3ffeabd2009-10-23 08:30:58 +0000231 q_space = EFX_TXQ_MASK - 1 - fill_level;
Ben Hutchings8ceee662008-04-27 12:55:59 +0100232 if (unlikely(q_space-- <= 0))
233 goto stop;
234 smp_mb();
235 --tx_queue->stopped;
236 }
237
Ben Hutchings3ffeabd2009-10-23 08:30:58 +0000238 insert_ptr = tx_queue->insert_count & EFX_TXQ_MASK;
Ben Hutchings8ceee662008-04-27 12:55:59 +0100239 buffer = &tx_queue->buffer[insert_ptr];
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100240 efx_tsoh_free(tx_queue, buffer);
241 EFX_BUG_ON_PARANOID(buffer->tsoh);
Ben Hutchings8ceee662008-04-27 12:55:59 +0100242 EFX_BUG_ON_PARANOID(buffer->skb);
243 EFX_BUG_ON_PARANOID(buffer->len);
Ben Hutchingsdc8cfa52008-09-01 12:46:50 +0100244 EFX_BUG_ON_PARANOID(!buffer->continuation);
Ben Hutchings8ceee662008-04-27 12:55:59 +0100245 EFX_BUG_ON_PARANOID(buffer->unmap_len);
246
Ben Hutchings63f19882009-10-23 08:31:20 +0000247 dma_len = efx_max_tx_len(efx, dma_addr);
248 if (likely(dma_len >= len))
Ben Hutchings8ceee662008-04-27 12:55:59 +0100249 dma_len = len;
250
Ben Hutchings8ceee662008-04-27 12:55:59 +0100251 /* Fill out per descriptor fields */
252 buffer->len = dma_len;
253 buffer->dma_addr = dma_addr;
254 len -= dma_len;
255 dma_addr += dma_len;
256 ++tx_queue->insert_count;
257 } while (len);
258
259 /* Transfer ownership of the unmapping to the final buffer */
Ben Hutchings8ceee662008-04-27 12:55:59 +0100260 buffer->unmap_single = unmap_single;
261 buffer->unmap_len = unmap_len;
262 unmap_len = 0;
263
264 /* Get address and size of next fragment */
265 if (i >= skb_shinfo(skb)->nr_frags)
266 break;
267 fragment = &skb_shinfo(skb)->frags[i];
268 len = fragment->size;
269 page = fragment->page;
270 page_offset = fragment->page_offset;
271 i++;
272 /* Map for DMA */
Ben Hutchingsdc8cfa52008-09-01 12:46:50 +0100273 unmap_single = false;
Ben Hutchings8ceee662008-04-27 12:55:59 +0100274 dma_addr = pci_map_page(pci_dev, page, page_offset, len,
275 PCI_DMA_TODEVICE);
276 }
277
278 /* Transfer ownership of the skb to the final buffer */
279 buffer->skb = skb;
Ben Hutchingsdc8cfa52008-09-01 12:46:50 +0100280 buffer->continuation = false;
Ben Hutchings8ceee662008-04-27 12:55:59 +0100281
282 /* Pass off to hardware */
Ben Hutchings152b6a62009-11-29 03:43:56 +0000283 efx_nic_push_buffers(tx_queue);
Ben Hutchings8ceee662008-04-27 12:55:59 +0100284
285 return NETDEV_TX_OK;
286
287 pci_err:
288 EFX_ERR_RL(efx, " TX queue %d could not map skb with %d bytes %d "
289 "fragments for DMA\n", tx_queue->queue, skb->len,
290 skb_shinfo(skb)->nr_frags + 1);
291
292 /* Mark the packet as transmitted, and free the SKB ourselves */
Ben Hutchings9bc183d2009-11-23 16:06:47 +0000293 dev_kfree_skb_any(skb);
Ben Hutchings8ceee662008-04-27 12:55:59 +0100294 goto unwind;
295
296 stop:
297 rc = NETDEV_TX_BUSY;
298
299 if (tx_queue->stopped == 1)
300 efx_stop_queue(efx);
301
302 unwind:
303 /* Work backwards until we hit the original insert pointer value */
304 while (tx_queue->insert_count != tx_queue->write_count) {
305 --tx_queue->insert_count;
Ben Hutchings3ffeabd2009-10-23 08:30:58 +0000306 insert_ptr = tx_queue->insert_count & EFX_TXQ_MASK;
Ben Hutchings8ceee662008-04-27 12:55:59 +0100307 buffer = &tx_queue->buffer[insert_ptr];
308 efx_dequeue_buffer(tx_queue, buffer);
309 buffer->len = 0;
310 }
311
312 /* Free the fragment we were mid-way through pushing */
Ben Hutchingsecbd95c2008-09-01 12:46:40 +0100313 if (unmap_len) {
314 if (unmap_single)
315 pci_unmap_single(pci_dev, unmap_addr, unmap_len,
316 PCI_DMA_TODEVICE);
317 else
318 pci_unmap_page(pci_dev, unmap_addr, unmap_len,
319 PCI_DMA_TODEVICE);
320 }
Ben Hutchings8ceee662008-04-27 12:55:59 +0100321
322 return rc;
323}
324
325/* Remove packets from the TX queue
326 *
327 * This removes packets from the TX queue, up to and including the
328 * specified index.
329 */
Ben Hutchings4d566062008-09-01 12:47:12 +0100330static void efx_dequeue_buffers(struct efx_tx_queue *tx_queue,
331 unsigned int index)
Ben Hutchings8ceee662008-04-27 12:55:59 +0100332{
333 struct efx_nic *efx = tx_queue->efx;
334 unsigned int stop_index, read_ptr;
Ben Hutchings8ceee662008-04-27 12:55:59 +0100335
Ben Hutchings3ffeabd2009-10-23 08:30:58 +0000336 stop_index = (index + 1) & EFX_TXQ_MASK;
337 read_ptr = tx_queue->read_count & EFX_TXQ_MASK;
Ben Hutchings8ceee662008-04-27 12:55:59 +0100338
339 while (read_ptr != stop_index) {
340 struct efx_tx_buffer *buffer = &tx_queue->buffer[read_ptr];
341 if (unlikely(buffer->len == 0)) {
342 EFX_ERR(tx_queue->efx, "TX queue %d spurious TX "
343 "completion id %x\n", tx_queue->queue,
344 read_ptr);
345 efx_schedule_reset(efx, RESET_TYPE_TX_SKIP);
346 return;
347 }
348
349 efx_dequeue_buffer(tx_queue, buffer);
Ben Hutchingsdc8cfa52008-09-01 12:46:50 +0100350 buffer->continuation = true;
Ben Hutchings8ceee662008-04-27 12:55:59 +0100351 buffer->len = 0;
352
353 ++tx_queue->read_count;
Ben Hutchings3ffeabd2009-10-23 08:30:58 +0000354 read_ptr = tx_queue->read_count & EFX_TXQ_MASK;
Ben Hutchings8ceee662008-04-27 12:55:59 +0100355 }
356}
357
Ben Hutchings8ceee662008-04-27 12:55:59 +0100358/* Initiate a packet transmission. We use one channel per CPU
359 * (sharing when we have more CPUs than channels). On Falcon, the TX
360 * completion events will be directed back to the CPU that transmitted
361 * the packet, which should be cache-efficient.
362 *
363 * Context: non-blocking.
364 * Note that returning anything other than NETDEV_TX_OK will cause the
365 * OS to free the skb.
366 */
Stephen Hemminger613573252009-08-31 19:50:58 +0000367netdev_tx_t efx_hard_start_xmit(struct sk_buff *skb,
368 struct net_device *net_dev)
Ben Hutchings8ceee662008-04-27 12:55:59 +0100369{
Ben Hutchings767e4682008-09-01 12:43:14 +0100370 struct efx_nic *efx = netdev_priv(net_dev);
Ben Hutchings60ac1062008-09-01 12:44:59 +0100371 struct efx_tx_queue *tx_queue;
372
Ben Hutchingsa7ef5932009-03-04 09:52:37 +0000373 if (unlikely(efx->port_inhibited))
374 return NETDEV_TX_BUSY;
375
Ben Hutchings60ac1062008-09-01 12:44:59 +0100376 if (likely(skb->ip_summed == CHECKSUM_PARTIAL))
377 tx_queue = &efx->tx_queue[EFX_TX_QUEUE_OFFLOAD_CSUM];
378 else
379 tx_queue = &efx->tx_queue[EFX_TX_QUEUE_NO_CSUM];
380
Ben Hutchings497f5ba2009-11-23 16:07:05 +0000381 return efx_enqueue_skb(tx_queue, skb);
Ben Hutchings8ceee662008-04-27 12:55:59 +0100382}
383
384void efx_xmit_done(struct efx_tx_queue *tx_queue, unsigned int index)
385{
386 unsigned fill_level;
387 struct efx_nic *efx = tx_queue->efx;
388
Ben Hutchings3ffeabd2009-10-23 08:30:58 +0000389 EFX_BUG_ON_PARANOID(index > EFX_TXQ_MASK);
Ben Hutchings8ceee662008-04-27 12:55:59 +0100390
391 efx_dequeue_buffers(tx_queue, index);
392
393 /* See if we need to restart the netif queue. This barrier
394 * separates the update of read_count from the test of
395 * stopped. */
396 smp_mb();
Ben Hutchings32d76002009-03-04 09:53:15 +0000397 if (unlikely(tx_queue->stopped) && likely(efx->port_enabled)) {
Ben Hutchings8ceee662008-04-27 12:55:59 +0100398 fill_level = tx_queue->insert_count - tx_queue->read_count;
Ben Hutchings3ffeabd2009-10-23 08:30:58 +0000399 if (fill_level < EFX_TXQ_THRESHOLD) {
Ben Hutchings55668612008-05-16 21:16:10 +0100400 EFX_BUG_ON_PARANOID(!efx_dev_registered(efx));
Ben Hutchings8ceee662008-04-27 12:55:59 +0100401
402 /* Do this under netif_tx_lock(), to avoid racing
403 * with efx_xmit(). */
404 netif_tx_lock(efx->net_dev);
405 if (tx_queue->stopped) {
406 tx_queue->stopped = 0;
407 efx_wake_queue(efx);
408 }
409 netif_tx_unlock(efx->net_dev);
410 }
411 }
412}
413
414int efx_probe_tx_queue(struct efx_tx_queue *tx_queue)
415{
416 struct efx_nic *efx = tx_queue->efx;
417 unsigned int txq_size;
418 int i, rc;
419
420 EFX_LOG(efx, "creating TX queue %d\n", tx_queue->queue);
421
422 /* Allocate software ring */
Ben Hutchings3ffeabd2009-10-23 08:30:58 +0000423 txq_size = EFX_TXQ_SIZE * sizeof(*tx_queue->buffer);
Ben Hutchings8ceee662008-04-27 12:55:59 +0100424 tx_queue->buffer = kzalloc(txq_size, GFP_KERNEL);
Ben Hutchings60ac1062008-09-01 12:44:59 +0100425 if (!tx_queue->buffer)
426 return -ENOMEM;
Ben Hutchings3ffeabd2009-10-23 08:30:58 +0000427 for (i = 0; i <= EFX_TXQ_MASK; ++i)
Ben Hutchingsdc8cfa52008-09-01 12:46:50 +0100428 tx_queue->buffer[i].continuation = true;
Ben Hutchings8ceee662008-04-27 12:55:59 +0100429
430 /* Allocate hardware ring */
Ben Hutchings152b6a62009-11-29 03:43:56 +0000431 rc = efx_nic_probe_tx(tx_queue);
Ben Hutchings8ceee662008-04-27 12:55:59 +0100432 if (rc)
Ben Hutchings60ac1062008-09-01 12:44:59 +0100433 goto fail;
Ben Hutchings8ceee662008-04-27 12:55:59 +0100434
435 return 0;
436
Ben Hutchings60ac1062008-09-01 12:44:59 +0100437 fail:
Ben Hutchings8ceee662008-04-27 12:55:59 +0100438 kfree(tx_queue->buffer);
439 tx_queue->buffer = NULL;
Ben Hutchings8ceee662008-04-27 12:55:59 +0100440 return rc;
441}
442
Ben Hutchingsbc3c90a2008-09-01 12:48:46 +0100443void efx_init_tx_queue(struct efx_tx_queue *tx_queue)
Ben Hutchings8ceee662008-04-27 12:55:59 +0100444{
445 EFX_LOG(tx_queue->efx, "initialising TX queue %d\n", tx_queue->queue);
446
447 tx_queue->insert_count = 0;
448 tx_queue->write_count = 0;
449 tx_queue->read_count = 0;
450 tx_queue->old_read_count = 0;
451 BUG_ON(tx_queue->stopped);
452
453 /* Set up TX descriptor ring */
Ben Hutchings152b6a62009-11-29 03:43:56 +0000454 efx_nic_init_tx(tx_queue);
Ben Hutchings8ceee662008-04-27 12:55:59 +0100455}
456
457void efx_release_tx_buffers(struct efx_tx_queue *tx_queue)
458{
459 struct efx_tx_buffer *buffer;
460
461 if (!tx_queue->buffer)
462 return;
463
464 /* Free any buffers left in the ring */
465 while (tx_queue->read_count != tx_queue->write_count) {
Ben Hutchings3ffeabd2009-10-23 08:30:58 +0000466 buffer = &tx_queue->buffer[tx_queue->read_count & EFX_TXQ_MASK];
Ben Hutchings8ceee662008-04-27 12:55:59 +0100467 efx_dequeue_buffer(tx_queue, buffer);
Ben Hutchingsdc8cfa52008-09-01 12:46:50 +0100468 buffer->continuation = true;
Ben Hutchings8ceee662008-04-27 12:55:59 +0100469 buffer->len = 0;
470
471 ++tx_queue->read_count;
472 }
473}
474
475void efx_fini_tx_queue(struct efx_tx_queue *tx_queue)
476{
477 EFX_LOG(tx_queue->efx, "shutting down TX queue %d\n", tx_queue->queue);
478
479 /* Flush TX queue, remove descriptor ring */
Ben Hutchings152b6a62009-11-29 03:43:56 +0000480 efx_nic_fini_tx(tx_queue);
Ben Hutchings8ceee662008-04-27 12:55:59 +0100481
482 efx_release_tx_buffers(tx_queue);
483
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100484 /* Free up TSO header cache */
485 efx_fini_tso(tx_queue);
486
Ben Hutchings8ceee662008-04-27 12:55:59 +0100487 /* Release queue's stop on port, if any */
488 if (tx_queue->stopped) {
489 tx_queue->stopped = 0;
490 efx_wake_queue(tx_queue->efx);
491 }
492}
493
494void efx_remove_tx_queue(struct efx_tx_queue *tx_queue)
495{
496 EFX_LOG(tx_queue->efx, "destroying TX queue %d\n", tx_queue->queue);
Ben Hutchings152b6a62009-11-29 03:43:56 +0000497 efx_nic_remove_tx(tx_queue);
Ben Hutchings8ceee662008-04-27 12:55:59 +0100498
499 kfree(tx_queue->buffer);
500 tx_queue->buffer = NULL;
Ben Hutchings8ceee662008-04-27 12:55:59 +0100501}
502
503
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100504/* Efx TCP segmentation acceleration.
505 *
506 * Why? Because by doing it here in the driver we can go significantly
507 * faster than the GSO.
508 *
509 * Requires TX checksum offload support.
510 */
511
512/* Number of bytes inserted at the start of a TSO header buffer,
513 * similar to NET_IP_ALIGN.
514 */
Ben Hutchings13e9ab12008-09-01 12:50:28 +0100515#ifdef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100516#define TSOH_OFFSET 0
517#else
518#define TSOH_OFFSET NET_IP_ALIGN
519#endif
520
521#define TSOH_BUFFER(tsoh) ((u8 *)(tsoh + 1) + TSOH_OFFSET)
522
523/* Total size of struct efx_tso_header, buffer and padding */
524#define TSOH_SIZE(hdr_len) \
525 (sizeof(struct efx_tso_header) + TSOH_OFFSET + hdr_len)
526
527/* Size of blocks on free list. Larger blocks must be allocated from
528 * the heap.
529 */
530#define TSOH_STD_SIZE 128
531
532#define PTR_DIFF(p1, p2) ((u8 *)(p1) - (u8 *)(p2))
533#define ETH_HDR_LEN(skb) (skb_network_header(skb) - (skb)->data)
534#define SKB_TCP_OFF(skb) PTR_DIFF(tcp_hdr(skb), (skb)->data)
535#define SKB_IPV4_OFF(skb) PTR_DIFF(ip_hdr(skb), (skb)->data)
Ben Hutchings738a8f42009-11-29 15:16:05 +0000536#define SKB_IPV6_OFF(skb) PTR_DIFF(ipv6_hdr(skb), (skb)->data)
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100537
538/**
539 * struct tso_state - TSO state for an SKB
Ben Hutchings23d9e602008-09-01 12:47:02 +0100540 * @out_len: Remaining length in current segment
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100541 * @seqnum: Current sequence number
Ben Hutchings23d9e602008-09-01 12:47:02 +0100542 * @ipv4_id: Current IPv4 ID, host endian
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100543 * @packet_space: Remaining space in current packet
Ben Hutchings23d9e602008-09-01 12:47:02 +0100544 * @dma_addr: DMA address of current position
545 * @in_len: Remaining length in current SKB fragment
546 * @unmap_len: Length of SKB fragment
547 * @unmap_addr: DMA address of SKB fragment
548 * @unmap_single: DMA single vs page mapping flag
Ben Hutchings738a8f42009-11-29 15:16:05 +0000549 * @protocol: Network protocol (after any VLAN header)
Ben Hutchings23d9e602008-09-01 12:47:02 +0100550 * @header_len: Number of bytes of header
551 * @full_packet_size: Number of bytes to put in each outgoing segment
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100552 *
553 * The state used during segmentation. It is put into this data structure
554 * just to make it easy to pass into inline functions.
555 */
556struct tso_state {
Ben Hutchings23d9e602008-09-01 12:47:02 +0100557 /* Output position */
558 unsigned out_len;
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100559 unsigned seqnum;
Ben Hutchings23d9e602008-09-01 12:47:02 +0100560 unsigned ipv4_id;
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100561 unsigned packet_space;
562
Ben Hutchings23d9e602008-09-01 12:47:02 +0100563 /* Input position */
564 dma_addr_t dma_addr;
565 unsigned in_len;
566 unsigned unmap_len;
567 dma_addr_t unmap_addr;
568 bool unmap_single;
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100569
Ben Hutchings738a8f42009-11-29 15:16:05 +0000570 __be16 protocol;
Ben Hutchings23d9e602008-09-01 12:47:02 +0100571 unsigned header_len;
572 int full_packet_size;
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100573};
574
575
576/*
577 * Verify that our various assumptions about sk_buffs and the conditions
Ben Hutchings738a8f42009-11-29 15:16:05 +0000578 * under which TSO will be attempted hold true. Return the protocol number.
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100579 */
Ben Hutchings738a8f42009-11-29 15:16:05 +0000580static __be16 efx_tso_check_protocol(struct sk_buff *skb)
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100581{
Ben Hutchings740847d2008-09-01 12:48:23 +0100582 __be16 protocol = skb->protocol;
583
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100584 EFX_BUG_ON_PARANOID(((struct ethhdr *)skb->data)->h_proto !=
Ben Hutchings740847d2008-09-01 12:48:23 +0100585 protocol);
586 if (protocol == htons(ETH_P_8021Q)) {
587 /* Find the encapsulated protocol; reset network header
588 * and transport header based on that. */
589 struct vlan_ethhdr *veh = (struct vlan_ethhdr *)skb->data;
590 protocol = veh->h_vlan_encapsulated_proto;
591 skb_set_network_header(skb, sizeof(*veh));
592 if (protocol == htons(ETH_P_IP))
593 skb_set_transport_header(skb, sizeof(*veh) +
594 4 * ip_hdr(skb)->ihl);
Ben Hutchings738a8f42009-11-29 15:16:05 +0000595 else if (protocol == htons(ETH_P_IPV6))
596 skb_set_transport_header(skb, sizeof(*veh) +
597 sizeof(struct ipv6hdr));
Ben Hutchings740847d2008-09-01 12:48:23 +0100598 }
599
Ben Hutchings738a8f42009-11-29 15:16:05 +0000600 if (protocol == htons(ETH_P_IP)) {
601 EFX_BUG_ON_PARANOID(ip_hdr(skb)->protocol != IPPROTO_TCP);
602 } else {
603 EFX_BUG_ON_PARANOID(protocol != htons(ETH_P_IPV6));
604 EFX_BUG_ON_PARANOID(ipv6_hdr(skb)->nexthdr != NEXTHDR_TCP);
605 }
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100606 EFX_BUG_ON_PARANOID((PTR_DIFF(tcp_hdr(skb), skb->data)
607 + (tcp_hdr(skb)->doff << 2u)) >
608 skb_headlen(skb));
Ben Hutchings738a8f42009-11-29 15:16:05 +0000609
610 return protocol;
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100611}
612
613
614/*
615 * Allocate a page worth of efx_tso_header structures, and string them
616 * into the tx_queue->tso_headers_free linked list. Return 0 or -ENOMEM.
617 */
618static int efx_tsoh_block_alloc(struct efx_tx_queue *tx_queue)
619{
620
621 struct pci_dev *pci_dev = tx_queue->efx->pci_dev;
622 struct efx_tso_header *tsoh;
623 dma_addr_t dma_addr;
624 u8 *base_kva, *kva;
625
626 base_kva = pci_alloc_consistent(pci_dev, PAGE_SIZE, &dma_addr);
627 if (base_kva == NULL) {
628 EFX_ERR(tx_queue->efx, "Unable to allocate page for TSO"
629 " headers\n");
630 return -ENOMEM;
631 }
632
633 /* pci_alloc_consistent() allocates pages. */
634 EFX_BUG_ON_PARANOID(dma_addr & (PAGE_SIZE - 1u));
635
636 for (kva = base_kva; kva < base_kva + PAGE_SIZE; kva += TSOH_STD_SIZE) {
637 tsoh = (struct efx_tso_header *)kva;
638 tsoh->dma_addr = dma_addr + (TSOH_BUFFER(tsoh) - base_kva);
639 tsoh->next = tx_queue->tso_headers_free;
640 tx_queue->tso_headers_free = tsoh;
641 }
642
643 return 0;
644}
645
646
647/* Free up a TSO header, and all others in the same page. */
648static void efx_tsoh_block_free(struct efx_tx_queue *tx_queue,
649 struct efx_tso_header *tsoh,
650 struct pci_dev *pci_dev)
651{
652 struct efx_tso_header **p;
653 unsigned long base_kva;
654 dma_addr_t base_dma;
655
656 base_kva = (unsigned long)tsoh & PAGE_MASK;
657 base_dma = tsoh->dma_addr & PAGE_MASK;
658
659 p = &tx_queue->tso_headers_free;
Ben Hutchingsb3475642008-05-16 21:15:49 +0100660 while (*p != NULL) {
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100661 if (((unsigned long)*p & PAGE_MASK) == base_kva)
662 *p = (*p)->next;
663 else
664 p = &(*p)->next;
Ben Hutchingsb3475642008-05-16 21:15:49 +0100665 }
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100666
667 pci_free_consistent(pci_dev, PAGE_SIZE, (void *)base_kva, base_dma);
668}
669
670static struct efx_tso_header *
671efx_tsoh_heap_alloc(struct efx_tx_queue *tx_queue, size_t header_len)
672{
673 struct efx_tso_header *tsoh;
674
675 tsoh = kmalloc(TSOH_SIZE(header_len), GFP_ATOMIC | GFP_DMA);
676 if (unlikely(!tsoh))
677 return NULL;
678
679 tsoh->dma_addr = pci_map_single(tx_queue->efx->pci_dev,
680 TSOH_BUFFER(tsoh), header_len,
681 PCI_DMA_TODEVICE);
FUJITA Tomonori8d8bb392008-07-25 19:44:49 -0700682 if (unlikely(pci_dma_mapping_error(tx_queue->efx->pci_dev,
683 tsoh->dma_addr))) {
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100684 kfree(tsoh);
685 return NULL;
686 }
687
688 tsoh->unmap_len = header_len;
689 return tsoh;
690}
691
692static void
693efx_tsoh_heap_free(struct efx_tx_queue *tx_queue, struct efx_tso_header *tsoh)
694{
695 pci_unmap_single(tx_queue->efx->pci_dev,
696 tsoh->dma_addr, tsoh->unmap_len,
697 PCI_DMA_TODEVICE);
698 kfree(tsoh);
699}
700
701/**
702 * efx_tx_queue_insert - push descriptors onto the TX queue
703 * @tx_queue: Efx TX queue
704 * @dma_addr: DMA address of fragment
705 * @len: Length of fragment
Ben Hutchingsecbd95c2008-09-01 12:46:40 +0100706 * @final_buffer: The final buffer inserted into the queue
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100707 *
708 * Push descriptors onto the TX queue. Return 0 on success or 1 if
709 * @tx_queue full.
710 */
711static int efx_tx_queue_insert(struct efx_tx_queue *tx_queue,
712 dma_addr_t dma_addr, unsigned len,
Ben Hutchingsecbd95c2008-09-01 12:46:40 +0100713 struct efx_tx_buffer **final_buffer)
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100714{
715 struct efx_tx_buffer *buffer;
716 struct efx_nic *efx = tx_queue->efx;
Ben Hutchings63f19882009-10-23 08:31:20 +0000717 unsigned dma_len, fill_level, insert_ptr;
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100718 int q_space;
719
720 EFX_BUG_ON_PARANOID(len <= 0);
721
722 fill_level = tx_queue->insert_count - tx_queue->old_read_count;
723 /* -1 as there is no way to represent all descriptors used */
Ben Hutchings3ffeabd2009-10-23 08:30:58 +0000724 q_space = EFX_TXQ_MASK - 1 - fill_level;
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100725
726 while (1) {
727 if (unlikely(q_space-- <= 0)) {
728 /* It might be that completions have happened
729 * since the xmit path last checked. Update
730 * the xmit path's copy of read_count.
731 */
732 ++tx_queue->stopped;
733 /* This memory barrier protects the change of
734 * stopped from the access of read_count. */
735 smp_mb();
736 tx_queue->old_read_count =
737 *(volatile unsigned *)&tx_queue->read_count;
738 fill_level = (tx_queue->insert_count
739 - tx_queue->old_read_count);
Ben Hutchings3ffeabd2009-10-23 08:30:58 +0000740 q_space = EFX_TXQ_MASK - 1 - fill_level;
Ben Hutchingsecbd95c2008-09-01 12:46:40 +0100741 if (unlikely(q_space-- <= 0)) {
742 *final_buffer = NULL;
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100743 return 1;
Ben Hutchingsecbd95c2008-09-01 12:46:40 +0100744 }
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100745 smp_mb();
746 --tx_queue->stopped;
747 }
748
Ben Hutchings3ffeabd2009-10-23 08:30:58 +0000749 insert_ptr = tx_queue->insert_count & EFX_TXQ_MASK;
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100750 buffer = &tx_queue->buffer[insert_ptr];
751 ++tx_queue->insert_count;
752
753 EFX_BUG_ON_PARANOID(tx_queue->insert_count -
754 tx_queue->read_count >
Ben Hutchings3ffeabd2009-10-23 08:30:58 +0000755 EFX_TXQ_MASK);
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100756
757 efx_tsoh_free(tx_queue, buffer);
758 EFX_BUG_ON_PARANOID(buffer->len);
759 EFX_BUG_ON_PARANOID(buffer->unmap_len);
760 EFX_BUG_ON_PARANOID(buffer->skb);
Ben Hutchingsdc8cfa52008-09-01 12:46:50 +0100761 EFX_BUG_ON_PARANOID(!buffer->continuation);
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100762 EFX_BUG_ON_PARANOID(buffer->tsoh);
763
764 buffer->dma_addr = dma_addr;
765
Ben Hutchings63f19882009-10-23 08:31:20 +0000766 dma_len = efx_max_tx_len(efx, dma_addr);
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100767
768 /* If there is enough space to send then do so */
769 if (dma_len >= len)
770 break;
771
772 buffer->len = dma_len; /* Don't set the other members */
773 dma_addr += dma_len;
774 len -= dma_len;
775 }
776
777 EFX_BUG_ON_PARANOID(!len);
778 buffer->len = len;
Ben Hutchingsecbd95c2008-09-01 12:46:40 +0100779 *final_buffer = buffer;
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100780 return 0;
781}
782
783
784/*
785 * Put a TSO header into the TX queue.
786 *
787 * This is special-cased because we know that it is small enough to fit in
788 * a single fragment, and we know it doesn't cross a page boundary. It
789 * also allows us to not worry about end-of-packet etc.
790 */
Ben Hutchings4d566062008-09-01 12:47:12 +0100791static void efx_tso_put_header(struct efx_tx_queue *tx_queue,
792 struct efx_tso_header *tsoh, unsigned len)
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100793{
794 struct efx_tx_buffer *buffer;
795
Ben Hutchings3ffeabd2009-10-23 08:30:58 +0000796 buffer = &tx_queue->buffer[tx_queue->insert_count & EFX_TXQ_MASK];
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100797 efx_tsoh_free(tx_queue, buffer);
798 EFX_BUG_ON_PARANOID(buffer->len);
799 EFX_BUG_ON_PARANOID(buffer->unmap_len);
800 EFX_BUG_ON_PARANOID(buffer->skb);
Ben Hutchingsdc8cfa52008-09-01 12:46:50 +0100801 EFX_BUG_ON_PARANOID(!buffer->continuation);
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100802 EFX_BUG_ON_PARANOID(buffer->tsoh);
803 buffer->len = len;
804 buffer->dma_addr = tsoh->dma_addr;
805 buffer->tsoh = tsoh;
806
807 ++tx_queue->insert_count;
808}
809
810
811/* Remove descriptors put into a tx_queue. */
812static void efx_enqueue_unwind(struct efx_tx_queue *tx_queue)
813{
814 struct efx_tx_buffer *buffer;
Ben Hutchingscc12dac2008-09-01 12:46:43 +0100815 dma_addr_t unmap_addr;
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100816
817 /* Work backwards until we hit the original insert pointer value */
818 while (tx_queue->insert_count != tx_queue->write_count) {
819 --tx_queue->insert_count;
820 buffer = &tx_queue->buffer[tx_queue->insert_count &
Ben Hutchings3ffeabd2009-10-23 08:30:58 +0000821 EFX_TXQ_MASK];
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100822 efx_tsoh_free(tx_queue, buffer);
823 EFX_BUG_ON_PARANOID(buffer->skb);
824 buffer->len = 0;
Ben Hutchingsdc8cfa52008-09-01 12:46:50 +0100825 buffer->continuation = true;
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100826 if (buffer->unmap_len) {
Ben Hutchingscc12dac2008-09-01 12:46:43 +0100827 unmap_addr = (buffer->dma_addr + buffer->len -
828 buffer->unmap_len);
Ben Hutchingsecbd95c2008-09-01 12:46:40 +0100829 if (buffer->unmap_single)
830 pci_unmap_single(tx_queue->efx->pci_dev,
Ben Hutchingscc12dac2008-09-01 12:46:43 +0100831 unmap_addr, buffer->unmap_len,
Ben Hutchingsecbd95c2008-09-01 12:46:40 +0100832 PCI_DMA_TODEVICE);
833 else
834 pci_unmap_page(tx_queue->efx->pci_dev,
Ben Hutchingscc12dac2008-09-01 12:46:43 +0100835 unmap_addr, buffer->unmap_len,
Ben Hutchingsecbd95c2008-09-01 12:46:40 +0100836 PCI_DMA_TODEVICE);
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100837 buffer->unmap_len = 0;
838 }
839 }
840}
841
842
843/* Parse the SKB header and initialise state. */
Ben Hutchings4d566062008-09-01 12:47:12 +0100844static void tso_start(struct tso_state *st, const struct sk_buff *skb)
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100845{
846 /* All ethernet/IP/TCP headers combined size is TCP header size
847 * plus offset of TCP header relative to start of packet.
848 */
Ben Hutchings23d9e602008-09-01 12:47:02 +0100849 st->header_len = ((tcp_hdr(skb)->doff << 2u)
850 + PTR_DIFF(tcp_hdr(skb), skb->data));
851 st->full_packet_size = st->header_len + skb_shinfo(skb)->gso_size;
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100852
Ben Hutchings738a8f42009-11-29 15:16:05 +0000853 if (st->protocol == htons(ETH_P_IP))
854 st->ipv4_id = ntohs(ip_hdr(skb)->id);
855 else
856 st->ipv4_id = 0;
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100857 st->seqnum = ntohl(tcp_hdr(skb)->seq);
858
859 EFX_BUG_ON_PARANOID(tcp_hdr(skb)->urg);
860 EFX_BUG_ON_PARANOID(tcp_hdr(skb)->syn);
861 EFX_BUG_ON_PARANOID(tcp_hdr(skb)->rst);
862
Ben Hutchings23d9e602008-09-01 12:47:02 +0100863 st->packet_space = st->full_packet_size;
864 st->out_len = skb->len - st->header_len;
865 st->unmap_len = 0;
866 st->unmap_single = false;
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100867}
868
Ben Hutchings4d566062008-09-01 12:47:12 +0100869static int tso_get_fragment(struct tso_state *st, struct efx_nic *efx,
870 skb_frag_t *frag)
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100871{
Ben Hutchings23d9e602008-09-01 12:47:02 +0100872 st->unmap_addr = pci_map_page(efx->pci_dev, frag->page,
873 frag->page_offset, frag->size,
874 PCI_DMA_TODEVICE);
875 if (likely(!pci_dma_mapping_error(efx->pci_dev, st->unmap_addr))) {
876 st->unmap_single = false;
877 st->unmap_len = frag->size;
878 st->in_len = frag->size;
879 st->dma_addr = st->unmap_addr;
Ben Hutchingsecbd95c2008-09-01 12:46:40 +0100880 return 0;
881 }
882 return -ENOMEM;
883}
884
Ben Hutchings4d566062008-09-01 12:47:12 +0100885static int tso_get_head_fragment(struct tso_state *st, struct efx_nic *efx,
886 const struct sk_buff *skb)
Ben Hutchingsecbd95c2008-09-01 12:46:40 +0100887{
Ben Hutchings23d9e602008-09-01 12:47:02 +0100888 int hl = st->header_len;
Ben Hutchingsecbd95c2008-09-01 12:46:40 +0100889 int len = skb_headlen(skb) - hl;
890
Ben Hutchings23d9e602008-09-01 12:47:02 +0100891 st->unmap_addr = pci_map_single(efx->pci_dev, skb->data + hl,
892 len, PCI_DMA_TODEVICE);
893 if (likely(!pci_dma_mapping_error(efx->pci_dev, st->unmap_addr))) {
894 st->unmap_single = true;
895 st->unmap_len = len;
896 st->in_len = len;
897 st->dma_addr = st->unmap_addr;
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100898 return 0;
899 }
900 return -ENOMEM;
901}
902
903
904/**
905 * tso_fill_packet_with_fragment - form descriptors for the current fragment
906 * @tx_queue: Efx TX queue
907 * @skb: Socket buffer
908 * @st: TSO state
909 *
910 * Form descriptors for the current fragment, until we reach the end
911 * of fragment or end-of-packet. Return 0 on success, 1 if not enough
912 * space in @tx_queue.
913 */
Ben Hutchings4d566062008-09-01 12:47:12 +0100914static int tso_fill_packet_with_fragment(struct efx_tx_queue *tx_queue,
915 const struct sk_buff *skb,
916 struct tso_state *st)
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100917{
Ben Hutchingsecbd95c2008-09-01 12:46:40 +0100918 struct efx_tx_buffer *buffer;
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100919 int n, end_of_packet, rc;
920
Ben Hutchings23d9e602008-09-01 12:47:02 +0100921 if (st->in_len == 0)
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100922 return 0;
923 if (st->packet_space == 0)
924 return 0;
925
Ben Hutchings23d9e602008-09-01 12:47:02 +0100926 EFX_BUG_ON_PARANOID(st->in_len <= 0);
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100927 EFX_BUG_ON_PARANOID(st->packet_space <= 0);
928
Ben Hutchings23d9e602008-09-01 12:47:02 +0100929 n = min(st->in_len, st->packet_space);
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100930
931 st->packet_space -= n;
Ben Hutchings23d9e602008-09-01 12:47:02 +0100932 st->out_len -= n;
933 st->in_len -= n;
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100934
Ben Hutchings23d9e602008-09-01 12:47:02 +0100935 rc = efx_tx_queue_insert(tx_queue, st->dma_addr, n, &buffer);
Ben Hutchingsecbd95c2008-09-01 12:46:40 +0100936 if (likely(rc == 0)) {
Ben Hutchings23d9e602008-09-01 12:47:02 +0100937 if (st->out_len == 0)
Ben Hutchingsecbd95c2008-09-01 12:46:40 +0100938 /* Transfer ownership of the skb */
939 buffer->skb = skb;
940
Ben Hutchings23d9e602008-09-01 12:47:02 +0100941 end_of_packet = st->out_len == 0 || st->packet_space == 0;
Ben Hutchingsecbd95c2008-09-01 12:46:40 +0100942 buffer->continuation = !end_of_packet;
943
Ben Hutchings23d9e602008-09-01 12:47:02 +0100944 if (st->in_len == 0) {
Ben Hutchingsecbd95c2008-09-01 12:46:40 +0100945 /* Transfer ownership of the pci mapping */
Ben Hutchings23d9e602008-09-01 12:47:02 +0100946 buffer->unmap_len = st->unmap_len;
947 buffer->unmap_single = st->unmap_single;
948 st->unmap_len = 0;
Ben Hutchingsecbd95c2008-09-01 12:46:40 +0100949 }
950 }
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100951
Ben Hutchings23d9e602008-09-01 12:47:02 +0100952 st->dma_addr += n;
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100953 return rc;
954}
955
956
957/**
958 * tso_start_new_packet - generate a new header and prepare for the new packet
959 * @tx_queue: Efx TX queue
960 * @skb: Socket buffer
961 * @st: TSO state
962 *
963 * Generate a new header and prepare for the new packet. Return 0 on
964 * success, or -1 if failed to alloc header.
965 */
Ben Hutchings4d566062008-09-01 12:47:12 +0100966static int tso_start_new_packet(struct efx_tx_queue *tx_queue,
967 const struct sk_buff *skb,
968 struct tso_state *st)
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100969{
970 struct efx_tso_header *tsoh;
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100971 struct tcphdr *tsoh_th;
972 unsigned ip_length;
973 u8 *header;
974
975 /* Allocate a DMA-mapped header buffer. */
Ben Hutchings23d9e602008-09-01 12:47:02 +0100976 if (likely(TSOH_SIZE(st->header_len) <= TSOH_STD_SIZE)) {
Ben Hutchingsb3475642008-05-16 21:15:49 +0100977 if (tx_queue->tso_headers_free == NULL) {
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100978 if (efx_tsoh_block_alloc(tx_queue))
979 return -1;
Ben Hutchingsb3475642008-05-16 21:15:49 +0100980 }
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100981 EFX_BUG_ON_PARANOID(!tx_queue->tso_headers_free);
982 tsoh = tx_queue->tso_headers_free;
983 tx_queue->tso_headers_free = tsoh->next;
984 tsoh->unmap_len = 0;
985 } else {
986 tx_queue->tso_long_headers++;
Ben Hutchings23d9e602008-09-01 12:47:02 +0100987 tsoh = efx_tsoh_heap_alloc(tx_queue, st->header_len);
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100988 if (unlikely(!tsoh))
989 return -1;
990 }
991
992 header = TSOH_BUFFER(tsoh);
993 tsoh_th = (struct tcphdr *)(header + SKB_TCP_OFF(skb));
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100994
995 /* Copy and update the headers. */
Ben Hutchings23d9e602008-09-01 12:47:02 +0100996 memcpy(header, skb->data, st->header_len);
Ben Hutchingsb9b39b62008-05-07 12:51:12 +0100997
998 tsoh_th->seq = htonl(st->seqnum);
999 st->seqnum += skb_shinfo(skb)->gso_size;
Ben Hutchings23d9e602008-09-01 12:47:02 +01001000 if (st->out_len > skb_shinfo(skb)->gso_size) {
Ben Hutchingsb9b39b62008-05-07 12:51:12 +01001001 /* This packet will not finish the TSO burst. */
Ben Hutchings23d9e602008-09-01 12:47:02 +01001002 ip_length = st->full_packet_size - ETH_HDR_LEN(skb);
Ben Hutchingsb9b39b62008-05-07 12:51:12 +01001003 tsoh_th->fin = 0;
1004 tsoh_th->psh = 0;
1005 } else {
1006 /* This packet will be the last in the TSO burst. */
Ben Hutchings23d9e602008-09-01 12:47:02 +01001007 ip_length = st->header_len - ETH_HDR_LEN(skb) + st->out_len;
Ben Hutchingsb9b39b62008-05-07 12:51:12 +01001008 tsoh_th->fin = tcp_hdr(skb)->fin;
1009 tsoh_th->psh = tcp_hdr(skb)->psh;
1010 }
Ben Hutchingsb9b39b62008-05-07 12:51:12 +01001011
Ben Hutchings738a8f42009-11-29 15:16:05 +00001012 if (st->protocol == htons(ETH_P_IP)) {
1013 struct iphdr *tsoh_iph =
1014 (struct iphdr *)(header + SKB_IPV4_OFF(skb));
1015
1016 tsoh_iph->tot_len = htons(ip_length);
1017
1018 /* Linux leaves suitable gaps in the IP ID space for us to fill. */
1019 tsoh_iph->id = htons(st->ipv4_id);
1020 st->ipv4_id++;
1021 } else {
1022 struct ipv6hdr *tsoh_iph =
1023 (struct ipv6hdr *)(header + SKB_IPV6_OFF(skb));
1024
1025 tsoh_iph->payload_len = htons(ip_length - sizeof(*tsoh_iph));
1026 }
Ben Hutchingsb9b39b62008-05-07 12:51:12 +01001027
1028 st->packet_space = skb_shinfo(skb)->gso_size;
1029 ++tx_queue->tso_packets;
1030
1031 /* Form a descriptor for this header. */
Ben Hutchings23d9e602008-09-01 12:47:02 +01001032 efx_tso_put_header(tx_queue, tsoh, st->header_len);
Ben Hutchingsb9b39b62008-05-07 12:51:12 +01001033
1034 return 0;
1035}
1036
1037
1038/**
1039 * efx_enqueue_skb_tso - segment and transmit a TSO socket buffer
1040 * @tx_queue: Efx TX queue
1041 * @skb: Socket buffer
1042 *
1043 * Context: You must hold netif_tx_lock() to call this function.
1044 *
1045 * Add socket buffer @skb to @tx_queue, doing TSO or return != 0 if
1046 * @skb was not enqueued. In all cases @skb is consumed. Return
1047 * %NETDEV_TX_OK or %NETDEV_TX_BUSY.
1048 */
1049static int efx_enqueue_skb_tso(struct efx_tx_queue *tx_queue,
Ben Hutchings740847d2008-09-01 12:48:23 +01001050 struct sk_buff *skb)
Ben Hutchingsb9b39b62008-05-07 12:51:12 +01001051{
Ben Hutchingsecbd95c2008-09-01 12:46:40 +01001052 struct efx_nic *efx = tx_queue->efx;
Ben Hutchingsb9b39b62008-05-07 12:51:12 +01001053 int frag_i, rc, rc2 = NETDEV_TX_OK;
1054 struct tso_state state;
Ben Hutchingsb9b39b62008-05-07 12:51:12 +01001055
Ben Hutchings738a8f42009-11-29 15:16:05 +00001056 /* Find the packet protocol and sanity-check it */
1057 state.protocol = efx_tso_check_protocol(skb);
Ben Hutchingsb9b39b62008-05-07 12:51:12 +01001058
1059 EFX_BUG_ON_PARANOID(tx_queue->write_count != tx_queue->insert_count);
1060
1061 tso_start(&state, skb);
1062
1063 /* Assume that skb header area contains exactly the headers, and
1064 * all payload is in the frag list.
1065 */
Ben Hutchings23d9e602008-09-01 12:47:02 +01001066 if (skb_headlen(skb) == state.header_len) {
Ben Hutchingsb9b39b62008-05-07 12:51:12 +01001067 /* Grab the first payload fragment. */
1068 EFX_BUG_ON_PARANOID(skb_shinfo(skb)->nr_frags < 1);
1069 frag_i = 0;
Ben Hutchingsecbd95c2008-09-01 12:46:40 +01001070 rc = tso_get_fragment(&state, efx,
1071 skb_shinfo(skb)->frags + frag_i);
Ben Hutchingsb9b39b62008-05-07 12:51:12 +01001072 if (rc)
1073 goto mem_err;
1074 } else {
Ben Hutchingsecbd95c2008-09-01 12:46:40 +01001075 rc = tso_get_head_fragment(&state, efx, skb);
Ben Hutchingsb9b39b62008-05-07 12:51:12 +01001076 if (rc)
1077 goto mem_err;
1078 frag_i = -1;
1079 }
1080
1081 if (tso_start_new_packet(tx_queue, skb, &state) < 0)
1082 goto mem_err;
1083
1084 while (1) {
1085 rc = tso_fill_packet_with_fragment(tx_queue, skb, &state);
1086 if (unlikely(rc))
1087 goto stop;
1088
1089 /* Move onto the next fragment? */
Ben Hutchings23d9e602008-09-01 12:47:02 +01001090 if (state.in_len == 0) {
Ben Hutchingsb9b39b62008-05-07 12:51:12 +01001091 if (++frag_i >= skb_shinfo(skb)->nr_frags)
1092 /* End of payload reached. */
1093 break;
Ben Hutchingsecbd95c2008-09-01 12:46:40 +01001094 rc = tso_get_fragment(&state, efx,
1095 skb_shinfo(skb)->frags + frag_i);
Ben Hutchingsb9b39b62008-05-07 12:51:12 +01001096 if (rc)
1097 goto mem_err;
1098 }
1099
1100 /* Start at new packet? */
1101 if (state.packet_space == 0 &&
1102 tso_start_new_packet(tx_queue, skb, &state) < 0)
1103 goto mem_err;
1104 }
1105
1106 /* Pass off to hardware */
Ben Hutchings152b6a62009-11-29 03:43:56 +00001107 efx_nic_push_buffers(tx_queue);
Ben Hutchingsb9b39b62008-05-07 12:51:12 +01001108
1109 tx_queue->tso_bursts++;
1110 return NETDEV_TX_OK;
1111
1112 mem_err:
Ben Hutchingsecbd95c2008-09-01 12:46:40 +01001113 EFX_ERR(efx, "Out of memory for TSO headers, or PCI mapping error\n");
Ben Hutchings9bc183d2009-11-23 16:06:47 +00001114 dev_kfree_skb_any(skb);
Ben Hutchingsb9b39b62008-05-07 12:51:12 +01001115 goto unwind;
1116
1117 stop:
1118 rc2 = NETDEV_TX_BUSY;
1119
1120 /* Stop the queue if it wasn't stopped before. */
1121 if (tx_queue->stopped == 1)
Ben Hutchingsecbd95c2008-09-01 12:46:40 +01001122 efx_stop_queue(efx);
Ben Hutchingsb9b39b62008-05-07 12:51:12 +01001123
1124 unwind:
Ben Hutchings5988b632008-09-01 12:46:36 +01001125 /* Free the DMA mapping we were in the process of writing out */
Ben Hutchings23d9e602008-09-01 12:47:02 +01001126 if (state.unmap_len) {
1127 if (state.unmap_single)
1128 pci_unmap_single(efx->pci_dev, state.unmap_addr,
1129 state.unmap_len, PCI_DMA_TODEVICE);
Ben Hutchingsecbd95c2008-09-01 12:46:40 +01001130 else
Ben Hutchings23d9e602008-09-01 12:47:02 +01001131 pci_unmap_page(efx->pci_dev, state.unmap_addr,
1132 state.unmap_len, PCI_DMA_TODEVICE);
Ben Hutchingsecbd95c2008-09-01 12:46:40 +01001133 }
Ben Hutchings5988b632008-09-01 12:46:36 +01001134
Ben Hutchingsb9b39b62008-05-07 12:51:12 +01001135 efx_enqueue_unwind(tx_queue);
1136 return rc2;
1137}
1138
1139
1140/*
1141 * Free up all TSO datastructures associated with tx_queue. This
1142 * routine should be called only once the tx_queue is both empty and
1143 * will no longer be used.
1144 */
1145static void efx_fini_tso(struct efx_tx_queue *tx_queue)
1146{
1147 unsigned i;
1148
Ben Hutchingsb3475642008-05-16 21:15:49 +01001149 if (tx_queue->buffer) {
Ben Hutchings3ffeabd2009-10-23 08:30:58 +00001150 for (i = 0; i <= EFX_TXQ_MASK; ++i)
Ben Hutchingsb9b39b62008-05-07 12:51:12 +01001151 efx_tsoh_free(tx_queue, &tx_queue->buffer[i]);
Ben Hutchingsb3475642008-05-16 21:15:49 +01001152 }
Ben Hutchingsb9b39b62008-05-07 12:51:12 +01001153
1154 while (tx_queue->tso_headers_free != NULL)
1155 efx_tsoh_block_free(tx_queue, tx_queue->tso_headers_free,
1156 tx_queue->efx->pci_dev);
1157}