Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 1 | /**************************************************************************** |
| 2 | * Driver for Solarflare Solarstorm network controllers and boards |
| 3 | * Copyright 2005-2006 Fen Systems Ltd. |
Ben Hutchings | 0a6f40c | 2011-02-25 00:01:34 +0000 | [diff] [blame] | 4 | * Copyright 2005-2011 Solarflare Communications Inc. |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 5 | * |
| 6 | * This program is free software; you can redistribute it and/or modify it |
| 7 | * under the terms of the GNU General Public License version 2 as published |
| 8 | * by the Free Software Foundation, incorporated herein by reference. |
| 9 | */ |
| 10 | |
| 11 | #include <linux/socket.h> |
| 12 | #include <linux/in.h> |
Tejun Heo | 5a0e3ad | 2010-03-24 17:04:11 +0900 | [diff] [blame] | 13 | #include <linux/slab.h> |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 14 | #include <linux/ip.h> |
| 15 | #include <linux/tcp.h> |
| 16 | #include <linux/udp.h> |
Paul Gortmaker | 70c7160 | 2011-05-22 16:47:17 -0400 | [diff] [blame] | 17 | #include <linux/prefetch.h> |
Paul Gortmaker | 6eb07ca | 2011-09-15 19:46:05 -0400 | [diff] [blame] | 18 | #include <linux/moduleparam.h> |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 19 | #include <linux/iommu.h> |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 20 | #include <net/ip.h> |
| 21 | #include <net/checksum.h> |
| 22 | #include "net_driver.h" |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 23 | #include "efx.h" |
Ben Hutchings | 744093c | 2009-11-29 15:12:08 +0000 | [diff] [blame] | 24 | #include "nic.h" |
Ben Hutchings | 3273c2e | 2008-05-07 13:36:19 +0100 | [diff] [blame] | 25 | #include "selftest.h" |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 26 | #include "workarounds.h" |
| 27 | |
| 28 | /* Number of RX descriptors pushed at once. */ |
| 29 | #define EFX_RX_BATCH 8 |
| 30 | |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 31 | /* Number of RX buffers to recycle pages for. When creating the RX page recycle |
| 32 | * ring, this number is divided by the number of buffers per page to calculate |
| 33 | * the number of pages to store in the RX page recycle ring. |
| 34 | */ |
| 35 | #define EFX_RECYCLE_RING_SIZE_IOMMU 4096 |
| 36 | #define EFX_RECYCLE_RING_SIZE_NOIOMMU (2 * EFX_RX_BATCH) |
| 37 | |
Ben Hutchings | 272baee | 2013-01-29 23:33:14 +0000 | [diff] [blame] | 38 | /* Maximum length for an RX descriptor sharing a page */ |
| 39 | #define EFX_RX_HALF_PAGE ((PAGE_SIZE >> 1) - sizeof(struct efx_rx_page_state) \ |
| 40 | - EFX_PAGE_IP_ALIGN) |
Steve Hodgson | 62b330b | 2010-06-01 11:20:53 +0000 | [diff] [blame] | 41 | |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 42 | /* Size of buffer allocated for skb header area. */ |
| 43 | #define EFX_SKB_HEADERS 64u |
| 44 | |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 45 | /* This is the percentage fill level below which new RX descriptors |
| 46 | * will be added to the RX descriptor ring. |
| 47 | */ |
David Riddoch | 6423518 | 2012-04-11 13:12:41 +0100 | [diff] [blame] | 48 | static unsigned int rx_refill_threshold; |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 49 | |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 50 | /* Each packet can consume up to ceil(max_frame_len / buffer_size) buffers */ |
| 51 | #define EFX_RX_MAX_FRAGS DIV_ROUND_UP(EFX_MAX_FRAME_LEN(EFX_MAX_MTU), \ |
| 52 | EFX_RX_USR_BUF_SIZE) |
| 53 | |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 54 | /* |
| 55 | * RX maximum head room required. |
| 56 | * |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 57 | * This must be at least 1 to prevent overflow, plus one packet-worth |
| 58 | * to allow pipelined receives. |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 59 | */ |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 60 | #define EFX_RXD_HEAD_ROOM (1 + EFX_RX_MAX_FRAGS) |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 61 | |
Ben Hutchings | b184f16 | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 62 | static inline u8 *efx_rx_buf_va(struct efx_rx_buffer *buf) |
Ben Hutchings | 39c9cf0 | 2010-06-23 11:31:28 +0000 | [diff] [blame] | 63 | { |
Ben Hutchings | b184f16 | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 64 | return page_address(buf->page) + buf->page_offset; |
Steve Hodgson | a526f14 | 2011-02-24 23:45:16 +0000 | [diff] [blame] | 65 | } |
| 66 | |
| 67 | static inline u32 efx_rx_buf_hash(const u8 *eh) |
| 68 | { |
| 69 | /* The ethernet header is always directly after any hash. */ |
Ben Hutchings | 39c9cf0 | 2010-06-23 11:31:28 +0000 | [diff] [blame] | 70 | #if defined(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) || NET_IP_ALIGN % 4 == 0 |
Steve Hodgson | a526f14 | 2011-02-24 23:45:16 +0000 | [diff] [blame] | 71 | return __le32_to_cpup((const __le32 *)(eh - 4)); |
Ben Hutchings | 39c9cf0 | 2010-06-23 11:31:28 +0000 | [diff] [blame] | 72 | #else |
Steve Hodgson | a526f14 | 2011-02-24 23:45:16 +0000 | [diff] [blame] | 73 | const u8 *data = eh - 4; |
Ben Hutchings | 0beaca2 | 2012-01-05 18:54:04 +0000 | [diff] [blame] | 74 | return (u32)data[0] | |
| 75 | (u32)data[1] << 8 | |
| 76 | (u32)data[2] << 16 | |
| 77 | (u32)data[3] << 24; |
Ben Hutchings | 39c9cf0 | 2010-06-23 11:31:28 +0000 | [diff] [blame] | 78 | #endif |
| 79 | } |
| 80 | |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 81 | static inline struct efx_rx_buffer * |
| 82 | efx_rx_buf_next(struct efx_rx_queue *rx_queue, struct efx_rx_buffer *rx_buf) |
| 83 | { |
| 84 | if (unlikely(rx_buf == efx_rx_buffer(rx_queue, rx_queue->ptr_mask))) |
| 85 | return efx_rx_buffer(rx_queue, 0); |
| 86 | else |
| 87 | return rx_buf + 1; |
| 88 | } |
| 89 | |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 90 | static inline void efx_sync_rx_buffer(struct efx_nic *efx, |
| 91 | struct efx_rx_buffer *rx_buf, |
| 92 | unsigned int len) |
| 93 | { |
| 94 | dma_sync_single_for_cpu(&efx->pci_dev->dev, rx_buf->dma_addr, len, |
| 95 | DMA_FROM_DEVICE); |
| 96 | } |
| 97 | |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 98 | /* Check the RX page recycle ring for a page that can be reused. */ |
| 99 | static struct page *efx_reuse_page(struct efx_rx_queue *rx_queue) |
| 100 | { |
| 101 | struct efx_nic *efx = rx_queue->efx; |
| 102 | struct page *page; |
| 103 | struct efx_rx_page_state *state; |
| 104 | unsigned index; |
| 105 | |
| 106 | index = rx_queue->page_remove & rx_queue->page_ptr_mask; |
| 107 | page = rx_queue->page_ring[index]; |
| 108 | if (page == NULL) |
| 109 | return NULL; |
| 110 | |
| 111 | rx_queue->page_ring[index] = NULL; |
| 112 | /* page_remove cannot exceed page_add. */ |
| 113 | if (rx_queue->page_remove != rx_queue->page_add) |
| 114 | ++rx_queue->page_remove; |
| 115 | |
| 116 | /* If page_count is 1 then we hold the only reference to this page. */ |
| 117 | if (page_count(page) == 1) { |
| 118 | ++rx_queue->page_recycle_count; |
| 119 | return page; |
| 120 | } else { |
| 121 | state = page_address(page); |
| 122 | dma_unmap_page(&efx->pci_dev->dev, state->dma_addr, |
| 123 | PAGE_SIZE << efx->rx_buffer_order, |
| 124 | DMA_FROM_DEVICE); |
| 125 | put_page(page); |
| 126 | ++rx_queue->page_recycle_failed; |
| 127 | } |
| 128 | |
| 129 | return NULL; |
| 130 | } |
| 131 | |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 132 | /** |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 133 | * efx_init_rx_buffers - create EFX_RX_BATCH page-based RX buffers |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 134 | * |
| 135 | * @rx_queue: Efx RX queue |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 136 | * |
Steve Hodgson | f7d6f37 | 2010-06-01 11:33:17 +0000 | [diff] [blame] | 137 | * This allocates memory for EFX_RX_BATCH receive buffers, maps them for DMA, |
| 138 | * and populates struct efx_rx_buffers for each one. Return a negative error |
| 139 | * code or 0 on success. If a single page can be split between two buffers, |
| 140 | * then the page will either be inserted fully, or not at at all. |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 141 | */ |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 142 | static int efx_init_rx_buffers(struct efx_rx_queue *rx_queue) |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 143 | { |
| 144 | struct efx_nic *efx = rx_queue->efx; |
Steve Hodgson | f7d6f37 | 2010-06-01 11:33:17 +0000 | [diff] [blame] | 145 | struct efx_rx_buffer *rx_buf; |
| 146 | struct page *page; |
Ben Hutchings | b590ace | 2013-01-10 23:51:54 +0000 | [diff] [blame] | 147 | unsigned int page_offset; |
Steve Hodgson | 62b330b | 2010-06-01 11:20:53 +0000 | [diff] [blame] | 148 | struct efx_rx_page_state *state; |
Steve Hodgson | f7d6f37 | 2010-06-01 11:33:17 +0000 | [diff] [blame] | 149 | dma_addr_t dma_addr; |
| 150 | unsigned index, count; |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 151 | |
Steve Hodgson | f7d6f37 | 2010-06-01 11:33:17 +0000 | [diff] [blame] | 152 | /* We can split a page between two buffers */ |
| 153 | BUILD_BUG_ON(EFX_RX_BATCH & 1); |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 154 | |
Steve Hodgson | f7d6f37 | 2010-06-01 11:33:17 +0000 | [diff] [blame] | 155 | for (count = 0; count < EFX_RX_BATCH; ++count) { |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 156 | page = efx_reuse_page(rx_queue); |
| 157 | if (page == NULL) { |
| 158 | page = alloc_pages(__GFP_COLD | __GFP_COMP | GFP_ATOMIC, |
| 159 | efx->rx_buffer_order); |
| 160 | if (unlikely(page == NULL)) |
| 161 | return -ENOMEM; |
| 162 | dma_addr = |
| 163 | dma_map_page(&efx->pci_dev->dev, page, 0, |
| 164 | PAGE_SIZE << efx->rx_buffer_order, |
| 165 | DMA_FROM_DEVICE); |
| 166 | if (unlikely(dma_mapping_error(&efx->pci_dev->dev, |
| 167 | dma_addr))) { |
| 168 | __free_pages(page, efx->rx_buffer_order); |
| 169 | return -EIO; |
| 170 | } |
| 171 | state = page_address(page); |
| 172 | state->dma_addr = dma_addr; |
| 173 | } else { |
| 174 | state = page_address(page); |
| 175 | dma_addr = state->dma_addr; |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 176 | } |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 177 | get_page(page); |
Steve Hodgson | 62b330b | 2010-06-01 11:20:53 +0000 | [diff] [blame] | 178 | |
Steve Hodgson | 62b330b | 2010-06-01 11:20:53 +0000 | [diff] [blame] | 179 | dma_addr += sizeof(struct efx_rx_page_state); |
Ben Hutchings | b590ace | 2013-01-10 23:51:54 +0000 | [diff] [blame] | 180 | page_offset = sizeof(struct efx_rx_page_state); |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 181 | |
Steve Hodgson | f7d6f37 | 2010-06-01 11:33:17 +0000 | [diff] [blame] | 182 | split: |
Steve Hodgson | ecc910f | 2010-09-10 06:42:22 +0000 | [diff] [blame] | 183 | index = rx_queue->added_count & rx_queue->ptr_mask; |
Steve Hodgson | f7d6f37 | 2010-06-01 11:33:17 +0000 | [diff] [blame] | 184 | rx_buf = efx_rx_buffer(rx_queue, index); |
Steve Hodgson | 62b330b | 2010-06-01 11:20:53 +0000 | [diff] [blame] | 185 | rx_buf->dma_addr = dma_addr + EFX_PAGE_IP_ALIGN; |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 186 | rx_buf->page = page; |
Ben Hutchings | c73e787 | 2013-03-05 17:49:39 +0000 | [diff] [blame] | 187 | rx_buf->page_offset = page_offset + EFX_PAGE_IP_ALIGN; |
Ben Hutchings | 272baee | 2013-01-29 23:33:14 +0000 | [diff] [blame] | 188 | rx_buf->len = efx->rx_dma_len; |
Steve Hodgson | f7d6f37 | 2010-06-01 11:33:17 +0000 | [diff] [blame] | 189 | ++rx_queue->added_count; |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 190 | |
Ben Hutchings | 272baee | 2013-01-29 23:33:14 +0000 | [diff] [blame] | 191 | if ((~count & 1) && (efx->rx_dma_len <= EFX_RX_HALF_PAGE)) { |
Steve Hodgson | f7d6f37 | 2010-06-01 11:33:17 +0000 | [diff] [blame] | 192 | /* Use the second half of the page */ |
| 193 | get_page(page); |
Ben Hutchings | 179ea7f | 2013-03-07 16:31:17 +0000 | [diff] [blame^] | 194 | rx_buf->flags = 0; |
Steve Hodgson | f7d6f37 | 2010-06-01 11:33:17 +0000 | [diff] [blame] | 195 | dma_addr += (PAGE_SIZE >> 1); |
Ben Hutchings | b590ace | 2013-01-10 23:51:54 +0000 | [diff] [blame] | 196 | page_offset += (PAGE_SIZE >> 1); |
Steve Hodgson | f7d6f37 | 2010-06-01 11:33:17 +0000 | [diff] [blame] | 197 | ++count; |
| 198 | goto split; |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 199 | } |
Ben Hutchings | 179ea7f | 2013-03-07 16:31:17 +0000 | [diff] [blame^] | 200 | |
| 201 | rx_buf->flags = EFX_RX_BUF_LAST_IN_PAGE; |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 202 | } |
| 203 | |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 204 | return 0; |
| 205 | } |
| 206 | |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 207 | /* Unmap a DMA-mapped page. This function is only called for the final RX |
| 208 | * buffer in a page. |
| 209 | */ |
Ben Hutchings | 4d56606 | 2008-09-01 12:47:12 +0100 | [diff] [blame] | 210 | static void efx_unmap_rx_buffer(struct efx_nic *efx, |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 211 | struct efx_rx_buffer *rx_buf) |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 212 | { |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 213 | struct page *page = rx_buf->page; |
Steve Hodgson | 62b330b | 2010-06-01 11:20:53 +0000 | [diff] [blame] | 214 | |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 215 | if (page) { |
| 216 | struct efx_rx_page_state *state = page_address(page); |
| 217 | dma_unmap_page(&efx->pci_dev->dev, |
| 218 | state->dma_addr, |
| 219 | PAGE_SIZE << efx->rx_buffer_order, |
| 220 | DMA_FROM_DEVICE); |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 221 | } |
| 222 | } |
| 223 | |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 224 | static void efx_free_rx_buffer(struct efx_rx_buffer *rx_buf) |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 225 | { |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 226 | if (rx_buf->page) { |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 227 | put_page(rx_buf->page); |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 228 | rx_buf->page = NULL; |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 229 | } |
| 230 | } |
| 231 | |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 232 | /* Attempt to recycle the page if there is an RX recycle ring; the page can |
| 233 | * only be added if this is the final RX buffer, to prevent pages being used in |
| 234 | * the descriptor ring and appearing in the recycle ring simultaneously. |
| 235 | */ |
| 236 | static void efx_recycle_rx_page(struct efx_channel *channel, |
| 237 | struct efx_rx_buffer *rx_buf) |
| 238 | { |
| 239 | struct page *page = rx_buf->page; |
| 240 | struct efx_rx_queue *rx_queue = efx_channel_get_rx_queue(channel); |
| 241 | struct efx_nic *efx = rx_queue->efx; |
| 242 | unsigned index; |
| 243 | |
| 244 | /* Only recycle the page after processing the final buffer. */ |
Ben Hutchings | 179ea7f | 2013-03-07 16:31:17 +0000 | [diff] [blame^] | 245 | if (!(rx_buf->flags & EFX_RX_BUF_LAST_IN_PAGE)) |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 246 | return; |
| 247 | |
| 248 | index = rx_queue->page_add & rx_queue->page_ptr_mask; |
| 249 | if (rx_queue->page_ring[index] == NULL) { |
| 250 | unsigned read_index = rx_queue->page_remove & |
| 251 | rx_queue->page_ptr_mask; |
| 252 | |
| 253 | /* The next slot in the recycle ring is available, but |
| 254 | * increment page_remove if the read pointer currently |
| 255 | * points here. |
| 256 | */ |
| 257 | if (read_index == index) |
| 258 | ++rx_queue->page_remove; |
| 259 | rx_queue->page_ring[index] = page; |
| 260 | ++rx_queue->page_add; |
| 261 | return; |
| 262 | } |
| 263 | ++rx_queue->page_recycle_full; |
| 264 | efx_unmap_rx_buffer(efx, rx_buf); |
| 265 | put_page(rx_buf->page); |
| 266 | } |
| 267 | |
Ben Hutchings | 4d56606 | 2008-09-01 12:47:12 +0100 | [diff] [blame] | 268 | static void efx_fini_rx_buffer(struct efx_rx_queue *rx_queue, |
| 269 | struct efx_rx_buffer *rx_buf) |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 270 | { |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 271 | /* Release the page reference we hold for the buffer. */ |
| 272 | if (rx_buf->page) |
| 273 | put_page(rx_buf->page); |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 274 | |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 275 | /* If this is the last buffer in a page, unmap and free it. */ |
Ben Hutchings | 179ea7f | 2013-03-07 16:31:17 +0000 | [diff] [blame^] | 276 | if (rx_buf->flags & EFX_RX_BUF_LAST_IN_PAGE) { |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 277 | efx_unmap_rx_buffer(rx_queue->efx, rx_buf); |
| 278 | efx_free_rx_buffer(rx_buf); |
Steve Hodgson | 62b330b | 2010-06-01 11:20:53 +0000 | [diff] [blame] | 279 | } |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 280 | rx_buf->page = NULL; |
Steve Hodgson | 2445580 | 2010-06-01 11:20:34 +0000 | [diff] [blame] | 281 | } |
| 282 | |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 283 | /* Recycle the pages that are used by buffers that have just been received. */ |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 284 | static void efx_recycle_rx_buffers(struct efx_channel *channel, |
| 285 | struct efx_rx_buffer *rx_buf, |
| 286 | unsigned int n_frags) |
Steve Hodgson | 2445580 | 2010-06-01 11:20:34 +0000 | [diff] [blame] | 287 | { |
Ben Hutchings | f7d12cd | 2010-09-10 06:41:47 +0000 | [diff] [blame] | 288 | struct efx_rx_queue *rx_queue = efx_channel_get_rx_queue(channel); |
Steve Hodgson | 2445580 | 2010-06-01 11:20:34 +0000 | [diff] [blame] | 289 | |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 290 | do { |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 291 | efx_recycle_rx_page(channel, rx_buf); |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 292 | rx_buf = efx_rx_buf_next(rx_queue, rx_buf); |
| 293 | } while (--n_frags); |
Steve Hodgson | 2445580 | 2010-06-01 11:20:34 +0000 | [diff] [blame] | 294 | } |
| 295 | |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 296 | /** |
| 297 | * efx_fast_push_rx_descriptors - push new RX descriptors quickly |
| 298 | * @rx_queue: RX descriptor queue |
Ben Hutchings | 49ce9c2 | 2012-07-10 10:56:00 +0000 | [diff] [blame] | 299 | * |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 300 | * This will aim to fill the RX descriptor queue up to |
David Riddoch | da9ca50 | 2012-04-11 13:09:24 +0100 | [diff] [blame] | 301 | * @rx_queue->@max_fill. If there is insufficient atomic |
Steve Hodgson | 90d683a | 2010-06-01 11:19:39 +0000 | [diff] [blame] | 302 | * memory to do so, a slow fill will be scheduled. |
| 303 | * |
| 304 | * The caller must provide serialisation (none is used here). In practise, |
| 305 | * this means this function must run from the NAPI handler, or be called |
| 306 | * when NAPI is disabled. |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 307 | */ |
Steve Hodgson | 90d683a | 2010-06-01 11:19:39 +0000 | [diff] [blame] | 308 | void efx_fast_push_rx_descriptors(struct efx_rx_queue *rx_queue) |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 309 | { |
Steve Hodgson | f7d6f37 | 2010-06-01 11:33:17 +0000 | [diff] [blame] | 310 | unsigned fill_level; |
| 311 | int space, rc = 0; |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 312 | |
Steve Hodgson | 90d683a | 2010-06-01 11:19:39 +0000 | [diff] [blame] | 313 | /* Calculate current fill level, and exit if we don't need to fill */ |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 314 | fill_level = (rx_queue->added_count - rx_queue->removed_count); |
Steve Hodgson | ecc910f | 2010-09-10 06:42:22 +0000 | [diff] [blame] | 315 | EFX_BUG_ON_PARANOID(fill_level > rx_queue->efx->rxq_entries); |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 316 | if (fill_level >= rx_queue->fast_fill_trigger) |
Steve Hodgson | 2445580 | 2010-06-01 11:20:34 +0000 | [diff] [blame] | 317 | goto out; |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 318 | |
| 319 | /* Record minimum fill level */ |
Ben Hutchings | b347564 | 2008-05-16 21:15:49 +0100 | [diff] [blame] | 320 | if (unlikely(fill_level < rx_queue->min_fill)) { |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 321 | if (fill_level) |
| 322 | rx_queue->min_fill = fill_level; |
Ben Hutchings | b347564 | 2008-05-16 21:15:49 +0100 | [diff] [blame] | 323 | } |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 324 | |
David Riddoch | da9ca50 | 2012-04-11 13:09:24 +0100 | [diff] [blame] | 325 | space = rx_queue->max_fill - fill_level; |
David Riddoch | 6423518 | 2012-04-11 13:12:41 +0100 | [diff] [blame] | 326 | EFX_BUG_ON_PARANOID(space < EFX_RX_BATCH); |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 327 | |
Ben Hutchings | 62776d0 | 2010-06-23 11:30:07 +0000 | [diff] [blame] | 328 | netif_vdbg(rx_queue->efx, rx_status, rx_queue->efx->net_dev, |
| 329 | "RX queue %d fast-filling descriptor ring from" |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 330 | " level %d to level %d\n", |
Ben Hutchings | ba1e8a3 | 2010-09-10 06:41:36 +0000 | [diff] [blame] | 331 | efx_rx_queue_index(rx_queue), fill_level, |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 332 | rx_queue->max_fill); |
| 333 | |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 334 | |
| 335 | do { |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 336 | rc = efx_init_rx_buffers(rx_queue); |
Steve Hodgson | f7d6f37 | 2010-06-01 11:33:17 +0000 | [diff] [blame] | 337 | if (unlikely(rc)) { |
| 338 | /* Ensure that we don't leave the rx queue empty */ |
| 339 | if (rx_queue->added_count == rx_queue->removed_count) |
| 340 | efx_schedule_slow_fill(rx_queue); |
| 341 | goto out; |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 342 | } |
| 343 | } while ((space -= EFX_RX_BATCH) >= EFX_RX_BATCH); |
| 344 | |
Ben Hutchings | 62776d0 | 2010-06-23 11:30:07 +0000 | [diff] [blame] | 345 | netif_vdbg(rx_queue->efx, rx_status, rx_queue->efx->net_dev, |
| 346 | "RX queue %d fast-filled descriptor ring " |
Ben Hutchings | ba1e8a3 | 2010-09-10 06:41:36 +0000 | [diff] [blame] | 347 | "to level %d\n", efx_rx_queue_index(rx_queue), |
Ben Hutchings | 62776d0 | 2010-06-23 11:30:07 +0000 | [diff] [blame] | 348 | rx_queue->added_count - rx_queue->removed_count); |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 349 | |
| 350 | out: |
Steve Hodgson | 2445580 | 2010-06-01 11:20:34 +0000 | [diff] [blame] | 351 | if (rx_queue->notified_count != rx_queue->added_count) |
| 352 | efx_nic_notify_rx_desc(rx_queue); |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 353 | } |
| 354 | |
Steve Hodgson | 90d683a | 2010-06-01 11:19:39 +0000 | [diff] [blame] | 355 | void efx_rx_slow_fill(unsigned long context) |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 356 | { |
Steve Hodgson | 90d683a | 2010-06-01 11:19:39 +0000 | [diff] [blame] | 357 | struct efx_rx_queue *rx_queue = (struct efx_rx_queue *)context; |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 358 | |
Steve Hodgson | 90d683a | 2010-06-01 11:19:39 +0000 | [diff] [blame] | 359 | /* Post an event to cause NAPI to run and refill the queue */ |
Ben Hutchings | 2ae75da | 2012-02-07 23:49:52 +0000 | [diff] [blame] | 360 | efx_nic_generate_fill_event(rx_queue); |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 361 | ++rx_queue->slow_fill_count; |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 362 | } |
| 363 | |
Ben Hutchings | 4d56606 | 2008-09-01 12:47:12 +0100 | [diff] [blame] | 364 | static void efx_rx_packet__check_len(struct efx_rx_queue *rx_queue, |
| 365 | struct efx_rx_buffer *rx_buf, |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 366 | int len) |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 367 | { |
| 368 | struct efx_nic *efx = rx_queue->efx; |
| 369 | unsigned max_len = rx_buf->len - efx->type->rx_buffer_padding; |
| 370 | |
| 371 | if (likely(len <= max_len)) |
| 372 | return; |
| 373 | |
| 374 | /* The packet must be discarded, but this is only a fatal error |
| 375 | * if the caller indicated it was |
| 376 | */ |
Ben Hutchings | db33956 | 2011-08-26 18:05:11 +0100 | [diff] [blame] | 377 | rx_buf->flags |= EFX_RX_PKT_DISCARD; |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 378 | |
| 379 | if ((len > rx_buf->len) && EFX_WORKAROUND_8071(efx)) { |
Ben Hutchings | 62776d0 | 2010-06-23 11:30:07 +0000 | [diff] [blame] | 380 | if (net_ratelimit()) |
| 381 | netif_err(efx, rx_err, efx->net_dev, |
| 382 | " RX queue %d seriously overlength " |
| 383 | "RX event (0x%x > 0x%x+0x%x). Leaking\n", |
Ben Hutchings | ba1e8a3 | 2010-09-10 06:41:36 +0000 | [diff] [blame] | 384 | efx_rx_queue_index(rx_queue), len, max_len, |
Ben Hutchings | 62776d0 | 2010-06-23 11:30:07 +0000 | [diff] [blame] | 385 | efx->type->rx_buffer_padding); |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 386 | efx_schedule_reset(efx, RESET_TYPE_RX_RECOVERY); |
| 387 | } else { |
Ben Hutchings | 62776d0 | 2010-06-23 11:30:07 +0000 | [diff] [blame] | 388 | if (net_ratelimit()) |
| 389 | netif_err(efx, rx_err, efx->net_dev, |
| 390 | " RX queue %d overlength RX event " |
| 391 | "(0x%x > 0x%x)\n", |
Ben Hutchings | ba1e8a3 | 2010-09-10 06:41:36 +0000 | [diff] [blame] | 392 | efx_rx_queue_index(rx_queue), len, max_len); |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 393 | } |
| 394 | |
Ben Hutchings | ba1e8a3 | 2010-09-10 06:41:36 +0000 | [diff] [blame] | 395 | efx_rx_queue_channel(rx_queue)->n_rx_overlength++; |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 396 | } |
| 397 | |
Ben Hutchings | 61321d9 | 2012-02-25 01:58:35 +0000 | [diff] [blame] | 398 | /* Pass a received packet up through GRO. GRO can handle pages |
| 399 | * regardless of checksum state and skbs with a good checksum. |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 400 | */ |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 401 | static void |
| 402 | efx_rx_packet_gro(struct efx_channel *channel, struct efx_rx_buffer *rx_buf, |
| 403 | unsigned int n_frags, u8 *eh) |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 404 | { |
Herbert Xu | da3bc07 | 2009-01-18 21:50:16 -0800 | [diff] [blame] | 405 | struct napi_struct *napi = &channel->napi_str; |
Ben Hutchings | 18e1d2b | 2009-10-29 07:21:24 +0000 | [diff] [blame] | 406 | gro_result_t gro_result; |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 407 | struct efx_nic *efx = channel->efx; |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 408 | struct sk_buff *skb; |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 409 | |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 410 | skb = napi_get_frags(napi); |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 411 | if (unlikely(!skb)) { |
| 412 | while (n_frags--) { |
| 413 | put_page(rx_buf->page); |
| 414 | rx_buf->page = NULL; |
| 415 | rx_buf = efx_rx_buf_next(&channel->rx_queue, rx_buf); |
| 416 | } |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 417 | return; |
| 418 | } |
Ben Hutchings | 1241e95 | 2009-11-23 16:02:25 +0000 | [diff] [blame] | 419 | |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 420 | if (efx->net_dev->features & NETIF_F_RXHASH) |
| 421 | skb->rxhash = efx_rx_buf_hash(eh); |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 422 | skb->ip_summed = ((rx_buf->flags & EFX_RX_PKT_CSUMMED) ? |
| 423 | CHECKSUM_UNNECESSARY : CHECKSUM_NONE); |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 424 | |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 425 | for (;;) { |
| 426 | skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags, |
| 427 | rx_buf->page, rx_buf->page_offset, |
| 428 | rx_buf->len); |
| 429 | rx_buf->page = NULL; |
| 430 | skb->len += rx_buf->len; |
| 431 | if (skb_shinfo(skb)->nr_frags == n_frags) |
| 432 | break; |
| 433 | |
| 434 | rx_buf = efx_rx_buf_next(&channel->rx_queue, rx_buf); |
| 435 | } |
| 436 | |
| 437 | skb->data_len = skb->len; |
| 438 | skb->truesize += n_frags * efx->rx_buffer_truesize; |
| 439 | |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 440 | skb_record_rx_queue(skb, channel->rx_queue.core_index); |
Ben Hutchings | 3eadb7b | 2009-11-23 16:02:40 +0000 | [diff] [blame] | 441 | |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 442 | gro_result = napi_gro_frags(napi); |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 443 | if (gro_result != GRO_DROP) |
Ben Hutchings | 18e1d2b | 2009-10-29 07:21:24 +0000 | [diff] [blame] | 444 | channel->irq_mod_score += 2; |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 445 | } |
| 446 | |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 447 | /* Allocate and construct an SKB around page fragments */ |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 448 | static struct sk_buff *efx_rx_mk_skb(struct efx_channel *channel, |
| 449 | struct efx_rx_buffer *rx_buf, |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 450 | unsigned int n_frags, |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 451 | u8 *eh, int hdr_len) |
| 452 | { |
| 453 | struct efx_nic *efx = channel->efx; |
| 454 | struct sk_buff *skb; |
| 455 | |
| 456 | /* Allocate an SKB to store the headers */ |
| 457 | skb = netdev_alloc_skb(efx->net_dev, hdr_len + EFX_PAGE_SKB_ALIGN); |
| 458 | if (unlikely(skb == NULL)) |
| 459 | return NULL; |
| 460 | |
| 461 | EFX_BUG_ON_PARANOID(rx_buf->len < hdr_len); |
| 462 | |
| 463 | skb_reserve(skb, EFX_PAGE_SKB_ALIGN); |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 464 | memcpy(__skb_put(skb, hdr_len), eh, hdr_len); |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 465 | |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 466 | /* Append the remaining page(s) onto the frag list */ |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 467 | if (rx_buf->len > hdr_len) { |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 468 | rx_buf->page_offset += hdr_len; |
| 469 | rx_buf->len -= hdr_len; |
| 470 | |
| 471 | for (;;) { |
| 472 | skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags, |
| 473 | rx_buf->page, rx_buf->page_offset, |
| 474 | rx_buf->len); |
| 475 | rx_buf->page = NULL; |
| 476 | skb->len += rx_buf->len; |
| 477 | skb->data_len += rx_buf->len; |
| 478 | if (skb_shinfo(skb)->nr_frags == n_frags) |
| 479 | break; |
| 480 | |
| 481 | rx_buf = efx_rx_buf_next(&channel->rx_queue, rx_buf); |
| 482 | } |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 483 | } else { |
| 484 | __free_pages(rx_buf->page, efx->rx_buffer_order); |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 485 | rx_buf->page = NULL; |
| 486 | n_frags = 0; |
Ben Hutchings | 18e1d2b | 2009-10-29 07:21:24 +0000 | [diff] [blame] | 487 | } |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 488 | |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 489 | skb->truesize += n_frags * efx->rx_buffer_truesize; |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 490 | |
| 491 | /* Move past the ethernet header */ |
| 492 | skb->protocol = eth_type_trans(skb, efx->net_dev); |
| 493 | |
| 494 | return skb; |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 495 | } |
| 496 | |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 497 | void efx_rx_packet(struct efx_rx_queue *rx_queue, unsigned int index, |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 498 | unsigned int n_frags, unsigned int len, u16 flags) |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 499 | { |
| 500 | struct efx_nic *efx = rx_queue->efx; |
Ben Hutchings | ba1e8a3 | 2010-09-10 06:41:36 +0000 | [diff] [blame] | 501 | struct efx_channel *channel = efx_rx_queue_channel(rx_queue); |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 502 | struct efx_rx_buffer *rx_buf; |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 503 | |
| 504 | rx_buf = efx_rx_buffer(rx_queue, index); |
Ben Hutchings | 179ea7f | 2013-03-07 16:31:17 +0000 | [diff] [blame^] | 505 | rx_buf->flags |= flags; |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 506 | |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 507 | /* Validate the number of fragments and completed length */ |
| 508 | if (n_frags == 1) { |
| 509 | efx_rx_packet__check_len(rx_queue, rx_buf, len); |
| 510 | } else if (unlikely(n_frags > EFX_RX_MAX_FRAGS) || |
| 511 | unlikely(len <= (n_frags - 1) * EFX_RX_USR_BUF_SIZE) || |
| 512 | unlikely(len > n_frags * EFX_RX_USR_BUF_SIZE) || |
| 513 | unlikely(!efx->rx_scatter)) { |
| 514 | /* If this isn't an explicit discard request, either |
| 515 | * the hardware or the driver is broken. |
| 516 | */ |
| 517 | WARN_ON(!(len == 0 && rx_buf->flags & EFX_RX_PKT_DISCARD)); |
| 518 | rx_buf->flags |= EFX_RX_PKT_DISCARD; |
| 519 | } |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 520 | |
Ben Hutchings | 62776d0 | 2010-06-23 11:30:07 +0000 | [diff] [blame] | 521 | netif_vdbg(efx, rx_status, efx->net_dev, |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 522 | "RX queue %d received ids %x-%x len %d %s%s\n", |
Ben Hutchings | ba1e8a3 | 2010-09-10 06:41:36 +0000 | [diff] [blame] | 523 | efx_rx_queue_index(rx_queue), index, |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 524 | (index + n_frags - 1) & rx_queue->ptr_mask, len, |
Ben Hutchings | db33956 | 2011-08-26 18:05:11 +0100 | [diff] [blame] | 525 | (rx_buf->flags & EFX_RX_PKT_CSUMMED) ? " [SUMMED]" : "", |
| 526 | (rx_buf->flags & EFX_RX_PKT_DISCARD) ? " [DISCARD]" : ""); |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 527 | |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 528 | /* Discard packet, if instructed to do so. Process the |
| 529 | * previous receive first. |
| 530 | */ |
Ben Hutchings | db33956 | 2011-08-26 18:05:11 +0100 | [diff] [blame] | 531 | if (unlikely(rx_buf->flags & EFX_RX_PKT_DISCARD)) { |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 532 | efx_rx_flush_packet(channel); |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 533 | put_page(rx_buf->page); |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 534 | efx_recycle_rx_buffers(channel, rx_buf, n_frags); |
| 535 | return; |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 536 | } |
| 537 | |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 538 | if (n_frags == 1) |
| 539 | rx_buf->len = len; |
| 540 | |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 541 | /* Release and/or sync the DMA mapping - assumes all RX buffers |
| 542 | * consumed in-order per RX queue. |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 543 | */ |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 544 | efx_sync_rx_buffer(efx, rx_buf, rx_buf->len); |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 545 | |
| 546 | /* Prefetch nice and early so data will (hopefully) be in cache by |
| 547 | * the time we look at it. |
| 548 | */ |
Ben Hutchings | 5036b7c | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 549 | prefetch(efx_rx_buf_va(rx_buf)); |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 550 | |
Ben Hutchings | b74e3e8 | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 551 | rx_buf->page_offset += efx->type->rx_buffer_hash_size; |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 552 | rx_buf->len -= efx->type->rx_buffer_hash_size; |
| 553 | |
| 554 | if (n_frags > 1) { |
| 555 | /* Release/sync DMA mapping for additional fragments. |
| 556 | * Fix length for last fragment. |
| 557 | */ |
| 558 | unsigned int tail_frags = n_frags - 1; |
| 559 | |
| 560 | for (;;) { |
| 561 | rx_buf = efx_rx_buf_next(rx_queue, rx_buf); |
| 562 | if (--tail_frags == 0) |
| 563 | break; |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 564 | efx_sync_rx_buffer(efx, rx_buf, EFX_RX_USR_BUF_SIZE); |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 565 | } |
| 566 | rx_buf->len = len - (n_frags - 1) * EFX_RX_USR_BUF_SIZE; |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 567 | efx_sync_rx_buffer(efx, rx_buf, rx_buf->len); |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 568 | } |
Ben Hutchings | b74e3e8 | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 569 | |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 570 | /* All fragments have been DMA-synced, so recycle buffers and pages. */ |
| 571 | rx_buf = efx_rx_buffer(rx_queue, index); |
| 572 | efx_recycle_rx_buffers(channel, rx_buf, n_frags); |
| 573 | |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 574 | /* Pipeline receives so that we give time for packet headers to be |
| 575 | * prefetched into cache. |
| 576 | */ |
Ben Hutchings | ff734ef | 2013-01-29 23:33:14 +0000 | [diff] [blame] | 577 | efx_rx_flush_packet(channel); |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 578 | channel->rx_pkt_n_frags = n_frags; |
| 579 | channel->rx_pkt_index = index; |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 580 | } |
| 581 | |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 582 | static void efx_rx_deliver(struct efx_channel *channel, u8 *eh, |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 583 | struct efx_rx_buffer *rx_buf, |
| 584 | unsigned int n_frags) |
Ben Hutchings | 1ddceb4 | 2012-01-23 22:41:30 +0000 | [diff] [blame] | 585 | { |
| 586 | struct sk_buff *skb; |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 587 | u16 hdr_len = min_t(u16, rx_buf->len, EFX_SKB_HEADERS); |
Ben Hutchings | 1ddceb4 | 2012-01-23 22:41:30 +0000 | [diff] [blame] | 588 | |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 589 | skb = efx_rx_mk_skb(channel, rx_buf, n_frags, eh, hdr_len); |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 590 | if (unlikely(skb == NULL)) { |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 591 | efx_free_rx_buffer(rx_buf); |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 592 | return; |
| 593 | } |
| 594 | skb_record_rx_queue(skb, channel->rx_queue.core_index); |
Ben Hutchings | 1ddceb4 | 2012-01-23 22:41:30 +0000 | [diff] [blame] | 595 | |
| 596 | /* Set the SKB flags */ |
| 597 | skb_checksum_none_assert(skb); |
| 598 | |
Stuart Hodgson | c31e5f9 | 2012-07-18 09:52:11 +0100 | [diff] [blame] | 599 | if (channel->type->receive_skb) |
Ben Hutchings | 4a74dc6 | 2013-03-05 20:13:54 +0000 | [diff] [blame] | 600 | if (channel->type->receive_skb(channel, skb)) |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 601 | return; |
Ben Hutchings | 1ddceb4 | 2012-01-23 22:41:30 +0000 | [diff] [blame] | 602 | |
Ben Hutchings | 4a74dc6 | 2013-03-05 20:13:54 +0000 | [diff] [blame] | 603 | /* Pass the packet up */ |
| 604 | netif_receive_skb(skb); |
Ben Hutchings | 1ddceb4 | 2012-01-23 22:41:30 +0000 | [diff] [blame] | 605 | } |
| 606 | |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 607 | /* Handle a received packet. Second half: Touches packet payload. */ |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 608 | void __efx_rx_packet(struct efx_channel *channel) |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 609 | { |
| 610 | struct efx_nic *efx = channel->efx; |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 611 | struct efx_rx_buffer *rx_buf = |
| 612 | efx_rx_buffer(&channel->rx_queue, channel->rx_pkt_index); |
Ben Hutchings | b74e3e8 | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 613 | u8 *eh = efx_rx_buf_va(rx_buf); |
Ben Hutchings | 604f604 | 2010-06-25 07:05:33 +0000 | [diff] [blame] | 614 | |
Ben Hutchings | 3273c2e | 2008-05-07 13:36:19 +0100 | [diff] [blame] | 615 | /* If we're in loopback test, then pass the packet directly to the |
| 616 | * loopback layer, and free the rx_buf here |
| 617 | */ |
| 618 | if (unlikely(efx->loopback_selftest)) { |
Steve Hodgson | a526f14 | 2011-02-24 23:45:16 +0000 | [diff] [blame] | 619 | efx_loopback_rx_packet(efx, eh, rx_buf->len); |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 620 | efx_free_rx_buffer(rx_buf); |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 621 | goto out; |
Ben Hutchings | 3273c2e | 2008-05-07 13:36:19 +0100 | [diff] [blame] | 622 | } |
| 623 | |
Ben Hutchings | abfe903 | 2011-04-05 15:00:02 +0100 | [diff] [blame] | 624 | if (unlikely(!(efx->net_dev->features & NETIF_F_RXCSUM))) |
Ben Hutchings | db33956 | 2011-08-26 18:05:11 +0100 | [diff] [blame] | 625 | rx_buf->flags &= ~EFX_RX_PKT_CSUMMED; |
Ben Hutchings | ab3cf6d | 2011-04-01 22:20:06 +0100 | [diff] [blame] | 626 | |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 627 | if (!channel->type->receive_skb) |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 628 | efx_rx_packet_gro(channel, rx_buf, channel->rx_pkt_n_frags, eh); |
Ben Hutchings | 1ddceb4 | 2012-01-23 22:41:30 +0000 | [diff] [blame] | 629 | else |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 630 | efx_rx_deliver(channel, eh, rx_buf, channel->rx_pkt_n_frags); |
| 631 | out: |
| 632 | channel->rx_pkt_n_frags = 0; |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 633 | } |
| 634 | |
| 635 | int efx_probe_rx_queue(struct efx_rx_queue *rx_queue) |
| 636 | { |
| 637 | struct efx_nic *efx = rx_queue->efx; |
Steve Hodgson | ecc910f | 2010-09-10 06:42:22 +0000 | [diff] [blame] | 638 | unsigned int entries; |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 639 | int rc; |
| 640 | |
Steve Hodgson | ecc910f | 2010-09-10 06:42:22 +0000 | [diff] [blame] | 641 | /* Create the smallest power-of-two aligned ring */ |
| 642 | entries = max(roundup_pow_of_two(efx->rxq_entries), EFX_MIN_DMAQ_SIZE); |
| 643 | EFX_BUG_ON_PARANOID(entries > EFX_MAX_DMAQ_SIZE); |
| 644 | rx_queue->ptr_mask = entries - 1; |
| 645 | |
Ben Hutchings | 62776d0 | 2010-06-23 11:30:07 +0000 | [diff] [blame] | 646 | netif_dbg(efx, probe, efx->net_dev, |
Steve Hodgson | ecc910f | 2010-09-10 06:42:22 +0000 | [diff] [blame] | 647 | "creating RX queue %d size %#x mask %#x\n", |
| 648 | efx_rx_queue_index(rx_queue), efx->rxq_entries, |
| 649 | rx_queue->ptr_mask); |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 650 | |
| 651 | /* Allocate RX buffers */ |
Thomas Meyer | c2e4e25 | 2011-12-02 12:36:13 +0000 | [diff] [blame] | 652 | rx_queue->buffer = kcalloc(entries, sizeof(*rx_queue->buffer), |
Steve Hodgson | ecc910f | 2010-09-10 06:42:22 +0000 | [diff] [blame] | 653 | GFP_KERNEL); |
Ben Hutchings | 8831da7 | 2008-09-01 12:47:48 +0100 | [diff] [blame] | 654 | if (!rx_queue->buffer) |
| 655 | return -ENOMEM; |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 656 | |
Ben Hutchings | 152b6a6 | 2009-11-29 03:43:56 +0000 | [diff] [blame] | 657 | rc = efx_nic_probe_rx(rx_queue); |
Ben Hutchings | 8831da7 | 2008-09-01 12:47:48 +0100 | [diff] [blame] | 658 | if (rc) { |
| 659 | kfree(rx_queue->buffer); |
| 660 | rx_queue->buffer = NULL; |
| 661 | } |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 662 | |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 663 | return rc; |
| 664 | } |
| 665 | |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 666 | void efx_init_rx_recycle_ring(struct efx_nic *efx, |
| 667 | struct efx_rx_queue *rx_queue) |
| 668 | { |
| 669 | unsigned int bufs_in_recycle_ring, page_ring_size; |
| 670 | |
| 671 | /* Set the RX recycle ring size */ |
| 672 | #ifdef CONFIG_PPC64 |
| 673 | bufs_in_recycle_ring = EFX_RECYCLE_RING_SIZE_IOMMU; |
| 674 | #else |
| 675 | if (efx->pci_dev->dev.iommu_group) |
| 676 | bufs_in_recycle_ring = EFX_RECYCLE_RING_SIZE_IOMMU; |
| 677 | else |
| 678 | bufs_in_recycle_ring = EFX_RECYCLE_RING_SIZE_NOIOMMU; |
| 679 | #endif /* CONFIG_PPC64 */ |
| 680 | |
| 681 | page_ring_size = roundup_pow_of_two(bufs_in_recycle_ring / |
| 682 | efx->rx_bufs_per_page); |
| 683 | rx_queue->page_ring = kcalloc(page_ring_size, |
| 684 | sizeof(*rx_queue->page_ring), GFP_KERNEL); |
| 685 | rx_queue->page_ptr_mask = page_ring_size - 1; |
| 686 | } |
| 687 | |
Ben Hutchings | bc3c90a | 2008-09-01 12:48:46 +0100 | [diff] [blame] | 688 | void efx_init_rx_queue(struct efx_rx_queue *rx_queue) |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 689 | { |
Steve Hodgson | ecc910f | 2010-09-10 06:42:22 +0000 | [diff] [blame] | 690 | struct efx_nic *efx = rx_queue->efx; |
David Riddoch | 6423518 | 2012-04-11 13:12:41 +0100 | [diff] [blame] | 691 | unsigned int max_fill, trigger, max_trigger; |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 692 | |
Ben Hutchings | 62776d0 | 2010-06-23 11:30:07 +0000 | [diff] [blame] | 693 | netif_dbg(rx_queue->efx, drv, rx_queue->efx->net_dev, |
Ben Hutchings | ba1e8a3 | 2010-09-10 06:41:36 +0000 | [diff] [blame] | 694 | "initialising RX queue %d\n", efx_rx_queue_index(rx_queue)); |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 695 | |
| 696 | /* Initialise ptr fields */ |
| 697 | rx_queue->added_count = 0; |
| 698 | rx_queue->notified_count = 0; |
| 699 | rx_queue->removed_count = 0; |
| 700 | rx_queue->min_fill = -1U; |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 701 | efx_init_rx_recycle_ring(efx, rx_queue); |
| 702 | |
| 703 | rx_queue->page_remove = 0; |
| 704 | rx_queue->page_add = rx_queue->page_ptr_mask + 1; |
| 705 | rx_queue->page_recycle_count = 0; |
| 706 | rx_queue->page_recycle_failed = 0; |
| 707 | rx_queue->page_recycle_full = 0; |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 708 | |
| 709 | /* Initialise limit fields */ |
Steve Hodgson | ecc910f | 2010-09-10 06:42:22 +0000 | [diff] [blame] | 710 | max_fill = efx->rxq_entries - EFX_RXD_HEAD_ROOM; |
David Riddoch | 6423518 | 2012-04-11 13:12:41 +0100 | [diff] [blame] | 711 | max_trigger = max_fill - EFX_RX_BATCH; |
| 712 | if (rx_refill_threshold != 0) { |
| 713 | trigger = max_fill * min(rx_refill_threshold, 100U) / 100U; |
| 714 | if (trigger > max_trigger) |
| 715 | trigger = max_trigger; |
| 716 | } else { |
| 717 | trigger = max_trigger; |
| 718 | } |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 719 | |
| 720 | rx_queue->max_fill = max_fill; |
| 721 | rx_queue->fast_fill_trigger = trigger; |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 722 | |
| 723 | /* Set up RX descriptor ring */ |
Ben Hutchings | 9f2cb71 | 2012-02-08 00:11:20 +0000 | [diff] [blame] | 724 | rx_queue->enabled = true; |
Ben Hutchings | 152b6a6 | 2009-11-29 03:43:56 +0000 | [diff] [blame] | 725 | efx_nic_init_rx(rx_queue); |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 726 | } |
| 727 | |
| 728 | void efx_fini_rx_queue(struct efx_rx_queue *rx_queue) |
| 729 | { |
| 730 | int i; |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 731 | struct efx_nic *efx = rx_queue->efx; |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 732 | struct efx_rx_buffer *rx_buf; |
| 733 | |
Ben Hutchings | 62776d0 | 2010-06-23 11:30:07 +0000 | [diff] [blame] | 734 | netif_dbg(rx_queue->efx, drv, rx_queue->efx->net_dev, |
Ben Hutchings | ba1e8a3 | 2010-09-10 06:41:36 +0000 | [diff] [blame] | 735 | "shutting down RX queue %d\n", efx_rx_queue_index(rx_queue)); |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 736 | |
Ben Hutchings | 9f2cb71 | 2012-02-08 00:11:20 +0000 | [diff] [blame] | 737 | /* A flush failure might have left rx_queue->enabled */ |
| 738 | rx_queue->enabled = false; |
| 739 | |
Steve Hodgson | 90d683a | 2010-06-01 11:19:39 +0000 | [diff] [blame] | 740 | del_timer_sync(&rx_queue->slow_fill); |
Ben Hutchings | 152b6a6 | 2009-11-29 03:43:56 +0000 | [diff] [blame] | 741 | efx_nic_fini_rx(rx_queue); |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 742 | |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 743 | /* Release RX buffers from the current read ptr to the write ptr */ |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 744 | if (rx_queue->buffer) { |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 745 | for (i = rx_queue->removed_count; i < rx_queue->added_count; |
| 746 | i++) { |
| 747 | unsigned index = i & rx_queue->ptr_mask; |
| 748 | rx_buf = efx_rx_buffer(rx_queue, index); |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 749 | efx_fini_rx_buffer(rx_queue, rx_buf); |
| 750 | } |
| 751 | } |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 752 | |
| 753 | /* Unmap and release the pages in the recycle ring. Remove the ring. */ |
| 754 | for (i = 0; i <= rx_queue->page_ptr_mask; i++) { |
| 755 | struct page *page = rx_queue->page_ring[i]; |
| 756 | struct efx_rx_page_state *state; |
| 757 | |
| 758 | if (page == NULL) |
| 759 | continue; |
| 760 | |
| 761 | state = page_address(page); |
| 762 | dma_unmap_page(&efx->pci_dev->dev, state->dma_addr, |
| 763 | PAGE_SIZE << efx->rx_buffer_order, |
| 764 | DMA_FROM_DEVICE); |
| 765 | put_page(page); |
| 766 | } |
| 767 | kfree(rx_queue->page_ring); |
| 768 | rx_queue->page_ring = NULL; |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 769 | } |
| 770 | |
| 771 | void efx_remove_rx_queue(struct efx_rx_queue *rx_queue) |
| 772 | { |
Ben Hutchings | 62776d0 | 2010-06-23 11:30:07 +0000 | [diff] [blame] | 773 | netif_dbg(rx_queue->efx, drv, rx_queue->efx->net_dev, |
Ben Hutchings | ba1e8a3 | 2010-09-10 06:41:36 +0000 | [diff] [blame] | 774 | "destroying RX queue %d\n", efx_rx_queue_index(rx_queue)); |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 775 | |
Ben Hutchings | 152b6a6 | 2009-11-29 03:43:56 +0000 | [diff] [blame] | 776 | efx_nic_remove_rx(rx_queue); |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 777 | |
| 778 | kfree(rx_queue->buffer); |
| 779 | rx_queue->buffer = NULL; |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 780 | } |
| 781 | |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 782 | |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 783 | module_param(rx_refill_threshold, uint, 0444); |
| 784 | MODULE_PARM_DESC(rx_refill_threshold, |
David Riddoch | 6423518 | 2012-04-11 13:12:41 +0100 | [diff] [blame] | 785 | "RX descriptor ring refill threshold (%)"); |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 786 | |