Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 1 | /**************************************************************************** |
| 2 | * Driver for Solarflare Solarstorm network controllers and boards |
| 3 | * Copyright 2005-2006 Fen Systems Ltd. |
Ben Hutchings | 0a6f40c | 2011-02-25 00:01:34 +0000 | [diff] [blame] | 4 | * Copyright 2005-2011 Solarflare Communications Inc. |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 5 | * |
| 6 | * This program is free software; you can redistribute it and/or modify it |
| 7 | * under the terms of the GNU General Public License version 2 as published |
| 8 | * by the Free Software Foundation, incorporated herein by reference. |
| 9 | */ |
| 10 | |
| 11 | #include <linux/socket.h> |
| 12 | #include <linux/in.h> |
Tejun Heo | 5a0e3ad | 2010-03-24 17:04:11 +0900 | [diff] [blame] | 13 | #include <linux/slab.h> |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 14 | #include <linux/ip.h> |
| 15 | #include <linux/tcp.h> |
| 16 | #include <linux/udp.h> |
Paul Gortmaker | 70c7160 | 2011-05-22 16:47:17 -0400 | [diff] [blame] | 17 | #include <linux/prefetch.h> |
Paul Gortmaker | 6eb07ca | 2011-09-15 19:46:05 -0400 | [diff] [blame] | 18 | #include <linux/moduleparam.h> |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 19 | #include <linux/iommu.h> |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 20 | #include <net/ip.h> |
| 21 | #include <net/checksum.h> |
| 22 | #include "net_driver.h" |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 23 | #include "efx.h" |
Ben Hutchings | add7247 | 2012-11-08 01:46:53 +0000 | [diff] [blame] | 24 | #include "filter.h" |
Ben Hutchings | 744093c | 2009-11-29 15:12:08 +0000 | [diff] [blame] | 25 | #include "nic.h" |
Ben Hutchings | 3273c2e | 2008-05-07 13:36:19 +0100 | [diff] [blame] | 26 | #include "selftest.h" |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 27 | #include "workarounds.h" |
| 28 | |
Daniel Pieczko | 1648a23 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 29 | /* Preferred number of descriptors to fill at once */ |
| 30 | #define EFX_RX_PREFERRED_BATCH 8U |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 31 | |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 32 | /* Number of RX buffers to recycle pages for. When creating the RX page recycle |
| 33 | * ring, this number is divided by the number of buffers per page to calculate |
| 34 | * the number of pages to store in the RX page recycle ring. |
| 35 | */ |
| 36 | #define EFX_RECYCLE_RING_SIZE_IOMMU 4096 |
Daniel Pieczko | 1648a23 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 37 | #define EFX_RECYCLE_RING_SIZE_NOIOMMU (2 * EFX_RX_PREFERRED_BATCH) |
Steve Hodgson | 62b330b | 2010-06-01 11:20:53 +0000 | [diff] [blame] | 38 | |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 39 | /* Size of buffer allocated for skb header area. */ |
Jon Cooper | d4ef5b6 | 2013-04-08 12:55:58 +0100 | [diff] [blame] | 40 | #define EFX_SKB_HEADERS 128u |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 41 | |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 42 | /* This is the percentage fill level below which new RX descriptors |
| 43 | * will be added to the RX descriptor ring. |
| 44 | */ |
David Riddoch | 6423518 | 2012-04-11 13:12:41 +0100 | [diff] [blame] | 45 | static unsigned int rx_refill_threshold; |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 46 | |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 47 | /* Each packet can consume up to ceil(max_frame_len / buffer_size) buffers */ |
| 48 | #define EFX_RX_MAX_FRAGS DIV_ROUND_UP(EFX_MAX_FRAME_LEN(EFX_MAX_MTU), \ |
| 49 | EFX_RX_USR_BUF_SIZE) |
| 50 | |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 51 | /* |
| 52 | * RX maximum head room required. |
| 53 | * |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 54 | * This must be at least 1 to prevent overflow, plus one packet-worth |
| 55 | * to allow pipelined receives. |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 56 | */ |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 57 | #define EFX_RXD_HEAD_ROOM (1 + EFX_RX_MAX_FRAGS) |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 58 | |
Ben Hutchings | b184f16 | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 59 | static inline u8 *efx_rx_buf_va(struct efx_rx_buffer *buf) |
Ben Hutchings | 39c9cf0 | 2010-06-23 11:31:28 +0000 | [diff] [blame] | 60 | { |
Ben Hutchings | b184f16 | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 61 | return page_address(buf->page) + buf->page_offset; |
Steve Hodgson | a526f14 | 2011-02-24 23:45:16 +0000 | [diff] [blame] | 62 | } |
| 63 | |
Jon Cooper | 43a3739 | 2012-10-18 15:49:54 +0100 | [diff] [blame] | 64 | static inline u32 efx_rx_buf_hash(struct efx_nic *efx, const u8 *eh) |
Steve Hodgson | a526f14 | 2011-02-24 23:45:16 +0000 | [diff] [blame] | 65 | { |
Jon Cooper | 43a3739 | 2012-10-18 15:49:54 +0100 | [diff] [blame] | 66 | #if defined(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) |
| 67 | return __le32_to_cpup((const __le32 *)(eh + efx->rx_packet_hash_offset)); |
Ben Hutchings | 39c9cf0 | 2010-06-23 11:31:28 +0000 | [diff] [blame] | 68 | #else |
Jon Cooper | 43a3739 | 2012-10-18 15:49:54 +0100 | [diff] [blame] | 69 | const u8 *data = eh + efx->rx_packet_hash_offset; |
Ben Hutchings | 0beaca2 | 2012-01-05 18:54:04 +0000 | [diff] [blame] | 70 | return (u32)data[0] | |
| 71 | (u32)data[1] << 8 | |
| 72 | (u32)data[2] << 16 | |
| 73 | (u32)data[3] << 24; |
Ben Hutchings | 39c9cf0 | 2010-06-23 11:31:28 +0000 | [diff] [blame] | 74 | #endif |
| 75 | } |
| 76 | |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 77 | static inline struct efx_rx_buffer * |
| 78 | efx_rx_buf_next(struct efx_rx_queue *rx_queue, struct efx_rx_buffer *rx_buf) |
| 79 | { |
| 80 | if (unlikely(rx_buf == efx_rx_buffer(rx_queue, rx_queue->ptr_mask))) |
| 81 | return efx_rx_buffer(rx_queue, 0); |
| 82 | else |
| 83 | return rx_buf + 1; |
| 84 | } |
| 85 | |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 86 | static inline void efx_sync_rx_buffer(struct efx_nic *efx, |
| 87 | struct efx_rx_buffer *rx_buf, |
| 88 | unsigned int len) |
| 89 | { |
| 90 | dma_sync_single_for_cpu(&efx->pci_dev->dev, rx_buf->dma_addr, len, |
| 91 | DMA_FROM_DEVICE); |
| 92 | } |
| 93 | |
Daniel Pieczko | 1648a23 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 94 | void efx_rx_config_page_split(struct efx_nic *efx) |
| 95 | { |
Ben Hutchings | c14ff2e | 2013-05-13 11:58:31 +0000 | [diff] [blame] | 96 | efx->rx_page_buf_step = ALIGN(efx->rx_dma_len + NET_IP_ALIGN, |
Ben Hutchings | 950c54d | 2013-05-13 12:01:22 +0000 | [diff] [blame] | 97 | EFX_RX_BUF_ALIGNMENT); |
Daniel Pieczko | 1648a23 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 98 | efx->rx_bufs_per_page = efx->rx_buffer_order ? 1 : |
| 99 | ((PAGE_SIZE - sizeof(struct efx_rx_page_state)) / |
| 100 | efx->rx_page_buf_step); |
| 101 | efx->rx_buffer_truesize = (PAGE_SIZE << efx->rx_buffer_order) / |
| 102 | efx->rx_bufs_per_page; |
| 103 | efx->rx_pages_per_batch = DIV_ROUND_UP(EFX_RX_PREFERRED_BATCH, |
| 104 | efx->rx_bufs_per_page); |
| 105 | } |
| 106 | |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 107 | /* Check the RX page recycle ring for a page that can be reused. */ |
| 108 | static struct page *efx_reuse_page(struct efx_rx_queue *rx_queue) |
| 109 | { |
| 110 | struct efx_nic *efx = rx_queue->efx; |
| 111 | struct page *page; |
| 112 | struct efx_rx_page_state *state; |
| 113 | unsigned index; |
| 114 | |
| 115 | index = rx_queue->page_remove & rx_queue->page_ptr_mask; |
| 116 | page = rx_queue->page_ring[index]; |
| 117 | if (page == NULL) |
| 118 | return NULL; |
| 119 | |
| 120 | rx_queue->page_ring[index] = NULL; |
| 121 | /* page_remove cannot exceed page_add. */ |
| 122 | if (rx_queue->page_remove != rx_queue->page_add) |
| 123 | ++rx_queue->page_remove; |
| 124 | |
| 125 | /* If page_count is 1 then we hold the only reference to this page. */ |
| 126 | if (page_count(page) == 1) { |
| 127 | ++rx_queue->page_recycle_count; |
| 128 | return page; |
| 129 | } else { |
| 130 | state = page_address(page); |
| 131 | dma_unmap_page(&efx->pci_dev->dev, state->dma_addr, |
| 132 | PAGE_SIZE << efx->rx_buffer_order, |
| 133 | DMA_FROM_DEVICE); |
| 134 | put_page(page); |
| 135 | ++rx_queue->page_recycle_failed; |
| 136 | } |
| 137 | |
| 138 | return NULL; |
| 139 | } |
| 140 | |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 141 | /** |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 142 | * efx_init_rx_buffers - create EFX_RX_BATCH page-based RX buffers |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 143 | * |
| 144 | * @rx_queue: Efx RX queue |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 145 | * |
Daniel Pieczko | 1648a23 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 146 | * This allocates a batch of pages, maps them for DMA, and populates |
| 147 | * struct efx_rx_buffers for each one. Return a negative error code or |
| 148 | * 0 on success. If a single page can be used for multiple buffers, |
| 149 | * then the page will either be inserted fully, or not at all. |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 150 | */ |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 151 | static int efx_init_rx_buffers(struct efx_rx_queue *rx_queue) |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 152 | { |
| 153 | struct efx_nic *efx = rx_queue->efx; |
Steve Hodgson | f7d6f37 | 2010-06-01 11:33:17 +0000 | [diff] [blame] | 154 | struct efx_rx_buffer *rx_buf; |
| 155 | struct page *page; |
Ben Hutchings | b590ace | 2013-01-10 23:51:54 +0000 | [diff] [blame] | 156 | unsigned int page_offset; |
Steve Hodgson | 62b330b | 2010-06-01 11:20:53 +0000 | [diff] [blame] | 157 | struct efx_rx_page_state *state; |
Steve Hodgson | f7d6f37 | 2010-06-01 11:33:17 +0000 | [diff] [blame] | 158 | dma_addr_t dma_addr; |
| 159 | unsigned index, count; |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 160 | |
Daniel Pieczko | 1648a23 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 161 | count = 0; |
| 162 | do { |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 163 | page = efx_reuse_page(rx_queue); |
| 164 | if (page == NULL) { |
| 165 | page = alloc_pages(__GFP_COLD | __GFP_COMP | GFP_ATOMIC, |
| 166 | efx->rx_buffer_order); |
| 167 | if (unlikely(page == NULL)) |
| 168 | return -ENOMEM; |
| 169 | dma_addr = |
| 170 | dma_map_page(&efx->pci_dev->dev, page, 0, |
| 171 | PAGE_SIZE << efx->rx_buffer_order, |
| 172 | DMA_FROM_DEVICE); |
| 173 | if (unlikely(dma_mapping_error(&efx->pci_dev->dev, |
| 174 | dma_addr))) { |
| 175 | __free_pages(page, efx->rx_buffer_order); |
| 176 | return -EIO; |
| 177 | } |
| 178 | state = page_address(page); |
| 179 | state->dma_addr = dma_addr; |
| 180 | } else { |
| 181 | state = page_address(page); |
| 182 | dma_addr = state->dma_addr; |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 183 | } |
Steve Hodgson | 62b330b | 2010-06-01 11:20:53 +0000 | [diff] [blame] | 184 | |
Steve Hodgson | 62b330b | 2010-06-01 11:20:53 +0000 | [diff] [blame] | 185 | dma_addr += sizeof(struct efx_rx_page_state); |
Ben Hutchings | b590ace | 2013-01-10 23:51:54 +0000 | [diff] [blame] | 186 | page_offset = sizeof(struct efx_rx_page_state); |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 187 | |
Daniel Pieczko | 1648a23 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 188 | do { |
| 189 | index = rx_queue->added_count & rx_queue->ptr_mask; |
| 190 | rx_buf = efx_rx_buffer(rx_queue, index); |
Ben Hutchings | c14ff2e | 2013-05-13 11:58:31 +0000 | [diff] [blame] | 191 | rx_buf->dma_addr = dma_addr + NET_IP_ALIGN; |
Daniel Pieczko | 1648a23 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 192 | rx_buf->page = page; |
Ben Hutchings | c14ff2e | 2013-05-13 11:58:31 +0000 | [diff] [blame] | 193 | rx_buf->page_offset = page_offset + NET_IP_ALIGN; |
Daniel Pieczko | 1648a23 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 194 | rx_buf->len = efx->rx_dma_len; |
Ben Hutchings | 179ea7f | 2013-03-07 16:31:17 +0000 | [diff] [blame] | 195 | rx_buf->flags = 0; |
Daniel Pieczko | 1648a23 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 196 | ++rx_queue->added_count; |
| 197 | get_page(page); |
| 198 | dma_addr += efx->rx_page_buf_step; |
| 199 | page_offset += efx->rx_page_buf_step; |
| 200 | } while (page_offset + efx->rx_page_buf_step <= PAGE_SIZE); |
Ben Hutchings | 179ea7f | 2013-03-07 16:31:17 +0000 | [diff] [blame] | 201 | |
| 202 | rx_buf->flags = EFX_RX_BUF_LAST_IN_PAGE; |
Daniel Pieczko | 1648a23 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 203 | } while (++count < efx->rx_pages_per_batch); |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 204 | |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 205 | return 0; |
| 206 | } |
| 207 | |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 208 | /* Unmap a DMA-mapped page. This function is only called for the final RX |
| 209 | * buffer in a page. |
| 210 | */ |
Ben Hutchings | 4d56606 | 2008-09-01 12:47:12 +0100 | [diff] [blame] | 211 | static void efx_unmap_rx_buffer(struct efx_nic *efx, |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 212 | struct efx_rx_buffer *rx_buf) |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 213 | { |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 214 | struct page *page = rx_buf->page; |
Steve Hodgson | 62b330b | 2010-06-01 11:20:53 +0000 | [diff] [blame] | 215 | |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 216 | if (page) { |
| 217 | struct efx_rx_page_state *state = page_address(page); |
| 218 | dma_unmap_page(&efx->pci_dev->dev, |
| 219 | state->dma_addr, |
| 220 | PAGE_SIZE << efx->rx_buffer_order, |
| 221 | DMA_FROM_DEVICE); |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 222 | } |
| 223 | } |
| 224 | |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 225 | static void efx_free_rx_buffer(struct efx_rx_buffer *rx_buf) |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 226 | { |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 227 | if (rx_buf->page) { |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 228 | put_page(rx_buf->page); |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 229 | rx_buf->page = NULL; |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 230 | } |
| 231 | } |
| 232 | |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 233 | /* Attempt to recycle the page if there is an RX recycle ring; the page can |
| 234 | * only be added if this is the final RX buffer, to prevent pages being used in |
| 235 | * the descriptor ring and appearing in the recycle ring simultaneously. |
| 236 | */ |
| 237 | static void efx_recycle_rx_page(struct efx_channel *channel, |
| 238 | struct efx_rx_buffer *rx_buf) |
| 239 | { |
| 240 | struct page *page = rx_buf->page; |
| 241 | struct efx_rx_queue *rx_queue = efx_channel_get_rx_queue(channel); |
| 242 | struct efx_nic *efx = rx_queue->efx; |
| 243 | unsigned index; |
| 244 | |
| 245 | /* Only recycle the page after processing the final buffer. */ |
Ben Hutchings | 179ea7f | 2013-03-07 16:31:17 +0000 | [diff] [blame] | 246 | if (!(rx_buf->flags & EFX_RX_BUF_LAST_IN_PAGE)) |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 247 | return; |
| 248 | |
| 249 | index = rx_queue->page_add & rx_queue->page_ptr_mask; |
| 250 | if (rx_queue->page_ring[index] == NULL) { |
| 251 | unsigned read_index = rx_queue->page_remove & |
| 252 | rx_queue->page_ptr_mask; |
| 253 | |
| 254 | /* The next slot in the recycle ring is available, but |
| 255 | * increment page_remove if the read pointer currently |
| 256 | * points here. |
| 257 | */ |
| 258 | if (read_index == index) |
| 259 | ++rx_queue->page_remove; |
| 260 | rx_queue->page_ring[index] = page; |
| 261 | ++rx_queue->page_add; |
| 262 | return; |
| 263 | } |
| 264 | ++rx_queue->page_recycle_full; |
| 265 | efx_unmap_rx_buffer(efx, rx_buf); |
| 266 | put_page(rx_buf->page); |
| 267 | } |
| 268 | |
Ben Hutchings | 4d56606 | 2008-09-01 12:47:12 +0100 | [diff] [blame] | 269 | static void efx_fini_rx_buffer(struct efx_rx_queue *rx_queue, |
| 270 | struct efx_rx_buffer *rx_buf) |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 271 | { |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 272 | /* Release the page reference we hold for the buffer. */ |
| 273 | if (rx_buf->page) |
| 274 | put_page(rx_buf->page); |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 275 | |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 276 | /* If this is the last buffer in a page, unmap and free it. */ |
Ben Hutchings | 179ea7f | 2013-03-07 16:31:17 +0000 | [diff] [blame] | 277 | if (rx_buf->flags & EFX_RX_BUF_LAST_IN_PAGE) { |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 278 | efx_unmap_rx_buffer(rx_queue->efx, rx_buf); |
| 279 | efx_free_rx_buffer(rx_buf); |
Steve Hodgson | 62b330b | 2010-06-01 11:20:53 +0000 | [diff] [blame] | 280 | } |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 281 | rx_buf->page = NULL; |
Steve Hodgson | 2445580 | 2010-06-01 11:20:34 +0000 | [diff] [blame] | 282 | } |
| 283 | |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 284 | /* Recycle the pages that are used by buffers that have just been received. */ |
Ben Hutchings | 734d4e1 | 2013-07-04 23:48:46 +0100 | [diff] [blame] | 285 | static void efx_recycle_rx_pages(struct efx_channel *channel, |
| 286 | struct efx_rx_buffer *rx_buf, |
| 287 | unsigned int n_frags) |
Steve Hodgson | 2445580 | 2010-06-01 11:20:34 +0000 | [diff] [blame] | 288 | { |
Ben Hutchings | f7d12cd | 2010-09-10 06:41:47 +0000 | [diff] [blame] | 289 | struct efx_rx_queue *rx_queue = efx_channel_get_rx_queue(channel); |
Steve Hodgson | 2445580 | 2010-06-01 11:20:34 +0000 | [diff] [blame] | 290 | |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 291 | do { |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 292 | efx_recycle_rx_page(channel, rx_buf); |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 293 | rx_buf = efx_rx_buf_next(rx_queue, rx_buf); |
| 294 | } while (--n_frags); |
Steve Hodgson | 2445580 | 2010-06-01 11:20:34 +0000 | [diff] [blame] | 295 | } |
| 296 | |
Ben Hutchings | 734d4e1 | 2013-07-04 23:48:46 +0100 | [diff] [blame] | 297 | static void efx_discard_rx_packet(struct efx_channel *channel, |
| 298 | struct efx_rx_buffer *rx_buf, |
| 299 | unsigned int n_frags) |
| 300 | { |
| 301 | struct efx_rx_queue *rx_queue = efx_channel_get_rx_queue(channel); |
| 302 | |
| 303 | efx_recycle_rx_pages(channel, rx_buf, n_frags); |
| 304 | |
| 305 | do { |
| 306 | efx_free_rx_buffer(rx_buf); |
| 307 | rx_buf = efx_rx_buf_next(rx_queue, rx_buf); |
| 308 | } while (--n_frags); |
| 309 | } |
| 310 | |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 311 | /** |
| 312 | * efx_fast_push_rx_descriptors - push new RX descriptors quickly |
| 313 | * @rx_queue: RX descriptor queue |
Ben Hutchings | 49ce9c2 | 2012-07-10 10:56:00 +0000 | [diff] [blame] | 314 | * |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 315 | * This will aim to fill the RX descriptor queue up to |
David Riddoch | da9ca50 | 2012-04-11 13:09:24 +0100 | [diff] [blame] | 316 | * @rx_queue->@max_fill. If there is insufficient atomic |
Steve Hodgson | 90d683a | 2010-06-01 11:19:39 +0000 | [diff] [blame] | 317 | * memory to do so, a slow fill will be scheduled. |
| 318 | * |
| 319 | * The caller must provide serialisation (none is used here). In practise, |
| 320 | * this means this function must run from the NAPI handler, or be called |
| 321 | * when NAPI is disabled. |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 322 | */ |
Steve Hodgson | 90d683a | 2010-06-01 11:19:39 +0000 | [diff] [blame] | 323 | void efx_fast_push_rx_descriptors(struct efx_rx_queue *rx_queue) |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 324 | { |
Daniel Pieczko | 1648a23 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 325 | struct efx_nic *efx = rx_queue->efx; |
| 326 | unsigned int fill_level, batch_size; |
Steve Hodgson | f7d6f37 | 2010-06-01 11:33:17 +0000 | [diff] [blame] | 327 | int space, rc = 0; |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 328 | |
Ben Hutchings | d8aec74 | 2013-05-27 16:52:54 +0100 | [diff] [blame] | 329 | if (!rx_queue->refill_enabled) |
| 330 | return; |
| 331 | |
Steve Hodgson | 90d683a | 2010-06-01 11:19:39 +0000 | [diff] [blame] | 332 | /* Calculate current fill level, and exit if we don't need to fill */ |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 333 | fill_level = (rx_queue->added_count - rx_queue->removed_count); |
Steve Hodgson | ecc910f | 2010-09-10 06:42:22 +0000 | [diff] [blame] | 334 | EFX_BUG_ON_PARANOID(fill_level > rx_queue->efx->rxq_entries); |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 335 | if (fill_level >= rx_queue->fast_fill_trigger) |
Steve Hodgson | 2445580 | 2010-06-01 11:20:34 +0000 | [diff] [blame] | 336 | goto out; |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 337 | |
| 338 | /* Record minimum fill level */ |
Ben Hutchings | b347564 | 2008-05-16 21:15:49 +0100 | [diff] [blame] | 339 | if (unlikely(fill_level < rx_queue->min_fill)) { |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 340 | if (fill_level) |
| 341 | rx_queue->min_fill = fill_level; |
Ben Hutchings | b347564 | 2008-05-16 21:15:49 +0100 | [diff] [blame] | 342 | } |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 343 | |
Daniel Pieczko | 1648a23 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 344 | batch_size = efx->rx_pages_per_batch * efx->rx_bufs_per_page; |
David Riddoch | da9ca50 | 2012-04-11 13:09:24 +0100 | [diff] [blame] | 345 | space = rx_queue->max_fill - fill_level; |
Daniel Pieczko | 1648a23 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 346 | EFX_BUG_ON_PARANOID(space < batch_size); |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 347 | |
Ben Hutchings | 62776d0 | 2010-06-23 11:30:07 +0000 | [diff] [blame] | 348 | netif_vdbg(rx_queue->efx, rx_status, rx_queue->efx->net_dev, |
| 349 | "RX queue %d fast-filling descriptor ring from" |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 350 | " level %d to level %d\n", |
Ben Hutchings | ba1e8a3 | 2010-09-10 06:41:36 +0000 | [diff] [blame] | 351 | efx_rx_queue_index(rx_queue), fill_level, |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 352 | rx_queue->max_fill); |
| 353 | |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 354 | |
| 355 | do { |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 356 | rc = efx_init_rx_buffers(rx_queue); |
Steve Hodgson | f7d6f37 | 2010-06-01 11:33:17 +0000 | [diff] [blame] | 357 | if (unlikely(rc)) { |
| 358 | /* Ensure that we don't leave the rx queue empty */ |
| 359 | if (rx_queue->added_count == rx_queue->removed_count) |
| 360 | efx_schedule_slow_fill(rx_queue); |
| 361 | goto out; |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 362 | } |
Daniel Pieczko | 1648a23 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 363 | } while ((space -= batch_size) >= batch_size); |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 364 | |
Ben Hutchings | 62776d0 | 2010-06-23 11:30:07 +0000 | [diff] [blame] | 365 | netif_vdbg(rx_queue->efx, rx_status, rx_queue->efx->net_dev, |
| 366 | "RX queue %d fast-filled descriptor ring " |
Ben Hutchings | ba1e8a3 | 2010-09-10 06:41:36 +0000 | [diff] [blame] | 367 | "to level %d\n", efx_rx_queue_index(rx_queue), |
Ben Hutchings | 62776d0 | 2010-06-23 11:30:07 +0000 | [diff] [blame] | 368 | rx_queue->added_count - rx_queue->removed_count); |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 369 | |
| 370 | out: |
Steve Hodgson | 2445580 | 2010-06-01 11:20:34 +0000 | [diff] [blame] | 371 | if (rx_queue->notified_count != rx_queue->added_count) |
| 372 | efx_nic_notify_rx_desc(rx_queue); |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 373 | } |
| 374 | |
Steve Hodgson | 90d683a | 2010-06-01 11:19:39 +0000 | [diff] [blame] | 375 | void efx_rx_slow_fill(unsigned long context) |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 376 | { |
Steve Hodgson | 90d683a | 2010-06-01 11:19:39 +0000 | [diff] [blame] | 377 | struct efx_rx_queue *rx_queue = (struct efx_rx_queue *)context; |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 378 | |
Steve Hodgson | 90d683a | 2010-06-01 11:19:39 +0000 | [diff] [blame] | 379 | /* Post an event to cause NAPI to run and refill the queue */ |
Ben Hutchings | 2ae75da | 2012-02-07 23:49:52 +0000 | [diff] [blame] | 380 | efx_nic_generate_fill_event(rx_queue); |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 381 | ++rx_queue->slow_fill_count; |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 382 | } |
| 383 | |
Ben Hutchings | 4d56606 | 2008-09-01 12:47:12 +0100 | [diff] [blame] | 384 | static void efx_rx_packet__check_len(struct efx_rx_queue *rx_queue, |
| 385 | struct efx_rx_buffer *rx_buf, |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 386 | int len) |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 387 | { |
| 388 | struct efx_nic *efx = rx_queue->efx; |
| 389 | unsigned max_len = rx_buf->len - efx->type->rx_buffer_padding; |
| 390 | |
| 391 | if (likely(len <= max_len)) |
| 392 | return; |
| 393 | |
| 394 | /* The packet must be discarded, but this is only a fatal error |
| 395 | * if the caller indicated it was |
| 396 | */ |
Ben Hutchings | db33956 | 2011-08-26 18:05:11 +0100 | [diff] [blame] | 397 | rx_buf->flags |= EFX_RX_PKT_DISCARD; |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 398 | |
| 399 | if ((len > rx_buf->len) && EFX_WORKAROUND_8071(efx)) { |
Ben Hutchings | 62776d0 | 2010-06-23 11:30:07 +0000 | [diff] [blame] | 400 | if (net_ratelimit()) |
| 401 | netif_err(efx, rx_err, efx->net_dev, |
| 402 | " RX queue %d seriously overlength " |
| 403 | "RX event (0x%x > 0x%x+0x%x). Leaking\n", |
Ben Hutchings | ba1e8a3 | 2010-09-10 06:41:36 +0000 | [diff] [blame] | 404 | efx_rx_queue_index(rx_queue), len, max_len, |
Ben Hutchings | 62776d0 | 2010-06-23 11:30:07 +0000 | [diff] [blame] | 405 | efx->type->rx_buffer_padding); |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 406 | efx_schedule_reset(efx, RESET_TYPE_RX_RECOVERY); |
| 407 | } else { |
Ben Hutchings | 62776d0 | 2010-06-23 11:30:07 +0000 | [diff] [blame] | 408 | if (net_ratelimit()) |
| 409 | netif_err(efx, rx_err, efx->net_dev, |
| 410 | " RX queue %d overlength RX event " |
| 411 | "(0x%x > 0x%x)\n", |
Ben Hutchings | ba1e8a3 | 2010-09-10 06:41:36 +0000 | [diff] [blame] | 412 | efx_rx_queue_index(rx_queue), len, max_len); |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 413 | } |
| 414 | |
Ben Hutchings | ba1e8a3 | 2010-09-10 06:41:36 +0000 | [diff] [blame] | 415 | efx_rx_queue_channel(rx_queue)->n_rx_overlength++; |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 416 | } |
| 417 | |
Ben Hutchings | 61321d9 | 2012-02-25 01:58:35 +0000 | [diff] [blame] | 418 | /* Pass a received packet up through GRO. GRO can handle pages |
| 419 | * regardless of checksum state and skbs with a good checksum. |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 420 | */ |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 421 | static void |
| 422 | efx_rx_packet_gro(struct efx_channel *channel, struct efx_rx_buffer *rx_buf, |
| 423 | unsigned int n_frags, u8 *eh) |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 424 | { |
Herbert Xu | da3bc07 | 2009-01-18 21:50:16 -0800 | [diff] [blame] | 425 | struct napi_struct *napi = &channel->napi_str; |
Ben Hutchings | 18e1d2b | 2009-10-29 07:21:24 +0000 | [diff] [blame] | 426 | gro_result_t gro_result; |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 427 | struct efx_nic *efx = channel->efx; |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 428 | struct sk_buff *skb; |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 429 | |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 430 | skb = napi_get_frags(napi); |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 431 | if (unlikely(!skb)) { |
| 432 | while (n_frags--) { |
| 433 | put_page(rx_buf->page); |
| 434 | rx_buf->page = NULL; |
| 435 | rx_buf = efx_rx_buf_next(&channel->rx_queue, rx_buf); |
| 436 | } |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 437 | return; |
| 438 | } |
Ben Hutchings | 1241e95 | 2009-11-23 16:02:25 +0000 | [diff] [blame] | 439 | |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 440 | if (efx->net_dev->features & NETIF_F_RXHASH) |
Jon Cooper | 43a3739 | 2012-10-18 15:49:54 +0100 | [diff] [blame] | 441 | skb->rxhash = efx_rx_buf_hash(efx, eh); |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 442 | skb->ip_summed = ((rx_buf->flags & EFX_RX_PKT_CSUMMED) ? |
| 443 | CHECKSUM_UNNECESSARY : CHECKSUM_NONE); |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 444 | |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 445 | for (;;) { |
| 446 | skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags, |
| 447 | rx_buf->page, rx_buf->page_offset, |
| 448 | rx_buf->len); |
| 449 | rx_buf->page = NULL; |
| 450 | skb->len += rx_buf->len; |
| 451 | if (skb_shinfo(skb)->nr_frags == n_frags) |
| 452 | break; |
| 453 | |
| 454 | rx_buf = efx_rx_buf_next(&channel->rx_queue, rx_buf); |
| 455 | } |
| 456 | |
| 457 | skb->data_len = skb->len; |
| 458 | skb->truesize += n_frags * efx->rx_buffer_truesize; |
| 459 | |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 460 | skb_record_rx_queue(skb, channel->rx_queue.core_index); |
Ben Hutchings | 3eadb7b | 2009-11-23 16:02:40 +0000 | [diff] [blame] | 461 | |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 462 | gro_result = napi_gro_frags(napi); |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 463 | if (gro_result != GRO_DROP) |
Ben Hutchings | 18e1d2b | 2009-10-29 07:21:24 +0000 | [diff] [blame] | 464 | channel->irq_mod_score += 2; |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 465 | } |
| 466 | |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 467 | /* Allocate and construct an SKB around page fragments */ |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 468 | static struct sk_buff *efx_rx_mk_skb(struct efx_channel *channel, |
| 469 | struct efx_rx_buffer *rx_buf, |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 470 | unsigned int n_frags, |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 471 | u8 *eh, int hdr_len) |
| 472 | { |
| 473 | struct efx_nic *efx = channel->efx; |
| 474 | struct sk_buff *skb; |
| 475 | |
| 476 | /* Allocate an SKB to store the headers */ |
| 477 | skb = netdev_alloc_skb(efx->net_dev, hdr_len + EFX_PAGE_SKB_ALIGN); |
| 478 | if (unlikely(skb == NULL)) |
| 479 | return NULL; |
| 480 | |
| 481 | EFX_BUG_ON_PARANOID(rx_buf->len < hdr_len); |
| 482 | |
| 483 | skb_reserve(skb, EFX_PAGE_SKB_ALIGN); |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 484 | memcpy(__skb_put(skb, hdr_len), eh, hdr_len); |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 485 | |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 486 | /* Append the remaining page(s) onto the frag list */ |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 487 | if (rx_buf->len > hdr_len) { |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 488 | rx_buf->page_offset += hdr_len; |
| 489 | rx_buf->len -= hdr_len; |
| 490 | |
| 491 | for (;;) { |
| 492 | skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags, |
| 493 | rx_buf->page, rx_buf->page_offset, |
| 494 | rx_buf->len); |
| 495 | rx_buf->page = NULL; |
| 496 | skb->len += rx_buf->len; |
| 497 | skb->data_len += rx_buf->len; |
| 498 | if (skb_shinfo(skb)->nr_frags == n_frags) |
| 499 | break; |
| 500 | |
| 501 | rx_buf = efx_rx_buf_next(&channel->rx_queue, rx_buf); |
| 502 | } |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 503 | } else { |
| 504 | __free_pages(rx_buf->page, efx->rx_buffer_order); |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 505 | rx_buf->page = NULL; |
| 506 | n_frags = 0; |
Ben Hutchings | 18e1d2b | 2009-10-29 07:21:24 +0000 | [diff] [blame] | 507 | } |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 508 | |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 509 | skb->truesize += n_frags * efx->rx_buffer_truesize; |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 510 | |
| 511 | /* Move past the ethernet header */ |
| 512 | skb->protocol = eth_type_trans(skb, efx->net_dev); |
| 513 | |
| 514 | return skb; |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 515 | } |
| 516 | |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 517 | void efx_rx_packet(struct efx_rx_queue *rx_queue, unsigned int index, |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 518 | unsigned int n_frags, unsigned int len, u16 flags) |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 519 | { |
| 520 | struct efx_nic *efx = rx_queue->efx; |
Ben Hutchings | ba1e8a3 | 2010-09-10 06:41:36 +0000 | [diff] [blame] | 521 | struct efx_channel *channel = efx_rx_queue_channel(rx_queue); |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 522 | struct efx_rx_buffer *rx_buf; |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 523 | |
| 524 | rx_buf = efx_rx_buffer(rx_queue, index); |
Ben Hutchings | 179ea7f | 2013-03-07 16:31:17 +0000 | [diff] [blame] | 525 | rx_buf->flags |= flags; |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 526 | |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 527 | /* Validate the number of fragments and completed length */ |
| 528 | if (n_frags == 1) { |
Ben Hutchings | 3dced74 | 2013-04-27 01:55:18 +0100 | [diff] [blame] | 529 | if (!(flags & EFX_RX_PKT_PREFIX_LEN)) |
| 530 | efx_rx_packet__check_len(rx_queue, rx_buf, len); |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 531 | } else if (unlikely(n_frags > EFX_RX_MAX_FRAGS) || |
| 532 | unlikely(len <= (n_frags - 1) * EFX_RX_USR_BUF_SIZE) || |
| 533 | unlikely(len > n_frags * EFX_RX_USR_BUF_SIZE) || |
| 534 | unlikely(!efx->rx_scatter)) { |
| 535 | /* If this isn't an explicit discard request, either |
| 536 | * the hardware or the driver is broken. |
| 537 | */ |
| 538 | WARN_ON(!(len == 0 && rx_buf->flags & EFX_RX_PKT_DISCARD)); |
| 539 | rx_buf->flags |= EFX_RX_PKT_DISCARD; |
| 540 | } |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 541 | |
Ben Hutchings | 62776d0 | 2010-06-23 11:30:07 +0000 | [diff] [blame] | 542 | netif_vdbg(efx, rx_status, efx->net_dev, |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 543 | "RX queue %d received ids %x-%x len %d %s%s\n", |
Ben Hutchings | ba1e8a3 | 2010-09-10 06:41:36 +0000 | [diff] [blame] | 544 | efx_rx_queue_index(rx_queue), index, |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 545 | (index + n_frags - 1) & rx_queue->ptr_mask, len, |
Ben Hutchings | db33956 | 2011-08-26 18:05:11 +0100 | [diff] [blame] | 546 | (rx_buf->flags & EFX_RX_PKT_CSUMMED) ? " [SUMMED]" : "", |
| 547 | (rx_buf->flags & EFX_RX_PKT_DISCARD) ? " [DISCARD]" : ""); |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 548 | |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 549 | /* Discard packet, if instructed to do so. Process the |
| 550 | * previous receive first. |
| 551 | */ |
Ben Hutchings | db33956 | 2011-08-26 18:05:11 +0100 | [diff] [blame] | 552 | if (unlikely(rx_buf->flags & EFX_RX_PKT_DISCARD)) { |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 553 | efx_rx_flush_packet(channel); |
Ben Hutchings | 734d4e1 | 2013-07-04 23:48:46 +0100 | [diff] [blame] | 554 | efx_discard_rx_packet(channel, rx_buf, n_frags); |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 555 | return; |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 556 | } |
| 557 | |
Ben Hutchings | 3dced74 | 2013-04-27 01:55:18 +0100 | [diff] [blame] | 558 | if (n_frags == 1 && !(flags & EFX_RX_PKT_PREFIX_LEN)) |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 559 | rx_buf->len = len; |
| 560 | |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 561 | /* Release and/or sync the DMA mapping - assumes all RX buffers |
| 562 | * consumed in-order per RX queue. |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 563 | */ |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 564 | efx_sync_rx_buffer(efx, rx_buf, rx_buf->len); |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 565 | |
| 566 | /* Prefetch nice and early so data will (hopefully) be in cache by |
| 567 | * the time we look at it. |
| 568 | */ |
Ben Hutchings | 5036b7c | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 569 | prefetch(efx_rx_buf_va(rx_buf)); |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 570 | |
Jon Cooper | 43a3739 | 2012-10-18 15:49:54 +0100 | [diff] [blame] | 571 | rx_buf->page_offset += efx->rx_prefix_size; |
| 572 | rx_buf->len -= efx->rx_prefix_size; |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 573 | |
| 574 | if (n_frags > 1) { |
| 575 | /* Release/sync DMA mapping for additional fragments. |
| 576 | * Fix length for last fragment. |
| 577 | */ |
| 578 | unsigned int tail_frags = n_frags - 1; |
| 579 | |
| 580 | for (;;) { |
| 581 | rx_buf = efx_rx_buf_next(rx_queue, rx_buf); |
| 582 | if (--tail_frags == 0) |
| 583 | break; |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 584 | efx_sync_rx_buffer(efx, rx_buf, EFX_RX_USR_BUF_SIZE); |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 585 | } |
| 586 | rx_buf->len = len - (n_frags - 1) * EFX_RX_USR_BUF_SIZE; |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 587 | efx_sync_rx_buffer(efx, rx_buf, rx_buf->len); |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 588 | } |
Ben Hutchings | b74e3e8 | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 589 | |
Ben Hutchings | 734d4e1 | 2013-07-04 23:48:46 +0100 | [diff] [blame] | 590 | /* All fragments have been DMA-synced, so recycle pages. */ |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 591 | rx_buf = efx_rx_buffer(rx_queue, index); |
Ben Hutchings | 734d4e1 | 2013-07-04 23:48:46 +0100 | [diff] [blame] | 592 | efx_recycle_rx_pages(channel, rx_buf, n_frags); |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 593 | |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 594 | /* Pipeline receives so that we give time for packet headers to be |
| 595 | * prefetched into cache. |
| 596 | */ |
Ben Hutchings | ff734ef | 2013-01-29 23:33:14 +0000 | [diff] [blame] | 597 | efx_rx_flush_packet(channel); |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 598 | channel->rx_pkt_n_frags = n_frags; |
| 599 | channel->rx_pkt_index = index; |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 600 | } |
| 601 | |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 602 | static void efx_rx_deliver(struct efx_channel *channel, u8 *eh, |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 603 | struct efx_rx_buffer *rx_buf, |
| 604 | unsigned int n_frags) |
Ben Hutchings | 1ddceb4 | 2012-01-23 22:41:30 +0000 | [diff] [blame] | 605 | { |
| 606 | struct sk_buff *skb; |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 607 | u16 hdr_len = min_t(u16, rx_buf->len, EFX_SKB_HEADERS); |
Ben Hutchings | 1ddceb4 | 2012-01-23 22:41:30 +0000 | [diff] [blame] | 608 | |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 609 | skb = efx_rx_mk_skb(channel, rx_buf, n_frags, eh, hdr_len); |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 610 | if (unlikely(skb == NULL)) { |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 611 | efx_free_rx_buffer(rx_buf); |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 612 | return; |
| 613 | } |
| 614 | skb_record_rx_queue(skb, channel->rx_queue.core_index); |
Ben Hutchings | 1ddceb4 | 2012-01-23 22:41:30 +0000 | [diff] [blame] | 615 | |
| 616 | /* Set the SKB flags */ |
| 617 | skb_checksum_none_assert(skb); |
Jon Cooper | c99dffc | 2013-04-08 12:49:48 +0100 | [diff] [blame] | 618 | if (likely(rx_buf->flags & EFX_RX_PKT_CSUMMED)) |
| 619 | skb->ip_summed = CHECKSUM_UNNECESSARY; |
Ben Hutchings | 1ddceb4 | 2012-01-23 22:41:30 +0000 | [diff] [blame] | 620 | |
Stuart Hodgson | c31e5f9 | 2012-07-18 09:52:11 +0100 | [diff] [blame] | 621 | if (channel->type->receive_skb) |
Ben Hutchings | 4a74dc6 | 2013-03-05 20:13:54 +0000 | [diff] [blame] | 622 | if (channel->type->receive_skb(channel, skb)) |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 623 | return; |
Ben Hutchings | 1ddceb4 | 2012-01-23 22:41:30 +0000 | [diff] [blame] | 624 | |
Ben Hutchings | 4a74dc6 | 2013-03-05 20:13:54 +0000 | [diff] [blame] | 625 | /* Pass the packet up */ |
| 626 | netif_receive_skb(skb); |
Ben Hutchings | 1ddceb4 | 2012-01-23 22:41:30 +0000 | [diff] [blame] | 627 | } |
| 628 | |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 629 | /* Handle a received packet. Second half: Touches packet payload. */ |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 630 | void __efx_rx_packet(struct efx_channel *channel) |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 631 | { |
| 632 | struct efx_nic *efx = channel->efx; |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 633 | struct efx_rx_buffer *rx_buf = |
| 634 | efx_rx_buffer(&channel->rx_queue, channel->rx_pkt_index); |
Ben Hutchings | b74e3e8 | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 635 | u8 *eh = efx_rx_buf_va(rx_buf); |
Ben Hutchings | 604f604 | 2010-06-25 07:05:33 +0000 | [diff] [blame] | 636 | |
Ben Hutchings | 3dced74 | 2013-04-27 01:55:18 +0100 | [diff] [blame] | 637 | /* Read length from the prefix if necessary. This already |
| 638 | * excludes the length of the prefix itself. |
| 639 | */ |
| 640 | if (rx_buf->flags & EFX_RX_PKT_PREFIX_LEN) |
| 641 | rx_buf->len = le16_to_cpup((__le16 *) |
| 642 | (eh + efx->rx_packet_len_offset)); |
| 643 | |
Ben Hutchings | 3273c2e | 2008-05-07 13:36:19 +0100 | [diff] [blame] | 644 | /* If we're in loopback test, then pass the packet directly to the |
| 645 | * loopback layer, and free the rx_buf here |
| 646 | */ |
| 647 | if (unlikely(efx->loopback_selftest)) { |
Steve Hodgson | a526f14 | 2011-02-24 23:45:16 +0000 | [diff] [blame] | 648 | efx_loopback_rx_packet(efx, eh, rx_buf->len); |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 649 | efx_free_rx_buffer(rx_buf); |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 650 | goto out; |
Ben Hutchings | 3273c2e | 2008-05-07 13:36:19 +0100 | [diff] [blame] | 651 | } |
| 652 | |
Ben Hutchings | abfe903 | 2011-04-05 15:00:02 +0100 | [diff] [blame] | 653 | if (unlikely(!(efx->net_dev->features & NETIF_F_RXCSUM))) |
Ben Hutchings | db33956 | 2011-08-26 18:05:11 +0100 | [diff] [blame] | 654 | rx_buf->flags &= ~EFX_RX_PKT_CSUMMED; |
Ben Hutchings | ab3cf6d | 2011-04-01 22:20:06 +0100 | [diff] [blame] | 655 | |
Ben Hutchings | e79255d | 2013-05-16 18:38:13 +0100 | [diff] [blame] | 656 | if ((rx_buf->flags & EFX_RX_PKT_TCP) && !channel->type->receive_skb) |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 657 | efx_rx_packet_gro(channel, rx_buf, channel->rx_pkt_n_frags, eh); |
Ben Hutchings | 1ddceb4 | 2012-01-23 22:41:30 +0000 | [diff] [blame] | 658 | else |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 659 | efx_rx_deliver(channel, eh, rx_buf, channel->rx_pkt_n_frags); |
| 660 | out: |
| 661 | channel->rx_pkt_n_frags = 0; |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 662 | } |
| 663 | |
| 664 | int efx_probe_rx_queue(struct efx_rx_queue *rx_queue) |
| 665 | { |
| 666 | struct efx_nic *efx = rx_queue->efx; |
Steve Hodgson | ecc910f | 2010-09-10 06:42:22 +0000 | [diff] [blame] | 667 | unsigned int entries; |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 668 | int rc; |
| 669 | |
Steve Hodgson | ecc910f | 2010-09-10 06:42:22 +0000 | [diff] [blame] | 670 | /* Create the smallest power-of-two aligned ring */ |
| 671 | entries = max(roundup_pow_of_two(efx->rxq_entries), EFX_MIN_DMAQ_SIZE); |
| 672 | EFX_BUG_ON_PARANOID(entries > EFX_MAX_DMAQ_SIZE); |
| 673 | rx_queue->ptr_mask = entries - 1; |
| 674 | |
Ben Hutchings | 62776d0 | 2010-06-23 11:30:07 +0000 | [diff] [blame] | 675 | netif_dbg(efx, probe, efx->net_dev, |
Steve Hodgson | ecc910f | 2010-09-10 06:42:22 +0000 | [diff] [blame] | 676 | "creating RX queue %d size %#x mask %#x\n", |
| 677 | efx_rx_queue_index(rx_queue), efx->rxq_entries, |
| 678 | rx_queue->ptr_mask); |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 679 | |
| 680 | /* Allocate RX buffers */ |
Thomas Meyer | c2e4e25 | 2011-12-02 12:36:13 +0000 | [diff] [blame] | 681 | rx_queue->buffer = kcalloc(entries, sizeof(*rx_queue->buffer), |
Steve Hodgson | ecc910f | 2010-09-10 06:42:22 +0000 | [diff] [blame] | 682 | GFP_KERNEL); |
Ben Hutchings | 8831da7 | 2008-09-01 12:47:48 +0100 | [diff] [blame] | 683 | if (!rx_queue->buffer) |
| 684 | return -ENOMEM; |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 685 | |
Ben Hutchings | 152b6a6 | 2009-11-29 03:43:56 +0000 | [diff] [blame] | 686 | rc = efx_nic_probe_rx(rx_queue); |
Ben Hutchings | 8831da7 | 2008-09-01 12:47:48 +0100 | [diff] [blame] | 687 | if (rc) { |
| 688 | kfree(rx_queue->buffer); |
| 689 | rx_queue->buffer = NULL; |
| 690 | } |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 691 | |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 692 | return rc; |
| 693 | } |
| 694 | |
stephen hemminger | debd003 | 2013-03-16 06:57:51 +0000 | [diff] [blame] | 695 | static void efx_init_rx_recycle_ring(struct efx_nic *efx, |
| 696 | struct efx_rx_queue *rx_queue) |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 697 | { |
| 698 | unsigned int bufs_in_recycle_ring, page_ring_size; |
| 699 | |
| 700 | /* Set the RX recycle ring size */ |
| 701 | #ifdef CONFIG_PPC64 |
| 702 | bufs_in_recycle_ring = EFX_RECYCLE_RING_SIZE_IOMMU; |
| 703 | #else |
Ben Hutchings | 636d73d | 2013-06-12 18:09:08 +0100 | [diff] [blame] | 704 | if (iommu_present(&pci_bus_type)) |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 705 | bufs_in_recycle_ring = EFX_RECYCLE_RING_SIZE_IOMMU; |
| 706 | else |
| 707 | bufs_in_recycle_ring = EFX_RECYCLE_RING_SIZE_NOIOMMU; |
| 708 | #endif /* CONFIG_PPC64 */ |
| 709 | |
| 710 | page_ring_size = roundup_pow_of_two(bufs_in_recycle_ring / |
| 711 | efx->rx_bufs_per_page); |
| 712 | rx_queue->page_ring = kcalloc(page_ring_size, |
| 713 | sizeof(*rx_queue->page_ring), GFP_KERNEL); |
| 714 | rx_queue->page_ptr_mask = page_ring_size - 1; |
| 715 | } |
| 716 | |
Ben Hutchings | bc3c90a | 2008-09-01 12:48:46 +0100 | [diff] [blame] | 717 | void efx_init_rx_queue(struct efx_rx_queue *rx_queue) |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 718 | { |
Steve Hodgson | ecc910f | 2010-09-10 06:42:22 +0000 | [diff] [blame] | 719 | struct efx_nic *efx = rx_queue->efx; |
David Riddoch | 6423518 | 2012-04-11 13:12:41 +0100 | [diff] [blame] | 720 | unsigned int max_fill, trigger, max_trigger; |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 721 | |
Ben Hutchings | 62776d0 | 2010-06-23 11:30:07 +0000 | [diff] [blame] | 722 | netif_dbg(rx_queue->efx, drv, rx_queue->efx->net_dev, |
Ben Hutchings | ba1e8a3 | 2010-09-10 06:41:36 +0000 | [diff] [blame] | 723 | "initialising RX queue %d\n", efx_rx_queue_index(rx_queue)); |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 724 | |
| 725 | /* Initialise ptr fields */ |
| 726 | rx_queue->added_count = 0; |
| 727 | rx_queue->notified_count = 0; |
| 728 | rx_queue->removed_count = 0; |
| 729 | rx_queue->min_fill = -1U; |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 730 | efx_init_rx_recycle_ring(efx, rx_queue); |
| 731 | |
| 732 | rx_queue->page_remove = 0; |
| 733 | rx_queue->page_add = rx_queue->page_ptr_mask + 1; |
| 734 | rx_queue->page_recycle_count = 0; |
| 735 | rx_queue->page_recycle_failed = 0; |
| 736 | rx_queue->page_recycle_full = 0; |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 737 | |
| 738 | /* Initialise limit fields */ |
Steve Hodgson | ecc910f | 2010-09-10 06:42:22 +0000 | [diff] [blame] | 739 | max_fill = efx->rxq_entries - EFX_RXD_HEAD_ROOM; |
Daniel Pieczko | 1648a23 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 740 | max_trigger = |
| 741 | max_fill - efx->rx_pages_per_batch * efx->rx_bufs_per_page; |
David Riddoch | 6423518 | 2012-04-11 13:12:41 +0100 | [diff] [blame] | 742 | if (rx_refill_threshold != 0) { |
| 743 | trigger = max_fill * min(rx_refill_threshold, 100U) / 100U; |
| 744 | if (trigger > max_trigger) |
| 745 | trigger = max_trigger; |
| 746 | } else { |
| 747 | trigger = max_trigger; |
| 748 | } |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 749 | |
| 750 | rx_queue->max_fill = max_fill; |
| 751 | rx_queue->fast_fill_trigger = trigger; |
Ben Hutchings | d8aec74 | 2013-05-27 16:52:54 +0100 | [diff] [blame] | 752 | rx_queue->refill_enabled = true; |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 753 | |
| 754 | /* Set up RX descriptor ring */ |
Ben Hutchings | 152b6a6 | 2009-11-29 03:43:56 +0000 | [diff] [blame] | 755 | efx_nic_init_rx(rx_queue); |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 756 | } |
| 757 | |
| 758 | void efx_fini_rx_queue(struct efx_rx_queue *rx_queue) |
| 759 | { |
| 760 | int i; |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 761 | struct efx_nic *efx = rx_queue->efx; |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 762 | struct efx_rx_buffer *rx_buf; |
| 763 | |
Ben Hutchings | 62776d0 | 2010-06-23 11:30:07 +0000 | [diff] [blame] | 764 | netif_dbg(rx_queue->efx, drv, rx_queue->efx->net_dev, |
Ben Hutchings | ba1e8a3 | 2010-09-10 06:41:36 +0000 | [diff] [blame] | 765 | "shutting down RX queue %d\n", efx_rx_queue_index(rx_queue)); |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 766 | |
Steve Hodgson | 90d683a | 2010-06-01 11:19:39 +0000 | [diff] [blame] | 767 | del_timer_sync(&rx_queue->slow_fill); |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 768 | |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 769 | /* Release RX buffers from the current read ptr to the write ptr */ |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 770 | if (rx_queue->buffer) { |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 771 | for (i = rx_queue->removed_count; i < rx_queue->added_count; |
| 772 | i++) { |
| 773 | unsigned index = i & rx_queue->ptr_mask; |
| 774 | rx_buf = efx_rx_buffer(rx_queue, index); |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 775 | efx_fini_rx_buffer(rx_queue, rx_buf); |
| 776 | } |
| 777 | } |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 778 | |
| 779 | /* Unmap and release the pages in the recycle ring. Remove the ring. */ |
| 780 | for (i = 0; i <= rx_queue->page_ptr_mask; i++) { |
| 781 | struct page *page = rx_queue->page_ring[i]; |
| 782 | struct efx_rx_page_state *state; |
| 783 | |
| 784 | if (page == NULL) |
| 785 | continue; |
| 786 | |
| 787 | state = page_address(page); |
| 788 | dma_unmap_page(&efx->pci_dev->dev, state->dma_addr, |
| 789 | PAGE_SIZE << efx->rx_buffer_order, |
| 790 | DMA_FROM_DEVICE); |
| 791 | put_page(page); |
| 792 | } |
| 793 | kfree(rx_queue->page_ring); |
| 794 | rx_queue->page_ring = NULL; |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 795 | } |
| 796 | |
| 797 | void efx_remove_rx_queue(struct efx_rx_queue *rx_queue) |
| 798 | { |
Ben Hutchings | 62776d0 | 2010-06-23 11:30:07 +0000 | [diff] [blame] | 799 | netif_dbg(rx_queue->efx, drv, rx_queue->efx->net_dev, |
Ben Hutchings | ba1e8a3 | 2010-09-10 06:41:36 +0000 | [diff] [blame] | 800 | "destroying RX queue %d\n", efx_rx_queue_index(rx_queue)); |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 801 | |
Ben Hutchings | 152b6a6 | 2009-11-29 03:43:56 +0000 | [diff] [blame] | 802 | efx_nic_remove_rx(rx_queue); |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 803 | |
| 804 | kfree(rx_queue->buffer); |
| 805 | rx_queue->buffer = NULL; |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 806 | } |
| 807 | |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 808 | |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 809 | module_param(rx_refill_threshold, uint, 0444); |
| 810 | MODULE_PARM_DESC(rx_refill_threshold, |
David Riddoch | 6423518 | 2012-04-11 13:12:41 +0100 | [diff] [blame] | 811 | "RX descriptor ring refill threshold (%)"); |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 812 | |
Ben Hutchings | add7247 | 2012-11-08 01:46:53 +0000 | [diff] [blame] | 813 | #ifdef CONFIG_RFS_ACCEL |
| 814 | |
| 815 | int efx_filter_rfs(struct net_device *net_dev, const struct sk_buff *skb, |
| 816 | u16 rxq_index, u32 flow_id) |
| 817 | { |
| 818 | struct efx_nic *efx = netdev_priv(net_dev); |
| 819 | struct efx_channel *channel; |
| 820 | struct efx_filter_spec spec; |
| 821 | const struct iphdr *ip; |
| 822 | const __be16 *ports; |
| 823 | int nhoff; |
| 824 | int rc; |
| 825 | |
| 826 | nhoff = skb_network_offset(skb); |
| 827 | |
| 828 | if (skb->protocol == htons(ETH_P_8021Q)) { |
| 829 | EFX_BUG_ON_PARANOID(skb_headlen(skb) < |
| 830 | nhoff + sizeof(struct vlan_hdr)); |
| 831 | if (((const struct vlan_hdr *)skb->data + nhoff)-> |
| 832 | h_vlan_encapsulated_proto != htons(ETH_P_IP)) |
| 833 | return -EPROTONOSUPPORT; |
| 834 | |
| 835 | /* This is IP over 802.1q VLAN. We can't filter on the |
| 836 | * IP 5-tuple and the vlan together, so just strip the |
| 837 | * vlan header and filter on the IP part. |
| 838 | */ |
| 839 | nhoff += sizeof(struct vlan_hdr); |
| 840 | } else if (skb->protocol != htons(ETH_P_IP)) { |
| 841 | return -EPROTONOSUPPORT; |
| 842 | } |
| 843 | |
| 844 | /* RFS must validate the IP header length before calling us */ |
| 845 | EFX_BUG_ON_PARANOID(skb_headlen(skb) < nhoff + sizeof(*ip)); |
| 846 | ip = (const struct iphdr *)(skb->data + nhoff); |
| 847 | if (ip_is_fragment(ip)) |
| 848 | return -EPROTONOSUPPORT; |
| 849 | EFX_BUG_ON_PARANOID(skb_headlen(skb) < nhoff + 4 * ip->ihl + 4); |
| 850 | ports = (const __be16 *)(skb->data + nhoff + 4 * ip->ihl); |
| 851 | |
| 852 | efx_filter_init_rx(&spec, EFX_FILTER_PRI_HINT, |
| 853 | efx->rx_scatter ? EFX_FILTER_FLAG_RX_SCATTER : 0, |
| 854 | rxq_index); |
| 855 | rc = efx_filter_set_ipv4_full(&spec, ip->protocol, |
| 856 | ip->daddr, ports[1], ip->saddr, ports[0]); |
| 857 | if (rc) |
| 858 | return rc; |
| 859 | |
| 860 | rc = efx->type->filter_rfs_insert(efx, &spec); |
| 861 | if (rc < 0) |
| 862 | return rc; |
| 863 | |
| 864 | /* Remember this so we can check whether to expire the filter later */ |
| 865 | efx->rps_flow_id[rc] = flow_id; |
| 866 | channel = efx_get_channel(efx, skb_get_rx_queue(skb)); |
| 867 | ++channel->rfs_filters_added; |
| 868 | |
| 869 | netif_info(efx, rx_status, efx->net_dev, |
| 870 | "steering %s %pI4:%u:%pI4:%u to queue %u [flow %u filter %d]\n", |
| 871 | (ip->protocol == IPPROTO_TCP) ? "TCP" : "UDP", |
| 872 | &ip->saddr, ntohs(ports[0]), &ip->daddr, ntohs(ports[1]), |
| 873 | rxq_index, flow_id, rc); |
| 874 | |
| 875 | return rc; |
| 876 | } |
| 877 | |
| 878 | bool __efx_filter_rfs_expire(struct efx_nic *efx, unsigned int quota) |
| 879 | { |
| 880 | bool (*expire_one)(struct efx_nic *efx, u32 flow_id, unsigned int index); |
| 881 | unsigned int index, size; |
| 882 | u32 flow_id; |
| 883 | |
| 884 | if (!spin_trylock_bh(&efx->filter_lock)) |
| 885 | return false; |
| 886 | |
| 887 | expire_one = efx->type->filter_rfs_expire_one; |
| 888 | index = efx->rps_expire_index; |
| 889 | size = efx->type->max_rx_ip_filters; |
| 890 | while (quota--) { |
| 891 | flow_id = efx->rps_flow_id[index]; |
| 892 | if (expire_one(efx, flow_id, index)) |
| 893 | netif_info(efx, rx_status, efx->net_dev, |
| 894 | "expired filter %d [flow %u]\n", |
| 895 | index, flow_id); |
| 896 | if (++index == size) |
| 897 | index = 0; |
| 898 | } |
| 899 | efx->rps_expire_index = index; |
| 900 | |
| 901 | spin_unlock_bh(&efx->filter_lock); |
| 902 | return true; |
| 903 | } |
| 904 | |
| 905 | #endif /* CONFIG_RFS_ACCEL */ |