Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 1 | /**************************************************************************** |
Ben Hutchings | f7a6d2c | 2013-08-29 23:32:48 +0100 | [diff] [blame] | 2 | * Driver for Solarflare network controllers and boards |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 3 | * Copyright 2005-2006 Fen Systems Ltd. |
Ben Hutchings | f7a6d2c | 2013-08-29 23:32:48 +0100 | [diff] [blame] | 4 | * Copyright 2005-2013 Solarflare Communications Inc. |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 5 | * |
| 6 | * This program is free software; you can redistribute it and/or modify it |
| 7 | * under the terms of the GNU General Public License version 2 as published |
| 8 | * by the Free Software Foundation, incorporated herein by reference. |
| 9 | */ |
| 10 | |
| 11 | #include <linux/socket.h> |
| 12 | #include <linux/in.h> |
Tejun Heo | 5a0e3ad | 2010-03-24 17:04:11 +0900 | [diff] [blame] | 13 | #include <linux/slab.h> |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 14 | #include <linux/ip.h> |
Ben Hutchings | c47b2d9 | 2013-09-03 17:22:23 +0100 | [diff] [blame] | 15 | #include <linux/ipv6.h> |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 16 | #include <linux/tcp.h> |
| 17 | #include <linux/udp.h> |
Paul Gortmaker | 70c7160 | 2011-05-22 16:47:17 -0400 | [diff] [blame] | 18 | #include <linux/prefetch.h> |
Paul Gortmaker | 6eb07ca | 2011-09-15 19:46:05 -0400 | [diff] [blame] | 19 | #include <linux/moduleparam.h> |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 20 | #include <linux/iommu.h> |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 21 | #include <net/ip.h> |
| 22 | #include <net/checksum.h> |
| 23 | #include "net_driver.h" |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 24 | #include "efx.h" |
Ben Hutchings | add7247 | 2012-11-08 01:46:53 +0000 | [diff] [blame] | 25 | #include "filter.h" |
Ben Hutchings | 744093c | 2009-11-29 15:12:08 +0000 | [diff] [blame] | 26 | #include "nic.h" |
Ben Hutchings | 3273c2e | 2008-05-07 13:36:19 +0100 | [diff] [blame] | 27 | #include "selftest.h" |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 28 | #include "workarounds.h" |
| 29 | |
Daniel Pieczko | 1648a23 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 30 | /* Preferred number of descriptors to fill at once */ |
| 31 | #define EFX_RX_PREFERRED_BATCH 8U |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 32 | |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 33 | /* Number of RX buffers to recycle pages for. When creating the RX page recycle |
| 34 | * ring, this number is divided by the number of buffers per page to calculate |
| 35 | * the number of pages to store in the RX page recycle ring. |
| 36 | */ |
| 37 | #define EFX_RECYCLE_RING_SIZE_IOMMU 4096 |
Daniel Pieczko | 1648a23 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 38 | #define EFX_RECYCLE_RING_SIZE_NOIOMMU (2 * EFX_RX_PREFERRED_BATCH) |
Steve Hodgson | 62b330b | 2010-06-01 11:20:53 +0000 | [diff] [blame] | 39 | |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 40 | /* Size of buffer allocated for skb header area. */ |
Jon Cooper | d4ef5b6 | 2013-04-08 12:55:58 +0100 | [diff] [blame] | 41 | #define EFX_SKB_HEADERS 128u |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 42 | |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 43 | /* This is the percentage fill level below which new RX descriptors |
| 44 | * will be added to the RX descriptor ring. |
| 45 | */ |
David Riddoch | 6423518 | 2012-04-11 13:12:41 +0100 | [diff] [blame] | 46 | static unsigned int rx_refill_threshold; |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 47 | |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 48 | /* Each packet can consume up to ceil(max_frame_len / buffer_size) buffers */ |
| 49 | #define EFX_RX_MAX_FRAGS DIV_ROUND_UP(EFX_MAX_FRAME_LEN(EFX_MAX_MTU), \ |
| 50 | EFX_RX_USR_BUF_SIZE) |
| 51 | |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 52 | /* |
| 53 | * RX maximum head room required. |
| 54 | * |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 55 | * This must be at least 1 to prevent overflow, plus one packet-worth |
| 56 | * to allow pipelined receives. |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 57 | */ |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 58 | #define EFX_RXD_HEAD_ROOM (1 + EFX_RX_MAX_FRAGS) |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 59 | |
Ben Hutchings | b184f16 | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 60 | static inline u8 *efx_rx_buf_va(struct efx_rx_buffer *buf) |
Ben Hutchings | 39c9cf0 | 2010-06-23 11:31:28 +0000 | [diff] [blame] | 61 | { |
Ben Hutchings | b184f16 | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 62 | return page_address(buf->page) + buf->page_offset; |
Steve Hodgson | a526f14 | 2011-02-24 23:45:16 +0000 | [diff] [blame] | 63 | } |
| 64 | |
Jon Cooper | 43a3739 | 2012-10-18 15:49:54 +0100 | [diff] [blame] | 65 | static inline u32 efx_rx_buf_hash(struct efx_nic *efx, const u8 *eh) |
Steve Hodgson | a526f14 | 2011-02-24 23:45:16 +0000 | [diff] [blame] | 66 | { |
Jon Cooper | 43a3739 | 2012-10-18 15:49:54 +0100 | [diff] [blame] | 67 | #if defined(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) |
| 68 | return __le32_to_cpup((const __le32 *)(eh + efx->rx_packet_hash_offset)); |
Ben Hutchings | 39c9cf0 | 2010-06-23 11:31:28 +0000 | [diff] [blame] | 69 | #else |
Jon Cooper | 43a3739 | 2012-10-18 15:49:54 +0100 | [diff] [blame] | 70 | const u8 *data = eh + efx->rx_packet_hash_offset; |
Ben Hutchings | 0beaca2 | 2012-01-05 18:54:04 +0000 | [diff] [blame] | 71 | return (u32)data[0] | |
| 72 | (u32)data[1] << 8 | |
| 73 | (u32)data[2] << 16 | |
| 74 | (u32)data[3] << 24; |
Ben Hutchings | 39c9cf0 | 2010-06-23 11:31:28 +0000 | [diff] [blame] | 75 | #endif |
| 76 | } |
| 77 | |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 78 | static inline struct efx_rx_buffer * |
| 79 | efx_rx_buf_next(struct efx_rx_queue *rx_queue, struct efx_rx_buffer *rx_buf) |
| 80 | { |
| 81 | if (unlikely(rx_buf == efx_rx_buffer(rx_queue, rx_queue->ptr_mask))) |
| 82 | return efx_rx_buffer(rx_queue, 0); |
| 83 | else |
| 84 | return rx_buf + 1; |
| 85 | } |
| 86 | |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 87 | static inline void efx_sync_rx_buffer(struct efx_nic *efx, |
| 88 | struct efx_rx_buffer *rx_buf, |
| 89 | unsigned int len) |
| 90 | { |
| 91 | dma_sync_single_for_cpu(&efx->pci_dev->dev, rx_buf->dma_addr, len, |
| 92 | DMA_FROM_DEVICE); |
| 93 | } |
| 94 | |
Daniel Pieczko | 1648a23 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 95 | void efx_rx_config_page_split(struct efx_nic *efx) |
| 96 | { |
Andrew Rybchenko | 2ec0301 | 2013-11-16 11:02:27 +0400 | [diff] [blame] | 97 | efx->rx_page_buf_step = ALIGN(efx->rx_dma_len + efx->rx_ip_align, |
Ben Hutchings | 950c54d | 2013-05-13 12:01:22 +0000 | [diff] [blame] | 98 | EFX_RX_BUF_ALIGNMENT); |
Daniel Pieczko | 1648a23 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 99 | efx->rx_bufs_per_page = efx->rx_buffer_order ? 1 : |
| 100 | ((PAGE_SIZE - sizeof(struct efx_rx_page_state)) / |
| 101 | efx->rx_page_buf_step); |
| 102 | efx->rx_buffer_truesize = (PAGE_SIZE << efx->rx_buffer_order) / |
| 103 | efx->rx_bufs_per_page; |
| 104 | efx->rx_pages_per_batch = DIV_ROUND_UP(EFX_RX_PREFERRED_BATCH, |
| 105 | efx->rx_bufs_per_page); |
| 106 | } |
| 107 | |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 108 | /* Check the RX page recycle ring for a page that can be reused. */ |
| 109 | static struct page *efx_reuse_page(struct efx_rx_queue *rx_queue) |
| 110 | { |
| 111 | struct efx_nic *efx = rx_queue->efx; |
| 112 | struct page *page; |
| 113 | struct efx_rx_page_state *state; |
| 114 | unsigned index; |
| 115 | |
| 116 | index = rx_queue->page_remove & rx_queue->page_ptr_mask; |
| 117 | page = rx_queue->page_ring[index]; |
| 118 | if (page == NULL) |
| 119 | return NULL; |
| 120 | |
| 121 | rx_queue->page_ring[index] = NULL; |
| 122 | /* page_remove cannot exceed page_add. */ |
| 123 | if (rx_queue->page_remove != rx_queue->page_add) |
| 124 | ++rx_queue->page_remove; |
| 125 | |
| 126 | /* If page_count is 1 then we hold the only reference to this page. */ |
| 127 | if (page_count(page) == 1) { |
| 128 | ++rx_queue->page_recycle_count; |
| 129 | return page; |
| 130 | } else { |
| 131 | state = page_address(page); |
| 132 | dma_unmap_page(&efx->pci_dev->dev, state->dma_addr, |
| 133 | PAGE_SIZE << efx->rx_buffer_order, |
| 134 | DMA_FROM_DEVICE); |
| 135 | put_page(page); |
| 136 | ++rx_queue->page_recycle_failed; |
| 137 | } |
| 138 | |
| 139 | return NULL; |
| 140 | } |
| 141 | |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 142 | /** |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 143 | * efx_init_rx_buffers - create EFX_RX_BATCH page-based RX buffers |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 144 | * |
| 145 | * @rx_queue: Efx RX queue |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 146 | * |
Daniel Pieczko | 1648a23 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 147 | * This allocates a batch of pages, maps them for DMA, and populates |
| 148 | * struct efx_rx_buffers for each one. Return a negative error code or |
| 149 | * 0 on success. If a single page can be used for multiple buffers, |
| 150 | * then the page will either be inserted fully, or not at all. |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 151 | */ |
Jon Cooper | cce2879 | 2013-10-02 11:04:14 +0100 | [diff] [blame] | 152 | static int efx_init_rx_buffers(struct efx_rx_queue *rx_queue, bool atomic) |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 153 | { |
| 154 | struct efx_nic *efx = rx_queue->efx; |
Steve Hodgson | f7d6f37 | 2010-06-01 11:33:17 +0000 | [diff] [blame] | 155 | struct efx_rx_buffer *rx_buf; |
| 156 | struct page *page; |
Ben Hutchings | b590ace | 2013-01-10 23:51:54 +0000 | [diff] [blame] | 157 | unsigned int page_offset; |
Steve Hodgson | 62b330b | 2010-06-01 11:20:53 +0000 | [diff] [blame] | 158 | struct efx_rx_page_state *state; |
Steve Hodgson | f7d6f37 | 2010-06-01 11:33:17 +0000 | [diff] [blame] | 159 | dma_addr_t dma_addr; |
| 160 | unsigned index, count; |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 161 | |
Daniel Pieczko | 1648a23 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 162 | count = 0; |
| 163 | do { |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 164 | page = efx_reuse_page(rx_queue); |
| 165 | if (page == NULL) { |
Jon Cooper | cce2879 | 2013-10-02 11:04:14 +0100 | [diff] [blame] | 166 | page = alloc_pages(__GFP_COLD | __GFP_COMP | |
| 167 | (atomic ? GFP_ATOMIC : GFP_KERNEL), |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 168 | efx->rx_buffer_order); |
| 169 | if (unlikely(page == NULL)) |
| 170 | return -ENOMEM; |
| 171 | dma_addr = |
| 172 | dma_map_page(&efx->pci_dev->dev, page, 0, |
| 173 | PAGE_SIZE << efx->rx_buffer_order, |
| 174 | DMA_FROM_DEVICE); |
| 175 | if (unlikely(dma_mapping_error(&efx->pci_dev->dev, |
| 176 | dma_addr))) { |
| 177 | __free_pages(page, efx->rx_buffer_order); |
| 178 | return -EIO; |
| 179 | } |
| 180 | state = page_address(page); |
| 181 | state->dma_addr = dma_addr; |
| 182 | } else { |
| 183 | state = page_address(page); |
| 184 | dma_addr = state->dma_addr; |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 185 | } |
Steve Hodgson | 62b330b | 2010-06-01 11:20:53 +0000 | [diff] [blame] | 186 | |
Steve Hodgson | 62b330b | 2010-06-01 11:20:53 +0000 | [diff] [blame] | 187 | dma_addr += sizeof(struct efx_rx_page_state); |
Ben Hutchings | b590ace | 2013-01-10 23:51:54 +0000 | [diff] [blame] | 188 | page_offset = sizeof(struct efx_rx_page_state); |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 189 | |
Daniel Pieczko | 1648a23 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 190 | do { |
| 191 | index = rx_queue->added_count & rx_queue->ptr_mask; |
| 192 | rx_buf = efx_rx_buffer(rx_queue, index); |
Andrew Rybchenko | 2ec0301 | 2013-11-16 11:02:27 +0400 | [diff] [blame] | 193 | rx_buf->dma_addr = dma_addr + efx->rx_ip_align; |
Daniel Pieczko | 1648a23 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 194 | rx_buf->page = page; |
Andrew Rybchenko | 2ec0301 | 2013-11-16 11:02:27 +0400 | [diff] [blame] | 195 | rx_buf->page_offset = page_offset + efx->rx_ip_align; |
Daniel Pieczko | 1648a23 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 196 | rx_buf->len = efx->rx_dma_len; |
Ben Hutchings | 179ea7f | 2013-03-07 16:31:17 +0000 | [diff] [blame] | 197 | rx_buf->flags = 0; |
Daniel Pieczko | 1648a23 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 198 | ++rx_queue->added_count; |
| 199 | get_page(page); |
| 200 | dma_addr += efx->rx_page_buf_step; |
| 201 | page_offset += efx->rx_page_buf_step; |
| 202 | } while (page_offset + efx->rx_page_buf_step <= PAGE_SIZE); |
Ben Hutchings | 179ea7f | 2013-03-07 16:31:17 +0000 | [diff] [blame] | 203 | |
| 204 | rx_buf->flags = EFX_RX_BUF_LAST_IN_PAGE; |
Daniel Pieczko | 1648a23 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 205 | } while (++count < efx->rx_pages_per_batch); |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 206 | |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 207 | return 0; |
| 208 | } |
| 209 | |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 210 | /* Unmap a DMA-mapped page. This function is only called for the final RX |
| 211 | * buffer in a page. |
| 212 | */ |
Ben Hutchings | 4d56606 | 2008-09-01 12:47:12 +0100 | [diff] [blame] | 213 | static void efx_unmap_rx_buffer(struct efx_nic *efx, |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 214 | struct efx_rx_buffer *rx_buf) |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 215 | { |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 216 | struct page *page = rx_buf->page; |
Steve Hodgson | 62b330b | 2010-06-01 11:20:53 +0000 | [diff] [blame] | 217 | |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 218 | if (page) { |
| 219 | struct efx_rx_page_state *state = page_address(page); |
| 220 | dma_unmap_page(&efx->pci_dev->dev, |
| 221 | state->dma_addr, |
| 222 | PAGE_SIZE << efx->rx_buffer_order, |
| 223 | DMA_FROM_DEVICE); |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 224 | } |
| 225 | } |
| 226 | |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 227 | static void efx_free_rx_buffer(struct efx_rx_buffer *rx_buf) |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 228 | { |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 229 | if (rx_buf->page) { |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 230 | put_page(rx_buf->page); |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 231 | rx_buf->page = NULL; |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 232 | } |
| 233 | } |
| 234 | |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 235 | /* Attempt to recycle the page if there is an RX recycle ring; the page can |
| 236 | * only be added if this is the final RX buffer, to prevent pages being used in |
| 237 | * the descriptor ring and appearing in the recycle ring simultaneously. |
| 238 | */ |
| 239 | static void efx_recycle_rx_page(struct efx_channel *channel, |
| 240 | struct efx_rx_buffer *rx_buf) |
| 241 | { |
| 242 | struct page *page = rx_buf->page; |
| 243 | struct efx_rx_queue *rx_queue = efx_channel_get_rx_queue(channel); |
| 244 | struct efx_nic *efx = rx_queue->efx; |
| 245 | unsigned index; |
| 246 | |
| 247 | /* Only recycle the page after processing the final buffer. */ |
Ben Hutchings | 179ea7f | 2013-03-07 16:31:17 +0000 | [diff] [blame] | 248 | if (!(rx_buf->flags & EFX_RX_BUF_LAST_IN_PAGE)) |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 249 | return; |
| 250 | |
| 251 | index = rx_queue->page_add & rx_queue->page_ptr_mask; |
| 252 | if (rx_queue->page_ring[index] == NULL) { |
| 253 | unsigned read_index = rx_queue->page_remove & |
| 254 | rx_queue->page_ptr_mask; |
| 255 | |
| 256 | /* The next slot in the recycle ring is available, but |
| 257 | * increment page_remove if the read pointer currently |
| 258 | * points here. |
| 259 | */ |
| 260 | if (read_index == index) |
| 261 | ++rx_queue->page_remove; |
| 262 | rx_queue->page_ring[index] = page; |
| 263 | ++rx_queue->page_add; |
| 264 | return; |
| 265 | } |
| 266 | ++rx_queue->page_recycle_full; |
| 267 | efx_unmap_rx_buffer(efx, rx_buf); |
| 268 | put_page(rx_buf->page); |
| 269 | } |
| 270 | |
Ben Hutchings | 4d56606 | 2008-09-01 12:47:12 +0100 | [diff] [blame] | 271 | static void efx_fini_rx_buffer(struct efx_rx_queue *rx_queue, |
| 272 | struct efx_rx_buffer *rx_buf) |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 273 | { |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 274 | /* Release the page reference we hold for the buffer. */ |
| 275 | if (rx_buf->page) |
| 276 | put_page(rx_buf->page); |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 277 | |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 278 | /* If this is the last buffer in a page, unmap and free it. */ |
Ben Hutchings | 179ea7f | 2013-03-07 16:31:17 +0000 | [diff] [blame] | 279 | if (rx_buf->flags & EFX_RX_BUF_LAST_IN_PAGE) { |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 280 | efx_unmap_rx_buffer(rx_queue->efx, rx_buf); |
| 281 | efx_free_rx_buffer(rx_buf); |
Steve Hodgson | 62b330b | 2010-06-01 11:20:53 +0000 | [diff] [blame] | 282 | } |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 283 | rx_buf->page = NULL; |
Steve Hodgson | 2445580 | 2010-06-01 11:20:34 +0000 | [diff] [blame] | 284 | } |
| 285 | |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 286 | /* Recycle the pages that are used by buffers that have just been received. */ |
Ben Hutchings | 734d4e1 | 2013-07-04 23:48:46 +0100 | [diff] [blame] | 287 | static void efx_recycle_rx_pages(struct efx_channel *channel, |
| 288 | struct efx_rx_buffer *rx_buf, |
| 289 | unsigned int n_frags) |
Steve Hodgson | 2445580 | 2010-06-01 11:20:34 +0000 | [diff] [blame] | 290 | { |
Ben Hutchings | f7d12cd | 2010-09-10 06:41:47 +0000 | [diff] [blame] | 291 | struct efx_rx_queue *rx_queue = efx_channel_get_rx_queue(channel); |
Steve Hodgson | 2445580 | 2010-06-01 11:20:34 +0000 | [diff] [blame] | 292 | |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 293 | do { |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 294 | efx_recycle_rx_page(channel, rx_buf); |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 295 | rx_buf = efx_rx_buf_next(rx_queue, rx_buf); |
| 296 | } while (--n_frags); |
Steve Hodgson | 2445580 | 2010-06-01 11:20:34 +0000 | [diff] [blame] | 297 | } |
| 298 | |
Ben Hutchings | 734d4e1 | 2013-07-04 23:48:46 +0100 | [diff] [blame] | 299 | static void efx_discard_rx_packet(struct efx_channel *channel, |
| 300 | struct efx_rx_buffer *rx_buf, |
| 301 | unsigned int n_frags) |
| 302 | { |
| 303 | struct efx_rx_queue *rx_queue = efx_channel_get_rx_queue(channel); |
| 304 | |
| 305 | efx_recycle_rx_pages(channel, rx_buf, n_frags); |
| 306 | |
| 307 | do { |
| 308 | efx_free_rx_buffer(rx_buf); |
| 309 | rx_buf = efx_rx_buf_next(rx_queue, rx_buf); |
| 310 | } while (--n_frags); |
| 311 | } |
| 312 | |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 313 | /** |
| 314 | * efx_fast_push_rx_descriptors - push new RX descriptors quickly |
| 315 | * @rx_queue: RX descriptor queue |
Ben Hutchings | 49ce9c2 | 2012-07-10 10:56:00 +0000 | [diff] [blame] | 316 | * |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 317 | * This will aim to fill the RX descriptor queue up to |
David Riddoch | da9ca50 | 2012-04-11 13:09:24 +0100 | [diff] [blame] | 318 | * @rx_queue->@max_fill. If there is insufficient atomic |
Steve Hodgson | 90d683a | 2010-06-01 11:19:39 +0000 | [diff] [blame] | 319 | * memory to do so, a slow fill will be scheduled. |
| 320 | * |
| 321 | * The caller must provide serialisation (none is used here). In practise, |
| 322 | * this means this function must run from the NAPI handler, or be called |
| 323 | * when NAPI is disabled. |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 324 | */ |
Jon Cooper | cce2879 | 2013-10-02 11:04:14 +0100 | [diff] [blame] | 325 | void efx_fast_push_rx_descriptors(struct efx_rx_queue *rx_queue, bool atomic) |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 326 | { |
Daniel Pieczko | 1648a23 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 327 | struct efx_nic *efx = rx_queue->efx; |
| 328 | unsigned int fill_level, batch_size; |
Steve Hodgson | f7d6f37 | 2010-06-01 11:33:17 +0000 | [diff] [blame] | 329 | int space, rc = 0; |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 330 | |
Ben Hutchings | d8aec74 | 2013-05-27 16:52:54 +0100 | [diff] [blame] | 331 | if (!rx_queue->refill_enabled) |
| 332 | return; |
| 333 | |
Steve Hodgson | 90d683a | 2010-06-01 11:19:39 +0000 | [diff] [blame] | 334 | /* Calculate current fill level, and exit if we don't need to fill */ |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 335 | fill_level = (rx_queue->added_count - rx_queue->removed_count); |
Steve Hodgson | ecc910f | 2010-09-10 06:42:22 +0000 | [diff] [blame] | 336 | EFX_BUG_ON_PARANOID(fill_level > rx_queue->efx->rxq_entries); |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 337 | if (fill_level >= rx_queue->fast_fill_trigger) |
Steve Hodgson | 2445580 | 2010-06-01 11:20:34 +0000 | [diff] [blame] | 338 | goto out; |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 339 | |
| 340 | /* Record minimum fill level */ |
Ben Hutchings | b347564 | 2008-05-16 21:15:49 +0100 | [diff] [blame] | 341 | if (unlikely(fill_level < rx_queue->min_fill)) { |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 342 | if (fill_level) |
| 343 | rx_queue->min_fill = fill_level; |
Ben Hutchings | b347564 | 2008-05-16 21:15:49 +0100 | [diff] [blame] | 344 | } |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 345 | |
Daniel Pieczko | 1648a23 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 346 | batch_size = efx->rx_pages_per_batch * efx->rx_bufs_per_page; |
David Riddoch | da9ca50 | 2012-04-11 13:09:24 +0100 | [diff] [blame] | 347 | space = rx_queue->max_fill - fill_level; |
Daniel Pieczko | 1648a23 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 348 | EFX_BUG_ON_PARANOID(space < batch_size); |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 349 | |
Ben Hutchings | 62776d0 | 2010-06-23 11:30:07 +0000 | [diff] [blame] | 350 | netif_vdbg(rx_queue->efx, rx_status, rx_queue->efx->net_dev, |
| 351 | "RX queue %d fast-filling descriptor ring from" |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 352 | " level %d to level %d\n", |
Ben Hutchings | ba1e8a3 | 2010-09-10 06:41:36 +0000 | [diff] [blame] | 353 | efx_rx_queue_index(rx_queue), fill_level, |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 354 | rx_queue->max_fill); |
| 355 | |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 356 | |
| 357 | do { |
Jon Cooper | cce2879 | 2013-10-02 11:04:14 +0100 | [diff] [blame] | 358 | rc = efx_init_rx_buffers(rx_queue, atomic); |
Steve Hodgson | f7d6f37 | 2010-06-01 11:33:17 +0000 | [diff] [blame] | 359 | if (unlikely(rc)) { |
| 360 | /* Ensure that we don't leave the rx queue empty */ |
| 361 | if (rx_queue->added_count == rx_queue->removed_count) |
| 362 | efx_schedule_slow_fill(rx_queue); |
| 363 | goto out; |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 364 | } |
Daniel Pieczko | 1648a23 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 365 | } while ((space -= batch_size) >= batch_size); |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 366 | |
Ben Hutchings | 62776d0 | 2010-06-23 11:30:07 +0000 | [diff] [blame] | 367 | netif_vdbg(rx_queue->efx, rx_status, rx_queue->efx->net_dev, |
| 368 | "RX queue %d fast-filled descriptor ring " |
Ben Hutchings | ba1e8a3 | 2010-09-10 06:41:36 +0000 | [diff] [blame] | 369 | "to level %d\n", efx_rx_queue_index(rx_queue), |
Ben Hutchings | 62776d0 | 2010-06-23 11:30:07 +0000 | [diff] [blame] | 370 | rx_queue->added_count - rx_queue->removed_count); |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 371 | |
| 372 | out: |
Steve Hodgson | 2445580 | 2010-06-01 11:20:34 +0000 | [diff] [blame] | 373 | if (rx_queue->notified_count != rx_queue->added_count) |
| 374 | efx_nic_notify_rx_desc(rx_queue); |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 375 | } |
| 376 | |
Steve Hodgson | 90d683a | 2010-06-01 11:19:39 +0000 | [diff] [blame] | 377 | void efx_rx_slow_fill(unsigned long context) |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 378 | { |
Steve Hodgson | 90d683a | 2010-06-01 11:19:39 +0000 | [diff] [blame] | 379 | struct efx_rx_queue *rx_queue = (struct efx_rx_queue *)context; |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 380 | |
Steve Hodgson | 90d683a | 2010-06-01 11:19:39 +0000 | [diff] [blame] | 381 | /* Post an event to cause NAPI to run and refill the queue */ |
Ben Hutchings | 2ae75da | 2012-02-07 23:49:52 +0000 | [diff] [blame] | 382 | efx_nic_generate_fill_event(rx_queue); |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 383 | ++rx_queue->slow_fill_count; |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 384 | } |
| 385 | |
Ben Hutchings | 4d56606 | 2008-09-01 12:47:12 +0100 | [diff] [blame] | 386 | static void efx_rx_packet__check_len(struct efx_rx_queue *rx_queue, |
| 387 | struct efx_rx_buffer *rx_buf, |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 388 | int len) |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 389 | { |
| 390 | struct efx_nic *efx = rx_queue->efx; |
| 391 | unsigned max_len = rx_buf->len - efx->type->rx_buffer_padding; |
| 392 | |
| 393 | if (likely(len <= max_len)) |
| 394 | return; |
| 395 | |
| 396 | /* The packet must be discarded, but this is only a fatal error |
| 397 | * if the caller indicated it was |
| 398 | */ |
Ben Hutchings | db33956 | 2011-08-26 18:05:11 +0100 | [diff] [blame] | 399 | rx_buf->flags |= EFX_RX_PKT_DISCARD; |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 400 | |
| 401 | if ((len > rx_buf->len) && EFX_WORKAROUND_8071(efx)) { |
Ben Hutchings | 62776d0 | 2010-06-23 11:30:07 +0000 | [diff] [blame] | 402 | if (net_ratelimit()) |
| 403 | netif_err(efx, rx_err, efx->net_dev, |
| 404 | " RX queue %d seriously overlength " |
| 405 | "RX event (0x%x > 0x%x+0x%x). Leaking\n", |
Ben Hutchings | ba1e8a3 | 2010-09-10 06:41:36 +0000 | [diff] [blame] | 406 | efx_rx_queue_index(rx_queue), len, max_len, |
Ben Hutchings | 62776d0 | 2010-06-23 11:30:07 +0000 | [diff] [blame] | 407 | efx->type->rx_buffer_padding); |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 408 | efx_schedule_reset(efx, RESET_TYPE_RX_RECOVERY); |
| 409 | } else { |
Ben Hutchings | 62776d0 | 2010-06-23 11:30:07 +0000 | [diff] [blame] | 410 | if (net_ratelimit()) |
| 411 | netif_err(efx, rx_err, efx->net_dev, |
| 412 | " RX queue %d overlength RX event " |
| 413 | "(0x%x > 0x%x)\n", |
Ben Hutchings | ba1e8a3 | 2010-09-10 06:41:36 +0000 | [diff] [blame] | 414 | efx_rx_queue_index(rx_queue), len, max_len); |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 415 | } |
| 416 | |
Ben Hutchings | ba1e8a3 | 2010-09-10 06:41:36 +0000 | [diff] [blame] | 417 | efx_rx_queue_channel(rx_queue)->n_rx_overlength++; |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 418 | } |
| 419 | |
Ben Hutchings | 61321d9 | 2012-02-25 01:58:35 +0000 | [diff] [blame] | 420 | /* Pass a received packet up through GRO. GRO can handle pages |
| 421 | * regardless of checksum state and skbs with a good checksum. |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 422 | */ |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 423 | static void |
| 424 | efx_rx_packet_gro(struct efx_channel *channel, struct efx_rx_buffer *rx_buf, |
| 425 | unsigned int n_frags, u8 *eh) |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 426 | { |
Herbert Xu | da3bc07 | 2009-01-18 21:50:16 -0800 | [diff] [blame] | 427 | struct napi_struct *napi = &channel->napi_str; |
Ben Hutchings | 18e1d2b | 2009-10-29 07:21:24 +0000 | [diff] [blame] | 428 | gro_result_t gro_result; |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 429 | struct efx_nic *efx = channel->efx; |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 430 | struct sk_buff *skb; |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 431 | |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 432 | skb = napi_get_frags(napi); |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 433 | if (unlikely(!skb)) { |
| 434 | while (n_frags--) { |
| 435 | put_page(rx_buf->page); |
| 436 | rx_buf->page = NULL; |
| 437 | rx_buf = efx_rx_buf_next(&channel->rx_queue, rx_buf); |
| 438 | } |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 439 | return; |
| 440 | } |
Ben Hutchings | 1241e95 | 2009-11-23 16:02:25 +0000 | [diff] [blame] | 441 | |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 442 | if (efx->net_dev->features & NETIF_F_RXHASH) |
Tom Herbert | c7cb38a | 2013-12-17 23:31:50 -0800 | [diff] [blame] | 443 | skb_set_hash(skb, efx_rx_buf_hash(efx, eh), |
| 444 | PKT_HASH_TYPE_L3); |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 445 | skb->ip_summed = ((rx_buf->flags & EFX_RX_PKT_CSUMMED) ? |
| 446 | CHECKSUM_UNNECESSARY : CHECKSUM_NONE); |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 447 | |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 448 | for (;;) { |
| 449 | skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags, |
| 450 | rx_buf->page, rx_buf->page_offset, |
| 451 | rx_buf->len); |
| 452 | rx_buf->page = NULL; |
| 453 | skb->len += rx_buf->len; |
| 454 | if (skb_shinfo(skb)->nr_frags == n_frags) |
| 455 | break; |
| 456 | |
| 457 | rx_buf = efx_rx_buf_next(&channel->rx_queue, rx_buf); |
| 458 | } |
| 459 | |
| 460 | skb->data_len = skb->len; |
| 461 | skb->truesize += n_frags * efx->rx_buffer_truesize; |
| 462 | |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 463 | skb_record_rx_queue(skb, channel->rx_queue.core_index); |
Ben Hutchings | 3eadb7b | 2009-11-23 16:02:40 +0000 | [diff] [blame] | 464 | |
Alexandre Rames | 3676326 | 2014-07-22 14:03:25 +0100 | [diff] [blame] | 465 | skb_mark_napi_id(skb, &channel->napi_str); |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 466 | gro_result = napi_gro_frags(napi); |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 467 | if (gro_result != GRO_DROP) |
Ben Hutchings | 18e1d2b | 2009-10-29 07:21:24 +0000 | [diff] [blame] | 468 | channel->irq_mod_score += 2; |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 469 | } |
| 470 | |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 471 | /* Allocate and construct an SKB around page fragments */ |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 472 | static struct sk_buff *efx_rx_mk_skb(struct efx_channel *channel, |
| 473 | struct efx_rx_buffer *rx_buf, |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 474 | unsigned int n_frags, |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 475 | u8 *eh, int hdr_len) |
| 476 | { |
| 477 | struct efx_nic *efx = channel->efx; |
| 478 | struct sk_buff *skb; |
| 479 | |
| 480 | /* Allocate an SKB to store the headers */ |
Ben Hutchings | 2ccd0b1 | 2013-11-28 18:58:11 +0000 | [diff] [blame] | 481 | skb = netdev_alloc_skb(efx->net_dev, |
| 482 | efx->rx_ip_align + efx->rx_prefix_size + |
| 483 | hdr_len); |
Edward Cree | e4d112e | 2014-07-15 11:58:12 +0100 | [diff] [blame] | 484 | if (unlikely(skb == NULL)) { |
| 485 | atomic_inc(&efx->n_rx_noskb_drops); |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 486 | return NULL; |
Edward Cree | e4d112e | 2014-07-15 11:58:12 +0100 | [diff] [blame] | 487 | } |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 488 | |
| 489 | EFX_BUG_ON_PARANOID(rx_buf->len < hdr_len); |
| 490 | |
Ben Hutchings | 2ccd0b1 | 2013-11-28 18:58:11 +0000 | [diff] [blame] | 491 | memcpy(skb->data + efx->rx_ip_align, eh - efx->rx_prefix_size, |
| 492 | efx->rx_prefix_size + hdr_len); |
| 493 | skb_reserve(skb, efx->rx_ip_align + efx->rx_prefix_size); |
| 494 | __skb_put(skb, hdr_len); |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 495 | |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 496 | /* Append the remaining page(s) onto the frag list */ |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 497 | if (rx_buf->len > hdr_len) { |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 498 | rx_buf->page_offset += hdr_len; |
| 499 | rx_buf->len -= hdr_len; |
| 500 | |
| 501 | for (;;) { |
| 502 | skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags, |
| 503 | rx_buf->page, rx_buf->page_offset, |
| 504 | rx_buf->len); |
| 505 | rx_buf->page = NULL; |
| 506 | skb->len += rx_buf->len; |
| 507 | skb->data_len += rx_buf->len; |
| 508 | if (skb_shinfo(skb)->nr_frags == n_frags) |
| 509 | break; |
| 510 | |
| 511 | rx_buf = efx_rx_buf_next(&channel->rx_queue, rx_buf); |
| 512 | } |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 513 | } else { |
| 514 | __free_pages(rx_buf->page, efx->rx_buffer_order); |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 515 | rx_buf->page = NULL; |
| 516 | n_frags = 0; |
Ben Hutchings | 18e1d2b | 2009-10-29 07:21:24 +0000 | [diff] [blame] | 517 | } |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 518 | |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 519 | skb->truesize += n_frags * efx->rx_buffer_truesize; |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 520 | |
| 521 | /* Move past the ethernet header */ |
| 522 | skb->protocol = eth_type_trans(skb, efx->net_dev); |
| 523 | |
Alexandre Rames | 3676326 | 2014-07-22 14:03:25 +0100 | [diff] [blame] | 524 | skb_mark_napi_id(skb, &channel->napi_str); |
| 525 | |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 526 | return skb; |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 527 | } |
| 528 | |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 529 | void efx_rx_packet(struct efx_rx_queue *rx_queue, unsigned int index, |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 530 | unsigned int n_frags, unsigned int len, u16 flags) |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 531 | { |
| 532 | struct efx_nic *efx = rx_queue->efx; |
Ben Hutchings | ba1e8a3 | 2010-09-10 06:41:36 +0000 | [diff] [blame] | 533 | struct efx_channel *channel = efx_rx_queue_channel(rx_queue); |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 534 | struct efx_rx_buffer *rx_buf; |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 535 | |
Andrew Rybchenko | 8ccf3800 | 2014-07-17 12:10:43 +0100 | [diff] [blame] | 536 | rx_queue->rx_packets++; |
| 537 | |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 538 | rx_buf = efx_rx_buffer(rx_queue, index); |
Ben Hutchings | 179ea7f | 2013-03-07 16:31:17 +0000 | [diff] [blame] | 539 | rx_buf->flags |= flags; |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 540 | |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 541 | /* Validate the number of fragments and completed length */ |
| 542 | if (n_frags == 1) { |
Ben Hutchings | 3dced74 | 2013-04-27 01:55:18 +0100 | [diff] [blame] | 543 | if (!(flags & EFX_RX_PKT_PREFIX_LEN)) |
| 544 | efx_rx_packet__check_len(rx_queue, rx_buf, len); |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 545 | } else if (unlikely(n_frags > EFX_RX_MAX_FRAGS) || |
Jon Cooper | e8c68c0 | 2013-03-08 10:18:28 +0000 | [diff] [blame] | 546 | unlikely(len <= (n_frags - 1) * efx->rx_dma_len) || |
| 547 | unlikely(len > n_frags * efx->rx_dma_len) || |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 548 | unlikely(!efx->rx_scatter)) { |
| 549 | /* If this isn't an explicit discard request, either |
| 550 | * the hardware or the driver is broken. |
| 551 | */ |
| 552 | WARN_ON(!(len == 0 && rx_buf->flags & EFX_RX_PKT_DISCARD)); |
| 553 | rx_buf->flags |= EFX_RX_PKT_DISCARD; |
| 554 | } |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 555 | |
Ben Hutchings | 62776d0 | 2010-06-23 11:30:07 +0000 | [diff] [blame] | 556 | netif_vdbg(efx, rx_status, efx->net_dev, |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 557 | "RX queue %d received ids %x-%x len %d %s%s\n", |
Ben Hutchings | ba1e8a3 | 2010-09-10 06:41:36 +0000 | [diff] [blame] | 558 | efx_rx_queue_index(rx_queue), index, |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 559 | (index + n_frags - 1) & rx_queue->ptr_mask, len, |
Ben Hutchings | db33956 | 2011-08-26 18:05:11 +0100 | [diff] [blame] | 560 | (rx_buf->flags & EFX_RX_PKT_CSUMMED) ? " [SUMMED]" : "", |
| 561 | (rx_buf->flags & EFX_RX_PKT_DISCARD) ? " [DISCARD]" : ""); |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 562 | |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 563 | /* Discard packet, if instructed to do so. Process the |
| 564 | * previous receive first. |
| 565 | */ |
Ben Hutchings | db33956 | 2011-08-26 18:05:11 +0100 | [diff] [blame] | 566 | if (unlikely(rx_buf->flags & EFX_RX_PKT_DISCARD)) { |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 567 | efx_rx_flush_packet(channel); |
Ben Hutchings | 734d4e1 | 2013-07-04 23:48:46 +0100 | [diff] [blame] | 568 | efx_discard_rx_packet(channel, rx_buf, n_frags); |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 569 | return; |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 570 | } |
| 571 | |
Ben Hutchings | 3dced74 | 2013-04-27 01:55:18 +0100 | [diff] [blame] | 572 | if (n_frags == 1 && !(flags & EFX_RX_PKT_PREFIX_LEN)) |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 573 | rx_buf->len = len; |
| 574 | |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 575 | /* Release and/or sync the DMA mapping - assumes all RX buffers |
| 576 | * consumed in-order per RX queue. |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 577 | */ |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 578 | efx_sync_rx_buffer(efx, rx_buf, rx_buf->len); |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 579 | |
| 580 | /* Prefetch nice and early so data will (hopefully) be in cache by |
| 581 | * the time we look at it. |
| 582 | */ |
Ben Hutchings | 5036b7c | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 583 | prefetch(efx_rx_buf_va(rx_buf)); |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 584 | |
Jon Cooper | 43a3739 | 2012-10-18 15:49:54 +0100 | [diff] [blame] | 585 | rx_buf->page_offset += efx->rx_prefix_size; |
| 586 | rx_buf->len -= efx->rx_prefix_size; |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 587 | |
| 588 | if (n_frags > 1) { |
| 589 | /* Release/sync DMA mapping for additional fragments. |
| 590 | * Fix length for last fragment. |
| 591 | */ |
| 592 | unsigned int tail_frags = n_frags - 1; |
| 593 | |
| 594 | for (;;) { |
| 595 | rx_buf = efx_rx_buf_next(rx_queue, rx_buf); |
| 596 | if (--tail_frags == 0) |
| 597 | break; |
Jon Cooper | e8c68c0 | 2013-03-08 10:18:28 +0000 | [diff] [blame] | 598 | efx_sync_rx_buffer(efx, rx_buf, efx->rx_dma_len); |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 599 | } |
Jon Cooper | e8c68c0 | 2013-03-08 10:18:28 +0000 | [diff] [blame] | 600 | rx_buf->len = len - (n_frags - 1) * efx->rx_dma_len; |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 601 | efx_sync_rx_buffer(efx, rx_buf, rx_buf->len); |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 602 | } |
Ben Hutchings | b74e3e8 | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 603 | |
Ben Hutchings | 734d4e1 | 2013-07-04 23:48:46 +0100 | [diff] [blame] | 604 | /* All fragments have been DMA-synced, so recycle pages. */ |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 605 | rx_buf = efx_rx_buffer(rx_queue, index); |
Ben Hutchings | 734d4e1 | 2013-07-04 23:48:46 +0100 | [diff] [blame] | 606 | efx_recycle_rx_pages(channel, rx_buf, n_frags); |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 607 | |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 608 | /* Pipeline receives so that we give time for packet headers to be |
| 609 | * prefetched into cache. |
| 610 | */ |
Ben Hutchings | ff734ef | 2013-01-29 23:33:14 +0000 | [diff] [blame] | 611 | efx_rx_flush_packet(channel); |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 612 | channel->rx_pkt_n_frags = n_frags; |
| 613 | channel->rx_pkt_index = index; |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 614 | } |
| 615 | |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 616 | static void efx_rx_deliver(struct efx_channel *channel, u8 *eh, |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 617 | struct efx_rx_buffer *rx_buf, |
| 618 | unsigned int n_frags) |
Ben Hutchings | 1ddceb4 | 2012-01-23 22:41:30 +0000 | [diff] [blame] | 619 | { |
| 620 | struct sk_buff *skb; |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 621 | u16 hdr_len = min_t(u16, rx_buf->len, EFX_SKB_HEADERS); |
Ben Hutchings | 1ddceb4 | 2012-01-23 22:41:30 +0000 | [diff] [blame] | 622 | |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 623 | skb = efx_rx_mk_skb(channel, rx_buf, n_frags, eh, hdr_len); |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 624 | if (unlikely(skb == NULL)) { |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 625 | efx_free_rx_buffer(rx_buf); |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 626 | return; |
| 627 | } |
| 628 | skb_record_rx_queue(skb, channel->rx_queue.core_index); |
Ben Hutchings | 1ddceb4 | 2012-01-23 22:41:30 +0000 | [diff] [blame] | 629 | |
| 630 | /* Set the SKB flags */ |
| 631 | skb_checksum_none_assert(skb); |
Jon Cooper | c99dffc | 2013-04-08 12:49:48 +0100 | [diff] [blame] | 632 | if (likely(rx_buf->flags & EFX_RX_PKT_CSUMMED)) |
| 633 | skb->ip_summed = CHECKSUM_UNNECESSARY; |
Ben Hutchings | 1ddceb4 | 2012-01-23 22:41:30 +0000 | [diff] [blame] | 634 | |
Jon Cooper | bd9a265 | 2013-11-18 12:54:41 +0000 | [diff] [blame] | 635 | efx_rx_skb_attach_timestamp(channel, skb); |
| 636 | |
Stuart Hodgson | c31e5f9 | 2012-07-18 09:52:11 +0100 | [diff] [blame] | 637 | if (channel->type->receive_skb) |
Ben Hutchings | 4a74dc6 | 2013-03-05 20:13:54 +0000 | [diff] [blame] | 638 | if (channel->type->receive_skb(channel, skb)) |
Alexandre Rames | 97d48a1 | 2013-01-11 12:26:21 +0000 | [diff] [blame] | 639 | return; |
Ben Hutchings | 1ddceb4 | 2012-01-23 22:41:30 +0000 | [diff] [blame] | 640 | |
Ben Hutchings | 4a74dc6 | 2013-03-05 20:13:54 +0000 | [diff] [blame] | 641 | /* Pass the packet up */ |
| 642 | netif_receive_skb(skb); |
Ben Hutchings | 1ddceb4 | 2012-01-23 22:41:30 +0000 | [diff] [blame] | 643 | } |
| 644 | |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 645 | /* Handle a received packet. Second half: Touches packet payload. */ |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 646 | void __efx_rx_packet(struct efx_channel *channel) |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 647 | { |
| 648 | struct efx_nic *efx = channel->efx; |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 649 | struct efx_rx_buffer *rx_buf = |
| 650 | efx_rx_buffer(&channel->rx_queue, channel->rx_pkt_index); |
Ben Hutchings | b74e3e8 | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 651 | u8 *eh = efx_rx_buf_va(rx_buf); |
Ben Hutchings | 604f604 | 2010-06-25 07:05:33 +0000 | [diff] [blame] | 652 | |
Ben Hutchings | 3dced74 | 2013-04-27 01:55:18 +0100 | [diff] [blame] | 653 | /* Read length from the prefix if necessary. This already |
| 654 | * excludes the length of the prefix itself. |
| 655 | */ |
| 656 | if (rx_buf->flags & EFX_RX_PKT_PREFIX_LEN) |
| 657 | rx_buf->len = le16_to_cpup((__le16 *) |
| 658 | (eh + efx->rx_packet_len_offset)); |
| 659 | |
Ben Hutchings | 3273c2e | 2008-05-07 13:36:19 +0100 | [diff] [blame] | 660 | /* If we're in loopback test, then pass the packet directly to the |
| 661 | * loopback layer, and free the rx_buf here |
| 662 | */ |
| 663 | if (unlikely(efx->loopback_selftest)) { |
Steve Hodgson | a526f14 | 2011-02-24 23:45:16 +0000 | [diff] [blame] | 664 | efx_loopback_rx_packet(efx, eh, rx_buf->len); |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 665 | efx_free_rx_buffer(rx_buf); |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 666 | goto out; |
Ben Hutchings | 3273c2e | 2008-05-07 13:36:19 +0100 | [diff] [blame] | 667 | } |
| 668 | |
Ben Hutchings | abfe903 | 2011-04-05 15:00:02 +0100 | [diff] [blame] | 669 | if (unlikely(!(efx->net_dev->features & NETIF_F_RXCSUM))) |
Ben Hutchings | db33956 | 2011-08-26 18:05:11 +0100 | [diff] [blame] | 670 | rx_buf->flags &= ~EFX_RX_PKT_CSUMMED; |
Ben Hutchings | ab3cf6d | 2011-04-01 22:20:06 +0100 | [diff] [blame] | 671 | |
Alexandre Rames | 3676326 | 2014-07-22 14:03:25 +0100 | [diff] [blame] | 672 | if ((rx_buf->flags & EFX_RX_PKT_TCP) && !channel->type->receive_skb && |
| 673 | !efx_channel_busy_polling(channel)) |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 674 | efx_rx_packet_gro(channel, rx_buf, channel->rx_pkt_n_frags, eh); |
Ben Hutchings | 1ddceb4 | 2012-01-23 22:41:30 +0000 | [diff] [blame] | 675 | else |
Ben Hutchings | 85740cdf | 2013-01-29 23:33:15 +0000 | [diff] [blame] | 676 | efx_rx_deliver(channel, eh, rx_buf, channel->rx_pkt_n_frags); |
| 677 | out: |
| 678 | channel->rx_pkt_n_frags = 0; |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 679 | } |
| 680 | |
| 681 | int efx_probe_rx_queue(struct efx_rx_queue *rx_queue) |
| 682 | { |
| 683 | struct efx_nic *efx = rx_queue->efx; |
Steve Hodgson | ecc910f | 2010-09-10 06:42:22 +0000 | [diff] [blame] | 684 | unsigned int entries; |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 685 | int rc; |
| 686 | |
Steve Hodgson | ecc910f | 2010-09-10 06:42:22 +0000 | [diff] [blame] | 687 | /* Create the smallest power-of-two aligned ring */ |
| 688 | entries = max(roundup_pow_of_two(efx->rxq_entries), EFX_MIN_DMAQ_SIZE); |
| 689 | EFX_BUG_ON_PARANOID(entries > EFX_MAX_DMAQ_SIZE); |
| 690 | rx_queue->ptr_mask = entries - 1; |
| 691 | |
Ben Hutchings | 62776d0 | 2010-06-23 11:30:07 +0000 | [diff] [blame] | 692 | netif_dbg(efx, probe, efx->net_dev, |
Steve Hodgson | ecc910f | 2010-09-10 06:42:22 +0000 | [diff] [blame] | 693 | "creating RX queue %d size %#x mask %#x\n", |
| 694 | efx_rx_queue_index(rx_queue), efx->rxq_entries, |
| 695 | rx_queue->ptr_mask); |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 696 | |
| 697 | /* Allocate RX buffers */ |
Thomas Meyer | c2e4e25 | 2011-12-02 12:36:13 +0000 | [diff] [blame] | 698 | rx_queue->buffer = kcalloc(entries, sizeof(*rx_queue->buffer), |
Steve Hodgson | ecc910f | 2010-09-10 06:42:22 +0000 | [diff] [blame] | 699 | GFP_KERNEL); |
Ben Hutchings | 8831da7 | 2008-09-01 12:47:48 +0100 | [diff] [blame] | 700 | if (!rx_queue->buffer) |
| 701 | return -ENOMEM; |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 702 | |
Ben Hutchings | 152b6a6 | 2009-11-29 03:43:56 +0000 | [diff] [blame] | 703 | rc = efx_nic_probe_rx(rx_queue); |
Ben Hutchings | 8831da7 | 2008-09-01 12:47:48 +0100 | [diff] [blame] | 704 | if (rc) { |
| 705 | kfree(rx_queue->buffer); |
| 706 | rx_queue->buffer = NULL; |
| 707 | } |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 708 | |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 709 | return rc; |
| 710 | } |
| 711 | |
stephen hemminger | debd003 | 2013-03-16 06:57:51 +0000 | [diff] [blame] | 712 | static void efx_init_rx_recycle_ring(struct efx_nic *efx, |
| 713 | struct efx_rx_queue *rx_queue) |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 714 | { |
| 715 | unsigned int bufs_in_recycle_ring, page_ring_size; |
| 716 | |
| 717 | /* Set the RX recycle ring size */ |
| 718 | #ifdef CONFIG_PPC64 |
| 719 | bufs_in_recycle_ring = EFX_RECYCLE_RING_SIZE_IOMMU; |
| 720 | #else |
Ben Hutchings | 636d73d | 2013-06-12 18:09:08 +0100 | [diff] [blame] | 721 | if (iommu_present(&pci_bus_type)) |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 722 | bufs_in_recycle_ring = EFX_RECYCLE_RING_SIZE_IOMMU; |
| 723 | else |
| 724 | bufs_in_recycle_ring = EFX_RECYCLE_RING_SIZE_NOIOMMU; |
| 725 | #endif /* CONFIG_PPC64 */ |
| 726 | |
| 727 | page_ring_size = roundup_pow_of_two(bufs_in_recycle_ring / |
| 728 | efx->rx_bufs_per_page); |
| 729 | rx_queue->page_ring = kcalloc(page_ring_size, |
| 730 | sizeof(*rx_queue->page_ring), GFP_KERNEL); |
| 731 | rx_queue->page_ptr_mask = page_ring_size - 1; |
| 732 | } |
| 733 | |
Ben Hutchings | bc3c90a | 2008-09-01 12:48:46 +0100 | [diff] [blame] | 734 | void efx_init_rx_queue(struct efx_rx_queue *rx_queue) |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 735 | { |
Steve Hodgson | ecc910f | 2010-09-10 06:42:22 +0000 | [diff] [blame] | 736 | struct efx_nic *efx = rx_queue->efx; |
David Riddoch | 6423518 | 2012-04-11 13:12:41 +0100 | [diff] [blame] | 737 | unsigned int max_fill, trigger, max_trigger; |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 738 | |
Ben Hutchings | 62776d0 | 2010-06-23 11:30:07 +0000 | [diff] [blame] | 739 | netif_dbg(rx_queue->efx, drv, rx_queue->efx->net_dev, |
Ben Hutchings | ba1e8a3 | 2010-09-10 06:41:36 +0000 | [diff] [blame] | 740 | "initialising RX queue %d\n", efx_rx_queue_index(rx_queue)); |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 741 | |
| 742 | /* Initialise ptr fields */ |
| 743 | rx_queue->added_count = 0; |
| 744 | rx_queue->notified_count = 0; |
| 745 | rx_queue->removed_count = 0; |
| 746 | rx_queue->min_fill = -1U; |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 747 | efx_init_rx_recycle_ring(efx, rx_queue); |
| 748 | |
| 749 | rx_queue->page_remove = 0; |
| 750 | rx_queue->page_add = rx_queue->page_ptr_mask + 1; |
| 751 | rx_queue->page_recycle_count = 0; |
| 752 | rx_queue->page_recycle_failed = 0; |
| 753 | rx_queue->page_recycle_full = 0; |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 754 | |
| 755 | /* Initialise limit fields */ |
Steve Hodgson | ecc910f | 2010-09-10 06:42:22 +0000 | [diff] [blame] | 756 | max_fill = efx->rxq_entries - EFX_RXD_HEAD_ROOM; |
Daniel Pieczko | 1648a23 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 757 | max_trigger = |
| 758 | max_fill - efx->rx_pages_per_batch * efx->rx_bufs_per_page; |
David Riddoch | 6423518 | 2012-04-11 13:12:41 +0100 | [diff] [blame] | 759 | if (rx_refill_threshold != 0) { |
| 760 | trigger = max_fill * min(rx_refill_threshold, 100U) / 100U; |
| 761 | if (trigger > max_trigger) |
| 762 | trigger = max_trigger; |
| 763 | } else { |
| 764 | trigger = max_trigger; |
| 765 | } |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 766 | |
| 767 | rx_queue->max_fill = max_fill; |
| 768 | rx_queue->fast_fill_trigger = trigger; |
Ben Hutchings | d8aec74 | 2013-05-27 16:52:54 +0100 | [diff] [blame] | 769 | rx_queue->refill_enabled = true; |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 770 | |
| 771 | /* Set up RX descriptor ring */ |
Ben Hutchings | 152b6a6 | 2009-11-29 03:43:56 +0000 | [diff] [blame] | 772 | efx_nic_init_rx(rx_queue); |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 773 | } |
| 774 | |
| 775 | void efx_fini_rx_queue(struct efx_rx_queue *rx_queue) |
| 776 | { |
| 777 | int i; |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 778 | struct efx_nic *efx = rx_queue->efx; |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 779 | struct efx_rx_buffer *rx_buf; |
| 780 | |
Ben Hutchings | 62776d0 | 2010-06-23 11:30:07 +0000 | [diff] [blame] | 781 | netif_dbg(rx_queue->efx, drv, rx_queue->efx->net_dev, |
Ben Hutchings | ba1e8a3 | 2010-09-10 06:41:36 +0000 | [diff] [blame] | 782 | "shutting down RX queue %d\n", efx_rx_queue_index(rx_queue)); |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 783 | |
Steve Hodgson | 90d683a | 2010-06-01 11:19:39 +0000 | [diff] [blame] | 784 | del_timer_sync(&rx_queue->slow_fill); |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 785 | |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 786 | /* Release RX buffers from the current read ptr to the write ptr */ |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 787 | if (rx_queue->buffer) { |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 788 | for (i = rx_queue->removed_count; i < rx_queue->added_count; |
| 789 | i++) { |
| 790 | unsigned index = i & rx_queue->ptr_mask; |
| 791 | rx_buf = efx_rx_buffer(rx_queue, index); |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 792 | efx_fini_rx_buffer(rx_queue, rx_buf); |
| 793 | } |
| 794 | } |
Daniel Pieczko | 2768935 | 2013-02-13 10:54:41 +0000 | [diff] [blame] | 795 | |
| 796 | /* Unmap and release the pages in the recycle ring. Remove the ring. */ |
| 797 | for (i = 0; i <= rx_queue->page_ptr_mask; i++) { |
| 798 | struct page *page = rx_queue->page_ring[i]; |
| 799 | struct efx_rx_page_state *state; |
| 800 | |
| 801 | if (page == NULL) |
| 802 | continue; |
| 803 | |
| 804 | state = page_address(page); |
| 805 | dma_unmap_page(&efx->pci_dev->dev, state->dma_addr, |
| 806 | PAGE_SIZE << efx->rx_buffer_order, |
| 807 | DMA_FROM_DEVICE); |
| 808 | put_page(page); |
| 809 | } |
| 810 | kfree(rx_queue->page_ring); |
| 811 | rx_queue->page_ring = NULL; |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 812 | } |
| 813 | |
| 814 | void efx_remove_rx_queue(struct efx_rx_queue *rx_queue) |
| 815 | { |
Ben Hutchings | 62776d0 | 2010-06-23 11:30:07 +0000 | [diff] [blame] | 816 | netif_dbg(rx_queue->efx, drv, rx_queue->efx->net_dev, |
Ben Hutchings | ba1e8a3 | 2010-09-10 06:41:36 +0000 | [diff] [blame] | 817 | "destroying RX queue %d\n", efx_rx_queue_index(rx_queue)); |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 818 | |
Ben Hutchings | 152b6a6 | 2009-11-29 03:43:56 +0000 | [diff] [blame] | 819 | efx_nic_remove_rx(rx_queue); |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 820 | |
| 821 | kfree(rx_queue->buffer); |
| 822 | rx_queue->buffer = NULL; |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 823 | } |
| 824 | |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 825 | |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 826 | module_param(rx_refill_threshold, uint, 0444); |
| 827 | MODULE_PARM_DESC(rx_refill_threshold, |
David Riddoch | 6423518 | 2012-04-11 13:12:41 +0100 | [diff] [blame] | 828 | "RX descriptor ring refill threshold (%)"); |
Ben Hutchings | 8ceee66 | 2008-04-27 12:55:59 +0100 | [diff] [blame] | 829 | |
Ben Hutchings | add7247 | 2012-11-08 01:46:53 +0000 | [diff] [blame] | 830 | #ifdef CONFIG_RFS_ACCEL |
| 831 | |
| 832 | int efx_filter_rfs(struct net_device *net_dev, const struct sk_buff *skb, |
| 833 | u16 rxq_index, u32 flow_id) |
| 834 | { |
| 835 | struct efx_nic *efx = netdev_priv(net_dev); |
| 836 | struct efx_channel *channel; |
| 837 | struct efx_filter_spec spec; |
Ben Hutchings | add7247 | 2012-11-08 01:46:53 +0000 | [diff] [blame] | 838 | const __be16 *ports; |
Ben Hutchings | c47b2d9 | 2013-09-03 17:22:23 +0100 | [diff] [blame] | 839 | __be16 ether_type; |
Ben Hutchings | add7247 | 2012-11-08 01:46:53 +0000 | [diff] [blame] | 840 | int nhoff; |
| 841 | int rc; |
| 842 | |
Ben Hutchings | c47b2d9 | 2013-09-03 17:22:23 +0100 | [diff] [blame] | 843 | /* The core RPS/RFS code has already parsed and validated |
| 844 | * VLAN, IP and transport headers. We assume they are in the |
| 845 | * header area. |
| 846 | */ |
Ben Hutchings | add7247 | 2012-11-08 01:46:53 +0000 | [diff] [blame] | 847 | |
| 848 | if (skb->protocol == htons(ETH_P_8021Q)) { |
Ben Hutchings | c47b2d9 | 2013-09-03 17:22:23 +0100 | [diff] [blame] | 849 | const struct vlan_hdr *vh = |
| 850 | (const struct vlan_hdr *)skb->data; |
Ben Hutchings | add7247 | 2012-11-08 01:46:53 +0000 | [diff] [blame] | 851 | |
Ben Hutchings | c47b2d9 | 2013-09-03 17:22:23 +0100 | [diff] [blame] | 852 | /* We can't filter on the IP 5-tuple and the vlan |
| 853 | * together, so just strip the vlan header and filter |
| 854 | * on the IP part. |
Ben Hutchings | add7247 | 2012-11-08 01:46:53 +0000 | [diff] [blame] | 855 | */ |
Ben Hutchings | c47b2d9 | 2013-09-03 17:22:23 +0100 | [diff] [blame] | 856 | EFX_BUG_ON_PARANOID(skb_headlen(skb) < sizeof(*vh)); |
| 857 | ether_type = vh->h_vlan_encapsulated_proto; |
| 858 | nhoff = sizeof(struct vlan_hdr); |
| 859 | } else { |
| 860 | ether_type = skb->protocol; |
| 861 | nhoff = 0; |
Ben Hutchings | add7247 | 2012-11-08 01:46:53 +0000 | [diff] [blame] | 862 | } |
| 863 | |
Ben Hutchings | c47b2d9 | 2013-09-03 17:22:23 +0100 | [diff] [blame] | 864 | if (ether_type != htons(ETH_P_IP) && ether_type != htons(ETH_P_IPV6)) |
Ben Hutchings | add7247 | 2012-11-08 01:46:53 +0000 | [diff] [blame] | 865 | return -EPROTONOSUPPORT; |
Ben Hutchings | add7247 | 2012-11-08 01:46:53 +0000 | [diff] [blame] | 866 | |
| 867 | efx_filter_init_rx(&spec, EFX_FILTER_PRI_HINT, |
| 868 | efx->rx_scatter ? EFX_FILTER_FLAG_RX_SCATTER : 0, |
| 869 | rxq_index); |
Ben Hutchings | c47b2d9 | 2013-09-03 17:22:23 +0100 | [diff] [blame] | 870 | spec.match_flags = |
| 871 | EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO | |
| 872 | EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT | |
| 873 | EFX_FILTER_MATCH_REM_HOST | EFX_FILTER_MATCH_REM_PORT; |
| 874 | spec.ether_type = ether_type; |
| 875 | |
| 876 | if (ether_type == htons(ETH_P_IP)) { |
| 877 | const struct iphdr *ip = |
| 878 | (const struct iphdr *)(skb->data + nhoff); |
| 879 | |
| 880 | EFX_BUG_ON_PARANOID(skb_headlen(skb) < nhoff + sizeof(*ip)); |
| 881 | if (ip_is_fragment(ip)) |
| 882 | return -EPROTONOSUPPORT; |
| 883 | spec.ip_proto = ip->protocol; |
| 884 | spec.rem_host[0] = ip->saddr; |
| 885 | spec.loc_host[0] = ip->daddr; |
| 886 | EFX_BUG_ON_PARANOID(skb_headlen(skb) < nhoff + 4 * ip->ihl + 4); |
| 887 | ports = (const __be16 *)(skb->data + nhoff + 4 * ip->ihl); |
| 888 | } else { |
| 889 | const struct ipv6hdr *ip6 = |
| 890 | (const struct ipv6hdr *)(skb->data + nhoff); |
| 891 | |
| 892 | EFX_BUG_ON_PARANOID(skb_headlen(skb) < |
| 893 | nhoff + sizeof(*ip6) + 4); |
| 894 | spec.ip_proto = ip6->nexthdr; |
| 895 | memcpy(spec.rem_host, &ip6->saddr, sizeof(ip6->saddr)); |
| 896 | memcpy(spec.loc_host, &ip6->daddr, sizeof(ip6->daddr)); |
| 897 | ports = (const __be16 *)(ip6 + 1); |
| 898 | } |
| 899 | |
| 900 | spec.rem_port = ports[0]; |
| 901 | spec.loc_port = ports[1]; |
Ben Hutchings | add7247 | 2012-11-08 01:46:53 +0000 | [diff] [blame] | 902 | |
| 903 | rc = efx->type->filter_rfs_insert(efx, &spec); |
| 904 | if (rc < 0) |
| 905 | return rc; |
| 906 | |
| 907 | /* Remember this so we can check whether to expire the filter later */ |
| 908 | efx->rps_flow_id[rc] = flow_id; |
| 909 | channel = efx_get_channel(efx, skb_get_rx_queue(skb)); |
| 910 | ++channel->rfs_filters_added; |
| 911 | |
Ben Hutchings | c47b2d9 | 2013-09-03 17:22:23 +0100 | [diff] [blame] | 912 | if (ether_type == htons(ETH_P_IP)) |
| 913 | netif_info(efx, rx_status, efx->net_dev, |
| 914 | "steering %s %pI4:%u:%pI4:%u to queue %u [flow %u filter %d]\n", |
| 915 | (spec.ip_proto == IPPROTO_TCP) ? "TCP" : "UDP", |
| 916 | spec.rem_host, ntohs(ports[0]), spec.loc_host, |
| 917 | ntohs(ports[1]), rxq_index, flow_id, rc); |
| 918 | else |
| 919 | netif_info(efx, rx_status, efx->net_dev, |
| 920 | "steering %s [%pI6]:%u:[%pI6]:%u to queue %u [flow %u filter %d]\n", |
| 921 | (spec.ip_proto == IPPROTO_TCP) ? "TCP" : "UDP", |
| 922 | spec.rem_host, ntohs(ports[0]), spec.loc_host, |
| 923 | ntohs(ports[1]), rxq_index, flow_id, rc); |
Ben Hutchings | add7247 | 2012-11-08 01:46:53 +0000 | [diff] [blame] | 924 | |
| 925 | return rc; |
| 926 | } |
| 927 | |
| 928 | bool __efx_filter_rfs_expire(struct efx_nic *efx, unsigned int quota) |
| 929 | { |
| 930 | bool (*expire_one)(struct efx_nic *efx, u32 flow_id, unsigned int index); |
| 931 | unsigned int index, size; |
| 932 | u32 flow_id; |
| 933 | |
| 934 | if (!spin_trylock_bh(&efx->filter_lock)) |
| 935 | return false; |
| 936 | |
| 937 | expire_one = efx->type->filter_rfs_expire_one; |
| 938 | index = efx->rps_expire_index; |
| 939 | size = efx->type->max_rx_ip_filters; |
| 940 | while (quota--) { |
| 941 | flow_id = efx->rps_flow_id[index]; |
| 942 | if (expire_one(efx, flow_id, index)) |
| 943 | netif_info(efx, rx_status, efx->net_dev, |
| 944 | "expired filter %d [flow %u]\n", |
| 945 | index, flow_id); |
| 946 | if (++index == size) |
| 947 | index = 0; |
| 948 | } |
| 949 | efx->rps_expire_index = index; |
| 950 | |
| 951 | spin_unlock_bh(&efx->filter_lock); |
| 952 | return true; |
| 953 | } |
| 954 | |
| 955 | #endif /* CONFIG_RFS_ACCEL */ |
Ben Hutchings | b883d0b | 2013-01-15 22:00:07 +0000 | [diff] [blame] | 956 | |
| 957 | /** |
| 958 | * efx_filter_is_mc_recipient - test whether spec is a multicast recipient |
| 959 | * @spec: Specification to test |
| 960 | * |
| 961 | * Return: %true if the specification is a non-drop RX filter that |
| 962 | * matches a local MAC address I/G bit value of 1 or matches a local |
| 963 | * IPv4 or IPv6 address value in the respective multicast address |
| 964 | * range. Otherwise %false. |
| 965 | */ |
| 966 | bool efx_filter_is_mc_recipient(const struct efx_filter_spec *spec) |
| 967 | { |
| 968 | if (!(spec->flags & EFX_FILTER_FLAG_RX) || |
| 969 | spec->dmaq_id == EFX_FILTER_RX_DMAQ_ID_DROP) |
| 970 | return false; |
| 971 | |
| 972 | if (spec->match_flags & |
| 973 | (EFX_FILTER_MATCH_LOC_MAC | EFX_FILTER_MATCH_LOC_MAC_IG) && |
| 974 | is_multicast_ether_addr(spec->loc_mac)) |
| 975 | return true; |
| 976 | |
| 977 | if ((spec->match_flags & |
| 978 | (EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_LOC_HOST)) == |
| 979 | (EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_LOC_HOST)) { |
| 980 | if (spec->ether_type == htons(ETH_P_IP) && |
| 981 | ipv4_is_multicast(spec->loc_host[0])) |
| 982 | return true; |
| 983 | if (spec->ether_type == htons(ETH_P_IPV6) && |
| 984 | ((const u8 *)spec->loc_host)[0] == 0xff) |
| 985 | return true; |
| 986 | } |
| 987 | |
| 988 | return false; |
| 989 | } |