Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 1 | /******************************************************************************* |
| 2 | * |
| 3 | * Intel Ethernet Controller XL710 Family Linux Virtual Function Driver |
Jesse Brandeburg | af1a2a9 | 2014-02-13 03:48:41 -0800 | [diff] [blame] | 4 | * Copyright(c) 2013 - 2014 Intel Corporation. |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 5 | * |
| 6 | * This program is free software; you can redistribute it and/or modify it |
| 7 | * under the terms and conditions of the GNU General Public License, |
| 8 | * version 2, as published by the Free Software Foundation. |
| 9 | * |
| 10 | * This program is distributed in the hope it will be useful, but WITHOUT |
| 11 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
| 12 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for |
| 13 | * more details. |
| 14 | * |
Jesse Brandeburg | b831607 | 2014-04-05 07:46:11 +0000 | [diff] [blame] | 15 | * You should have received a copy of the GNU General Public License along |
| 16 | * with this program. If not, see <http://www.gnu.org/licenses/>. |
| 17 | * |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 18 | * The full GNU General Public License is included in this distribution in |
| 19 | * the file called "COPYING". |
| 20 | * |
| 21 | * Contact Information: |
| 22 | * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net> |
| 23 | * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497 |
| 24 | * |
| 25 | ******************************************************************************/ |
| 26 | |
Paul Gortmaker | 7ed3f5f | 2014-01-11 04:00:31 +0000 | [diff] [blame] | 27 | #include <linux/prefetch.h> |
Mitch Williams | a132af2 | 2015-01-24 09:58:35 +0000 | [diff] [blame] | 28 | #include <net/busy_poll.h> |
Paul Gortmaker | 7ed3f5f | 2014-01-11 04:00:31 +0000 | [diff] [blame] | 29 | |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 30 | #include "i40evf.h" |
Jesse Brandeburg | 206812b | 2014-02-12 01:45:33 +0000 | [diff] [blame] | 31 | #include "i40e_prototype.h" |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 32 | |
| 33 | static inline __le64 build_ctob(u32 td_cmd, u32 td_offset, unsigned int size, |
| 34 | u32 td_tag) |
| 35 | { |
| 36 | return cpu_to_le64(I40E_TX_DESC_DTYPE_DATA | |
| 37 | ((u64)td_cmd << I40E_TXD_QW1_CMD_SHIFT) | |
| 38 | ((u64)td_offset << I40E_TXD_QW1_OFFSET_SHIFT) | |
| 39 | ((u64)size << I40E_TXD_QW1_TX_BUF_SZ_SHIFT) | |
| 40 | ((u64)td_tag << I40E_TXD_QW1_L2TAG1_SHIFT)); |
| 41 | } |
| 42 | |
| 43 | #define I40E_TXD_CMD (I40E_TX_DESC_CMD_EOP | I40E_TX_DESC_CMD_RS) |
| 44 | |
| 45 | /** |
| 46 | * i40e_unmap_and_free_tx_resource - Release a Tx buffer |
| 47 | * @ring: the ring that owns the buffer |
| 48 | * @tx_buffer: the buffer to free |
| 49 | **/ |
| 50 | static void i40e_unmap_and_free_tx_resource(struct i40e_ring *ring, |
| 51 | struct i40e_tx_buffer *tx_buffer) |
| 52 | { |
| 53 | if (tx_buffer->skb) { |
Anjali Singhai Jain | 49d7d93 | 2014-06-04 08:45:15 +0000 | [diff] [blame] | 54 | if (tx_buffer->tx_flags & I40E_TX_FLAGS_FD_SB) |
| 55 | kfree(tx_buffer->raw_buf); |
| 56 | else |
| 57 | dev_kfree_skb_any(tx_buffer->skb); |
| 58 | |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 59 | if (dma_unmap_len(tx_buffer, len)) |
| 60 | dma_unmap_single(ring->dev, |
| 61 | dma_unmap_addr(tx_buffer, dma), |
| 62 | dma_unmap_len(tx_buffer, len), |
| 63 | DMA_TO_DEVICE); |
| 64 | } else if (dma_unmap_len(tx_buffer, len)) { |
| 65 | dma_unmap_page(ring->dev, |
| 66 | dma_unmap_addr(tx_buffer, dma), |
| 67 | dma_unmap_len(tx_buffer, len), |
| 68 | DMA_TO_DEVICE); |
| 69 | } |
| 70 | tx_buffer->next_to_watch = NULL; |
| 71 | tx_buffer->skb = NULL; |
| 72 | dma_unmap_len_set(tx_buffer, len, 0); |
| 73 | /* tx_buffer must be completely set up in the transmit path */ |
| 74 | } |
| 75 | |
| 76 | /** |
| 77 | * i40evf_clean_tx_ring - Free any empty Tx buffers |
| 78 | * @tx_ring: ring to be cleaned |
| 79 | **/ |
| 80 | void i40evf_clean_tx_ring(struct i40e_ring *tx_ring) |
| 81 | { |
| 82 | unsigned long bi_size; |
| 83 | u16 i; |
| 84 | |
| 85 | /* ring already cleared, nothing to do */ |
| 86 | if (!tx_ring->tx_bi) |
| 87 | return; |
| 88 | |
| 89 | /* Free all the Tx ring sk_buffs */ |
| 90 | for (i = 0; i < tx_ring->count; i++) |
| 91 | i40e_unmap_and_free_tx_resource(tx_ring, &tx_ring->tx_bi[i]); |
| 92 | |
| 93 | bi_size = sizeof(struct i40e_tx_buffer) * tx_ring->count; |
| 94 | memset(tx_ring->tx_bi, 0, bi_size); |
| 95 | |
| 96 | /* Zero out the descriptor ring */ |
| 97 | memset(tx_ring->desc, 0, tx_ring->size); |
| 98 | |
| 99 | tx_ring->next_to_use = 0; |
| 100 | tx_ring->next_to_clean = 0; |
| 101 | |
| 102 | if (!tx_ring->netdev) |
| 103 | return; |
| 104 | |
| 105 | /* cleanup Tx queue statistics */ |
| 106 | netdev_tx_reset_queue(netdev_get_tx_queue(tx_ring->netdev, |
| 107 | tx_ring->queue_index)); |
| 108 | } |
| 109 | |
| 110 | /** |
| 111 | * i40evf_free_tx_resources - Free Tx resources per queue |
| 112 | * @tx_ring: Tx descriptor ring for a specific queue |
| 113 | * |
| 114 | * Free all transmit software resources |
| 115 | **/ |
| 116 | void i40evf_free_tx_resources(struct i40e_ring *tx_ring) |
| 117 | { |
| 118 | i40evf_clean_tx_ring(tx_ring); |
| 119 | kfree(tx_ring->tx_bi); |
| 120 | tx_ring->tx_bi = NULL; |
| 121 | |
| 122 | if (tx_ring->desc) { |
| 123 | dma_free_coherent(tx_ring->dev, tx_ring->size, |
| 124 | tx_ring->desc, tx_ring->dma); |
| 125 | tx_ring->desc = NULL; |
| 126 | } |
| 127 | } |
| 128 | |
| 129 | /** |
Jesse Brandeburg | a68de58 | 2015-02-24 05:26:03 +0000 | [diff] [blame] | 130 | * i40e_get_head - Retrieve head from head writeback |
| 131 | * @tx_ring: tx ring to fetch head of |
| 132 | * |
| 133 | * Returns value of Tx ring head based on value stored |
| 134 | * in head write-back location |
| 135 | **/ |
| 136 | static inline u32 i40e_get_head(struct i40e_ring *tx_ring) |
| 137 | { |
| 138 | void *head = (struct i40e_tx_desc *)tx_ring->desc + tx_ring->count; |
| 139 | |
| 140 | return le32_to_cpu(*(volatile __le32 *)head); |
| 141 | } |
| 142 | |
| 143 | /** |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 144 | * i40e_get_tx_pending - how many tx descriptors not processed |
| 145 | * @tx_ring: the ring of descriptors |
| 146 | * |
| 147 | * Since there is no access to the ring head register |
| 148 | * in XL710, we need to use our local copies |
| 149 | **/ |
| 150 | static u32 i40e_get_tx_pending(struct i40e_ring *ring) |
| 151 | { |
Jesse Brandeburg | a68de58 | 2015-02-24 05:26:03 +0000 | [diff] [blame] | 152 | u32 head, tail; |
| 153 | |
| 154 | head = i40e_get_head(ring); |
| 155 | tail = readl(ring->tail); |
| 156 | |
| 157 | if (head != tail) |
| 158 | return (head < tail) ? |
| 159 | tail - head : (tail + ring->count - head); |
| 160 | |
| 161 | return 0; |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 162 | } |
| 163 | |
| 164 | /** |
| 165 | * i40e_check_tx_hang - Is there a hang in the Tx queue |
| 166 | * @tx_ring: the ring of descriptors |
| 167 | **/ |
| 168 | static bool i40e_check_tx_hang(struct i40e_ring *tx_ring) |
| 169 | { |
Jesse Brandeburg | a68de58 | 2015-02-24 05:26:03 +0000 | [diff] [blame] | 170 | u32 tx_done = tx_ring->stats.packets; |
| 171 | u32 tx_done_old = tx_ring->tx_stats.tx_done_old; |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 172 | u32 tx_pending = i40e_get_tx_pending(tx_ring); |
| 173 | bool ret = false; |
| 174 | |
| 175 | clear_check_for_tx_hang(tx_ring); |
| 176 | |
| 177 | /* Check for a hung queue, but be thorough. This verifies |
| 178 | * that a transmit has been completed since the previous |
| 179 | * check AND there is at least one packet pending. The |
| 180 | * ARMED bit is set to indicate a potential hang. The |
| 181 | * bit is cleared if a pause frame is received to remove |
| 182 | * false hang detection due to PFC or 802.3x frames. By |
| 183 | * requiring this to fail twice we avoid races with |
| 184 | * PFC clearing the ARMED bit and conditions where we |
| 185 | * run the check_tx_hang logic with a transmit completion |
| 186 | * pending but without time to complete it yet. |
| 187 | */ |
Jesse Brandeburg | a68de58 | 2015-02-24 05:26:03 +0000 | [diff] [blame] | 188 | if ((tx_done_old == tx_done) && tx_pending) { |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 189 | /* make sure it is true for two checks in a row */ |
| 190 | ret = test_and_set_bit(__I40E_HANG_CHECK_ARMED, |
| 191 | &tx_ring->state); |
Jesse Brandeburg | a68de58 | 2015-02-24 05:26:03 +0000 | [diff] [blame] | 192 | } else if (tx_done_old == tx_done && |
| 193 | (tx_pending < I40E_MIN_DESC_PENDING) && (tx_pending > 0)) { |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 194 | /* update completed stats and disarm the hang check */ |
Jesse Brandeburg | a68de58 | 2015-02-24 05:26:03 +0000 | [diff] [blame] | 195 | tx_ring->tx_stats.tx_done_old = tx_done; |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 196 | clear_bit(__I40E_HANG_CHECK_ARMED, &tx_ring->state); |
| 197 | } |
| 198 | |
| 199 | return ret; |
| 200 | } |
| 201 | |
Anjali Singhai Jain | c29af37 | 2015-01-10 01:07:19 +0000 | [diff] [blame] | 202 | #define WB_STRIDE 0x3 |
| 203 | |
Jesse Brandeburg | 1943d8b | 2014-02-14 02:14:40 +0000 | [diff] [blame] | 204 | /** |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 205 | * i40e_clean_tx_irq - Reclaim resources after transmit completes |
| 206 | * @tx_ring: tx ring to clean |
| 207 | * @budget: how many cleans we're allowed |
| 208 | * |
| 209 | * Returns true if there's any budget left (e.g. the clean is finished) |
| 210 | **/ |
| 211 | static bool i40e_clean_tx_irq(struct i40e_ring *tx_ring, int budget) |
| 212 | { |
| 213 | u16 i = tx_ring->next_to_clean; |
| 214 | struct i40e_tx_buffer *tx_buf; |
Jesse Brandeburg | 1943d8b | 2014-02-14 02:14:40 +0000 | [diff] [blame] | 215 | struct i40e_tx_desc *tx_head; |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 216 | struct i40e_tx_desc *tx_desc; |
| 217 | unsigned int total_packets = 0; |
| 218 | unsigned int total_bytes = 0; |
| 219 | |
| 220 | tx_buf = &tx_ring->tx_bi[i]; |
| 221 | tx_desc = I40E_TX_DESC(tx_ring, i); |
| 222 | i -= tx_ring->count; |
| 223 | |
Jesse Brandeburg | 1943d8b | 2014-02-14 02:14:40 +0000 | [diff] [blame] | 224 | tx_head = I40E_TX_DESC(tx_ring, i40e_get_head(tx_ring)); |
| 225 | |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 226 | do { |
| 227 | struct i40e_tx_desc *eop_desc = tx_buf->next_to_watch; |
| 228 | |
| 229 | /* if next_to_watch is not set then there is no work pending */ |
| 230 | if (!eop_desc) |
| 231 | break; |
| 232 | |
| 233 | /* prevent any other reads prior to eop_desc */ |
| 234 | read_barrier_depends(); |
| 235 | |
Jesse Brandeburg | 1943d8b | 2014-02-14 02:14:40 +0000 | [diff] [blame] | 236 | /* we have caught up to head, no work left to do */ |
| 237 | if (tx_head == tx_desc) |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 238 | break; |
| 239 | |
| 240 | /* clear next_to_watch to prevent false hangs */ |
| 241 | tx_buf->next_to_watch = NULL; |
| 242 | |
| 243 | /* update the statistics for this packet */ |
| 244 | total_bytes += tx_buf->bytecount; |
| 245 | total_packets += tx_buf->gso_segs; |
| 246 | |
| 247 | /* free the skb */ |
| 248 | dev_kfree_skb_any(tx_buf->skb); |
| 249 | |
| 250 | /* unmap skb header data */ |
| 251 | dma_unmap_single(tx_ring->dev, |
| 252 | dma_unmap_addr(tx_buf, dma), |
| 253 | dma_unmap_len(tx_buf, len), |
| 254 | DMA_TO_DEVICE); |
| 255 | |
| 256 | /* clear tx_buffer data */ |
| 257 | tx_buf->skb = NULL; |
| 258 | dma_unmap_len_set(tx_buf, len, 0); |
| 259 | |
| 260 | /* unmap remaining buffers */ |
| 261 | while (tx_desc != eop_desc) { |
| 262 | |
| 263 | tx_buf++; |
| 264 | tx_desc++; |
| 265 | i++; |
| 266 | if (unlikely(!i)) { |
| 267 | i -= tx_ring->count; |
| 268 | tx_buf = tx_ring->tx_bi; |
| 269 | tx_desc = I40E_TX_DESC(tx_ring, 0); |
| 270 | } |
| 271 | |
| 272 | /* unmap any remaining paged data */ |
| 273 | if (dma_unmap_len(tx_buf, len)) { |
| 274 | dma_unmap_page(tx_ring->dev, |
| 275 | dma_unmap_addr(tx_buf, dma), |
| 276 | dma_unmap_len(tx_buf, len), |
| 277 | DMA_TO_DEVICE); |
| 278 | dma_unmap_len_set(tx_buf, len, 0); |
| 279 | } |
| 280 | } |
| 281 | |
| 282 | /* move us one more past the eop_desc for start of next pkt */ |
| 283 | tx_buf++; |
| 284 | tx_desc++; |
| 285 | i++; |
| 286 | if (unlikely(!i)) { |
| 287 | i -= tx_ring->count; |
| 288 | tx_buf = tx_ring->tx_bi; |
| 289 | tx_desc = I40E_TX_DESC(tx_ring, 0); |
| 290 | } |
| 291 | |
Jesse Brandeburg | 016890b | 2015-02-27 09:15:31 +0000 | [diff] [blame] | 292 | prefetch(tx_desc); |
| 293 | |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 294 | /* update budget accounting */ |
| 295 | budget--; |
| 296 | } while (likely(budget)); |
| 297 | |
| 298 | i += tx_ring->count; |
| 299 | tx_ring->next_to_clean = i; |
| 300 | u64_stats_update_begin(&tx_ring->syncp); |
| 301 | tx_ring->stats.bytes += total_bytes; |
| 302 | tx_ring->stats.packets += total_packets; |
| 303 | u64_stats_update_end(&tx_ring->syncp); |
| 304 | tx_ring->q_vector->tx.total_bytes += total_bytes; |
| 305 | tx_ring->q_vector->tx.total_packets += total_packets; |
| 306 | |
Anjali Singhai Jain | c29af37 | 2015-01-10 01:07:19 +0000 | [diff] [blame] | 307 | if (budget && |
| 308 | !((i & WB_STRIDE) == WB_STRIDE) && |
| 309 | !test_bit(__I40E_DOWN, &tx_ring->vsi->state) && |
| 310 | (I40E_DESC_UNUSED(tx_ring) != tx_ring->count)) |
| 311 | tx_ring->arm_wb = true; |
| 312 | else |
| 313 | tx_ring->arm_wb = false; |
| 314 | |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 315 | if (check_for_tx_hang(tx_ring) && i40e_check_tx_hang(tx_ring)) { |
| 316 | /* schedule immediate reset if we believe we hung */ |
| 317 | dev_info(tx_ring->dev, "Detected Tx Unit Hang\n" |
| 318 | " VSI <%d>\n" |
| 319 | " Tx Queue <%d>\n" |
| 320 | " next_to_use <%x>\n" |
| 321 | " next_to_clean <%x>\n", |
| 322 | tx_ring->vsi->seid, |
| 323 | tx_ring->queue_index, |
| 324 | tx_ring->next_to_use, i); |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 325 | |
| 326 | netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index); |
| 327 | |
| 328 | dev_info(tx_ring->dev, |
| 329 | "tx hang detected on queue %d, resetting adapter\n", |
| 330 | tx_ring->queue_index); |
| 331 | |
| 332 | tx_ring->netdev->netdev_ops->ndo_tx_timeout(tx_ring->netdev); |
| 333 | |
| 334 | /* the adapter is about to reset, no point in enabling stuff */ |
| 335 | return true; |
| 336 | } |
| 337 | |
| 338 | netdev_tx_completed_queue(netdev_get_tx_queue(tx_ring->netdev, |
| 339 | tx_ring->queue_index), |
| 340 | total_packets, total_bytes); |
| 341 | |
| 342 | #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2) |
| 343 | if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) && |
| 344 | (I40E_DESC_UNUSED(tx_ring) >= TX_WAKE_THRESHOLD))) { |
| 345 | /* Make sure that anybody stopping the queue after this |
| 346 | * sees the new next_to_clean. |
| 347 | */ |
| 348 | smp_mb(); |
| 349 | if (__netif_subqueue_stopped(tx_ring->netdev, |
| 350 | tx_ring->queue_index) && |
| 351 | !test_bit(__I40E_DOWN, &tx_ring->vsi->state)) { |
| 352 | netif_wake_subqueue(tx_ring->netdev, |
| 353 | tx_ring->queue_index); |
| 354 | ++tx_ring->tx_stats.restart_queue; |
| 355 | } |
| 356 | } |
| 357 | |
| 358 | return budget > 0; |
| 359 | } |
| 360 | |
| 361 | /** |
Anjali Singhai Jain | c29af37 | 2015-01-10 01:07:19 +0000 | [diff] [blame] | 362 | * i40e_force_wb -Arm hardware to do a wb on noncache aligned descriptors |
| 363 | * @vsi: the VSI we care about |
| 364 | * @q_vector: the vector on which to force writeback |
| 365 | * |
| 366 | **/ |
| 367 | static void i40e_force_wb(struct i40e_vsi *vsi, struct i40e_q_vector *q_vector) |
| 368 | { |
| 369 | u32 val = I40E_VFINT_DYN_CTLN_INTENA_MASK | |
Jesse Brandeburg | 97bf75f | 2015-02-27 09:18:32 +0000 | [diff] [blame] | 370 | I40E_VFINT_DYN_CTLN1_ITR_INDX_MASK | /* set noitr */ |
Anjali Singhai Jain | c29af37 | 2015-01-10 01:07:19 +0000 | [diff] [blame] | 371 | I40E_VFINT_DYN_CTLN_SWINT_TRIG_MASK | |
| 372 | I40E_VFINT_DYN_CTLN_SW_ITR_INDX_ENA_MASK; |
| 373 | /* allow 00 to be written to the index */ |
| 374 | |
| 375 | wr32(&vsi->back->hw, |
| 376 | I40E_VFINT_DYN_CTLN1(q_vector->v_idx + vsi->base_vector - 1), |
| 377 | val); |
| 378 | } |
| 379 | |
| 380 | /** |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 381 | * i40e_set_new_dynamic_itr - Find new ITR level |
| 382 | * @rc: structure containing ring performance data |
| 383 | * |
| 384 | * Stores a new ITR value based on packets and byte counts during |
| 385 | * the last interrupt. The advantage of per interrupt computation |
| 386 | * is faster updates and more accurate ITR for the current traffic |
| 387 | * pattern. Constants in this function were computed based on |
| 388 | * theoretical maximum wire speed and thresholds were set based on |
| 389 | * testing data as well as attempting to minimize response time |
| 390 | * while increasing bulk throughput. |
| 391 | **/ |
| 392 | static void i40e_set_new_dynamic_itr(struct i40e_ring_container *rc) |
| 393 | { |
| 394 | enum i40e_latency_range new_latency_range = rc->latency_range; |
| 395 | u32 new_itr = rc->itr; |
| 396 | int bytes_per_int; |
| 397 | |
| 398 | if (rc->total_packets == 0 || !rc->itr) |
| 399 | return; |
| 400 | |
| 401 | /* simple throttlerate management |
| 402 | * 0-10MB/s lowest (100000 ints/s) |
| 403 | * 10-20MB/s low (20000 ints/s) |
| 404 | * 20-1249MB/s bulk (8000 ints/s) |
| 405 | */ |
| 406 | bytes_per_int = rc->total_bytes / rc->itr; |
| 407 | switch (rc->itr) { |
| 408 | case I40E_LOWEST_LATENCY: |
| 409 | if (bytes_per_int > 10) |
| 410 | new_latency_range = I40E_LOW_LATENCY; |
| 411 | break; |
| 412 | case I40E_LOW_LATENCY: |
| 413 | if (bytes_per_int > 20) |
| 414 | new_latency_range = I40E_BULK_LATENCY; |
| 415 | else if (bytes_per_int <= 10) |
| 416 | new_latency_range = I40E_LOWEST_LATENCY; |
| 417 | break; |
| 418 | case I40E_BULK_LATENCY: |
| 419 | if (bytes_per_int <= 20) |
| 420 | rc->latency_range = I40E_LOW_LATENCY; |
| 421 | break; |
| 422 | } |
| 423 | |
| 424 | switch (new_latency_range) { |
| 425 | case I40E_LOWEST_LATENCY: |
| 426 | new_itr = I40E_ITR_100K; |
| 427 | break; |
| 428 | case I40E_LOW_LATENCY: |
| 429 | new_itr = I40E_ITR_20K; |
| 430 | break; |
| 431 | case I40E_BULK_LATENCY: |
| 432 | new_itr = I40E_ITR_8K; |
| 433 | break; |
| 434 | default: |
| 435 | break; |
| 436 | } |
| 437 | |
| 438 | if (new_itr != rc->itr) { |
| 439 | /* do an exponential smoothing */ |
| 440 | new_itr = (10 * new_itr * rc->itr) / |
| 441 | ((9 * new_itr) + rc->itr); |
| 442 | rc->itr = new_itr & I40E_MAX_ITR; |
| 443 | } |
| 444 | |
| 445 | rc->total_bytes = 0; |
| 446 | rc->total_packets = 0; |
| 447 | } |
| 448 | |
| 449 | /** |
| 450 | * i40e_update_dynamic_itr - Adjust ITR based on bytes per int |
| 451 | * @q_vector: the vector to adjust |
| 452 | **/ |
| 453 | static void i40e_update_dynamic_itr(struct i40e_q_vector *q_vector) |
| 454 | { |
| 455 | u16 vector = q_vector->vsi->base_vector + q_vector->v_idx; |
| 456 | struct i40e_hw *hw = &q_vector->vsi->back->hw; |
| 457 | u32 reg_addr; |
| 458 | u16 old_itr; |
| 459 | |
| 460 | reg_addr = I40E_VFINT_ITRN1(I40E_RX_ITR, vector - 1); |
| 461 | old_itr = q_vector->rx.itr; |
| 462 | i40e_set_new_dynamic_itr(&q_vector->rx); |
| 463 | if (old_itr != q_vector->rx.itr) |
| 464 | wr32(hw, reg_addr, q_vector->rx.itr); |
| 465 | |
| 466 | reg_addr = I40E_VFINT_ITRN1(I40E_TX_ITR, vector - 1); |
| 467 | old_itr = q_vector->tx.itr; |
| 468 | i40e_set_new_dynamic_itr(&q_vector->tx); |
| 469 | if (old_itr != q_vector->tx.itr) |
| 470 | wr32(hw, reg_addr, q_vector->tx.itr); |
| 471 | } |
| 472 | |
| 473 | /** |
| 474 | * i40evf_setup_tx_descriptors - Allocate the Tx descriptors |
| 475 | * @tx_ring: the tx ring to set up |
| 476 | * |
| 477 | * Return 0 on success, negative on error |
| 478 | **/ |
| 479 | int i40evf_setup_tx_descriptors(struct i40e_ring *tx_ring) |
| 480 | { |
| 481 | struct device *dev = tx_ring->dev; |
| 482 | int bi_size; |
| 483 | |
| 484 | if (!dev) |
| 485 | return -ENOMEM; |
| 486 | |
Mitch Williams | 67c818a | 2015-06-19 08:56:30 -0700 | [diff] [blame^] | 487 | /* warn if we are about to overwrite the pointer */ |
| 488 | WARN_ON(tx_ring->tx_bi); |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 489 | bi_size = sizeof(struct i40e_tx_buffer) * tx_ring->count; |
| 490 | tx_ring->tx_bi = kzalloc(bi_size, GFP_KERNEL); |
| 491 | if (!tx_ring->tx_bi) |
| 492 | goto err; |
| 493 | |
| 494 | /* round up to nearest 4K */ |
| 495 | tx_ring->size = tx_ring->count * sizeof(struct i40e_tx_desc); |
Jesse Brandeburg | 1943d8b | 2014-02-14 02:14:40 +0000 | [diff] [blame] | 496 | /* add u32 for head writeback, align after this takes care of |
| 497 | * guaranteeing this is at least one cache line in size |
| 498 | */ |
| 499 | tx_ring->size += sizeof(u32); |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 500 | tx_ring->size = ALIGN(tx_ring->size, 4096); |
| 501 | tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size, |
| 502 | &tx_ring->dma, GFP_KERNEL); |
| 503 | if (!tx_ring->desc) { |
| 504 | dev_info(dev, "Unable to allocate memory for the Tx descriptor ring, size=%d\n", |
| 505 | tx_ring->size); |
| 506 | goto err; |
| 507 | } |
| 508 | |
| 509 | tx_ring->next_to_use = 0; |
| 510 | tx_ring->next_to_clean = 0; |
| 511 | return 0; |
| 512 | |
| 513 | err: |
| 514 | kfree(tx_ring->tx_bi); |
| 515 | tx_ring->tx_bi = NULL; |
| 516 | return -ENOMEM; |
| 517 | } |
| 518 | |
| 519 | /** |
| 520 | * i40evf_clean_rx_ring - Free Rx buffers |
| 521 | * @rx_ring: ring to be cleaned |
| 522 | **/ |
| 523 | void i40evf_clean_rx_ring(struct i40e_ring *rx_ring) |
| 524 | { |
| 525 | struct device *dev = rx_ring->dev; |
| 526 | struct i40e_rx_buffer *rx_bi; |
| 527 | unsigned long bi_size; |
| 528 | u16 i; |
| 529 | |
| 530 | /* ring already cleared, nothing to do */ |
| 531 | if (!rx_ring->rx_bi) |
| 532 | return; |
| 533 | |
Mitch Williams | a132af2 | 2015-01-24 09:58:35 +0000 | [diff] [blame] | 534 | if (ring_is_ps_enabled(rx_ring)) { |
| 535 | int bufsz = ALIGN(rx_ring->rx_hdr_len, 256) * rx_ring->count; |
| 536 | |
| 537 | rx_bi = &rx_ring->rx_bi[0]; |
| 538 | if (rx_bi->hdr_buf) { |
| 539 | dma_free_coherent(dev, |
| 540 | bufsz, |
| 541 | rx_bi->hdr_buf, |
| 542 | rx_bi->dma); |
| 543 | for (i = 0; i < rx_ring->count; i++) { |
| 544 | rx_bi = &rx_ring->rx_bi[i]; |
| 545 | rx_bi->dma = 0; |
Shannon Nelson | 37a2973 | 2015-02-27 09:15:19 +0000 | [diff] [blame] | 546 | rx_bi->hdr_buf = NULL; |
Mitch Williams | a132af2 | 2015-01-24 09:58:35 +0000 | [diff] [blame] | 547 | } |
| 548 | } |
| 549 | } |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 550 | /* Free all the Rx ring sk_buffs */ |
| 551 | for (i = 0; i < rx_ring->count; i++) { |
| 552 | rx_bi = &rx_ring->rx_bi[i]; |
| 553 | if (rx_bi->dma) { |
| 554 | dma_unmap_single(dev, |
| 555 | rx_bi->dma, |
| 556 | rx_ring->rx_buf_len, |
| 557 | DMA_FROM_DEVICE); |
| 558 | rx_bi->dma = 0; |
| 559 | } |
| 560 | if (rx_bi->skb) { |
| 561 | dev_kfree_skb(rx_bi->skb); |
| 562 | rx_bi->skb = NULL; |
| 563 | } |
| 564 | if (rx_bi->page) { |
| 565 | if (rx_bi->page_dma) { |
| 566 | dma_unmap_page(dev, |
| 567 | rx_bi->page_dma, |
| 568 | PAGE_SIZE / 2, |
| 569 | DMA_FROM_DEVICE); |
| 570 | rx_bi->page_dma = 0; |
| 571 | } |
| 572 | __free_page(rx_bi->page); |
| 573 | rx_bi->page = NULL; |
| 574 | rx_bi->page_offset = 0; |
| 575 | } |
| 576 | } |
| 577 | |
| 578 | bi_size = sizeof(struct i40e_rx_buffer) * rx_ring->count; |
| 579 | memset(rx_ring->rx_bi, 0, bi_size); |
| 580 | |
| 581 | /* Zero out the descriptor ring */ |
| 582 | memset(rx_ring->desc, 0, rx_ring->size); |
| 583 | |
| 584 | rx_ring->next_to_clean = 0; |
| 585 | rx_ring->next_to_use = 0; |
| 586 | } |
| 587 | |
| 588 | /** |
| 589 | * i40evf_free_rx_resources - Free Rx resources |
| 590 | * @rx_ring: ring to clean the resources from |
| 591 | * |
| 592 | * Free all receive software resources |
| 593 | **/ |
| 594 | void i40evf_free_rx_resources(struct i40e_ring *rx_ring) |
| 595 | { |
| 596 | i40evf_clean_rx_ring(rx_ring); |
| 597 | kfree(rx_ring->rx_bi); |
| 598 | rx_ring->rx_bi = NULL; |
| 599 | |
| 600 | if (rx_ring->desc) { |
| 601 | dma_free_coherent(rx_ring->dev, rx_ring->size, |
| 602 | rx_ring->desc, rx_ring->dma); |
| 603 | rx_ring->desc = NULL; |
| 604 | } |
| 605 | } |
| 606 | |
| 607 | /** |
Mitch Williams | a132af2 | 2015-01-24 09:58:35 +0000 | [diff] [blame] | 608 | * i40evf_alloc_rx_headers - allocate rx header buffers |
| 609 | * @rx_ring: ring to alloc buffers |
| 610 | * |
| 611 | * Allocate rx header buffers for the entire ring. As these are static, |
| 612 | * this is only called when setting up a new ring. |
| 613 | **/ |
| 614 | void i40evf_alloc_rx_headers(struct i40e_ring *rx_ring) |
| 615 | { |
| 616 | struct device *dev = rx_ring->dev; |
| 617 | struct i40e_rx_buffer *rx_bi; |
| 618 | dma_addr_t dma; |
| 619 | void *buffer; |
| 620 | int buf_size; |
| 621 | int i; |
| 622 | |
| 623 | if (rx_ring->rx_bi[0].hdr_buf) |
| 624 | return; |
| 625 | /* Make sure the buffers don't cross cache line boundaries. */ |
| 626 | buf_size = ALIGN(rx_ring->rx_hdr_len, 256); |
| 627 | buffer = dma_alloc_coherent(dev, buf_size * rx_ring->count, |
| 628 | &dma, GFP_KERNEL); |
| 629 | if (!buffer) |
| 630 | return; |
| 631 | for (i = 0; i < rx_ring->count; i++) { |
| 632 | rx_bi = &rx_ring->rx_bi[i]; |
| 633 | rx_bi->dma = dma + (i * buf_size); |
| 634 | rx_bi->hdr_buf = buffer + (i * buf_size); |
| 635 | } |
| 636 | } |
| 637 | |
| 638 | /** |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 639 | * i40evf_setup_rx_descriptors - Allocate Rx descriptors |
| 640 | * @rx_ring: Rx descriptor ring (for a specific queue) to setup |
| 641 | * |
| 642 | * Returns 0 on success, negative on failure |
| 643 | **/ |
| 644 | int i40evf_setup_rx_descriptors(struct i40e_ring *rx_ring) |
| 645 | { |
| 646 | struct device *dev = rx_ring->dev; |
| 647 | int bi_size; |
| 648 | |
Mitch Williams | 67c818a | 2015-06-19 08:56:30 -0700 | [diff] [blame^] | 649 | /* warn if we are about to overwrite the pointer */ |
| 650 | WARN_ON(rx_ring->rx_bi); |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 651 | bi_size = sizeof(struct i40e_rx_buffer) * rx_ring->count; |
| 652 | rx_ring->rx_bi = kzalloc(bi_size, GFP_KERNEL); |
| 653 | if (!rx_ring->rx_bi) |
| 654 | goto err; |
| 655 | |
Carolyn Wyborny | f217d6c | 2015-02-09 17:42:31 -0800 | [diff] [blame] | 656 | u64_stats_init(&rx_ring->syncp); |
Carolyn Wyborny | 638702b | 2015-01-24 09:58:32 +0000 | [diff] [blame] | 657 | |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 658 | /* Round up to nearest 4K */ |
| 659 | rx_ring->size = ring_is_16byte_desc_enabled(rx_ring) |
| 660 | ? rx_ring->count * sizeof(union i40e_16byte_rx_desc) |
| 661 | : rx_ring->count * sizeof(union i40e_32byte_rx_desc); |
| 662 | rx_ring->size = ALIGN(rx_ring->size, 4096); |
| 663 | rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size, |
| 664 | &rx_ring->dma, GFP_KERNEL); |
| 665 | |
| 666 | if (!rx_ring->desc) { |
| 667 | dev_info(dev, "Unable to allocate memory for the Rx descriptor ring, size=%d\n", |
| 668 | rx_ring->size); |
| 669 | goto err; |
| 670 | } |
| 671 | |
| 672 | rx_ring->next_to_clean = 0; |
| 673 | rx_ring->next_to_use = 0; |
| 674 | |
| 675 | return 0; |
| 676 | err: |
| 677 | kfree(rx_ring->rx_bi); |
| 678 | rx_ring->rx_bi = NULL; |
| 679 | return -ENOMEM; |
| 680 | } |
| 681 | |
| 682 | /** |
| 683 | * i40e_release_rx_desc - Store the new tail and head values |
| 684 | * @rx_ring: ring to bump |
| 685 | * @val: new head index |
| 686 | **/ |
| 687 | static inline void i40e_release_rx_desc(struct i40e_ring *rx_ring, u32 val) |
| 688 | { |
| 689 | rx_ring->next_to_use = val; |
| 690 | /* Force memory writes to complete before letting h/w |
| 691 | * know there are new descriptors to fetch. (Only |
| 692 | * applicable for weak-ordered memory model archs, |
| 693 | * such as IA-64). |
| 694 | */ |
| 695 | wmb(); |
| 696 | writel(val, rx_ring->tail); |
| 697 | } |
| 698 | |
| 699 | /** |
Mitch Williams | a132af2 | 2015-01-24 09:58:35 +0000 | [diff] [blame] | 700 | * i40evf_alloc_rx_buffers_ps - Replace used receive buffers; packet split |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 701 | * @rx_ring: ring to place buffers on |
| 702 | * @cleaned_count: number of buffers to replace |
| 703 | **/ |
Mitch Williams | a132af2 | 2015-01-24 09:58:35 +0000 | [diff] [blame] | 704 | void i40evf_alloc_rx_buffers_ps(struct i40e_ring *rx_ring, u16 cleaned_count) |
| 705 | { |
| 706 | u16 i = rx_ring->next_to_use; |
| 707 | union i40e_rx_desc *rx_desc; |
| 708 | struct i40e_rx_buffer *bi; |
| 709 | |
| 710 | /* do nothing if no valid netdev defined */ |
| 711 | if (!rx_ring->netdev || !cleaned_count) |
| 712 | return; |
| 713 | |
| 714 | while (cleaned_count--) { |
| 715 | rx_desc = I40E_RX_DESC(rx_ring, i); |
| 716 | bi = &rx_ring->rx_bi[i]; |
| 717 | |
| 718 | if (bi->skb) /* desc is in use */ |
| 719 | goto no_buffers; |
| 720 | if (!bi->page) { |
| 721 | bi->page = alloc_page(GFP_ATOMIC); |
| 722 | if (!bi->page) { |
| 723 | rx_ring->rx_stats.alloc_page_failed++; |
| 724 | goto no_buffers; |
| 725 | } |
| 726 | } |
| 727 | |
| 728 | if (!bi->page_dma) { |
| 729 | /* use a half page if we're re-using */ |
| 730 | bi->page_offset ^= PAGE_SIZE / 2; |
| 731 | bi->page_dma = dma_map_page(rx_ring->dev, |
| 732 | bi->page, |
| 733 | bi->page_offset, |
| 734 | PAGE_SIZE / 2, |
| 735 | DMA_FROM_DEVICE); |
| 736 | if (dma_mapping_error(rx_ring->dev, |
| 737 | bi->page_dma)) { |
| 738 | rx_ring->rx_stats.alloc_page_failed++; |
| 739 | bi->page_dma = 0; |
| 740 | goto no_buffers; |
| 741 | } |
| 742 | } |
| 743 | |
| 744 | dma_sync_single_range_for_device(rx_ring->dev, |
| 745 | bi->dma, |
| 746 | 0, |
| 747 | rx_ring->rx_hdr_len, |
| 748 | DMA_FROM_DEVICE); |
| 749 | /* Refresh the desc even if buffer_addrs didn't change |
| 750 | * because each write-back erases this info. |
| 751 | */ |
| 752 | rx_desc->read.pkt_addr = cpu_to_le64(bi->page_dma); |
| 753 | rx_desc->read.hdr_addr = cpu_to_le64(bi->dma); |
| 754 | i++; |
| 755 | if (i == rx_ring->count) |
| 756 | i = 0; |
| 757 | } |
| 758 | |
| 759 | no_buffers: |
| 760 | if (rx_ring->next_to_use != i) |
| 761 | i40e_release_rx_desc(rx_ring, i); |
| 762 | } |
| 763 | |
| 764 | /** |
| 765 | * i40evf_alloc_rx_buffers_1buf - Replace used receive buffers; single buffer |
| 766 | * @rx_ring: ring to place buffers on |
| 767 | * @cleaned_count: number of buffers to replace |
| 768 | **/ |
| 769 | void i40evf_alloc_rx_buffers_1buf(struct i40e_ring *rx_ring, u16 cleaned_count) |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 770 | { |
| 771 | u16 i = rx_ring->next_to_use; |
| 772 | union i40e_rx_desc *rx_desc; |
| 773 | struct i40e_rx_buffer *bi; |
| 774 | struct sk_buff *skb; |
| 775 | |
| 776 | /* do nothing if no valid netdev defined */ |
| 777 | if (!rx_ring->netdev || !cleaned_count) |
| 778 | return; |
| 779 | |
| 780 | while (cleaned_count--) { |
| 781 | rx_desc = I40E_RX_DESC(rx_ring, i); |
| 782 | bi = &rx_ring->rx_bi[i]; |
| 783 | skb = bi->skb; |
| 784 | |
| 785 | if (!skb) { |
| 786 | skb = netdev_alloc_skb_ip_align(rx_ring->netdev, |
| 787 | rx_ring->rx_buf_len); |
| 788 | if (!skb) { |
| 789 | rx_ring->rx_stats.alloc_buff_failed++; |
| 790 | goto no_buffers; |
| 791 | } |
| 792 | /* initialize queue mapping */ |
| 793 | skb_record_rx_queue(skb, rx_ring->queue_index); |
| 794 | bi->skb = skb; |
| 795 | } |
| 796 | |
| 797 | if (!bi->dma) { |
| 798 | bi->dma = dma_map_single(rx_ring->dev, |
| 799 | skb->data, |
| 800 | rx_ring->rx_buf_len, |
| 801 | DMA_FROM_DEVICE); |
| 802 | if (dma_mapping_error(rx_ring->dev, bi->dma)) { |
| 803 | rx_ring->rx_stats.alloc_buff_failed++; |
| 804 | bi->dma = 0; |
| 805 | goto no_buffers; |
| 806 | } |
| 807 | } |
| 808 | |
Mitch Williams | a132af2 | 2015-01-24 09:58:35 +0000 | [diff] [blame] | 809 | rx_desc->read.pkt_addr = cpu_to_le64(bi->dma); |
| 810 | rx_desc->read.hdr_addr = 0; |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 811 | i++; |
| 812 | if (i == rx_ring->count) |
| 813 | i = 0; |
| 814 | } |
| 815 | |
| 816 | no_buffers: |
| 817 | if (rx_ring->next_to_use != i) |
| 818 | i40e_release_rx_desc(rx_ring, i); |
| 819 | } |
| 820 | |
| 821 | /** |
| 822 | * i40e_receive_skb - Send a completed packet up the stack |
| 823 | * @rx_ring: rx ring in play |
| 824 | * @skb: packet to send up |
| 825 | * @vlan_tag: vlan tag for packet |
| 826 | **/ |
| 827 | static void i40e_receive_skb(struct i40e_ring *rx_ring, |
| 828 | struct sk_buff *skb, u16 vlan_tag) |
| 829 | { |
| 830 | struct i40e_q_vector *q_vector = rx_ring->q_vector; |
| 831 | struct i40e_vsi *vsi = rx_ring->vsi; |
| 832 | u64 flags = vsi->back->flags; |
| 833 | |
| 834 | if (vlan_tag & VLAN_VID_MASK) |
| 835 | __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlan_tag); |
| 836 | |
| 837 | if (flags & I40E_FLAG_IN_NETPOLL) |
| 838 | netif_rx(skb); |
| 839 | else |
| 840 | napi_gro_receive(&q_vector->napi, skb); |
| 841 | } |
| 842 | |
| 843 | /** |
| 844 | * i40e_rx_checksum - Indicate in skb if hw indicated a good cksum |
| 845 | * @vsi: the VSI we care about |
| 846 | * @skb: skb currently being received and modified |
| 847 | * @rx_status: status value of last descriptor in packet |
| 848 | * @rx_error: error value of last descriptor in packet |
| 849 | * @rx_ptype: ptype value of last descriptor in packet |
| 850 | **/ |
| 851 | static inline void i40e_rx_checksum(struct i40e_vsi *vsi, |
| 852 | struct sk_buff *skb, |
| 853 | u32 rx_status, |
| 854 | u32 rx_error, |
| 855 | u16 rx_ptype) |
| 856 | { |
Jesse Brandeburg | 8a3c91c | 2014-05-20 08:01:43 +0000 | [diff] [blame] | 857 | struct i40e_rx_ptype_decoded decoded = decode_rx_desc_ptype(rx_ptype); |
| 858 | bool ipv4 = false, ipv6 = false; |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 859 | bool ipv4_tunnel, ipv6_tunnel; |
| 860 | __wsum rx_udp_csum; |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 861 | struct iphdr *iph; |
Jesse Brandeburg | 8a3c91c | 2014-05-20 08:01:43 +0000 | [diff] [blame] | 862 | __sum16 csum; |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 863 | |
Anjali Singhai Jain | f8faaa4 | 2015-02-24 06:58:48 +0000 | [diff] [blame] | 864 | ipv4_tunnel = (rx_ptype >= I40E_RX_PTYPE_GRENAT4_MAC_PAY3) && |
| 865 | (rx_ptype <= I40E_RX_PTYPE_GRENAT4_MACVLAN_IPV6_ICMP_PAY4); |
| 866 | ipv6_tunnel = (rx_ptype >= I40E_RX_PTYPE_GRENAT6_MAC_PAY3) && |
| 867 | (rx_ptype <= I40E_RX_PTYPE_GRENAT6_MACVLAN_IPV6_ICMP_PAY4); |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 868 | |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 869 | skb->ip_summed = CHECKSUM_NONE; |
| 870 | |
| 871 | /* Rx csum enabled and ip headers found? */ |
Jesse Brandeburg | 8a3c91c | 2014-05-20 08:01:43 +0000 | [diff] [blame] | 872 | if (!(vsi->netdev->features & NETIF_F_RXCSUM)) |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 873 | return; |
| 874 | |
Jesse Brandeburg | 8a3c91c | 2014-05-20 08:01:43 +0000 | [diff] [blame] | 875 | /* did the hardware decode the packet and checksum? */ |
| 876 | if (!(rx_status & (1 << I40E_RX_DESC_STATUS_L3L4P_SHIFT))) |
| 877 | return; |
| 878 | |
| 879 | /* both known and outer_ip must be set for the below code to work */ |
| 880 | if (!(decoded.known && decoded.outer_ip)) |
| 881 | return; |
| 882 | |
| 883 | if (decoded.outer_ip == I40E_RX_PTYPE_OUTER_IP && |
| 884 | decoded.outer_ip_ver == I40E_RX_PTYPE_OUTER_IPV4) |
| 885 | ipv4 = true; |
| 886 | else if (decoded.outer_ip == I40E_RX_PTYPE_OUTER_IP && |
| 887 | decoded.outer_ip_ver == I40E_RX_PTYPE_OUTER_IPV6) |
| 888 | ipv6 = true; |
| 889 | |
| 890 | if (ipv4 && |
| 891 | (rx_error & ((1 << I40E_RX_DESC_ERROR_IPE_SHIFT) | |
| 892 | (1 << I40E_RX_DESC_ERROR_EIPE_SHIFT)))) |
| 893 | goto checksum_fail; |
| 894 | |
Jesse Brandeburg | ddf1d0d | 2014-02-13 03:48:39 -0800 | [diff] [blame] | 895 | /* likely incorrect csum if alternate IP extension headers found */ |
Jesse Brandeburg | 8a3c91c | 2014-05-20 08:01:43 +0000 | [diff] [blame] | 896 | if (ipv6 && |
Jesse Brandeburg | 8a3c91c | 2014-05-20 08:01:43 +0000 | [diff] [blame] | 897 | rx_status & (1 << I40E_RX_DESC_STATUS_IPV6EXADD_SHIFT)) |
| 898 | /* don't increment checksum err here, non-fatal err */ |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 899 | return; |
| 900 | |
Jesse Brandeburg | 8a3c91c | 2014-05-20 08:01:43 +0000 | [diff] [blame] | 901 | /* there was some L4 error, count error and punt packet to the stack */ |
| 902 | if (rx_error & (1 << I40E_RX_DESC_ERROR_L4E_SHIFT)) |
| 903 | goto checksum_fail; |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 904 | |
Jesse Brandeburg | 8a3c91c | 2014-05-20 08:01:43 +0000 | [diff] [blame] | 905 | /* handle packets that were not able to be checksummed due |
| 906 | * to arrival speed, in this case the stack can compute |
| 907 | * the csum. |
| 908 | */ |
| 909 | if (rx_error & (1 << I40E_RX_DESC_ERROR_PPRS_SHIFT)) |
| 910 | return; |
| 911 | |
| 912 | /* If VXLAN traffic has an outer UDPv4 checksum we need to check |
| 913 | * it in the driver, hardware does not do it for us. |
| 914 | * Since L3L4P bit was set we assume a valid IHL value (>=5) |
| 915 | * so the total length of IPv4 header is IHL*4 bytes |
| 916 | * The UDP_0 bit *may* bet set if the *inner* header is UDP |
| 917 | */ |
Anjali Singhai Jain | 818f2e7 | 2015-03-31 00:44:59 -0700 | [diff] [blame] | 918 | if (ipv4_tunnel) { |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 919 | skb->transport_header = skb->mac_header + |
| 920 | sizeof(struct ethhdr) + |
| 921 | (ip_hdr(skb)->ihl * 4); |
| 922 | |
| 923 | /* Add 4 bytes for VLAN tagged packets */ |
| 924 | skb->transport_header += (skb->protocol == htons(ETH_P_8021Q) || |
| 925 | skb->protocol == htons(ETH_P_8021AD)) |
| 926 | ? VLAN_HLEN : 0; |
| 927 | |
Anjali Singhai Jain | 818f2e7 | 2015-03-31 00:44:59 -0700 | [diff] [blame] | 928 | if ((ip_hdr(skb)->protocol == IPPROTO_UDP) && |
| 929 | (udp_hdr(skb)->check != 0)) { |
| 930 | rx_udp_csum = udp_csum(skb); |
| 931 | iph = ip_hdr(skb); |
| 932 | csum = csum_tcpudp_magic(iph->saddr, iph->daddr, |
| 933 | (skb->len - |
| 934 | skb_transport_offset(skb)), |
| 935 | IPPROTO_UDP, rx_udp_csum); |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 936 | |
Anjali Singhai Jain | 818f2e7 | 2015-03-31 00:44:59 -0700 | [diff] [blame] | 937 | if (udp_hdr(skb)->check != csum) |
| 938 | goto checksum_fail; |
| 939 | |
| 940 | } /* else its GRE and so no outer UDP header */ |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 941 | } |
| 942 | |
| 943 | skb->ip_summed = CHECKSUM_UNNECESSARY; |
Tom Herbert | 407fa08 | 2014-08-27 21:27:43 -0700 | [diff] [blame] | 944 | skb->csum_level = ipv4_tunnel || ipv6_tunnel; |
Jesse Brandeburg | 8a3c91c | 2014-05-20 08:01:43 +0000 | [diff] [blame] | 945 | |
| 946 | return; |
| 947 | |
| 948 | checksum_fail: |
| 949 | vsi->back->hw_csum_rx_error++; |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 950 | } |
| 951 | |
| 952 | /** |
| 953 | * i40e_rx_hash - returns the hash value from the Rx descriptor |
| 954 | * @ring: descriptor ring |
| 955 | * @rx_desc: specific descriptor |
| 956 | **/ |
| 957 | static inline u32 i40e_rx_hash(struct i40e_ring *ring, |
| 958 | union i40e_rx_desc *rx_desc) |
| 959 | { |
| 960 | const __le64 rss_mask = |
| 961 | cpu_to_le64((u64)I40E_RX_DESC_FLTSTAT_RSS_HASH << |
| 962 | I40E_RX_DESC_STATUS_FLTSTAT_SHIFT); |
| 963 | |
| 964 | if ((ring->netdev->features & NETIF_F_RXHASH) && |
| 965 | (rx_desc->wb.qword1.status_error_len & rss_mask) == rss_mask) |
| 966 | return le32_to_cpu(rx_desc->wb.qword0.hi_dword.rss); |
| 967 | else |
| 968 | return 0; |
| 969 | } |
| 970 | |
| 971 | /** |
Jesse Brandeburg | 206812b | 2014-02-12 01:45:33 +0000 | [diff] [blame] | 972 | * i40e_ptype_to_hash - get a hash type |
| 973 | * @ptype: the ptype value from the descriptor |
| 974 | * |
| 975 | * Returns a hash type to be used by skb_set_hash |
| 976 | **/ |
| 977 | static inline enum pkt_hash_types i40e_ptype_to_hash(u8 ptype) |
| 978 | { |
| 979 | struct i40e_rx_ptype_decoded decoded = decode_rx_desc_ptype(ptype); |
| 980 | |
| 981 | if (!decoded.known) |
| 982 | return PKT_HASH_TYPE_NONE; |
| 983 | |
| 984 | if (decoded.outer_ip == I40E_RX_PTYPE_OUTER_IP && |
| 985 | decoded.payload_layer == I40E_RX_PTYPE_PAYLOAD_LAYER_PAY4) |
| 986 | return PKT_HASH_TYPE_L4; |
| 987 | else if (decoded.outer_ip == I40E_RX_PTYPE_OUTER_IP && |
| 988 | decoded.payload_layer == I40E_RX_PTYPE_PAYLOAD_LAYER_PAY3) |
| 989 | return PKT_HASH_TYPE_L3; |
| 990 | else |
| 991 | return PKT_HASH_TYPE_L2; |
| 992 | } |
| 993 | |
| 994 | /** |
Mitch Williams | a132af2 | 2015-01-24 09:58:35 +0000 | [diff] [blame] | 995 | * i40e_clean_rx_irq_ps - Reclaim resources after receive; packet split |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 996 | * @rx_ring: rx ring to clean |
| 997 | * @budget: how many cleans we're allowed |
| 998 | * |
| 999 | * Returns true if there's any budget left (e.g. the clean is finished) |
| 1000 | **/ |
Mitch Williams | a132af2 | 2015-01-24 09:58:35 +0000 | [diff] [blame] | 1001 | static int i40e_clean_rx_irq_ps(struct i40e_ring *rx_ring, int budget) |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 1002 | { |
| 1003 | unsigned int total_rx_bytes = 0, total_rx_packets = 0; |
| 1004 | u16 rx_packet_len, rx_header_len, rx_sph, rx_hbo; |
| 1005 | u16 cleaned_count = I40E_DESC_UNUSED(rx_ring); |
| 1006 | const int current_node = numa_node_id(); |
| 1007 | struct i40e_vsi *vsi = rx_ring->vsi; |
| 1008 | u16 i = rx_ring->next_to_clean; |
| 1009 | union i40e_rx_desc *rx_desc; |
| 1010 | u32 rx_error, rx_status; |
Jesse Brandeburg | 206812b | 2014-02-12 01:45:33 +0000 | [diff] [blame] | 1011 | u8 rx_ptype; |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 1012 | u64 qword; |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 1013 | |
Mitch Williams | a132af2 | 2015-01-24 09:58:35 +0000 | [diff] [blame] | 1014 | do { |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 1015 | struct i40e_rx_buffer *rx_bi; |
| 1016 | struct sk_buff *skb; |
| 1017 | u16 vlan_tag; |
Mitch Williams | a132af2 | 2015-01-24 09:58:35 +0000 | [diff] [blame] | 1018 | /* return some buffers to hardware, one at a time is too slow */ |
| 1019 | if (cleaned_count >= I40E_RX_BUFFER_WRITE) { |
| 1020 | i40evf_alloc_rx_buffers_ps(rx_ring, cleaned_count); |
| 1021 | cleaned_count = 0; |
| 1022 | } |
| 1023 | |
| 1024 | i = rx_ring->next_to_clean; |
| 1025 | rx_desc = I40E_RX_DESC(rx_ring, i); |
| 1026 | qword = le64_to_cpu(rx_desc->wb.qword1.status_error_len); |
| 1027 | rx_status = (qword & I40E_RXD_QW1_STATUS_MASK) >> |
| 1028 | I40E_RXD_QW1_STATUS_SHIFT; |
| 1029 | |
| 1030 | if (!(rx_status & (1 << I40E_RX_DESC_STATUS_DD_SHIFT))) |
| 1031 | break; |
| 1032 | |
| 1033 | /* This memory barrier is needed to keep us from reading |
| 1034 | * any other fields out of the rx_desc until we know the |
| 1035 | * DD bit is set. |
| 1036 | */ |
Alexander Duyck | 6731716 | 2015-04-08 18:49:43 -0700 | [diff] [blame] | 1037 | dma_rmb(); |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 1038 | rx_bi = &rx_ring->rx_bi[i]; |
| 1039 | skb = rx_bi->skb; |
Mitch Williams | a132af2 | 2015-01-24 09:58:35 +0000 | [diff] [blame] | 1040 | if (likely(!skb)) { |
| 1041 | skb = netdev_alloc_skb_ip_align(rx_ring->netdev, |
| 1042 | rx_ring->rx_hdr_len); |
Jesse Brandeburg | 8b6ed9c | 2015-03-31 00:45:01 -0700 | [diff] [blame] | 1043 | if (!skb) { |
Mitch Williams | a132af2 | 2015-01-24 09:58:35 +0000 | [diff] [blame] | 1044 | rx_ring->rx_stats.alloc_buff_failed++; |
Jesse Brandeburg | 8b6ed9c | 2015-03-31 00:45:01 -0700 | [diff] [blame] | 1045 | break; |
| 1046 | } |
| 1047 | |
Mitch Williams | a132af2 | 2015-01-24 09:58:35 +0000 | [diff] [blame] | 1048 | /* initialize queue mapping */ |
| 1049 | skb_record_rx_queue(skb, rx_ring->queue_index); |
| 1050 | /* we are reusing so sync this buffer for CPU use */ |
| 1051 | dma_sync_single_range_for_cpu(rx_ring->dev, |
| 1052 | rx_bi->dma, |
| 1053 | 0, |
| 1054 | rx_ring->rx_hdr_len, |
| 1055 | DMA_FROM_DEVICE); |
| 1056 | } |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 1057 | rx_packet_len = (qword & I40E_RXD_QW1_LENGTH_PBUF_MASK) >> |
| 1058 | I40E_RXD_QW1_LENGTH_PBUF_SHIFT; |
| 1059 | rx_header_len = (qword & I40E_RXD_QW1_LENGTH_HBUF_MASK) >> |
| 1060 | I40E_RXD_QW1_LENGTH_HBUF_SHIFT; |
| 1061 | rx_sph = (qword & I40E_RXD_QW1_LENGTH_SPH_MASK) >> |
| 1062 | I40E_RXD_QW1_LENGTH_SPH_SHIFT; |
| 1063 | |
| 1064 | rx_error = (qword & I40E_RXD_QW1_ERROR_MASK) >> |
| 1065 | I40E_RXD_QW1_ERROR_SHIFT; |
| 1066 | rx_hbo = rx_error & (1 << I40E_RX_DESC_ERROR_HBO_SHIFT); |
| 1067 | rx_error &= ~(1 << I40E_RX_DESC_ERROR_HBO_SHIFT); |
| 1068 | |
| 1069 | rx_ptype = (qword & I40E_RXD_QW1_PTYPE_MASK) >> |
| 1070 | I40E_RXD_QW1_PTYPE_SHIFT; |
Mitch Williams | a132af2 | 2015-01-24 09:58:35 +0000 | [diff] [blame] | 1071 | prefetch(rx_bi->page); |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 1072 | rx_bi->skb = NULL; |
Mitch Williams | a132af2 | 2015-01-24 09:58:35 +0000 | [diff] [blame] | 1073 | cleaned_count++; |
| 1074 | if (rx_hbo || rx_sph) { |
| 1075 | int len; |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 1076 | if (rx_hbo) |
| 1077 | len = I40E_RX_HDR_SIZE; |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 1078 | else |
Mitch Williams | a132af2 | 2015-01-24 09:58:35 +0000 | [diff] [blame] | 1079 | len = rx_header_len; |
| 1080 | memcpy(__skb_put(skb, len), rx_bi->hdr_buf, len); |
| 1081 | } else if (skb->len == 0) { |
| 1082 | int len; |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 1083 | |
Mitch Williams | a132af2 | 2015-01-24 09:58:35 +0000 | [diff] [blame] | 1084 | len = (rx_packet_len > skb_headlen(skb) ? |
| 1085 | skb_headlen(skb) : rx_packet_len); |
| 1086 | memcpy(__skb_put(skb, len), |
| 1087 | rx_bi->page + rx_bi->page_offset, |
| 1088 | len); |
| 1089 | rx_bi->page_offset += len; |
| 1090 | rx_packet_len -= len; |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 1091 | } |
| 1092 | |
| 1093 | /* Get the rest of the data if this was a header split */ |
Mitch Williams | a132af2 | 2015-01-24 09:58:35 +0000 | [diff] [blame] | 1094 | if (rx_packet_len) { |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 1095 | skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags, |
| 1096 | rx_bi->page, |
| 1097 | rx_bi->page_offset, |
| 1098 | rx_packet_len); |
| 1099 | |
| 1100 | skb->len += rx_packet_len; |
| 1101 | skb->data_len += rx_packet_len; |
| 1102 | skb->truesize += rx_packet_len; |
| 1103 | |
| 1104 | if ((page_count(rx_bi->page) == 1) && |
| 1105 | (page_to_nid(rx_bi->page) == current_node)) |
| 1106 | get_page(rx_bi->page); |
| 1107 | else |
| 1108 | rx_bi->page = NULL; |
| 1109 | |
| 1110 | dma_unmap_page(rx_ring->dev, |
| 1111 | rx_bi->page_dma, |
| 1112 | PAGE_SIZE / 2, |
| 1113 | DMA_FROM_DEVICE); |
| 1114 | rx_bi->page_dma = 0; |
| 1115 | } |
Mitch Williams | a132af2 | 2015-01-24 09:58:35 +0000 | [diff] [blame] | 1116 | I40E_RX_INCREMENT(rx_ring, i); |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 1117 | |
| 1118 | if (unlikely( |
| 1119 | !(rx_status & (1 << I40E_RX_DESC_STATUS_EOF_SHIFT)))) { |
| 1120 | struct i40e_rx_buffer *next_buffer; |
| 1121 | |
| 1122 | next_buffer = &rx_ring->rx_bi[i]; |
Mitch Williams | a132af2 | 2015-01-24 09:58:35 +0000 | [diff] [blame] | 1123 | next_buffer->skb = skb; |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 1124 | rx_ring->rx_stats.non_eop_descs++; |
Mitch Williams | a132af2 | 2015-01-24 09:58:35 +0000 | [diff] [blame] | 1125 | continue; |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 1126 | } |
| 1127 | |
| 1128 | /* ERR_MASK will only have valid bits if EOP set */ |
| 1129 | if (unlikely(rx_error & (1 << I40E_RX_DESC_ERROR_RXE_SHIFT))) { |
| 1130 | dev_kfree_skb_any(skb); |
Mitch Williams | a132af2 | 2015-01-24 09:58:35 +0000 | [diff] [blame] | 1131 | continue; |
| 1132 | } |
| 1133 | |
| 1134 | skb_set_hash(skb, i40e_rx_hash(rx_ring, rx_desc), |
| 1135 | i40e_ptype_to_hash(rx_ptype)); |
| 1136 | /* probably a little skewed due to removing CRC */ |
| 1137 | total_rx_bytes += skb->len; |
| 1138 | total_rx_packets++; |
| 1139 | |
| 1140 | skb->protocol = eth_type_trans(skb, rx_ring->netdev); |
| 1141 | |
| 1142 | i40e_rx_checksum(vsi, skb, rx_status, rx_error, rx_ptype); |
| 1143 | |
| 1144 | vlan_tag = rx_status & (1 << I40E_RX_DESC_STATUS_L2TAG1P_SHIFT) |
| 1145 | ? le16_to_cpu(rx_desc->wb.qword0.lo_dword.l2tag1) |
| 1146 | : 0; |
| 1147 | #ifdef I40E_FCOE |
| 1148 | if (!i40e_fcoe_handle_offload(rx_ring, rx_desc, skb)) { |
| 1149 | dev_kfree_skb_any(skb); |
| 1150 | continue; |
| 1151 | } |
| 1152 | #endif |
| 1153 | skb_mark_napi_id(skb, &rx_ring->q_vector->napi); |
| 1154 | i40e_receive_skb(rx_ring, skb, vlan_tag); |
| 1155 | |
Mitch Williams | a132af2 | 2015-01-24 09:58:35 +0000 | [diff] [blame] | 1156 | rx_desc->wb.qword1.status_error_len = 0; |
| 1157 | |
| 1158 | } while (likely(total_rx_packets < budget)); |
| 1159 | |
| 1160 | u64_stats_update_begin(&rx_ring->syncp); |
| 1161 | rx_ring->stats.packets += total_rx_packets; |
| 1162 | rx_ring->stats.bytes += total_rx_bytes; |
| 1163 | u64_stats_update_end(&rx_ring->syncp); |
| 1164 | rx_ring->q_vector->rx.total_packets += total_rx_packets; |
| 1165 | rx_ring->q_vector->rx.total_bytes += total_rx_bytes; |
| 1166 | |
| 1167 | return total_rx_packets; |
| 1168 | } |
| 1169 | |
| 1170 | /** |
| 1171 | * i40e_clean_rx_irq_1buf - Reclaim resources after receive; single buffer |
| 1172 | * @rx_ring: rx ring to clean |
| 1173 | * @budget: how many cleans we're allowed |
| 1174 | * |
| 1175 | * Returns number of packets cleaned |
| 1176 | **/ |
| 1177 | static int i40e_clean_rx_irq_1buf(struct i40e_ring *rx_ring, int budget) |
| 1178 | { |
| 1179 | unsigned int total_rx_bytes = 0, total_rx_packets = 0; |
| 1180 | u16 cleaned_count = I40E_DESC_UNUSED(rx_ring); |
| 1181 | struct i40e_vsi *vsi = rx_ring->vsi; |
| 1182 | union i40e_rx_desc *rx_desc; |
| 1183 | u32 rx_error, rx_status; |
| 1184 | u16 rx_packet_len; |
| 1185 | u8 rx_ptype; |
| 1186 | u64 qword; |
| 1187 | u16 i; |
| 1188 | |
| 1189 | do { |
| 1190 | struct i40e_rx_buffer *rx_bi; |
| 1191 | struct sk_buff *skb; |
| 1192 | u16 vlan_tag; |
| 1193 | /* return some buffers to hardware, one at a time is too slow */ |
| 1194 | if (cleaned_count >= I40E_RX_BUFFER_WRITE) { |
| 1195 | i40evf_alloc_rx_buffers_1buf(rx_ring, cleaned_count); |
| 1196 | cleaned_count = 0; |
| 1197 | } |
| 1198 | |
| 1199 | i = rx_ring->next_to_clean; |
| 1200 | rx_desc = I40E_RX_DESC(rx_ring, i); |
| 1201 | qword = le64_to_cpu(rx_desc->wb.qword1.status_error_len); |
| 1202 | rx_status = (qword & I40E_RXD_QW1_STATUS_MASK) >> |
| 1203 | I40E_RXD_QW1_STATUS_SHIFT; |
| 1204 | |
| 1205 | if (!(rx_status & (1 << I40E_RX_DESC_STATUS_DD_SHIFT))) |
| 1206 | break; |
| 1207 | |
| 1208 | /* This memory barrier is needed to keep us from reading |
| 1209 | * any other fields out of the rx_desc until we know the |
| 1210 | * DD bit is set. |
| 1211 | */ |
Alexander Duyck | 6731716 | 2015-04-08 18:49:43 -0700 | [diff] [blame] | 1212 | dma_rmb(); |
Mitch Williams | a132af2 | 2015-01-24 09:58:35 +0000 | [diff] [blame] | 1213 | |
| 1214 | rx_bi = &rx_ring->rx_bi[i]; |
| 1215 | skb = rx_bi->skb; |
| 1216 | prefetch(skb->data); |
| 1217 | |
| 1218 | rx_packet_len = (qword & I40E_RXD_QW1_LENGTH_PBUF_MASK) >> |
| 1219 | I40E_RXD_QW1_LENGTH_PBUF_SHIFT; |
| 1220 | |
| 1221 | rx_error = (qword & I40E_RXD_QW1_ERROR_MASK) >> |
| 1222 | I40E_RXD_QW1_ERROR_SHIFT; |
| 1223 | rx_error &= ~(1 << I40E_RX_DESC_ERROR_HBO_SHIFT); |
| 1224 | |
| 1225 | rx_ptype = (qword & I40E_RXD_QW1_PTYPE_MASK) >> |
| 1226 | I40E_RXD_QW1_PTYPE_SHIFT; |
| 1227 | rx_bi->skb = NULL; |
| 1228 | cleaned_count++; |
| 1229 | |
| 1230 | /* Get the header and possibly the whole packet |
| 1231 | * If this is an skb from previous receive dma will be 0 |
| 1232 | */ |
| 1233 | skb_put(skb, rx_packet_len); |
| 1234 | dma_unmap_single(rx_ring->dev, rx_bi->dma, rx_ring->rx_buf_len, |
| 1235 | DMA_FROM_DEVICE); |
| 1236 | rx_bi->dma = 0; |
| 1237 | |
| 1238 | I40E_RX_INCREMENT(rx_ring, i); |
| 1239 | |
| 1240 | if (unlikely( |
| 1241 | !(rx_status & (1 << I40E_RX_DESC_STATUS_EOF_SHIFT)))) { |
| 1242 | rx_ring->rx_stats.non_eop_descs++; |
| 1243 | continue; |
| 1244 | } |
| 1245 | |
| 1246 | /* ERR_MASK will only have valid bits if EOP set */ |
| 1247 | if (unlikely(rx_error & (1 << I40E_RX_DESC_ERROR_RXE_SHIFT))) { |
| 1248 | dev_kfree_skb_any(skb); |
| 1249 | /* TODO: shouldn't we increment a counter indicating the |
| 1250 | * drop? |
| 1251 | */ |
| 1252 | continue; |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 1253 | } |
| 1254 | |
Jesse Brandeburg | 206812b | 2014-02-12 01:45:33 +0000 | [diff] [blame] | 1255 | skb_set_hash(skb, i40e_rx_hash(rx_ring, rx_desc), |
| 1256 | i40e_ptype_to_hash(rx_ptype)); |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 1257 | /* probably a little skewed due to removing CRC */ |
| 1258 | total_rx_bytes += skb->len; |
| 1259 | total_rx_packets++; |
| 1260 | |
| 1261 | skb->protocol = eth_type_trans(skb, rx_ring->netdev); |
| 1262 | |
| 1263 | i40e_rx_checksum(vsi, skb, rx_status, rx_error, rx_ptype); |
| 1264 | |
| 1265 | vlan_tag = rx_status & (1 << I40E_RX_DESC_STATUS_L2TAG1P_SHIFT) |
| 1266 | ? le16_to_cpu(rx_desc->wb.qword0.lo_dword.l2tag1) |
| 1267 | : 0; |
| 1268 | i40e_receive_skb(rx_ring, skb, vlan_tag); |
| 1269 | |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 1270 | rx_desc->wb.qword1.status_error_len = 0; |
Mitch Williams | a132af2 | 2015-01-24 09:58:35 +0000 | [diff] [blame] | 1271 | } while (likely(total_rx_packets < budget)); |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 1272 | |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 1273 | u64_stats_update_begin(&rx_ring->syncp); |
| 1274 | rx_ring->stats.packets += total_rx_packets; |
| 1275 | rx_ring->stats.bytes += total_rx_bytes; |
| 1276 | u64_stats_update_end(&rx_ring->syncp); |
| 1277 | rx_ring->q_vector->rx.total_packets += total_rx_packets; |
| 1278 | rx_ring->q_vector->rx.total_bytes += total_rx_bytes; |
| 1279 | |
Mitch Williams | a132af2 | 2015-01-24 09:58:35 +0000 | [diff] [blame] | 1280 | return total_rx_packets; |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 1281 | } |
| 1282 | |
| 1283 | /** |
| 1284 | * i40evf_napi_poll - NAPI polling Rx/Tx cleanup routine |
| 1285 | * @napi: napi struct with our devices info in it |
| 1286 | * @budget: amount of work driver is allowed to do this pass, in packets |
| 1287 | * |
| 1288 | * This function will clean all queues associated with a q_vector. |
| 1289 | * |
| 1290 | * Returns the amount of work done |
| 1291 | **/ |
| 1292 | int i40evf_napi_poll(struct napi_struct *napi, int budget) |
| 1293 | { |
| 1294 | struct i40e_q_vector *q_vector = |
| 1295 | container_of(napi, struct i40e_q_vector, napi); |
| 1296 | struct i40e_vsi *vsi = q_vector->vsi; |
| 1297 | struct i40e_ring *ring; |
| 1298 | bool clean_complete = true; |
Anjali Singhai Jain | c29af37 | 2015-01-10 01:07:19 +0000 | [diff] [blame] | 1299 | bool arm_wb = false; |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 1300 | int budget_per_ring; |
Mitch Williams | a132af2 | 2015-01-24 09:58:35 +0000 | [diff] [blame] | 1301 | int cleaned; |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 1302 | |
| 1303 | if (test_bit(__I40E_DOWN, &vsi->state)) { |
| 1304 | napi_complete(napi); |
| 1305 | return 0; |
| 1306 | } |
| 1307 | |
| 1308 | /* Since the actual Tx work is minimal, we can give the Tx a larger |
| 1309 | * budget and be more aggressive about cleaning up the Tx descriptors. |
| 1310 | */ |
Anjali Singhai Jain | c29af37 | 2015-01-10 01:07:19 +0000 | [diff] [blame] | 1311 | i40e_for_each_ring(ring, q_vector->tx) { |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 1312 | clean_complete &= i40e_clean_tx_irq(ring, vsi->work_limit); |
Anjali Singhai Jain | c29af37 | 2015-01-10 01:07:19 +0000 | [diff] [blame] | 1313 | arm_wb |= ring->arm_wb; |
| 1314 | } |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 1315 | |
| 1316 | /* We attempt to distribute budget to each Rx queue fairly, but don't |
| 1317 | * allow the budget to go below 1 because that would exit polling early. |
| 1318 | */ |
| 1319 | budget_per_ring = max(budget/q_vector->num_ringpairs, 1); |
| 1320 | |
Mitch Williams | a132af2 | 2015-01-24 09:58:35 +0000 | [diff] [blame] | 1321 | i40e_for_each_ring(ring, q_vector->rx) { |
| 1322 | if (ring_is_ps_enabled(ring)) |
| 1323 | cleaned = i40e_clean_rx_irq_ps(ring, budget_per_ring); |
| 1324 | else |
| 1325 | cleaned = i40e_clean_rx_irq_1buf(ring, budget_per_ring); |
| 1326 | /* if we didn't clean as many as budgeted, we must be done */ |
| 1327 | clean_complete &= (budget_per_ring != cleaned); |
| 1328 | } |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 1329 | |
| 1330 | /* If work not completed, return budget and polling will return */ |
Anjali Singhai Jain | c29af37 | 2015-01-10 01:07:19 +0000 | [diff] [blame] | 1331 | if (!clean_complete) { |
| 1332 | if (arm_wb) |
| 1333 | i40e_force_wb(vsi, q_vector); |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 1334 | return budget; |
Anjali Singhai Jain | c29af37 | 2015-01-10 01:07:19 +0000 | [diff] [blame] | 1335 | } |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 1336 | |
| 1337 | /* Work is done so exit the polling mode and re-enable the interrupt */ |
| 1338 | napi_complete(napi); |
| 1339 | if (ITR_IS_DYNAMIC(vsi->rx_itr_setting) || |
| 1340 | ITR_IS_DYNAMIC(vsi->tx_itr_setting)) |
| 1341 | i40e_update_dynamic_itr(q_vector); |
| 1342 | |
| 1343 | if (!test_bit(__I40E_DOWN, &vsi->state)) |
| 1344 | i40evf_irq_enable_queues(vsi->back, 1 << q_vector->v_idx); |
| 1345 | |
| 1346 | return 0; |
| 1347 | } |
| 1348 | |
| 1349 | /** |
Jesse Brandeburg | 3e587cf | 2015-04-16 20:06:10 -0400 | [diff] [blame] | 1350 | * i40evf_tx_prepare_vlan_flags - prepare generic TX VLAN tagging flags for HW |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 1351 | * @skb: send buffer |
| 1352 | * @tx_ring: ring to send buffer on |
| 1353 | * @flags: the tx flags to be set |
| 1354 | * |
| 1355 | * Checks the skb and set up correspondingly several generic transmit flags |
| 1356 | * related to VLAN tagging for the HW, such as VLAN, DCB, etc. |
| 1357 | * |
| 1358 | * Returns error code indicate the frame should be dropped upon error and the |
| 1359 | * otherwise returns 0 to indicate the flags has been set properly. |
| 1360 | **/ |
Jesse Brandeburg | 3e587cf | 2015-04-16 20:06:10 -0400 | [diff] [blame] | 1361 | static inline int i40evf_tx_prepare_vlan_flags(struct sk_buff *skb, |
| 1362 | struct i40e_ring *tx_ring, |
| 1363 | u32 *flags) |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 1364 | { |
| 1365 | __be16 protocol = skb->protocol; |
| 1366 | u32 tx_flags = 0; |
| 1367 | |
Greg Rose | 31eaacc | 2015-03-31 00:45:03 -0700 | [diff] [blame] | 1368 | if (protocol == htons(ETH_P_8021Q) && |
| 1369 | !(tx_ring->netdev->features & NETIF_F_HW_VLAN_CTAG_TX)) { |
| 1370 | /* When HW VLAN acceleration is turned off by the user the |
| 1371 | * stack sets the protocol to 8021q so that the driver |
| 1372 | * can take any steps required to support the SW only |
| 1373 | * VLAN handling. In our case the driver doesn't need |
| 1374 | * to take any further steps so just set the protocol |
| 1375 | * to the encapsulated ethertype. |
| 1376 | */ |
| 1377 | skb->protocol = vlan_get_protocol(skb); |
| 1378 | goto out; |
| 1379 | } |
| 1380 | |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 1381 | /* if we have a HW VLAN tag being added, default to the HW one */ |
Jiri Pirko | df8a39d | 2015-01-13 17:13:44 +0100 | [diff] [blame] | 1382 | if (skb_vlan_tag_present(skb)) { |
| 1383 | tx_flags |= skb_vlan_tag_get(skb) << I40E_TX_FLAGS_VLAN_SHIFT; |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 1384 | tx_flags |= I40E_TX_FLAGS_HW_VLAN; |
| 1385 | /* else if it is a SW VLAN, check the next protocol and store the tag */ |
| 1386 | } else if (protocol == htons(ETH_P_8021Q)) { |
| 1387 | struct vlan_hdr *vhdr, _vhdr; |
| 1388 | vhdr = skb_header_pointer(skb, ETH_HLEN, sizeof(_vhdr), &_vhdr); |
| 1389 | if (!vhdr) |
| 1390 | return -EINVAL; |
| 1391 | |
| 1392 | protocol = vhdr->h_vlan_encapsulated_proto; |
| 1393 | tx_flags |= ntohs(vhdr->h_vlan_TCI) << I40E_TX_FLAGS_VLAN_SHIFT; |
| 1394 | tx_flags |= I40E_TX_FLAGS_SW_VLAN; |
| 1395 | } |
| 1396 | |
Greg Rose | 31eaacc | 2015-03-31 00:45:03 -0700 | [diff] [blame] | 1397 | out: |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 1398 | *flags = tx_flags; |
| 1399 | return 0; |
| 1400 | } |
| 1401 | |
| 1402 | /** |
| 1403 | * i40e_tso - set up the tso context descriptor |
| 1404 | * @tx_ring: ptr to the ring to send |
| 1405 | * @skb: ptr to the skb we're sending |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 1406 | * @hdr_len: ptr to the size of the packet header |
| 1407 | * @cd_tunneling: ptr to context descriptor bits |
| 1408 | * |
| 1409 | * Returns 0 if no TSO can happen, 1 if tso is going, or error |
| 1410 | **/ |
| 1411 | static int i40e_tso(struct i40e_ring *tx_ring, struct sk_buff *skb, |
Anjali Singhai Jain | 89232c3 | 2015-04-16 20:06:00 -0400 | [diff] [blame] | 1412 | u8 *hdr_len, u64 *cd_type_cmd_tso_mss, |
| 1413 | u32 *cd_tunneling) |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 1414 | { |
| 1415 | u32 cd_cmd, cd_tso_len, cd_mss; |
Francois Romieu | fe6d4aa | 2014-03-30 03:14:53 +0000 | [diff] [blame] | 1416 | struct ipv6hdr *ipv6h; |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 1417 | struct tcphdr *tcph; |
| 1418 | struct iphdr *iph; |
| 1419 | u32 l4len; |
| 1420 | int err; |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 1421 | |
| 1422 | if (!skb_is_gso(skb)) |
| 1423 | return 0; |
| 1424 | |
Francois Romieu | fe6d4aa | 2014-03-30 03:14:53 +0000 | [diff] [blame] | 1425 | err = skb_cow_head(skb, 0); |
| 1426 | if (err < 0) |
| 1427 | return err; |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 1428 | |
Anjali Singhai | 85e76d0 | 2015-02-21 06:44:16 +0000 | [diff] [blame] | 1429 | iph = skb->encapsulation ? inner_ip_hdr(skb) : ip_hdr(skb); |
| 1430 | ipv6h = skb->encapsulation ? inner_ipv6_hdr(skb) : ipv6_hdr(skb); |
| 1431 | |
| 1432 | if (iph->version == 4) { |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 1433 | tcph = skb->encapsulation ? inner_tcp_hdr(skb) : tcp_hdr(skb); |
| 1434 | iph->tot_len = 0; |
| 1435 | iph->check = 0; |
| 1436 | tcph->check = ~csum_tcpudp_magic(iph->saddr, iph->daddr, |
| 1437 | 0, IPPROTO_TCP, 0); |
Anjali Singhai | 85e76d0 | 2015-02-21 06:44:16 +0000 | [diff] [blame] | 1438 | } else if (ipv6h->version == 6) { |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 1439 | tcph = skb->encapsulation ? inner_tcp_hdr(skb) : tcp_hdr(skb); |
| 1440 | ipv6h->payload_len = 0; |
| 1441 | tcph->check = ~csum_ipv6_magic(&ipv6h->saddr, &ipv6h->daddr, |
| 1442 | 0, IPPROTO_TCP, 0); |
| 1443 | } |
| 1444 | |
| 1445 | l4len = skb->encapsulation ? inner_tcp_hdrlen(skb) : tcp_hdrlen(skb); |
| 1446 | *hdr_len = (skb->encapsulation |
| 1447 | ? (skb_inner_transport_header(skb) - skb->data) |
| 1448 | : skb_transport_offset(skb)) + l4len; |
| 1449 | |
| 1450 | /* find the field values */ |
| 1451 | cd_cmd = I40E_TX_CTX_DESC_TSO; |
| 1452 | cd_tso_len = skb->len - *hdr_len; |
| 1453 | cd_mss = skb_shinfo(skb)->gso_size; |
| 1454 | *cd_type_cmd_tso_mss |= ((u64)cd_cmd << I40E_TXD_CTX_QW1_CMD_SHIFT) | |
| 1455 | ((u64)cd_tso_len << |
| 1456 | I40E_TXD_CTX_QW1_TSO_LEN_SHIFT) | |
| 1457 | ((u64)cd_mss << I40E_TXD_CTX_QW1_MSS_SHIFT); |
| 1458 | return 1; |
| 1459 | } |
| 1460 | |
| 1461 | /** |
| 1462 | * i40e_tx_enable_csum - Enable Tx checksum offloads |
| 1463 | * @skb: send buffer |
Anjali Singhai Jain | 89232c3 | 2015-04-16 20:06:00 -0400 | [diff] [blame] | 1464 | * @tx_flags: pointer to Tx flags currently set |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 1465 | * @td_cmd: Tx descriptor command bits to set |
| 1466 | * @td_offset: Tx descriptor header offsets to set |
| 1467 | * @cd_tunneling: ptr to context desc bits |
| 1468 | **/ |
Anjali Singhai Jain | 89232c3 | 2015-04-16 20:06:00 -0400 | [diff] [blame] | 1469 | static void i40e_tx_enable_csum(struct sk_buff *skb, u32 *tx_flags, |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 1470 | u32 *td_cmd, u32 *td_offset, |
| 1471 | struct i40e_ring *tx_ring, |
| 1472 | u32 *cd_tunneling) |
| 1473 | { |
| 1474 | struct ipv6hdr *this_ipv6_hdr; |
| 1475 | unsigned int this_tcp_hdrlen; |
| 1476 | struct iphdr *this_ip_hdr; |
| 1477 | u32 network_hdr_len; |
| 1478 | u8 l4_hdr = 0; |
Anjali Singhai Jain | 4599120 | 2015-02-27 09:15:29 +0000 | [diff] [blame] | 1479 | u32 l4_tunnel = 0; |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 1480 | |
| 1481 | if (skb->encapsulation) { |
Anjali Singhai Jain | 4599120 | 2015-02-27 09:15:29 +0000 | [diff] [blame] | 1482 | switch (ip_hdr(skb)->protocol) { |
| 1483 | case IPPROTO_UDP: |
| 1484 | l4_tunnel = I40E_TXD_CTX_UDP_TUNNELING; |
Anjali Singhai Jain | 89232c3 | 2015-04-16 20:06:00 -0400 | [diff] [blame] | 1485 | *tx_flags |= I40E_TX_FLAGS_VXLAN_TUNNEL; |
Anjali Singhai Jain | 4599120 | 2015-02-27 09:15:29 +0000 | [diff] [blame] | 1486 | break; |
| 1487 | default: |
| 1488 | return; |
| 1489 | } |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 1490 | network_hdr_len = skb_inner_network_header_len(skb); |
| 1491 | this_ip_hdr = inner_ip_hdr(skb); |
| 1492 | this_ipv6_hdr = inner_ipv6_hdr(skb); |
| 1493 | this_tcp_hdrlen = inner_tcp_hdrlen(skb); |
| 1494 | |
Anjali Singhai Jain | 89232c3 | 2015-04-16 20:06:00 -0400 | [diff] [blame] | 1495 | if (*tx_flags & I40E_TX_FLAGS_IPV4) { |
| 1496 | if (*tx_flags & I40E_TX_FLAGS_TSO) { |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 1497 | *cd_tunneling |= I40E_TX_CTX_EXT_IP_IPV4; |
| 1498 | ip_hdr(skb)->check = 0; |
| 1499 | } else { |
| 1500 | *cd_tunneling |= |
| 1501 | I40E_TX_CTX_EXT_IP_IPV4_NO_CSUM; |
| 1502 | } |
Anjali Singhai Jain | 89232c3 | 2015-04-16 20:06:00 -0400 | [diff] [blame] | 1503 | } else if (*tx_flags & I40E_TX_FLAGS_IPV6) { |
Anjali Singhai | 85e76d0 | 2015-02-21 06:44:16 +0000 | [diff] [blame] | 1504 | *cd_tunneling |= I40E_TX_CTX_EXT_IP_IPV6; |
Anjali Singhai Jain | 89232c3 | 2015-04-16 20:06:00 -0400 | [diff] [blame] | 1505 | if (*tx_flags & I40E_TX_FLAGS_TSO) |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 1506 | ip_hdr(skb)->check = 0; |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 1507 | } |
| 1508 | |
| 1509 | /* Now set the ctx descriptor fields */ |
| 1510 | *cd_tunneling |= (skb_network_header_len(skb) >> 2) << |
Anjali Singhai Jain | 4599120 | 2015-02-27 09:15:29 +0000 | [diff] [blame] | 1511 | I40E_TXD_CTX_QW0_EXT_IPLEN_SHIFT | |
| 1512 | l4_tunnel | |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 1513 | ((skb_inner_network_offset(skb) - |
| 1514 | skb_transport_offset(skb)) >> 1) << |
| 1515 | I40E_TXD_CTX_QW0_NATLEN_SHIFT; |
Anjali Singhai | 85e76d0 | 2015-02-21 06:44:16 +0000 | [diff] [blame] | 1516 | if (this_ip_hdr->version == 6) { |
Anjali Singhai Jain | 89232c3 | 2015-04-16 20:06:00 -0400 | [diff] [blame] | 1517 | *tx_flags &= ~I40E_TX_FLAGS_IPV4; |
| 1518 | *tx_flags |= I40E_TX_FLAGS_IPV6; |
Anjali Singhai | 85e76d0 | 2015-02-21 06:44:16 +0000 | [diff] [blame] | 1519 | } |
| 1520 | |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 1521 | |
| 1522 | } else { |
| 1523 | network_hdr_len = skb_network_header_len(skb); |
| 1524 | this_ip_hdr = ip_hdr(skb); |
| 1525 | this_ipv6_hdr = ipv6_hdr(skb); |
| 1526 | this_tcp_hdrlen = tcp_hdrlen(skb); |
| 1527 | } |
| 1528 | |
| 1529 | /* Enable IP checksum offloads */ |
Anjali Singhai Jain | 89232c3 | 2015-04-16 20:06:00 -0400 | [diff] [blame] | 1530 | if (*tx_flags & I40E_TX_FLAGS_IPV4) { |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 1531 | l4_hdr = this_ip_hdr->protocol; |
| 1532 | /* the stack computes the IP header already, the only time we |
| 1533 | * need the hardware to recompute it is in the case of TSO. |
| 1534 | */ |
Anjali Singhai Jain | 89232c3 | 2015-04-16 20:06:00 -0400 | [diff] [blame] | 1535 | if (*tx_flags & I40E_TX_FLAGS_TSO) { |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 1536 | *td_cmd |= I40E_TX_DESC_CMD_IIPT_IPV4_CSUM; |
| 1537 | this_ip_hdr->check = 0; |
| 1538 | } else { |
| 1539 | *td_cmd |= I40E_TX_DESC_CMD_IIPT_IPV4; |
| 1540 | } |
| 1541 | /* Now set the td_offset for IP header length */ |
| 1542 | *td_offset = (network_hdr_len >> 2) << |
| 1543 | I40E_TX_DESC_LENGTH_IPLEN_SHIFT; |
Anjali Singhai Jain | 89232c3 | 2015-04-16 20:06:00 -0400 | [diff] [blame] | 1544 | } else if (*tx_flags & I40E_TX_FLAGS_IPV6) { |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 1545 | l4_hdr = this_ipv6_hdr->nexthdr; |
| 1546 | *td_cmd |= I40E_TX_DESC_CMD_IIPT_IPV6; |
| 1547 | /* Now set the td_offset for IP header length */ |
| 1548 | *td_offset = (network_hdr_len >> 2) << |
| 1549 | I40E_TX_DESC_LENGTH_IPLEN_SHIFT; |
| 1550 | } |
| 1551 | /* words in MACLEN + dwords in IPLEN + dwords in L4Len */ |
| 1552 | *td_offset |= (skb_network_offset(skb) >> 1) << |
| 1553 | I40E_TX_DESC_LENGTH_MACLEN_SHIFT; |
| 1554 | |
| 1555 | /* Enable L4 checksum offloads */ |
| 1556 | switch (l4_hdr) { |
| 1557 | case IPPROTO_TCP: |
| 1558 | /* enable checksum offloads */ |
| 1559 | *td_cmd |= I40E_TX_DESC_CMD_L4T_EOFT_TCP; |
| 1560 | *td_offset |= (this_tcp_hdrlen >> 2) << |
| 1561 | I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT; |
| 1562 | break; |
| 1563 | case IPPROTO_SCTP: |
| 1564 | /* enable SCTP checksum offload */ |
| 1565 | *td_cmd |= I40E_TX_DESC_CMD_L4T_EOFT_SCTP; |
| 1566 | *td_offset |= (sizeof(struct sctphdr) >> 2) << |
| 1567 | I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT; |
| 1568 | break; |
| 1569 | case IPPROTO_UDP: |
| 1570 | /* enable UDP checksum offload */ |
| 1571 | *td_cmd |= I40E_TX_DESC_CMD_L4T_EOFT_UDP; |
| 1572 | *td_offset |= (sizeof(struct udphdr) >> 2) << |
| 1573 | I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT; |
| 1574 | break; |
| 1575 | default: |
| 1576 | break; |
| 1577 | } |
| 1578 | } |
| 1579 | |
| 1580 | /** |
| 1581 | * i40e_create_tx_ctx Build the Tx context descriptor |
| 1582 | * @tx_ring: ring to create the descriptor on |
| 1583 | * @cd_type_cmd_tso_mss: Quad Word 1 |
| 1584 | * @cd_tunneling: Quad Word 0 - bits 0-31 |
| 1585 | * @cd_l2tag2: Quad Word 0 - bits 32-63 |
| 1586 | **/ |
| 1587 | static void i40e_create_tx_ctx(struct i40e_ring *tx_ring, |
| 1588 | const u64 cd_type_cmd_tso_mss, |
| 1589 | const u32 cd_tunneling, const u32 cd_l2tag2) |
| 1590 | { |
| 1591 | struct i40e_tx_context_desc *context_desc; |
| 1592 | int i = tx_ring->next_to_use; |
| 1593 | |
Jesse Brandeburg | ff40dd5 | 2014-02-14 02:14:41 +0000 | [diff] [blame] | 1594 | if ((cd_type_cmd_tso_mss == I40E_TX_DESC_DTYPE_CONTEXT) && |
| 1595 | !cd_tunneling && !cd_l2tag2) |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 1596 | return; |
| 1597 | |
| 1598 | /* grab the next descriptor */ |
| 1599 | context_desc = I40E_TX_CTXTDESC(tx_ring, i); |
| 1600 | |
| 1601 | i++; |
| 1602 | tx_ring->next_to_use = (i < tx_ring->count) ? i : 0; |
| 1603 | |
| 1604 | /* cpu_to_le32 and assign to struct fields */ |
| 1605 | context_desc->tunneling_params = cpu_to_le32(cd_tunneling); |
| 1606 | context_desc->l2tag2 = cpu_to_le16(cd_l2tag2); |
Jesse Brandeburg | 3efbbb2 | 2014-06-04 20:41:54 +0000 | [diff] [blame] | 1607 | context_desc->rsvd = cpu_to_le16(0); |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 1608 | context_desc->type_cmd_tso_mss = cpu_to_le64(cd_type_cmd_tso_mss); |
| 1609 | } |
| 1610 | |
Anjali Singhai | 71da619 | 2015-02-21 06:42:35 +0000 | [diff] [blame] | 1611 | /** |
| 1612 | * i40e_chk_linearize - Check if there are more than 8 fragments per packet |
| 1613 | * @skb: send buffer |
| 1614 | * @tx_flags: collected send information |
Anjali Singhai | 71da619 | 2015-02-21 06:42:35 +0000 | [diff] [blame] | 1615 | * |
| 1616 | * Note: Our HW can't scatter-gather more than 8 fragments to build |
| 1617 | * a packet on the wire and so we need to figure out the cases where we |
| 1618 | * need to linearize the skb. |
| 1619 | **/ |
Anjali Singhai Jain | 3052083 | 2015-05-08 15:35:52 -0700 | [diff] [blame] | 1620 | static bool i40e_chk_linearize(struct sk_buff *skb, u32 tx_flags) |
Anjali Singhai | 71da619 | 2015-02-21 06:42:35 +0000 | [diff] [blame] | 1621 | { |
| 1622 | struct skb_frag_struct *frag; |
| 1623 | bool linearize = false; |
| 1624 | unsigned int size = 0; |
| 1625 | u16 num_frags; |
| 1626 | u16 gso_segs; |
| 1627 | |
| 1628 | num_frags = skb_shinfo(skb)->nr_frags; |
| 1629 | gso_segs = skb_shinfo(skb)->gso_segs; |
| 1630 | |
| 1631 | if (tx_flags & (I40E_TX_FLAGS_TSO | I40E_TX_FLAGS_FSO)) { |
Anjali Singhai Jain | 3052083 | 2015-05-08 15:35:52 -0700 | [diff] [blame] | 1632 | u16 j = 0; |
Anjali Singhai | 71da619 | 2015-02-21 06:42:35 +0000 | [diff] [blame] | 1633 | |
| 1634 | if (num_frags < (I40E_MAX_BUFFER_TXD)) |
| 1635 | goto linearize_chk_done; |
| 1636 | /* try the simple math, if we have too many frags per segment */ |
| 1637 | if (DIV_ROUND_UP((num_frags + gso_segs), gso_segs) > |
| 1638 | I40E_MAX_BUFFER_TXD) { |
| 1639 | linearize = true; |
| 1640 | goto linearize_chk_done; |
| 1641 | } |
| 1642 | frag = &skb_shinfo(skb)->frags[0]; |
Anjali Singhai | 71da619 | 2015-02-21 06:42:35 +0000 | [diff] [blame] | 1643 | /* we might still have more fragments per segment */ |
| 1644 | do { |
| 1645 | size += skb_frag_size(frag); |
| 1646 | frag++; j++; |
Anjali Singhai Jain | 3052083 | 2015-05-08 15:35:52 -0700 | [diff] [blame] | 1647 | if ((size >= skb_shinfo(skb)->gso_size) && |
| 1648 | (j < I40E_MAX_BUFFER_TXD)) { |
| 1649 | size = (size % skb_shinfo(skb)->gso_size); |
| 1650 | j = (size) ? 1 : 0; |
| 1651 | } |
Anjali Singhai | 71da619 | 2015-02-21 06:42:35 +0000 | [diff] [blame] | 1652 | if (j == I40E_MAX_BUFFER_TXD) { |
Anjali Singhai Jain | 3052083 | 2015-05-08 15:35:52 -0700 | [diff] [blame] | 1653 | linearize = true; |
| 1654 | break; |
Anjali Singhai | 71da619 | 2015-02-21 06:42:35 +0000 | [diff] [blame] | 1655 | } |
| 1656 | num_frags--; |
| 1657 | } while (num_frags); |
| 1658 | } else { |
| 1659 | if (num_frags >= I40E_MAX_BUFFER_TXD) |
| 1660 | linearize = true; |
| 1661 | } |
| 1662 | |
| 1663 | linearize_chk_done: |
| 1664 | return linearize; |
| 1665 | } |
| 1666 | |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 1667 | /** |
Jesse Brandeburg | 8f6a2b0 | 2015-04-16 20:06:09 -0400 | [diff] [blame] | 1668 | * __i40evf_maybe_stop_tx - 2nd level check for tx stop conditions |
| 1669 | * @tx_ring: the ring to be checked |
| 1670 | * @size: the size buffer we want to assure is available |
| 1671 | * |
| 1672 | * Returns -EBUSY if a stop is needed, else 0 |
| 1673 | **/ |
| 1674 | static inline int __i40evf_maybe_stop_tx(struct i40e_ring *tx_ring, int size) |
| 1675 | { |
| 1676 | netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index); |
| 1677 | /* Memory barrier before checking head and tail */ |
| 1678 | smp_mb(); |
| 1679 | |
| 1680 | /* Check again in a case another CPU has just made room available. */ |
| 1681 | if (likely(I40E_DESC_UNUSED(tx_ring) < size)) |
| 1682 | return -EBUSY; |
| 1683 | |
| 1684 | /* A reprieve! - use start_queue because it doesn't call schedule */ |
| 1685 | netif_start_subqueue(tx_ring->netdev, tx_ring->queue_index); |
| 1686 | ++tx_ring->tx_stats.restart_queue; |
| 1687 | return 0; |
| 1688 | } |
| 1689 | |
| 1690 | /** |
| 1691 | * i40evf_maybe_stop_tx - 1st level check for tx stop conditions |
| 1692 | * @tx_ring: the ring to be checked |
| 1693 | * @size: the size buffer we want to assure is available |
| 1694 | * |
| 1695 | * Returns 0 if stop is not needed |
| 1696 | **/ |
Jesse Brandeburg | 3e587cf | 2015-04-16 20:06:10 -0400 | [diff] [blame] | 1697 | static inline int i40evf_maybe_stop_tx(struct i40e_ring *tx_ring, int size) |
Jesse Brandeburg | 8f6a2b0 | 2015-04-16 20:06:09 -0400 | [diff] [blame] | 1698 | { |
| 1699 | if (likely(I40E_DESC_UNUSED(tx_ring) >= size)) |
| 1700 | return 0; |
| 1701 | return __i40evf_maybe_stop_tx(tx_ring, size); |
| 1702 | } |
| 1703 | |
| 1704 | /** |
Jesse Brandeburg | 3e587cf | 2015-04-16 20:06:10 -0400 | [diff] [blame] | 1705 | * i40evf_tx_map - Build the Tx descriptor |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 1706 | * @tx_ring: ring to send buffer on |
| 1707 | * @skb: send buffer |
| 1708 | * @first: first buffer info buffer to use |
| 1709 | * @tx_flags: collected send information |
| 1710 | * @hdr_len: size of the packet header |
| 1711 | * @td_cmd: the command field in the descriptor |
| 1712 | * @td_offset: offset for checksum or crc |
| 1713 | **/ |
Jesse Brandeburg | 3e587cf | 2015-04-16 20:06:10 -0400 | [diff] [blame] | 1714 | static inline void i40evf_tx_map(struct i40e_ring *tx_ring, struct sk_buff *skb, |
| 1715 | struct i40e_tx_buffer *first, u32 tx_flags, |
| 1716 | const u8 hdr_len, u32 td_cmd, u32 td_offset) |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 1717 | { |
| 1718 | unsigned int data_len = skb->data_len; |
| 1719 | unsigned int size = skb_headlen(skb); |
| 1720 | struct skb_frag_struct *frag; |
| 1721 | struct i40e_tx_buffer *tx_bi; |
| 1722 | struct i40e_tx_desc *tx_desc; |
| 1723 | u16 i = tx_ring->next_to_use; |
| 1724 | u32 td_tag = 0; |
| 1725 | dma_addr_t dma; |
| 1726 | u16 gso_segs; |
| 1727 | |
| 1728 | if (tx_flags & I40E_TX_FLAGS_HW_VLAN) { |
| 1729 | td_cmd |= I40E_TX_DESC_CMD_IL2TAG1; |
| 1730 | td_tag = (tx_flags & I40E_TX_FLAGS_VLAN_MASK) >> |
| 1731 | I40E_TX_FLAGS_VLAN_SHIFT; |
| 1732 | } |
| 1733 | |
| 1734 | if (tx_flags & (I40E_TX_FLAGS_TSO | I40E_TX_FLAGS_FSO)) |
| 1735 | gso_segs = skb_shinfo(skb)->gso_segs; |
| 1736 | else |
| 1737 | gso_segs = 1; |
| 1738 | |
| 1739 | /* multiply data chunks by size of headers */ |
| 1740 | first->bytecount = skb->len - hdr_len + (gso_segs * hdr_len); |
| 1741 | first->gso_segs = gso_segs; |
| 1742 | first->skb = skb; |
| 1743 | first->tx_flags = tx_flags; |
| 1744 | |
| 1745 | dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE); |
| 1746 | |
| 1747 | tx_desc = I40E_TX_DESC(tx_ring, i); |
| 1748 | tx_bi = first; |
| 1749 | |
| 1750 | for (frag = &skb_shinfo(skb)->frags[0];; frag++) { |
| 1751 | if (dma_mapping_error(tx_ring->dev, dma)) |
| 1752 | goto dma_error; |
| 1753 | |
| 1754 | /* record length, and DMA address */ |
| 1755 | dma_unmap_len_set(tx_bi, len, size); |
| 1756 | dma_unmap_addr_set(tx_bi, dma, dma); |
| 1757 | |
| 1758 | tx_desc->buffer_addr = cpu_to_le64(dma); |
| 1759 | |
| 1760 | while (unlikely(size > I40E_MAX_DATA_PER_TXD)) { |
| 1761 | tx_desc->cmd_type_offset_bsz = |
| 1762 | build_ctob(td_cmd, td_offset, |
| 1763 | I40E_MAX_DATA_PER_TXD, td_tag); |
| 1764 | |
| 1765 | tx_desc++; |
| 1766 | i++; |
| 1767 | if (i == tx_ring->count) { |
| 1768 | tx_desc = I40E_TX_DESC(tx_ring, 0); |
| 1769 | i = 0; |
| 1770 | } |
| 1771 | |
| 1772 | dma += I40E_MAX_DATA_PER_TXD; |
| 1773 | size -= I40E_MAX_DATA_PER_TXD; |
| 1774 | |
| 1775 | tx_desc->buffer_addr = cpu_to_le64(dma); |
| 1776 | } |
| 1777 | |
| 1778 | if (likely(!data_len)) |
| 1779 | break; |
| 1780 | |
| 1781 | tx_desc->cmd_type_offset_bsz = build_ctob(td_cmd, td_offset, |
| 1782 | size, td_tag); |
| 1783 | |
| 1784 | tx_desc++; |
| 1785 | i++; |
| 1786 | if (i == tx_ring->count) { |
| 1787 | tx_desc = I40E_TX_DESC(tx_ring, 0); |
| 1788 | i = 0; |
| 1789 | } |
| 1790 | |
| 1791 | size = skb_frag_size(frag); |
| 1792 | data_len -= size; |
| 1793 | |
| 1794 | dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size, |
| 1795 | DMA_TO_DEVICE); |
| 1796 | |
| 1797 | tx_bi = &tx_ring->tx_bi[i]; |
| 1798 | } |
| 1799 | |
Jesse Brandeburg | 1943d8b | 2014-02-14 02:14:40 +0000 | [diff] [blame] | 1800 | /* Place RS bit on last descriptor of any packet that spans across the |
| 1801 | * 4th descriptor (WB_STRIDE aka 0x3) in a 64B cacheline. |
| 1802 | */ |
| 1803 | #define WB_STRIDE 0x3 |
| 1804 | if (((i & WB_STRIDE) != WB_STRIDE) && |
| 1805 | (first <= &tx_ring->tx_bi[i]) && |
| 1806 | (first >= &tx_ring->tx_bi[i & ~WB_STRIDE])) { |
| 1807 | tx_desc->cmd_type_offset_bsz = |
| 1808 | build_ctob(td_cmd, td_offset, size, td_tag) | |
| 1809 | cpu_to_le64((u64)I40E_TX_DESC_CMD_EOP << |
| 1810 | I40E_TXD_QW1_CMD_SHIFT); |
| 1811 | } else { |
| 1812 | tx_desc->cmd_type_offset_bsz = |
| 1813 | build_ctob(td_cmd, td_offset, size, td_tag) | |
| 1814 | cpu_to_le64((u64)I40E_TXD_CMD << |
| 1815 | I40E_TXD_QW1_CMD_SHIFT); |
| 1816 | } |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 1817 | |
| 1818 | netdev_tx_sent_queue(netdev_get_tx_queue(tx_ring->netdev, |
| 1819 | tx_ring->queue_index), |
| 1820 | first->bytecount); |
| 1821 | |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 1822 | /* Force memory writes to complete before letting h/w |
| 1823 | * know there are new descriptors to fetch. (Only |
| 1824 | * applicable for weak-ordered memory model archs, |
| 1825 | * such as IA-64). |
| 1826 | */ |
| 1827 | wmb(); |
| 1828 | |
| 1829 | /* set next_to_watch value indicating a packet is present */ |
| 1830 | first->next_to_watch = tx_desc; |
| 1831 | |
| 1832 | i++; |
| 1833 | if (i == tx_ring->count) |
| 1834 | i = 0; |
| 1835 | |
| 1836 | tx_ring->next_to_use = i; |
| 1837 | |
Jesse Brandeburg | 8f6a2b0 | 2015-04-16 20:06:09 -0400 | [diff] [blame] | 1838 | i40evf_maybe_stop_tx(tx_ring, DESC_NEEDED); |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 1839 | /* notify HW of packet */ |
Jesse Brandeburg | 8f6a2b0 | 2015-04-16 20:06:09 -0400 | [diff] [blame] | 1840 | if (!skb->xmit_more || |
| 1841 | netif_xmit_stopped(netdev_get_tx_queue(tx_ring->netdev, |
| 1842 | tx_ring->queue_index))) |
| 1843 | writel(i, tx_ring->tail); |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 1844 | |
| 1845 | return; |
| 1846 | |
| 1847 | dma_error: |
| 1848 | dev_info(tx_ring->dev, "TX DMA map failed\n"); |
| 1849 | |
| 1850 | /* clear dma mappings for failed tx_bi map */ |
| 1851 | for (;;) { |
| 1852 | tx_bi = &tx_ring->tx_bi[i]; |
| 1853 | i40e_unmap_and_free_tx_resource(tx_ring, tx_bi); |
| 1854 | if (tx_bi == first) |
| 1855 | break; |
| 1856 | if (i == 0) |
| 1857 | i = tx_ring->count; |
| 1858 | i--; |
| 1859 | } |
| 1860 | |
| 1861 | tx_ring->next_to_use = i; |
| 1862 | } |
| 1863 | |
| 1864 | /** |
Jesse Brandeburg | 3e587cf | 2015-04-16 20:06:10 -0400 | [diff] [blame] | 1865 | * i40evf_xmit_descriptor_count - calculate number of tx descriptors needed |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 1866 | * @skb: send buffer |
| 1867 | * @tx_ring: ring to send buffer on |
| 1868 | * |
| 1869 | * Returns number of data descriptors needed for this skb. Returns 0 to indicate |
| 1870 | * there is not enough descriptors available in this ring since we need at least |
| 1871 | * one descriptor. |
| 1872 | **/ |
Jesse Brandeburg | 3e587cf | 2015-04-16 20:06:10 -0400 | [diff] [blame] | 1873 | static inline int i40evf_xmit_descriptor_count(struct sk_buff *skb, |
| 1874 | struct i40e_ring *tx_ring) |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 1875 | { |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 1876 | unsigned int f; |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 1877 | int count = 0; |
| 1878 | |
| 1879 | /* need: 1 descriptor per page * PAGE_SIZE/I40E_MAX_DATA_PER_TXD, |
| 1880 | * + 1 desc for skb_head_len/I40E_MAX_DATA_PER_TXD, |
Jesse Brandeburg | be56052 | 2014-02-06 05:51:13 +0000 | [diff] [blame] | 1881 | * + 4 desc gap to avoid the cache line where head is, |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 1882 | * + 1 desc for context descriptor, |
| 1883 | * otherwise try next time |
| 1884 | */ |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 1885 | for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) |
| 1886 | count += TXD_USE_COUNT(skb_shinfo(skb)->frags[f].size); |
Jesse Brandeburg | 980093e | 2014-05-10 04:49:12 +0000 | [diff] [blame] | 1887 | |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 1888 | count += TXD_USE_COUNT(skb_headlen(skb)); |
Jesse Brandeburg | 8f6a2b0 | 2015-04-16 20:06:09 -0400 | [diff] [blame] | 1889 | if (i40evf_maybe_stop_tx(tx_ring, count + 4 + 1)) { |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 1890 | tx_ring->tx_stats.tx_busy++; |
| 1891 | return 0; |
| 1892 | } |
| 1893 | return count; |
| 1894 | } |
| 1895 | |
| 1896 | /** |
| 1897 | * i40e_xmit_frame_ring - Sends buffer on Tx ring |
| 1898 | * @skb: send buffer |
| 1899 | * @tx_ring: ring to send buffer on |
| 1900 | * |
| 1901 | * Returns NETDEV_TX_OK if sent, else an error code |
| 1902 | **/ |
| 1903 | static netdev_tx_t i40e_xmit_frame_ring(struct sk_buff *skb, |
| 1904 | struct i40e_ring *tx_ring) |
| 1905 | { |
| 1906 | u64 cd_type_cmd_tso_mss = I40E_TX_DESC_DTYPE_CONTEXT; |
| 1907 | u32 cd_tunneling = 0, cd_l2tag2 = 0; |
| 1908 | struct i40e_tx_buffer *first; |
| 1909 | u32 td_offset = 0; |
| 1910 | u32 tx_flags = 0; |
| 1911 | __be16 protocol; |
| 1912 | u32 td_cmd = 0; |
| 1913 | u8 hdr_len = 0; |
| 1914 | int tso; |
Jesse Brandeburg | 3e587cf | 2015-04-16 20:06:10 -0400 | [diff] [blame] | 1915 | if (0 == i40evf_xmit_descriptor_count(skb, tx_ring)) |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 1916 | return NETDEV_TX_BUSY; |
| 1917 | |
| 1918 | /* prepare the xmit flags */ |
Jesse Brandeburg | 3e587cf | 2015-04-16 20:06:10 -0400 | [diff] [blame] | 1919 | if (i40evf_tx_prepare_vlan_flags(skb, tx_ring, &tx_flags)) |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 1920 | goto out_drop; |
| 1921 | |
| 1922 | /* obtain protocol of skb */ |
Vlad Yasevich | a12c415 | 2014-08-25 10:34:53 -0400 | [diff] [blame] | 1923 | protocol = vlan_get_protocol(skb); |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 1924 | |
| 1925 | /* record the location of the first descriptor for this packet */ |
| 1926 | first = &tx_ring->tx_bi[tx_ring->next_to_use]; |
| 1927 | |
| 1928 | /* setup IPv4/IPv6 offloads */ |
| 1929 | if (protocol == htons(ETH_P_IP)) |
| 1930 | tx_flags |= I40E_TX_FLAGS_IPV4; |
| 1931 | else if (protocol == htons(ETH_P_IPV6)) |
| 1932 | tx_flags |= I40E_TX_FLAGS_IPV6; |
| 1933 | |
Anjali Singhai Jain | 89232c3 | 2015-04-16 20:06:00 -0400 | [diff] [blame] | 1934 | tso = i40e_tso(tx_ring, skb, &hdr_len, |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 1935 | &cd_type_cmd_tso_mss, &cd_tunneling); |
| 1936 | |
| 1937 | if (tso < 0) |
| 1938 | goto out_drop; |
| 1939 | else if (tso) |
| 1940 | tx_flags |= I40E_TX_FLAGS_TSO; |
| 1941 | |
Anjali Singhai Jain | 3052083 | 2015-05-08 15:35:52 -0700 | [diff] [blame] | 1942 | if (i40e_chk_linearize(skb, tx_flags)) |
Anjali Singhai | 71da619 | 2015-02-21 06:42:35 +0000 | [diff] [blame] | 1943 | if (skb_linearize(skb)) |
| 1944 | goto out_drop; |
| 1945 | |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 1946 | skb_tx_timestamp(skb); |
| 1947 | |
| 1948 | /* always enable CRC insertion offload */ |
| 1949 | td_cmd |= I40E_TX_DESC_CMD_ICRC; |
| 1950 | |
| 1951 | /* Always offload the checksum, since it's in the data descriptor */ |
| 1952 | if (skb->ip_summed == CHECKSUM_PARTIAL) { |
| 1953 | tx_flags |= I40E_TX_FLAGS_CSUM; |
| 1954 | |
Anjali Singhai Jain | 89232c3 | 2015-04-16 20:06:00 -0400 | [diff] [blame] | 1955 | i40e_tx_enable_csum(skb, &tx_flags, &td_cmd, &td_offset, |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 1956 | tx_ring, &cd_tunneling); |
| 1957 | } |
| 1958 | |
| 1959 | i40e_create_tx_ctx(tx_ring, cd_type_cmd_tso_mss, |
| 1960 | cd_tunneling, cd_l2tag2); |
| 1961 | |
Jesse Brandeburg | 3e587cf | 2015-04-16 20:06:10 -0400 | [diff] [blame] | 1962 | i40evf_tx_map(tx_ring, skb, first, tx_flags, hdr_len, |
| 1963 | td_cmd, td_offset); |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 1964 | |
Greg Rose | 7f12ad7 | 2013-12-21 06:12:51 +0000 | [diff] [blame] | 1965 | return NETDEV_TX_OK; |
| 1966 | |
| 1967 | out_drop: |
| 1968 | dev_kfree_skb_any(skb); |
| 1969 | return NETDEV_TX_OK; |
| 1970 | } |
| 1971 | |
| 1972 | /** |
| 1973 | * i40evf_xmit_frame - Selects the correct VSI and Tx queue to send buffer |
| 1974 | * @skb: send buffer |
| 1975 | * @netdev: network interface device structure |
| 1976 | * |
| 1977 | * Returns NETDEV_TX_OK if sent, else an error code |
| 1978 | **/ |
| 1979 | netdev_tx_t i40evf_xmit_frame(struct sk_buff *skb, struct net_device *netdev) |
| 1980 | { |
| 1981 | struct i40evf_adapter *adapter = netdev_priv(netdev); |
| 1982 | struct i40e_ring *tx_ring = adapter->tx_rings[skb->queue_mapping]; |
| 1983 | |
| 1984 | /* hardware can't handle really short frames, hardware padding works |
| 1985 | * beyond this point |
| 1986 | */ |
| 1987 | if (unlikely(skb->len < I40E_MIN_TX_LEN)) { |
| 1988 | if (skb_pad(skb, I40E_MIN_TX_LEN - skb->len)) |
| 1989 | return NETDEV_TX_OK; |
| 1990 | skb->len = I40E_MIN_TX_LEN; |
| 1991 | skb_set_tail_pointer(skb, I40E_MIN_TX_LEN); |
| 1992 | } |
| 1993 | |
| 1994 | return i40e_xmit_frame_ring(skb, tx_ring); |
| 1995 | } |