blob: 88666adb07436faf2ffae7a8fac2be319ef03ace [file] [log] [blame]
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +00001/*******************************************************************************
2 *
3 * Intel Ethernet Controller XL710 Family Linux Driver
Greg Rosedc641b72013-12-18 13:45:51 +00004 * Copyright(c) 2013 - 2014 Intel Corporation.
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +00005 *
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms and conditions of the GNU General Public License,
8 * version 2, as published by the Free Software Foundation.
9 *
10 * This program is distributed in the hope it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 * more details.
14 *
Greg Rosedc641b72013-12-18 13:45:51 +000015 * You should have received a copy of the GNU General Public License along
16 * with this program. If not, see <http://www.gnu.org/licenses/>.
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +000017 *
18 * The full GNU General Public License is included in this distribution in
19 * the file called "COPYING".
20 *
21 * Contact Information:
22 * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
23 * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
24 *
25 ******************************************************************************/
26
27#include "i40e.h"
Jesse Brandeburg206812b2014-02-12 01:45:33 +000028#include "i40e_prototype.h"
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +000029
30static inline __le64 build_ctob(u32 td_cmd, u32 td_offset, unsigned int size,
31 u32 td_tag)
32{
33 return cpu_to_le64(I40E_TX_DESC_DTYPE_DATA |
34 ((u64)td_cmd << I40E_TXD_QW1_CMD_SHIFT) |
35 ((u64)td_offset << I40E_TXD_QW1_OFFSET_SHIFT) |
36 ((u64)size << I40E_TXD_QW1_TX_BUF_SZ_SHIFT) |
37 ((u64)td_tag << I40E_TXD_QW1_L2TAG1_SHIFT));
38}
39
Jesse Brandeburgeaefbd02013-09-28 07:13:54 +000040#define I40E_TXD_CMD (I40E_TX_DESC_CMD_EOP | I40E_TX_DESC_CMD_RS)
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +000041/**
42 * i40e_program_fdir_filter - Program a Flow Director filter
Joseph Gasparakis17a73f62014-02-12 01:45:30 +000043 * @fdir_data: Packet data that will be filter parameters
44 * @raw_packet: the pre-allocated packet buffer for FDir
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +000045 * @pf: The pf pointer
46 * @add: True for add/update, False for remove
47 **/
Joseph Gasparakis17a73f62014-02-12 01:45:30 +000048int i40e_program_fdir_filter(struct i40e_fdir_filter *fdir_data, u8 *raw_packet,
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +000049 struct i40e_pf *pf, bool add)
50{
51 struct i40e_filter_program_desc *fdir_desc;
52 struct i40e_tx_buffer *tx_buf;
53 struct i40e_tx_desc *tx_desc;
54 struct i40e_ring *tx_ring;
Jesse Brandeburgeaefbd02013-09-28 07:13:54 +000055 unsigned int fpt, dcc;
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +000056 struct i40e_vsi *vsi;
57 struct device *dev;
58 dma_addr_t dma;
59 u32 td_cmd = 0;
60 u16 i;
61
62 /* find existing FDIR VSI */
63 vsi = NULL;
64 for (i = 0; i < pf->hw.func_caps.num_vsis; i++)
65 if (pf->vsi[i] && pf->vsi[i]->type == I40E_VSI_FDIR)
66 vsi = pf->vsi[i];
67 if (!vsi)
68 return -ENOENT;
69
Alexander Duyck9f65e152013-09-28 06:00:58 +000070 tx_ring = vsi->tx_rings[0];
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +000071 dev = tx_ring->dev;
72
Joseph Gasparakis17a73f62014-02-12 01:45:30 +000073 dma = dma_map_single(dev, raw_packet,
74 I40E_FDIR_MAX_RAW_PACKET_SIZE, DMA_TO_DEVICE);
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +000075 if (dma_mapping_error(dev, dma))
76 goto dma_fail;
77
78 /* grab the next descriptor */
Alexander Duyckfc4ac672013-09-28 06:00:22 +000079 i = tx_ring->next_to_use;
80 fdir_desc = I40E_TX_FDIRDESC(tx_ring, i);
Alexander Duyckfc4ac672013-09-28 06:00:22 +000081
Jesse Brandeburgeaefbd02013-09-28 07:13:54 +000082 tx_ring->next_to_use = (i + 1 < tx_ring->count) ? i + 1 : 0;
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +000083
Jesse Brandeburgeaefbd02013-09-28 07:13:54 +000084 fpt = (fdir_data->q_index << I40E_TXD_FLTR_QW0_QINDEX_SHIFT) &
85 I40E_TXD_FLTR_QW0_QINDEX_MASK;
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +000086
Jesse Brandeburgeaefbd02013-09-28 07:13:54 +000087 fpt |= (fdir_data->flex_off << I40E_TXD_FLTR_QW0_FLEXOFF_SHIFT) &
88 I40E_TXD_FLTR_QW0_FLEXOFF_MASK;
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +000089
Jesse Brandeburgeaefbd02013-09-28 07:13:54 +000090 fpt |= (fdir_data->pctype << I40E_TXD_FLTR_QW0_PCTYPE_SHIFT) &
91 I40E_TXD_FLTR_QW0_PCTYPE_MASK;
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +000092
93 /* Use LAN VSI Id if not programmed by user */
94 if (fdir_data->dest_vsi == 0)
Jesse Brandeburgeaefbd02013-09-28 07:13:54 +000095 fpt |= (pf->vsi[pf->lan_vsi]->id) <<
96 I40E_TXD_FLTR_QW0_DEST_VSI_SHIFT;
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +000097 else
Jesse Brandeburgeaefbd02013-09-28 07:13:54 +000098 fpt |= ((u32)fdir_data->dest_vsi <<
99 I40E_TXD_FLTR_QW0_DEST_VSI_SHIFT) &
100 I40E_TXD_FLTR_QW0_DEST_VSI_MASK;
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +0000101
Jesse Brandeburgeaefbd02013-09-28 07:13:54 +0000102 fdir_desc->qindex_flex_ptype_vsi = cpu_to_le32(fpt);
103
104 dcc = I40E_TX_DESC_DTYPE_FILTER_PROG;
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +0000105
106 if (add)
Jesse Brandeburgeaefbd02013-09-28 07:13:54 +0000107 dcc |= I40E_FILTER_PROGRAM_DESC_PCMD_ADD_UPDATE <<
108 I40E_TXD_FLTR_QW1_PCMD_SHIFT;
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +0000109 else
Jesse Brandeburgeaefbd02013-09-28 07:13:54 +0000110 dcc |= I40E_FILTER_PROGRAM_DESC_PCMD_REMOVE <<
111 I40E_TXD_FLTR_QW1_PCMD_SHIFT;
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +0000112
Jesse Brandeburgeaefbd02013-09-28 07:13:54 +0000113 dcc |= (fdir_data->dest_ctl << I40E_TXD_FLTR_QW1_DEST_SHIFT) &
114 I40E_TXD_FLTR_QW1_DEST_MASK;
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +0000115
Jesse Brandeburgeaefbd02013-09-28 07:13:54 +0000116 dcc |= (fdir_data->fd_status << I40E_TXD_FLTR_QW1_FD_STATUS_SHIFT) &
117 I40E_TXD_FLTR_QW1_FD_STATUS_MASK;
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +0000118
119 if (fdir_data->cnt_index != 0) {
Jesse Brandeburgeaefbd02013-09-28 07:13:54 +0000120 dcc |= I40E_TXD_FLTR_QW1_CNT_ENA_MASK;
121 dcc |= ((u32)fdir_data->cnt_index <<
122 I40E_TXD_FLTR_QW1_CNTINDEX_SHIFT) &
123 I40E_TXD_FLTR_QW1_CNTINDEX_MASK;
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +0000124 }
125
Jesse Brandeburgeaefbd02013-09-28 07:13:54 +0000126 fdir_desc->dtype_cmd_cntindex = cpu_to_le32(dcc);
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +0000127 fdir_desc->fd_id = cpu_to_le32(fdir_data->fd_id);
128
129 /* Now program a dummy descriptor */
Alexander Duyckfc4ac672013-09-28 06:00:22 +0000130 i = tx_ring->next_to_use;
131 tx_desc = I40E_TX_DESC(tx_ring, i);
Anjali Singhai Jain298deef2013-11-28 06:39:33 +0000132 tx_buf = &tx_ring->tx_bi[i];
Alexander Duyckfc4ac672013-09-28 06:00:22 +0000133
Jesse Brandeburgeaefbd02013-09-28 07:13:54 +0000134 tx_ring->next_to_use = (i + 1 < tx_ring->count) ? i + 1 : 0;
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +0000135
Anjali Singhai Jain298deef2013-11-28 06:39:33 +0000136 /* record length, and DMA address */
Joseph Gasparakis17a73f62014-02-12 01:45:30 +0000137 dma_unmap_len_set(tx_buf, len, I40E_FDIR_MAX_RAW_PACKET_SIZE);
Anjali Singhai Jain298deef2013-11-28 06:39:33 +0000138 dma_unmap_addr_set(tx_buf, dma, dma);
139
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +0000140 tx_desc->buffer_addr = cpu_to_le64(dma);
Jesse Brandeburgeaefbd02013-09-28 07:13:54 +0000141 td_cmd = I40E_TXD_CMD | I40E_TX_DESC_CMD_DUMMY;
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +0000142
143 tx_desc->cmd_type_offset_bsz =
Joseph Gasparakis17a73f62014-02-12 01:45:30 +0000144 build_ctob(td_cmd, 0, I40E_FDIR_MAX_RAW_PACKET_SIZE, 0);
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +0000145
Anjali Singhai Jain298deef2013-11-28 06:39:33 +0000146 /* set the timestamp */
147 tx_buf->time_stamp = jiffies;
148
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +0000149 /* Force memory writes to complete before letting h/w
150 * know there are new descriptors to fetch. (Only
151 * applicable for weak-ordered memory model archs,
152 * such as IA-64).
153 */
154 wmb();
155
Alexander Duyckfc4ac672013-09-28 06:00:22 +0000156 /* Mark the data descriptor to be watched */
157 tx_buf->next_to_watch = tx_desc;
158
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +0000159 writel(tx_ring->next_to_use, tx_ring->tail);
160 return 0;
161
162dma_fail:
163 return -1;
164}
165
Joseph Gasparakis17a73f62014-02-12 01:45:30 +0000166#define IP_HEADER_OFFSET 14
167#define I40E_UDPIP_DUMMY_PACKET_LEN 42
168/**
169 * i40e_add_del_fdir_udpv4 - Add/Remove UDPv4 filters
170 * @vsi: pointer to the targeted VSI
171 * @fd_data: the flow director data required for the FDir descriptor
172 * @raw_packet: the pre-allocated packet buffer for FDir
173 * @add: true adds a filter, false removes it
174 *
175 * Returns 0 if the filters were successfully added or removed
176 **/
177static int i40e_add_del_fdir_udpv4(struct i40e_vsi *vsi,
178 struct i40e_fdir_filter *fd_data,
179 u8 *raw_packet, bool add)
180{
181 struct i40e_pf *pf = vsi->back;
182 struct udphdr *udp;
183 struct iphdr *ip;
184 bool err = false;
185 int ret;
186 int i;
187 static char packet[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x08, 0,
188 0x45, 0, 0, 0x1c, 0, 0, 0x40, 0, 0x40, 0x11, 0, 0, 0, 0, 0, 0,
189 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
190
191 memcpy(raw_packet, packet, I40E_UDPIP_DUMMY_PACKET_LEN);
192
193 ip = (struct iphdr *)(raw_packet + IP_HEADER_OFFSET);
194 udp = (struct udphdr *)(raw_packet + IP_HEADER_OFFSET
195 + sizeof(struct iphdr));
196
197 ip->daddr = fd_data->dst_ip[0];
198 udp->dest = fd_data->dst_port;
199 ip->saddr = fd_data->src_ip[0];
200 udp->source = fd_data->src_port;
201
202 for (i = I40E_FILTER_PCTYPE_NONF_UNICAST_IPV4_UDP;
203 i <= I40E_FILTER_PCTYPE_NONF_IPV4_UDP; i++) {
204 fd_data->pctype = i;
205 ret = i40e_program_fdir_filter(fd_data, raw_packet, pf, add);
206
207 if (ret) {
208 dev_info(&pf->pdev->dev,
209 "Filter command send failed for PCTYPE %d (ret = %d)\n",
210 fd_data->pctype, ret);
211 err = true;
212 } else {
213 dev_info(&pf->pdev->dev,
214 "Filter OK for PCTYPE %d (ret = %d)\n",
215 fd_data->pctype, ret);
216 }
217 }
218
219 return err ? -EOPNOTSUPP : 0;
220}
221
222#define I40E_TCPIP_DUMMY_PACKET_LEN 54
223/**
224 * i40e_add_del_fdir_tcpv4 - Add/Remove TCPv4 filters
225 * @vsi: pointer to the targeted VSI
226 * @fd_data: the flow director data required for the FDir descriptor
227 * @raw_packet: the pre-allocated packet buffer for FDir
228 * @add: true adds a filter, false removes it
229 *
230 * Returns 0 if the filters were successfully added or removed
231 **/
232static int i40e_add_del_fdir_tcpv4(struct i40e_vsi *vsi,
233 struct i40e_fdir_filter *fd_data,
234 u8 *raw_packet, bool add)
235{
236 struct i40e_pf *pf = vsi->back;
237 struct tcphdr *tcp;
238 struct iphdr *ip;
239 bool err = false;
240 int ret;
241 /* Dummy packet */
242 static char packet[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x08, 0,
243 0x45, 0, 0, 0x28, 0, 0, 0x40, 0, 0x40, 0x6, 0, 0, 0, 0, 0, 0,
244 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x80, 0x11,
245 0x0, 0x72, 0, 0, 0, 0};
246
247 memcpy(raw_packet, packet, I40E_TCPIP_DUMMY_PACKET_LEN);
248
249 ip = (struct iphdr *)(raw_packet + IP_HEADER_OFFSET);
250 tcp = (struct tcphdr *)(raw_packet + IP_HEADER_OFFSET
251 + sizeof(struct iphdr));
252
253 ip->daddr = fd_data->dst_ip[0];
254 tcp->dest = fd_data->dst_port;
255 ip->saddr = fd_data->src_ip[0];
256 tcp->source = fd_data->src_port;
257
258 if (add) {
259 if (pf->flags & I40E_FLAG_FD_ATR_ENABLED) {
260 dev_info(&pf->pdev->dev, "Forcing ATR off, sideband rules for TCP/IPv4 flow being applied\n");
261 pf->flags &= ~I40E_FLAG_FD_ATR_ENABLED;
262 }
263 }
264
265 fd_data->pctype = I40E_FILTER_PCTYPE_NONF_IPV4_TCP_SYN;
266 ret = i40e_program_fdir_filter(fd_data, raw_packet, pf, add);
267
268 if (ret) {
269 dev_info(&pf->pdev->dev,
270 "Filter command send failed for PCTYPE %d (ret = %d)\n",
271 fd_data->pctype, ret);
272 err = true;
273 } else {
274 dev_info(&pf->pdev->dev, "Filter OK for PCTYPE %d (ret = %d)\n",
275 fd_data->pctype, ret);
276 }
277
278 fd_data->pctype = I40E_FILTER_PCTYPE_NONF_IPV4_TCP;
279
280 ret = i40e_program_fdir_filter(fd_data, raw_packet, pf, add);
281 if (ret) {
282 dev_info(&pf->pdev->dev,
283 "Filter command send failed for PCTYPE %d (ret = %d)\n",
284 fd_data->pctype, ret);
285 err = true;
286 } else {
287 dev_info(&pf->pdev->dev, "Filter OK for PCTYPE %d (ret = %d)\n",
288 fd_data->pctype, ret);
289 }
290
291 return err ? -EOPNOTSUPP : 0;
292}
293
294/**
295 * i40e_add_del_fdir_sctpv4 - Add/Remove SCTPv4 Flow Director filters for
296 * a specific flow spec
297 * @vsi: pointer to the targeted VSI
298 * @fd_data: the flow director data required for the FDir descriptor
299 * @raw_packet: the pre-allocated packet buffer for FDir
300 * @add: true adds a filter, false removes it
301 *
302 * Returns 0 if the filters were successfully added or removed
303 **/
304static int i40e_add_del_fdir_sctpv4(struct i40e_vsi *vsi,
305 struct i40e_fdir_filter *fd_data,
306 u8 *raw_packet, bool add)
307{
308 return -EOPNOTSUPP;
309}
310
311#define I40E_IP_DUMMY_PACKET_LEN 34
312/**
313 * i40e_add_del_fdir_ipv4 - Add/Remove IPv4 Flow Director filters for
314 * a specific flow spec
315 * @vsi: pointer to the targeted VSI
316 * @fd_data: the flow director data required for the FDir descriptor
317 * @raw_packet: the pre-allocated packet buffer for FDir
318 * @add: true adds a filter, false removes it
319 *
320 * Returns 0 if the filters were successfully added or removed
321 **/
322static int i40e_add_del_fdir_ipv4(struct i40e_vsi *vsi,
323 struct i40e_fdir_filter *fd_data,
324 u8 *raw_packet, bool add)
325{
326 struct i40e_pf *pf = vsi->back;
327 struct iphdr *ip;
328 bool err = false;
329 int ret;
330 int i;
331 static char packet[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x08, 0,
332 0x45, 0, 0, 0x14, 0, 0, 0x40, 0, 0x40, 0x10, 0, 0, 0, 0, 0, 0,
333 0, 0, 0, 0};
334
335 memcpy(raw_packet, packet, I40E_IP_DUMMY_PACKET_LEN);
336 ip = (struct iphdr *)(raw_packet + IP_HEADER_OFFSET);
337
338 ip->saddr = fd_data->src_ip[0];
339 ip->daddr = fd_data->dst_ip[0];
340 ip->protocol = 0;
341
342 for (i = I40E_FILTER_PCTYPE_NONF_IPV4_OTHER;
343 i <= I40E_FILTER_PCTYPE_FRAG_IPV4; i++) {
344 fd_data->pctype = i;
345 ret = i40e_program_fdir_filter(fd_data, raw_packet, pf, add);
346
347 if (ret) {
348 dev_info(&pf->pdev->dev,
349 "Filter command send failed for PCTYPE %d (ret = %d)\n",
350 fd_data->pctype, ret);
351 err = true;
352 } else {
353 dev_info(&pf->pdev->dev,
354 "Filter OK for PCTYPE %d (ret = %d)\n",
355 fd_data->pctype, ret);
356 }
357 }
358
359 return err ? -EOPNOTSUPP : 0;
360}
361
362/**
363 * i40e_add_del_fdir - Build raw packets to add/del fdir filter
364 * @vsi: pointer to the targeted VSI
365 * @cmd: command to get or set RX flow classification rules
366 * @add: true adds a filter, false removes it
367 *
368 **/
369int i40e_add_del_fdir(struct i40e_vsi *vsi,
370 struct i40e_fdir_filter *input, bool add)
371{
372 struct i40e_pf *pf = vsi->back;
373 u8 *raw_packet;
374 int ret;
375
376 /* Populate the Flow Director that we have at the moment
377 * and allocate the raw packet buffer for the calling functions
378 */
379 raw_packet = kzalloc(I40E_FDIR_MAX_RAW_PACKET_SIZE, GFP_KERNEL);
380 if (!raw_packet)
381 return -ENOMEM;
382
383 switch (input->flow_type & ~FLOW_EXT) {
384 case TCP_V4_FLOW:
385 ret = i40e_add_del_fdir_tcpv4(vsi, input, raw_packet,
386 add);
387 break;
388 case UDP_V4_FLOW:
389 ret = i40e_add_del_fdir_udpv4(vsi, input, raw_packet,
390 add);
391 break;
392 case SCTP_V4_FLOW:
393 ret = i40e_add_del_fdir_sctpv4(vsi, input, raw_packet,
394 add);
395 break;
396 case IPV4_FLOW:
397 ret = i40e_add_del_fdir_ipv4(vsi, input, raw_packet,
398 add);
399 break;
400 case IP_USER_FLOW:
401 switch (input->ip4_proto) {
402 case IPPROTO_TCP:
403 ret = i40e_add_del_fdir_tcpv4(vsi, input,
404 raw_packet, add);
405 break;
406 case IPPROTO_UDP:
407 ret = i40e_add_del_fdir_udpv4(vsi, input,
408 raw_packet, add);
409 break;
410 case IPPROTO_SCTP:
411 ret = i40e_add_del_fdir_sctpv4(vsi, input,
412 raw_packet, add);
413 break;
414 default:
415 ret = i40e_add_del_fdir_ipv4(vsi, input,
416 raw_packet, add);
417 break;
418 }
419 break;
420 default:
421 dev_info(&pf->pdev->dev, "Could not specify spec type %d",
422 input->flow_type);
423 ret = -EINVAL;
424 }
425
426 kfree(raw_packet);
427 return ret;
428}
429
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +0000430/**
431 * i40e_fd_handle_status - check the Programming Status for FD
432 * @rx_ring: the Rx ring for this descriptor
Anjali Singhai Jain55a5e602014-02-12 06:33:25 +0000433 * @rx_desc: the Rx descriptor for programming Status, not a packet descriptor.
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +0000434 * @prog_id: the id originally used for programming
435 *
436 * This is used to verify if the FD programming or invalidation
437 * requested by SW to the HW is successful or not and take actions accordingly.
438 **/
Anjali Singhai Jain55a5e602014-02-12 06:33:25 +0000439static void i40e_fd_handle_status(struct i40e_ring *rx_ring,
440 union i40e_rx_desc *rx_desc, u8 prog_id)
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +0000441{
Anjali Singhai Jain55a5e602014-02-12 06:33:25 +0000442 struct i40e_pf *pf = rx_ring->vsi->back;
443 struct pci_dev *pdev = pf->pdev;
444 u32 fcnt_prog, fcnt_avail;
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +0000445 u32 error;
Anjali Singhai Jain55a5e602014-02-12 06:33:25 +0000446 u64 qw;
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +0000447
Anjali Singhai Jain55a5e602014-02-12 06:33:25 +0000448 qw = le64_to_cpu(rx_desc->wb.qword1.status_error_len);
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +0000449 error = (qw & I40E_RX_PROG_STATUS_DESC_QW1_ERROR_MASK) >>
450 I40E_RX_PROG_STATUS_DESC_QW1_ERROR_SHIFT;
451
Anjali Singhai Jain55a5e602014-02-12 06:33:25 +0000452 if (error == (0x1 << I40E_RX_PROG_STATUS_DESC_FD_TBL_FULL_SHIFT)) {
453 dev_warn(&pdev->dev, "ntuple filter loc = %d, could not be added\n",
454 rx_desc->wb.qword0.hi_dword.fd_id);
455
456 /* filter programming failed most likely due to table full */
457 fcnt_prog = i40e_get_current_fd_count(pf);
458 fcnt_avail = pf->hw.fdir_shared_filter_count +
459 pf->fdir_pf_filter_count;
460
461 /* If ATR is running fcnt_prog can quickly change,
462 * if we are very close to full, it makes sense to disable
463 * FD ATR/SB and then re-enable it when there is room.
464 */
465 if (fcnt_prog >= (fcnt_avail - I40E_FDIR_BUFFER_FULL_MARGIN)) {
466 /* Turn off ATR first */
467 if (pf->flags | I40E_FLAG_FD_ATR_ENABLED) {
468 pf->flags &= ~I40E_FLAG_FD_ATR_ENABLED;
469 dev_warn(&pdev->dev, "FD filter space full, ATR for further flows will be turned off\n");
470 pf->auto_disable_flags |=
471 I40E_FLAG_FD_ATR_ENABLED;
472 pf->flags |= I40E_FLAG_FDIR_REQUIRES_REINIT;
473 } else if (pf->flags | I40E_FLAG_FD_SB_ENABLED) {
474 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
475 dev_warn(&pdev->dev, "FD filter space full, new ntuple rules will not be added\n");
476 pf->auto_disable_flags |=
477 I40E_FLAG_FD_SB_ENABLED;
478 pf->flags |= I40E_FLAG_FDIR_REQUIRES_REINIT;
479 }
480 } else {
481 dev_info(&pdev->dev, "FD filter programming error");
482 }
483 } else if (error ==
484 (0x1 << I40E_RX_PROG_STATUS_DESC_NO_FD_ENTRY_SHIFT)) {
485 netdev_info(rx_ring->vsi->netdev, "ntuple filter loc = %d, could not be removed\n",
486 rx_desc->wb.qword0.hi_dword.fd_id);
487 }
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +0000488}
489
490/**
Alexander Duycka5e9c572013-09-28 06:00:27 +0000491 * i40e_unmap_and_free_tx_resource - Release a Tx buffer
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +0000492 * @ring: the ring that owns the buffer
493 * @tx_buffer: the buffer to free
494 **/
Alexander Duycka5e9c572013-09-28 06:00:27 +0000495static void i40e_unmap_and_free_tx_resource(struct i40e_ring *ring,
496 struct i40e_tx_buffer *tx_buffer)
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +0000497{
Alexander Duycka5e9c572013-09-28 06:00:27 +0000498 if (tx_buffer->skb) {
499 dev_kfree_skb_any(tx_buffer->skb);
500 if (dma_unmap_len(tx_buffer, len))
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +0000501 dma_unmap_single(ring->dev,
Alexander Duyck35a1e2a2013-09-28 06:00:17 +0000502 dma_unmap_addr(tx_buffer, dma),
503 dma_unmap_len(tx_buffer, len),
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +0000504 DMA_TO_DEVICE);
Alexander Duycka5e9c572013-09-28 06:00:27 +0000505 } else if (dma_unmap_len(tx_buffer, len)) {
506 dma_unmap_page(ring->dev,
507 dma_unmap_addr(tx_buffer, dma),
508 dma_unmap_len(tx_buffer, len),
509 DMA_TO_DEVICE);
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +0000510 }
Alexander Duycka5e9c572013-09-28 06:00:27 +0000511 tx_buffer->next_to_watch = NULL;
512 tx_buffer->skb = NULL;
Alexander Duyck35a1e2a2013-09-28 06:00:17 +0000513 dma_unmap_len_set(tx_buffer, len, 0);
Alexander Duycka5e9c572013-09-28 06:00:27 +0000514 /* tx_buffer must be completely set up in the transmit path */
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +0000515}
516
517/**
518 * i40e_clean_tx_ring - Free any empty Tx buffers
519 * @tx_ring: ring to be cleaned
520 **/
521void i40e_clean_tx_ring(struct i40e_ring *tx_ring)
522{
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +0000523 unsigned long bi_size;
524 u16 i;
525
526 /* ring already cleared, nothing to do */
527 if (!tx_ring->tx_bi)
528 return;
529
530 /* Free all the Tx ring sk_buffs */
Alexander Duycka5e9c572013-09-28 06:00:27 +0000531 for (i = 0; i < tx_ring->count; i++)
532 i40e_unmap_and_free_tx_resource(tx_ring, &tx_ring->tx_bi[i]);
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +0000533
534 bi_size = sizeof(struct i40e_tx_buffer) * tx_ring->count;
535 memset(tx_ring->tx_bi, 0, bi_size);
536
537 /* Zero out the descriptor ring */
538 memset(tx_ring->desc, 0, tx_ring->size);
539
540 tx_ring->next_to_use = 0;
541 tx_ring->next_to_clean = 0;
Alexander Duyck7070ce02013-09-28 06:00:37 +0000542
543 if (!tx_ring->netdev)
544 return;
545
546 /* cleanup Tx queue statistics */
547 netdev_tx_reset_queue(netdev_get_tx_queue(tx_ring->netdev,
548 tx_ring->queue_index));
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +0000549}
550
551/**
552 * i40e_free_tx_resources - Free Tx resources per queue
553 * @tx_ring: Tx descriptor ring for a specific queue
554 *
555 * Free all transmit software resources
556 **/
557void i40e_free_tx_resources(struct i40e_ring *tx_ring)
558{
559 i40e_clean_tx_ring(tx_ring);
560 kfree(tx_ring->tx_bi);
561 tx_ring->tx_bi = NULL;
562
563 if (tx_ring->desc) {
564 dma_free_coherent(tx_ring->dev, tx_ring->size,
565 tx_ring->desc, tx_ring->dma);
566 tx_ring->desc = NULL;
567 }
568}
569
570/**
571 * i40e_get_tx_pending - how many tx descriptors not processed
572 * @tx_ring: the ring of descriptors
573 *
574 * Since there is no access to the ring head register
575 * in XL710, we need to use our local copies
576 **/
577static u32 i40e_get_tx_pending(struct i40e_ring *ring)
578{
579 u32 ntu = ((ring->next_to_clean <= ring->next_to_use)
580 ? ring->next_to_use
581 : ring->next_to_use + ring->count);
582 return ntu - ring->next_to_clean;
583}
584
585/**
586 * i40e_check_tx_hang - Is there a hang in the Tx queue
587 * @tx_ring: the ring of descriptors
588 **/
589static bool i40e_check_tx_hang(struct i40e_ring *tx_ring)
590{
591 u32 tx_pending = i40e_get_tx_pending(tx_ring);
592 bool ret = false;
593
594 clear_check_for_tx_hang(tx_ring);
595
596 /* Check for a hung queue, but be thorough. This verifies
597 * that a transmit has been completed since the previous
598 * check AND there is at least one packet pending. The
599 * ARMED bit is set to indicate a potential hang. The
600 * bit is cleared if a pause frame is received to remove
601 * false hang detection due to PFC or 802.3x frames. By
602 * requiring this to fail twice we avoid races with
603 * PFC clearing the ARMED bit and conditions where we
604 * run the check_tx_hang logic with a transmit completion
605 * pending but without time to complete it yet.
606 */
Alexander Duycka114d0a2013-09-28 06:00:43 +0000607 if ((tx_ring->tx_stats.tx_done_old == tx_ring->stats.packets) &&
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +0000608 tx_pending) {
609 /* make sure it is true for two checks in a row */
610 ret = test_and_set_bit(__I40E_HANG_CHECK_ARMED,
611 &tx_ring->state);
612 } else {
613 /* update completed stats and disarm the hang check */
Alexander Duycka114d0a2013-09-28 06:00:43 +0000614 tx_ring->tx_stats.tx_done_old = tx_ring->stats.packets;
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +0000615 clear_bit(__I40E_HANG_CHECK_ARMED, &tx_ring->state);
616 }
617
618 return ret;
619}
620
621/**
622 * i40e_clean_tx_irq - Reclaim resources after transmit completes
623 * @tx_ring: tx ring to clean
624 * @budget: how many cleans we're allowed
625 *
626 * Returns true if there's any budget left (e.g. the clean is finished)
627 **/
628static bool i40e_clean_tx_irq(struct i40e_ring *tx_ring, int budget)
629{
630 u16 i = tx_ring->next_to_clean;
631 struct i40e_tx_buffer *tx_buf;
632 struct i40e_tx_desc *tx_desc;
633 unsigned int total_packets = 0;
634 unsigned int total_bytes = 0;
635
636 tx_buf = &tx_ring->tx_bi[i];
637 tx_desc = I40E_TX_DESC(tx_ring, i);
Alexander Duycka5e9c572013-09-28 06:00:27 +0000638 i -= tx_ring->count;
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +0000639
Alexander Duycka5e9c572013-09-28 06:00:27 +0000640 do {
641 struct i40e_tx_desc *eop_desc = tx_buf->next_to_watch;
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +0000642
643 /* if next_to_watch is not set then there is no work pending */
644 if (!eop_desc)
645 break;
646
Alexander Duycka5e9c572013-09-28 06:00:27 +0000647 /* prevent any other reads prior to eop_desc */
648 read_barrier_depends();
649
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +0000650 /* if the descriptor isn't done, no work yet to do */
651 if (!(eop_desc->cmd_type_offset_bsz &
652 cpu_to_le64(I40E_TX_DESC_DTYPE_DESC_DONE)))
653 break;
654
Alexander Duyckc304fda2013-09-28 06:00:12 +0000655 /* clear next_to_watch to prevent false hangs */
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +0000656 tx_buf->next_to_watch = NULL;
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +0000657
Alexander Duycka5e9c572013-09-28 06:00:27 +0000658 /* update the statistics for this packet */
659 total_bytes += tx_buf->bytecount;
660 total_packets += tx_buf->gso_segs;
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +0000661
Alexander Duycka5e9c572013-09-28 06:00:27 +0000662 /* free the skb */
663 dev_kfree_skb_any(tx_buf->skb);
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +0000664
Alexander Duycka5e9c572013-09-28 06:00:27 +0000665 /* unmap skb header data */
666 dma_unmap_single(tx_ring->dev,
667 dma_unmap_addr(tx_buf, dma),
668 dma_unmap_len(tx_buf, len),
669 DMA_TO_DEVICE);
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +0000670
Alexander Duycka5e9c572013-09-28 06:00:27 +0000671 /* clear tx_buffer data */
672 tx_buf->skb = NULL;
673 dma_unmap_len_set(tx_buf, len, 0);
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +0000674
Alexander Duycka5e9c572013-09-28 06:00:27 +0000675 /* unmap remaining buffers */
676 while (tx_desc != eop_desc) {
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +0000677
678 tx_buf++;
679 tx_desc++;
680 i++;
Alexander Duycka5e9c572013-09-28 06:00:27 +0000681 if (unlikely(!i)) {
682 i -= tx_ring->count;
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +0000683 tx_buf = tx_ring->tx_bi;
684 tx_desc = I40E_TX_DESC(tx_ring, 0);
685 }
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +0000686
Alexander Duycka5e9c572013-09-28 06:00:27 +0000687 /* unmap any remaining paged data */
688 if (dma_unmap_len(tx_buf, len)) {
689 dma_unmap_page(tx_ring->dev,
690 dma_unmap_addr(tx_buf, dma),
691 dma_unmap_len(tx_buf, len),
692 DMA_TO_DEVICE);
693 dma_unmap_len_set(tx_buf, len, 0);
694 }
695 }
696
697 /* move us one more past the eop_desc for start of next pkt */
698 tx_buf++;
699 tx_desc++;
700 i++;
701 if (unlikely(!i)) {
702 i -= tx_ring->count;
703 tx_buf = tx_ring->tx_bi;
704 tx_desc = I40E_TX_DESC(tx_ring, 0);
705 }
706
707 /* update budget accounting */
708 budget--;
709 } while (likely(budget));
710
711 i += tx_ring->count;
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +0000712 tx_ring->next_to_clean = i;
Alexander Duyck980e9b12013-09-28 06:01:03 +0000713 u64_stats_update_begin(&tx_ring->syncp);
Alexander Duycka114d0a2013-09-28 06:00:43 +0000714 tx_ring->stats.bytes += total_bytes;
715 tx_ring->stats.packets += total_packets;
Alexander Duyck980e9b12013-09-28 06:01:03 +0000716 u64_stats_update_end(&tx_ring->syncp);
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +0000717 tx_ring->q_vector->tx.total_bytes += total_bytes;
718 tx_ring->q_vector->tx.total_packets += total_packets;
Alexander Duycka5e9c572013-09-28 06:00:27 +0000719
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +0000720 if (check_for_tx_hang(tx_ring) && i40e_check_tx_hang(tx_ring)) {
721 /* schedule immediate reset if we believe we hung */
722 dev_info(tx_ring->dev, "Detected Tx Unit Hang\n"
723 " VSI <%d>\n"
724 " Tx Queue <%d>\n"
725 " next_to_use <%x>\n"
726 " next_to_clean <%x>\n",
727 tx_ring->vsi->seid,
728 tx_ring->queue_index,
729 tx_ring->next_to_use, i);
730 dev_info(tx_ring->dev, "tx_bi[next_to_clean]\n"
731 " time_stamp <%lx>\n"
732 " jiffies <%lx>\n",
733 tx_ring->tx_bi[i].time_stamp, jiffies);
734
735 netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index);
736
737 dev_info(tx_ring->dev,
738 "tx hang detected on queue %d, resetting adapter\n",
739 tx_ring->queue_index);
740
741 tx_ring->netdev->netdev_ops->ndo_tx_timeout(tx_ring->netdev);
742
743 /* the adapter is about to reset, no point in enabling stuff */
744 return true;
745 }
746
Alexander Duyck7070ce02013-09-28 06:00:37 +0000747 netdev_tx_completed_queue(netdev_get_tx_queue(tx_ring->netdev,
748 tx_ring->queue_index),
749 total_packets, total_bytes);
750
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +0000751#define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
752 if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) &&
753 (I40E_DESC_UNUSED(tx_ring) >= TX_WAKE_THRESHOLD))) {
754 /* Make sure that anybody stopping the queue after this
755 * sees the new next_to_clean.
756 */
757 smp_mb();
758 if (__netif_subqueue_stopped(tx_ring->netdev,
759 tx_ring->queue_index) &&
760 !test_bit(__I40E_DOWN, &tx_ring->vsi->state)) {
761 netif_wake_subqueue(tx_ring->netdev,
762 tx_ring->queue_index);
763 ++tx_ring->tx_stats.restart_queue;
764 }
765 }
766
767 return budget > 0;
768}
769
770/**
771 * i40e_set_new_dynamic_itr - Find new ITR level
772 * @rc: structure containing ring performance data
773 *
774 * Stores a new ITR value based on packets and byte counts during
775 * the last interrupt. The advantage of per interrupt computation
776 * is faster updates and more accurate ITR for the current traffic
777 * pattern. Constants in this function were computed based on
778 * theoretical maximum wire speed and thresholds were set based on
779 * testing data as well as attempting to minimize response time
780 * while increasing bulk throughput.
781 **/
782static void i40e_set_new_dynamic_itr(struct i40e_ring_container *rc)
783{
784 enum i40e_latency_range new_latency_range = rc->latency_range;
785 u32 new_itr = rc->itr;
786 int bytes_per_int;
787
788 if (rc->total_packets == 0 || !rc->itr)
789 return;
790
791 /* simple throttlerate management
792 * 0-10MB/s lowest (100000 ints/s)
793 * 10-20MB/s low (20000 ints/s)
794 * 20-1249MB/s bulk (8000 ints/s)
795 */
796 bytes_per_int = rc->total_bytes / rc->itr;
797 switch (rc->itr) {
798 case I40E_LOWEST_LATENCY:
799 if (bytes_per_int > 10)
800 new_latency_range = I40E_LOW_LATENCY;
801 break;
802 case I40E_LOW_LATENCY:
803 if (bytes_per_int > 20)
804 new_latency_range = I40E_BULK_LATENCY;
805 else if (bytes_per_int <= 10)
806 new_latency_range = I40E_LOWEST_LATENCY;
807 break;
808 case I40E_BULK_LATENCY:
809 if (bytes_per_int <= 20)
810 rc->latency_range = I40E_LOW_LATENCY;
811 break;
812 }
813
814 switch (new_latency_range) {
815 case I40E_LOWEST_LATENCY:
816 new_itr = I40E_ITR_100K;
817 break;
818 case I40E_LOW_LATENCY:
819 new_itr = I40E_ITR_20K;
820 break;
821 case I40E_BULK_LATENCY:
822 new_itr = I40E_ITR_8K;
823 break;
824 default:
825 break;
826 }
827
828 if (new_itr != rc->itr) {
829 /* do an exponential smoothing */
830 new_itr = (10 * new_itr * rc->itr) /
831 ((9 * new_itr) + rc->itr);
832 rc->itr = new_itr & I40E_MAX_ITR;
833 }
834
835 rc->total_bytes = 0;
836 rc->total_packets = 0;
837}
838
839/**
840 * i40e_update_dynamic_itr - Adjust ITR based on bytes per int
841 * @q_vector: the vector to adjust
842 **/
843static void i40e_update_dynamic_itr(struct i40e_q_vector *q_vector)
844{
845 u16 vector = q_vector->vsi->base_vector + q_vector->v_idx;
846 struct i40e_hw *hw = &q_vector->vsi->back->hw;
847 u32 reg_addr;
848 u16 old_itr;
849
850 reg_addr = I40E_PFINT_ITRN(I40E_RX_ITR, vector - 1);
851 old_itr = q_vector->rx.itr;
852 i40e_set_new_dynamic_itr(&q_vector->rx);
853 if (old_itr != q_vector->rx.itr)
854 wr32(hw, reg_addr, q_vector->rx.itr);
855
856 reg_addr = I40E_PFINT_ITRN(I40E_TX_ITR, vector - 1);
857 old_itr = q_vector->tx.itr;
858 i40e_set_new_dynamic_itr(&q_vector->tx);
859 if (old_itr != q_vector->tx.itr)
860 wr32(hw, reg_addr, q_vector->tx.itr);
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +0000861}
862
863/**
864 * i40e_clean_programming_status - clean the programming status descriptor
865 * @rx_ring: the rx ring that has this descriptor
866 * @rx_desc: the rx descriptor written back by HW
867 *
868 * Flow director should handle FD_FILTER_STATUS to check its filter programming
869 * status being successful or not and take actions accordingly. FCoE should
870 * handle its context/filter programming/invalidation status and take actions.
871 *
872 **/
873static void i40e_clean_programming_status(struct i40e_ring *rx_ring,
874 union i40e_rx_desc *rx_desc)
875{
876 u64 qw;
877 u8 id;
878
879 qw = le64_to_cpu(rx_desc->wb.qword1.status_error_len);
880 id = (qw & I40E_RX_PROG_STATUS_DESC_QW1_PROGID_MASK) >>
881 I40E_RX_PROG_STATUS_DESC_QW1_PROGID_SHIFT;
882
883 if (id == I40E_RX_PROG_STATUS_DESC_FD_FILTER_STATUS)
Anjali Singhai Jain55a5e602014-02-12 06:33:25 +0000884 i40e_fd_handle_status(rx_ring, rx_desc, id);
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +0000885}
886
887/**
888 * i40e_setup_tx_descriptors - Allocate the Tx descriptors
889 * @tx_ring: the tx ring to set up
890 *
891 * Return 0 on success, negative on error
892 **/
893int i40e_setup_tx_descriptors(struct i40e_ring *tx_ring)
894{
895 struct device *dev = tx_ring->dev;
896 int bi_size;
897
898 if (!dev)
899 return -ENOMEM;
900
901 bi_size = sizeof(struct i40e_tx_buffer) * tx_ring->count;
902 tx_ring->tx_bi = kzalloc(bi_size, GFP_KERNEL);
903 if (!tx_ring->tx_bi)
904 goto err;
905
906 /* round up to nearest 4K */
907 tx_ring->size = tx_ring->count * sizeof(struct i40e_tx_desc);
908 tx_ring->size = ALIGN(tx_ring->size, 4096);
909 tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size,
910 &tx_ring->dma, GFP_KERNEL);
911 if (!tx_ring->desc) {
912 dev_info(dev, "Unable to allocate memory for the Tx descriptor ring, size=%d\n",
913 tx_ring->size);
914 goto err;
915 }
916
917 tx_ring->next_to_use = 0;
918 tx_ring->next_to_clean = 0;
919 return 0;
920
921err:
922 kfree(tx_ring->tx_bi);
923 tx_ring->tx_bi = NULL;
924 return -ENOMEM;
925}
926
927/**
928 * i40e_clean_rx_ring - Free Rx buffers
929 * @rx_ring: ring to be cleaned
930 **/
931void i40e_clean_rx_ring(struct i40e_ring *rx_ring)
932{
933 struct device *dev = rx_ring->dev;
934 struct i40e_rx_buffer *rx_bi;
935 unsigned long bi_size;
936 u16 i;
937
938 /* ring already cleared, nothing to do */
939 if (!rx_ring->rx_bi)
940 return;
941
942 /* Free all the Rx ring sk_buffs */
943 for (i = 0; i < rx_ring->count; i++) {
944 rx_bi = &rx_ring->rx_bi[i];
945 if (rx_bi->dma) {
946 dma_unmap_single(dev,
947 rx_bi->dma,
948 rx_ring->rx_buf_len,
949 DMA_FROM_DEVICE);
950 rx_bi->dma = 0;
951 }
952 if (rx_bi->skb) {
953 dev_kfree_skb(rx_bi->skb);
954 rx_bi->skb = NULL;
955 }
956 if (rx_bi->page) {
957 if (rx_bi->page_dma) {
958 dma_unmap_page(dev,
959 rx_bi->page_dma,
960 PAGE_SIZE / 2,
961 DMA_FROM_DEVICE);
962 rx_bi->page_dma = 0;
963 }
964 __free_page(rx_bi->page);
965 rx_bi->page = NULL;
966 rx_bi->page_offset = 0;
967 }
968 }
969
970 bi_size = sizeof(struct i40e_rx_buffer) * rx_ring->count;
971 memset(rx_ring->rx_bi, 0, bi_size);
972
973 /* Zero out the descriptor ring */
974 memset(rx_ring->desc, 0, rx_ring->size);
975
976 rx_ring->next_to_clean = 0;
977 rx_ring->next_to_use = 0;
978}
979
980/**
981 * i40e_free_rx_resources - Free Rx resources
982 * @rx_ring: ring to clean the resources from
983 *
984 * Free all receive software resources
985 **/
986void i40e_free_rx_resources(struct i40e_ring *rx_ring)
987{
988 i40e_clean_rx_ring(rx_ring);
989 kfree(rx_ring->rx_bi);
990 rx_ring->rx_bi = NULL;
991
992 if (rx_ring->desc) {
993 dma_free_coherent(rx_ring->dev, rx_ring->size,
994 rx_ring->desc, rx_ring->dma);
995 rx_ring->desc = NULL;
996 }
997}
998
999/**
1000 * i40e_setup_rx_descriptors - Allocate Rx descriptors
1001 * @rx_ring: Rx descriptor ring (for a specific queue) to setup
1002 *
1003 * Returns 0 on success, negative on failure
1004 **/
1005int i40e_setup_rx_descriptors(struct i40e_ring *rx_ring)
1006{
1007 struct device *dev = rx_ring->dev;
1008 int bi_size;
1009
1010 bi_size = sizeof(struct i40e_rx_buffer) * rx_ring->count;
1011 rx_ring->rx_bi = kzalloc(bi_size, GFP_KERNEL);
1012 if (!rx_ring->rx_bi)
1013 goto err;
1014
1015 /* Round up to nearest 4K */
1016 rx_ring->size = ring_is_16byte_desc_enabled(rx_ring)
1017 ? rx_ring->count * sizeof(union i40e_16byte_rx_desc)
1018 : rx_ring->count * sizeof(union i40e_32byte_rx_desc);
1019 rx_ring->size = ALIGN(rx_ring->size, 4096);
1020 rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size,
1021 &rx_ring->dma, GFP_KERNEL);
1022
1023 if (!rx_ring->desc) {
1024 dev_info(dev, "Unable to allocate memory for the Rx descriptor ring, size=%d\n",
1025 rx_ring->size);
1026 goto err;
1027 }
1028
1029 rx_ring->next_to_clean = 0;
1030 rx_ring->next_to_use = 0;
1031
1032 return 0;
1033err:
1034 kfree(rx_ring->rx_bi);
1035 rx_ring->rx_bi = NULL;
1036 return -ENOMEM;
1037}
1038
1039/**
1040 * i40e_release_rx_desc - Store the new tail and head values
1041 * @rx_ring: ring to bump
1042 * @val: new head index
1043 **/
1044static inline void i40e_release_rx_desc(struct i40e_ring *rx_ring, u32 val)
1045{
1046 rx_ring->next_to_use = val;
1047 /* Force memory writes to complete before letting h/w
1048 * know there are new descriptors to fetch. (Only
1049 * applicable for weak-ordered memory model archs,
1050 * such as IA-64).
1051 */
1052 wmb();
1053 writel(val, rx_ring->tail);
1054}
1055
1056/**
1057 * i40e_alloc_rx_buffers - Replace used receive buffers; packet split
1058 * @rx_ring: ring to place buffers on
1059 * @cleaned_count: number of buffers to replace
1060 **/
1061void i40e_alloc_rx_buffers(struct i40e_ring *rx_ring, u16 cleaned_count)
1062{
1063 u16 i = rx_ring->next_to_use;
1064 union i40e_rx_desc *rx_desc;
1065 struct i40e_rx_buffer *bi;
1066 struct sk_buff *skb;
1067
1068 /* do nothing if no valid netdev defined */
1069 if (!rx_ring->netdev || !cleaned_count)
1070 return;
1071
1072 while (cleaned_count--) {
1073 rx_desc = I40E_RX_DESC(rx_ring, i);
1074 bi = &rx_ring->rx_bi[i];
1075 skb = bi->skb;
1076
1077 if (!skb) {
1078 skb = netdev_alloc_skb_ip_align(rx_ring->netdev,
1079 rx_ring->rx_buf_len);
1080 if (!skb) {
Mitch Williams420136c2013-12-18 13:45:59 +00001081 rx_ring->rx_stats.alloc_buff_failed++;
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +00001082 goto no_buffers;
1083 }
1084 /* initialize queue mapping */
1085 skb_record_rx_queue(skb, rx_ring->queue_index);
1086 bi->skb = skb;
1087 }
1088
1089 if (!bi->dma) {
1090 bi->dma = dma_map_single(rx_ring->dev,
1091 skb->data,
1092 rx_ring->rx_buf_len,
1093 DMA_FROM_DEVICE);
1094 if (dma_mapping_error(rx_ring->dev, bi->dma)) {
Mitch Williams420136c2013-12-18 13:45:59 +00001095 rx_ring->rx_stats.alloc_buff_failed++;
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +00001096 bi->dma = 0;
1097 goto no_buffers;
1098 }
1099 }
1100
1101 if (ring_is_ps_enabled(rx_ring)) {
1102 if (!bi->page) {
1103 bi->page = alloc_page(GFP_ATOMIC);
1104 if (!bi->page) {
Mitch Williams420136c2013-12-18 13:45:59 +00001105 rx_ring->rx_stats.alloc_page_failed++;
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +00001106 goto no_buffers;
1107 }
1108 }
1109
1110 if (!bi->page_dma) {
1111 /* use a half page if we're re-using */
1112 bi->page_offset ^= PAGE_SIZE / 2;
1113 bi->page_dma = dma_map_page(rx_ring->dev,
1114 bi->page,
1115 bi->page_offset,
1116 PAGE_SIZE / 2,
1117 DMA_FROM_DEVICE);
1118 if (dma_mapping_error(rx_ring->dev,
1119 bi->page_dma)) {
Mitch Williams420136c2013-12-18 13:45:59 +00001120 rx_ring->rx_stats.alloc_page_failed++;
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +00001121 bi->page_dma = 0;
1122 goto no_buffers;
1123 }
1124 }
1125
1126 /* Refresh the desc even if buffer_addrs didn't change
1127 * because each write-back erases this info.
1128 */
1129 rx_desc->read.pkt_addr = cpu_to_le64(bi->page_dma);
1130 rx_desc->read.hdr_addr = cpu_to_le64(bi->dma);
1131 } else {
1132 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma);
1133 rx_desc->read.hdr_addr = 0;
1134 }
1135 i++;
1136 if (i == rx_ring->count)
1137 i = 0;
1138 }
1139
1140no_buffers:
1141 if (rx_ring->next_to_use != i)
1142 i40e_release_rx_desc(rx_ring, i);
1143}
1144
1145/**
1146 * i40e_receive_skb - Send a completed packet up the stack
1147 * @rx_ring: rx ring in play
1148 * @skb: packet to send up
1149 * @vlan_tag: vlan tag for packet
1150 **/
1151static void i40e_receive_skb(struct i40e_ring *rx_ring,
1152 struct sk_buff *skb, u16 vlan_tag)
1153{
1154 struct i40e_q_vector *q_vector = rx_ring->q_vector;
1155 struct i40e_vsi *vsi = rx_ring->vsi;
1156 u64 flags = vsi->back->flags;
1157
1158 if (vlan_tag & VLAN_VID_MASK)
1159 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlan_tag);
1160
1161 if (flags & I40E_FLAG_IN_NETPOLL)
1162 netif_rx(skb);
1163 else
1164 napi_gro_receive(&q_vector->napi, skb);
1165}
1166
1167/**
1168 * i40e_rx_checksum - Indicate in skb if hw indicated a good cksum
1169 * @vsi: the VSI we care about
1170 * @skb: skb currently being received and modified
1171 * @rx_status: status value of last descriptor in packet
1172 * @rx_error: error value of last descriptor in packet
Joseph Gasparakis8144f0f2013-12-28 05:27:57 +00001173 * @rx_ptype: ptype value of last descriptor in packet
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +00001174 **/
1175static inline void i40e_rx_checksum(struct i40e_vsi *vsi,
1176 struct sk_buff *skb,
1177 u32 rx_status,
Joseph Gasparakis8144f0f2013-12-28 05:27:57 +00001178 u32 rx_error,
1179 u16 rx_ptype)
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +00001180{
Joseph Gasparakis8144f0f2013-12-28 05:27:57 +00001181 bool ipv4_tunnel, ipv6_tunnel;
1182 __wsum rx_udp_csum;
1183 __sum16 csum;
1184 struct iphdr *iph;
1185
1186 ipv4_tunnel = (rx_ptype > I40E_RX_PTYPE_GRENAT4_MAC_PAY3) &&
1187 (rx_ptype < I40E_RX_PTYPE_GRENAT4_MACVLAN_IPV6_ICMP_PAY4);
1188 ipv6_tunnel = (rx_ptype > I40E_RX_PTYPE_GRENAT6_MAC_PAY3) &&
1189 (rx_ptype < I40E_RX_PTYPE_GRENAT6_MACVLAN_IPV6_ICMP_PAY4);
1190
1191 skb->encapsulation = ipv4_tunnel || ipv6_tunnel;
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +00001192 skb->ip_summed = CHECKSUM_NONE;
1193
1194 /* Rx csum enabled and ip headers found? */
1195 if (!(vsi->netdev->features & NETIF_F_RXCSUM &&
1196 rx_status & (1 << I40E_RX_DESC_STATUS_L3L4P_SHIFT)))
1197 return;
1198
Jesse Brandeburgddf1d0d2014-02-13 03:48:39 -08001199 /* likely incorrect csum if alternate IP extension headers found */
Shannon Nelson8ee75a82013-12-21 05:44:46 +00001200 if (rx_status & (1 << I40E_RX_DESC_STATUS_IPV6EXADD_SHIFT))
1201 return;
1202
Joseph Gasparakis8144f0f2013-12-28 05:27:57 +00001203 /* IP or L4 or outmost IP checksum error */
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +00001204 if (rx_error & ((1 << I40E_RX_DESC_ERROR_IPE_SHIFT) |
Joseph Gasparakis8144f0f2013-12-28 05:27:57 +00001205 (1 << I40E_RX_DESC_ERROR_L4E_SHIFT) |
1206 (1 << I40E_RX_DESC_ERROR_EIPE_SHIFT))) {
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +00001207 vsi->back->hw_csum_rx_error++;
1208 return;
1209 }
1210
Joseph Gasparakis8144f0f2013-12-28 05:27:57 +00001211 if (ipv4_tunnel &&
1212 !(rx_status & (1 << I40E_RX_DESC_STATUS_UDP_0_SHIFT))) {
1213 /* If VXLAN traffic has an outer UDPv4 checksum we need to check
1214 * it in the driver, hardware does not do it for us.
1215 * Since L3L4P bit was set we assume a valid IHL value (>=5)
1216 * so the total length of IPv4 header is IHL*4 bytes
1217 */
1218 skb->transport_header = skb->mac_header +
1219 sizeof(struct ethhdr) +
1220 (ip_hdr(skb)->ihl * 4);
1221
1222 /* Add 4 bytes for VLAN tagged packets */
1223 skb->transport_header += (skb->protocol == htons(ETH_P_8021Q) ||
1224 skb->protocol == htons(ETH_P_8021AD))
1225 ? VLAN_HLEN : 0;
1226
1227 rx_udp_csum = udp_csum(skb);
1228 iph = ip_hdr(skb);
1229 csum = csum_tcpudp_magic(
1230 iph->saddr, iph->daddr,
1231 (skb->len - skb_transport_offset(skb)),
1232 IPPROTO_UDP, rx_udp_csum);
1233
1234 if (udp_hdr(skb)->check != csum) {
1235 vsi->back->hw_csum_rx_error++;
1236 return;
1237 }
1238 }
1239
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +00001240 skb->ip_summed = CHECKSUM_UNNECESSARY;
1241}
1242
1243/**
1244 * i40e_rx_hash - returns the hash value from the Rx descriptor
1245 * @ring: descriptor ring
1246 * @rx_desc: specific descriptor
1247 **/
1248static inline u32 i40e_rx_hash(struct i40e_ring *ring,
1249 union i40e_rx_desc *rx_desc)
1250{
Jesse Brandeburg8a494922013-11-20 10:02:49 +00001251 const __le64 rss_mask =
1252 cpu_to_le64((u64)I40E_RX_DESC_FLTSTAT_RSS_HASH <<
1253 I40E_RX_DESC_STATUS_FLTSTAT_SHIFT);
1254
1255 if ((ring->netdev->features & NETIF_F_RXHASH) &&
1256 (rx_desc->wb.qword1.status_error_len & rss_mask) == rss_mask)
1257 return le32_to_cpu(rx_desc->wb.qword0.hi_dword.rss);
1258 else
1259 return 0;
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +00001260}
1261
1262/**
Jesse Brandeburg206812b2014-02-12 01:45:33 +00001263 * i40e_ptype_to_hash - get a hash type
1264 * @ptype: the ptype value from the descriptor
1265 *
1266 * Returns a hash type to be used by skb_set_hash
1267 **/
1268static inline enum pkt_hash_types i40e_ptype_to_hash(u8 ptype)
1269{
1270 struct i40e_rx_ptype_decoded decoded = decode_rx_desc_ptype(ptype);
1271
1272 if (!decoded.known)
1273 return PKT_HASH_TYPE_NONE;
1274
1275 if (decoded.outer_ip == I40E_RX_PTYPE_OUTER_IP &&
1276 decoded.payload_layer == I40E_RX_PTYPE_PAYLOAD_LAYER_PAY4)
1277 return PKT_HASH_TYPE_L4;
1278 else if (decoded.outer_ip == I40E_RX_PTYPE_OUTER_IP &&
1279 decoded.payload_layer == I40E_RX_PTYPE_PAYLOAD_LAYER_PAY3)
1280 return PKT_HASH_TYPE_L3;
1281 else
1282 return PKT_HASH_TYPE_L2;
1283}
1284
1285/**
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +00001286 * i40e_clean_rx_irq - Reclaim resources after receive completes
1287 * @rx_ring: rx ring to clean
1288 * @budget: how many cleans we're allowed
1289 *
1290 * Returns true if there's any budget left (e.g. the clean is finished)
1291 **/
1292static int i40e_clean_rx_irq(struct i40e_ring *rx_ring, int budget)
1293{
1294 unsigned int total_rx_bytes = 0, total_rx_packets = 0;
1295 u16 rx_packet_len, rx_header_len, rx_sph, rx_hbo;
1296 u16 cleaned_count = I40E_DESC_UNUSED(rx_ring);
1297 const int current_node = numa_node_id();
1298 struct i40e_vsi *vsi = rx_ring->vsi;
1299 u16 i = rx_ring->next_to_clean;
1300 union i40e_rx_desc *rx_desc;
1301 u32 rx_error, rx_status;
Jesse Brandeburg206812b2014-02-12 01:45:33 +00001302 u8 rx_ptype;
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +00001303 u64 qword;
1304
Eric W. Biederman390f86d2014-03-14 17:59:10 -07001305 if (budget <= 0)
1306 return 0;
1307
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +00001308 rx_desc = I40E_RX_DESC(rx_ring, i);
1309 qword = le64_to_cpu(rx_desc->wb.qword1.status_error_len);
Jesse Brandeburg6838b532014-01-14 00:49:52 -08001310 rx_status = (qword & I40E_RXD_QW1_STATUS_MASK) >>
1311 I40E_RXD_QW1_STATUS_SHIFT;
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +00001312
1313 while (rx_status & (1 << I40E_RX_DESC_STATUS_DD_SHIFT)) {
1314 union i40e_rx_desc *next_rxd;
1315 struct i40e_rx_buffer *rx_bi;
1316 struct sk_buff *skb;
1317 u16 vlan_tag;
1318 if (i40e_rx_is_programming_status(qword)) {
1319 i40e_clean_programming_status(rx_ring, rx_desc);
1320 I40E_RX_NEXT_DESC_PREFETCH(rx_ring, i, next_rxd);
1321 goto next_desc;
1322 }
1323 rx_bi = &rx_ring->rx_bi[i];
1324 skb = rx_bi->skb;
1325 prefetch(skb->data);
1326
Mitch Williams829af3ac2013-12-18 13:46:00 +00001327 rx_packet_len = (qword & I40E_RXD_QW1_LENGTH_PBUF_MASK) >>
1328 I40E_RXD_QW1_LENGTH_PBUF_SHIFT;
1329 rx_header_len = (qword & I40E_RXD_QW1_LENGTH_HBUF_MASK) >>
1330 I40E_RXD_QW1_LENGTH_HBUF_SHIFT;
1331 rx_sph = (qword & I40E_RXD_QW1_LENGTH_SPH_MASK) >>
1332 I40E_RXD_QW1_LENGTH_SPH_SHIFT;
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +00001333
Mitch Williams829af3ac2013-12-18 13:46:00 +00001334 rx_error = (qword & I40E_RXD_QW1_ERROR_MASK) >>
1335 I40E_RXD_QW1_ERROR_SHIFT;
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +00001336 rx_hbo = rx_error & (1 << I40E_RX_DESC_ERROR_HBO_SHIFT);
1337 rx_error &= ~(1 << I40E_RX_DESC_ERROR_HBO_SHIFT);
1338
Joseph Gasparakis8144f0f2013-12-28 05:27:57 +00001339 rx_ptype = (qword & I40E_RXD_QW1_PTYPE_MASK) >>
1340 I40E_RXD_QW1_PTYPE_SHIFT;
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +00001341 rx_bi->skb = NULL;
1342
1343 /* This memory barrier is needed to keep us from reading
1344 * any other fields out of the rx_desc until we know the
1345 * STATUS_DD bit is set
1346 */
1347 rmb();
1348
1349 /* Get the header and possibly the whole packet
1350 * If this is an skb from previous receive dma will be 0
1351 */
1352 if (rx_bi->dma) {
1353 u16 len;
1354
1355 if (rx_hbo)
1356 len = I40E_RX_HDR_SIZE;
1357 else if (rx_sph)
1358 len = rx_header_len;
1359 else if (rx_packet_len)
1360 len = rx_packet_len; /* 1buf/no split found */
1361 else
1362 len = rx_header_len; /* split always mode */
1363
1364 skb_put(skb, len);
1365 dma_unmap_single(rx_ring->dev,
1366 rx_bi->dma,
1367 rx_ring->rx_buf_len,
1368 DMA_FROM_DEVICE);
1369 rx_bi->dma = 0;
1370 }
1371
1372 /* Get the rest of the data if this was a header split */
1373 if (ring_is_ps_enabled(rx_ring) && rx_packet_len) {
1374
1375 skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags,
1376 rx_bi->page,
1377 rx_bi->page_offset,
1378 rx_packet_len);
1379
1380 skb->len += rx_packet_len;
1381 skb->data_len += rx_packet_len;
1382 skb->truesize += rx_packet_len;
1383
1384 if ((page_count(rx_bi->page) == 1) &&
1385 (page_to_nid(rx_bi->page) == current_node))
1386 get_page(rx_bi->page);
1387 else
1388 rx_bi->page = NULL;
1389
1390 dma_unmap_page(rx_ring->dev,
1391 rx_bi->page_dma,
1392 PAGE_SIZE / 2,
1393 DMA_FROM_DEVICE);
1394 rx_bi->page_dma = 0;
1395 }
1396 I40E_RX_NEXT_DESC_PREFETCH(rx_ring, i, next_rxd);
1397
1398 if (unlikely(
1399 !(rx_status & (1 << I40E_RX_DESC_STATUS_EOF_SHIFT)))) {
1400 struct i40e_rx_buffer *next_buffer;
1401
1402 next_buffer = &rx_ring->rx_bi[i];
1403
1404 if (ring_is_ps_enabled(rx_ring)) {
1405 rx_bi->skb = next_buffer->skb;
1406 rx_bi->dma = next_buffer->dma;
1407 next_buffer->skb = skb;
1408 next_buffer->dma = 0;
1409 }
1410 rx_ring->rx_stats.non_eop_descs++;
1411 goto next_desc;
1412 }
1413
1414 /* ERR_MASK will only have valid bits if EOP set */
1415 if (unlikely(rx_error & (1 << I40E_RX_DESC_ERROR_RXE_SHIFT))) {
1416 dev_kfree_skb_any(skb);
1417 goto next_desc;
1418 }
1419
Jesse Brandeburg206812b2014-02-12 01:45:33 +00001420 skb_set_hash(skb, i40e_rx_hash(rx_ring, rx_desc),
1421 i40e_ptype_to_hash(rx_ptype));
Jacob Kellerbeb0dff2014-01-11 05:43:19 +00001422 if (unlikely(rx_status & I40E_RXD_QW1_STATUS_TSYNVALID_MASK)) {
1423 i40e_ptp_rx_hwtstamp(vsi->back, skb, (rx_status &
1424 I40E_RXD_QW1_STATUS_TSYNINDX_MASK) >>
1425 I40E_RXD_QW1_STATUS_TSYNINDX_SHIFT);
1426 rx_ring->last_rx_timestamp = jiffies;
1427 }
1428
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +00001429 /* probably a little skewed due to removing CRC */
1430 total_rx_bytes += skb->len;
1431 total_rx_packets++;
1432
1433 skb->protocol = eth_type_trans(skb, rx_ring->netdev);
Joseph Gasparakis8144f0f2013-12-28 05:27:57 +00001434
1435 i40e_rx_checksum(vsi, skb, rx_status, rx_error, rx_ptype);
1436
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +00001437 vlan_tag = rx_status & (1 << I40E_RX_DESC_STATUS_L2TAG1P_SHIFT)
1438 ? le16_to_cpu(rx_desc->wb.qword0.lo_dword.l2tag1)
1439 : 0;
1440 i40e_receive_skb(rx_ring, skb, vlan_tag);
1441
1442 rx_ring->netdev->last_rx = jiffies;
1443 budget--;
1444next_desc:
1445 rx_desc->wb.qword1.status_error_len = 0;
1446 if (!budget)
1447 break;
1448
1449 cleaned_count++;
1450 /* return some buffers to hardware, one at a time is too slow */
1451 if (cleaned_count >= I40E_RX_BUFFER_WRITE) {
1452 i40e_alloc_rx_buffers(rx_ring, cleaned_count);
1453 cleaned_count = 0;
1454 }
1455
1456 /* use prefetched values */
1457 rx_desc = next_rxd;
1458 qword = le64_to_cpu(rx_desc->wb.qword1.status_error_len);
Mitch Williams829af3ac2013-12-18 13:46:00 +00001459 rx_status = (qword & I40E_RXD_QW1_STATUS_MASK) >>
1460 I40E_RXD_QW1_STATUS_SHIFT;
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +00001461 }
1462
1463 rx_ring->next_to_clean = i;
Alexander Duyck980e9b12013-09-28 06:01:03 +00001464 u64_stats_update_begin(&rx_ring->syncp);
Alexander Duycka114d0a2013-09-28 06:00:43 +00001465 rx_ring->stats.packets += total_rx_packets;
1466 rx_ring->stats.bytes += total_rx_bytes;
Alexander Duyck980e9b12013-09-28 06:01:03 +00001467 u64_stats_update_end(&rx_ring->syncp);
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +00001468 rx_ring->q_vector->rx.total_packets += total_rx_packets;
1469 rx_ring->q_vector->rx.total_bytes += total_rx_bytes;
1470
1471 if (cleaned_count)
1472 i40e_alloc_rx_buffers(rx_ring, cleaned_count);
1473
1474 return budget > 0;
1475}
1476
1477/**
1478 * i40e_napi_poll - NAPI polling Rx/Tx cleanup routine
1479 * @napi: napi struct with our devices info in it
1480 * @budget: amount of work driver is allowed to do this pass, in packets
1481 *
1482 * This function will clean all queues associated with a q_vector.
1483 *
1484 * Returns the amount of work done
1485 **/
1486int i40e_napi_poll(struct napi_struct *napi, int budget)
1487{
1488 struct i40e_q_vector *q_vector =
1489 container_of(napi, struct i40e_q_vector, napi);
1490 struct i40e_vsi *vsi = q_vector->vsi;
Alexander Duyckcd0b6fa2013-09-28 06:00:53 +00001491 struct i40e_ring *ring;
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +00001492 bool clean_complete = true;
1493 int budget_per_ring;
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +00001494
1495 if (test_bit(__I40E_DOWN, &vsi->state)) {
1496 napi_complete(napi);
1497 return 0;
1498 }
1499
Alexander Duyckcd0b6fa2013-09-28 06:00:53 +00001500 /* Since the actual Tx work is minimal, we can give the Tx a larger
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +00001501 * budget and be more aggressive about cleaning up the Tx descriptors.
1502 */
Alexander Duyckcd0b6fa2013-09-28 06:00:53 +00001503 i40e_for_each_ring(ring, q_vector->tx)
1504 clean_complete &= i40e_clean_tx_irq(ring, vsi->work_limit);
1505
1506 /* We attempt to distribute budget to each Rx queue fairly, but don't
1507 * allow the budget to go below 1 because that would exit polling early.
1508 */
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +00001509 budget_per_ring = max(budget/q_vector->num_ringpairs, 1);
Alexander Duyckcd0b6fa2013-09-28 06:00:53 +00001510
1511 i40e_for_each_ring(ring, q_vector->rx)
1512 clean_complete &= i40e_clean_rx_irq(ring, budget_per_ring);
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +00001513
1514 /* If work not completed, return budget and polling will return */
1515 if (!clean_complete)
1516 return budget;
1517
1518 /* Work is done so exit the polling mode and re-enable the interrupt */
1519 napi_complete(napi);
1520 if (ITR_IS_DYNAMIC(vsi->rx_itr_setting) ||
1521 ITR_IS_DYNAMIC(vsi->tx_itr_setting))
1522 i40e_update_dynamic_itr(q_vector);
1523
1524 if (!test_bit(__I40E_DOWN, &vsi->state)) {
1525 if (vsi->back->flags & I40E_FLAG_MSIX_ENABLED) {
1526 i40e_irq_dynamic_enable(vsi,
1527 q_vector->v_idx + vsi->base_vector);
1528 } else {
1529 struct i40e_hw *hw = &vsi->back->hw;
1530 /* We re-enable the queue 0 cause, but
1531 * don't worry about dynamic_enable
1532 * because we left it on for the other
1533 * possible interrupts during napi
1534 */
1535 u32 qval = rd32(hw, I40E_QINT_RQCTL(0));
1536 qval |= I40E_QINT_RQCTL_CAUSE_ENA_MASK;
1537 wr32(hw, I40E_QINT_RQCTL(0), qval);
1538
1539 qval = rd32(hw, I40E_QINT_TQCTL(0));
1540 qval |= I40E_QINT_TQCTL_CAUSE_ENA_MASK;
1541 wr32(hw, I40E_QINT_TQCTL(0), qval);
Shannon Nelson116a57d2013-09-28 07:13:59 +00001542
1543 i40e_irq_dynamic_enable_icr0(vsi->back);
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +00001544 }
1545 }
1546
1547 return 0;
1548}
1549
1550/**
1551 * i40e_atr - Add a Flow Director ATR filter
1552 * @tx_ring: ring to add programming descriptor to
1553 * @skb: send buffer
1554 * @flags: send flags
1555 * @protocol: wire protocol
1556 **/
1557static void i40e_atr(struct i40e_ring *tx_ring, struct sk_buff *skb,
1558 u32 flags, __be16 protocol)
1559{
1560 struct i40e_filter_program_desc *fdir_desc;
1561 struct i40e_pf *pf = tx_ring->vsi->back;
1562 union {
1563 unsigned char *network;
1564 struct iphdr *ipv4;
1565 struct ipv6hdr *ipv6;
1566 } hdr;
1567 struct tcphdr *th;
1568 unsigned int hlen;
1569 u32 flex_ptype, dtype_cmd;
Alexander Duyckfc4ac672013-09-28 06:00:22 +00001570 u16 i;
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +00001571
1572 /* make sure ATR is enabled */
Jesse Brandeburg60ea5f82014-01-17 15:36:34 -08001573 if (!(pf->flags & I40E_FLAG_FD_ATR_ENABLED))
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +00001574 return;
1575
1576 /* if sampling is disabled do nothing */
1577 if (!tx_ring->atr_sample_rate)
1578 return;
1579
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +00001580 /* snag network header to get L4 type and address */
1581 hdr.network = skb_network_header(skb);
1582
1583 /* Currently only IPv4/IPv6 with TCP is supported */
1584 if (protocol == htons(ETH_P_IP)) {
1585 if (hdr.ipv4->protocol != IPPROTO_TCP)
1586 return;
1587
1588 /* access ihl as a u8 to avoid unaligned access on ia64 */
1589 hlen = (hdr.network[0] & 0x0F) << 2;
1590 } else if (protocol == htons(ETH_P_IPV6)) {
1591 if (hdr.ipv6->nexthdr != IPPROTO_TCP)
1592 return;
1593
1594 hlen = sizeof(struct ipv6hdr);
1595 } else {
1596 return;
1597 }
1598
1599 th = (struct tcphdr *)(hdr.network + hlen);
1600
Anjali Singhai Jain55a5e602014-02-12 06:33:25 +00001601 /* Due to lack of space, no more new filters can be programmed */
1602 if (th->syn && (pf->auto_disable_flags & I40E_FLAG_FD_ATR_ENABLED))
1603 return;
1604
1605 tx_ring->atr_count++;
1606
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +00001607 /* sample on all syn/fin packets or once every atr sample rate */
1608 if (!th->fin && !th->syn && (tx_ring->atr_count < tx_ring->atr_sample_rate))
1609 return;
1610
1611 tx_ring->atr_count = 0;
1612
1613 /* grab the next descriptor */
Alexander Duyckfc4ac672013-09-28 06:00:22 +00001614 i = tx_ring->next_to_use;
1615 fdir_desc = I40E_TX_FDIRDESC(tx_ring, i);
1616
1617 i++;
1618 tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +00001619
1620 flex_ptype = (tx_ring->queue_index << I40E_TXD_FLTR_QW0_QINDEX_SHIFT) &
1621 I40E_TXD_FLTR_QW0_QINDEX_MASK;
1622 flex_ptype |= (protocol == htons(ETH_P_IP)) ?
1623 (I40E_FILTER_PCTYPE_NONF_IPV4_TCP <<
1624 I40E_TXD_FLTR_QW0_PCTYPE_SHIFT) :
1625 (I40E_FILTER_PCTYPE_NONF_IPV6_TCP <<
1626 I40E_TXD_FLTR_QW0_PCTYPE_SHIFT);
1627
1628 flex_ptype |= tx_ring->vsi->id << I40E_TXD_FLTR_QW0_DEST_VSI_SHIFT;
1629
1630 dtype_cmd = I40E_TX_DESC_DTYPE_FILTER_PROG;
1631
1632 dtype_cmd |= th->fin ?
1633 (I40E_FILTER_PROGRAM_DESC_PCMD_REMOVE <<
1634 I40E_TXD_FLTR_QW1_PCMD_SHIFT) :
1635 (I40E_FILTER_PROGRAM_DESC_PCMD_ADD_UPDATE <<
1636 I40E_TXD_FLTR_QW1_PCMD_SHIFT);
1637
1638 dtype_cmd |= I40E_FILTER_PROGRAM_DESC_DEST_DIRECT_PACKET_QINDEX <<
1639 I40E_TXD_FLTR_QW1_DEST_SHIFT;
1640
1641 dtype_cmd |= I40E_FILTER_PROGRAM_DESC_FD_STATUS_FD_ID <<
1642 I40E_TXD_FLTR_QW1_FD_STATUS_SHIFT;
1643
1644 fdir_desc->qindex_flex_ptype_vsi = cpu_to_le32(flex_ptype);
1645 fdir_desc->dtype_cmd_cntindex = cpu_to_le32(dtype_cmd);
1646}
1647
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +00001648/**
1649 * i40e_tx_prepare_vlan_flags - prepare generic TX VLAN tagging flags for HW
1650 * @skb: send buffer
1651 * @tx_ring: ring to send buffer on
1652 * @flags: the tx flags to be set
1653 *
1654 * Checks the skb and set up correspondingly several generic transmit flags
1655 * related to VLAN tagging for the HW, such as VLAN, DCB, etc.
1656 *
1657 * Returns error code indicate the frame should be dropped upon error and the
1658 * otherwise returns 0 to indicate the flags has been set properly.
1659 **/
1660static int i40e_tx_prepare_vlan_flags(struct sk_buff *skb,
1661 struct i40e_ring *tx_ring,
1662 u32 *flags)
1663{
1664 __be16 protocol = skb->protocol;
1665 u32 tx_flags = 0;
1666
1667 /* if we have a HW VLAN tag being added, default to the HW one */
1668 if (vlan_tx_tag_present(skb)) {
1669 tx_flags |= vlan_tx_tag_get(skb) << I40E_TX_FLAGS_VLAN_SHIFT;
1670 tx_flags |= I40E_TX_FLAGS_HW_VLAN;
1671 /* else if it is a SW VLAN, check the next protocol and store the tag */
Jesse Brandeburg0e2fe46c2013-11-28 06:39:29 +00001672 } else if (protocol == htons(ETH_P_8021Q)) {
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +00001673 struct vlan_hdr *vhdr, _vhdr;
1674 vhdr = skb_header_pointer(skb, ETH_HLEN, sizeof(_vhdr), &_vhdr);
1675 if (!vhdr)
1676 return -EINVAL;
1677
1678 protocol = vhdr->h_vlan_encapsulated_proto;
1679 tx_flags |= ntohs(vhdr->h_vlan_TCI) << I40E_TX_FLAGS_VLAN_SHIFT;
1680 tx_flags |= I40E_TX_FLAGS_SW_VLAN;
1681 }
1682
1683 /* Insert 802.1p priority into VLAN header */
1684 if ((tx_ring->vsi->back->flags & I40E_FLAG_DCB_ENABLED) &&
1685 ((tx_flags & (I40E_TX_FLAGS_HW_VLAN | I40E_TX_FLAGS_SW_VLAN)) ||
1686 (skb->priority != TC_PRIO_CONTROL))) {
1687 tx_flags &= ~I40E_TX_FLAGS_VLAN_PRIO_MASK;
1688 tx_flags |= (skb->priority & 0x7) <<
1689 I40E_TX_FLAGS_VLAN_PRIO_SHIFT;
1690 if (tx_flags & I40E_TX_FLAGS_SW_VLAN) {
1691 struct vlan_ethhdr *vhdr;
1692 if (skb_header_cloned(skb) &&
1693 pskb_expand_head(skb, 0, 0, GFP_ATOMIC))
1694 return -ENOMEM;
1695 vhdr = (struct vlan_ethhdr *)skb->data;
1696 vhdr->h_vlan_TCI = htons(tx_flags >>
1697 I40E_TX_FLAGS_VLAN_SHIFT);
1698 } else {
1699 tx_flags |= I40E_TX_FLAGS_HW_VLAN;
1700 }
1701 }
1702 *flags = tx_flags;
1703 return 0;
1704}
1705
1706/**
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +00001707 * i40e_tso - set up the tso context descriptor
1708 * @tx_ring: ptr to the ring to send
1709 * @skb: ptr to the skb we're sending
1710 * @tx_flags: the collected send information
1711 * @protocol: the send protocol
1712 * @hdr_len: ptr to the size of the packet header
1713 * @cd_tunneling: ptr to context descriptor bits
1714 *
1715 * Returns 0 if no TSO can happen, 1 if tso is going, or error
1716 **/
1717static int i40e_tso(struct i40e_ring *tx_ring, struct sk_buff *skb,
1718 u32 tx_flags, __be16 protocol, u8 *hdr_len,
1719 u64 *cd_type_cmd_tso_mss, u32 *cd_tunneling)
1720{
1721 u32 cd_cmd, cd_tso_len, cd_mss;
1722 struct tcphdr *tcph;
1723 struct iphdr *iph;
1724 u32 l4len;
1725 int err;
1726 struct ipv6hdr *ipv6h;
1727
1728 if (!skb_is_gso(skb))
1729 return 0;
1730
1731 if (skb_header_cloned(skb)) {
1732 err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
1733 if (err)
1734 return err;
1735 }
1736
Jesse Brandeburg0e2fe46c2013-11-28 06:39:29 +00001737 if (protocol == htons(ETH_P_IP)) {
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +00001738 iph = skb->encapsulation ? inner_ip_hdr(skb) : ip_hdr(skb);
1739 tcph = skb->encapsulation ? inner_tcp_hdr(skb) : tcp_hdr(skb);
1740 iph->tot_len = 0;
1741 iph->check = 0;
1742 tcph->check = ~csum_tcpudp_magic(iph->saddr, iph->daddr,
1743 0, IPPROTO_TCP, 0);
1744 } else if (skb_is_gso_v6(skb)) {
1745
1746 ipv6h = skb->encapsulation ? inner_ipv6_hdr(skb)
1747 : ipv6_hdr(skb);
1748 tcph = skb->encapsulation ? inner_tcp_hdr(skb) : tcp_hdr(skb);
1749 ipv6h->payload_len = 0;
1750 tcph->check = ~csum_ipv6_magic(&ipv6h->saddr, &ipv6h->daddr,
1751 0, IPPROTO_TCP, 0);
1752 }
1753
1754 l4len = skb->encapsulation ? inner_tcp_hdrlen(skb) : tcp_hdrlen(skb);
1755 *hdr_len = (skb->encapsulation
1756 ? (skb_inner_transport_header(skb) - skb->data)
1757 : skb_transport_offset(skb)) + l4len;
1758
1759 /* find the field values */
1760 cd_cmd = I40E_TX_CTX_DESC_TSO;
1761 cd_tso_len = skb->len - *hdr_len;
1762 cd_mss = skb_shinfo(skb)->gso_size;
Mitch Williams829af3ac2013-12-18 13:46:00 +00001763 *cd_type_cmd_tso_mss |= ((u64)cd_cmd << I40E_TXD_CTX_QW1_CMD_SHIFT) |
1764 ((u64)cd_tso_len <<
1765 I40E_TXD_CTX_QW1_TSO_LEN_SHIFT) |
1766 ((u64)cd_mss << I40E_TXD_CTX_QW1_MSS_SHIFT);
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +00001767 return 1;
1768}
1769
1770/**
Jacob Kellerbeb0dff2014-01-11 05:43:19 +00001771 * i40e_tsyn - set up the tsyn context descriptor
1772 * @tx_ring: ptr to the ring to send
1773 * @skb: ptr to the skb we're sending
1774 * @tx_flags: the collected send information
1775 *
1776 * Returns 0 if no Tx timestamp can happen and 1 if the timestamp will happen
1777 **/
1778static int i40e_tsyn(struct i40e_ring *tx_ring, struct sk_buff *skb,
1779 u32 tx_flags, u64 *cd_type_cmd_tso_mss)
1780{
1781 struct i40e_pf *pf;
1782
1783 if (likely(!(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)))
1784 return 0;
1785
1786 /* Tx timestamps cannot be sampled when doing TSO */
1787 if (tx_flags & I40E_TX_FLAGS_TSO)
1788 return 0;
1789
1790 /* only timestamp the outbound packet if the user has requested it and
1791 * we are not already transmitting a packet to be timestamped
1792 */
1793 pf = i40e_netdev_to_pf(tx_ring->netdev);
1794 if (pf->ptp_tx && !pf->ptp_tx_skb) {
1795 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
1796 pf->ptp_tx_skb = skb_get(skb);
1797 } else {
1798 return 0;
1799 }
1800
1801 *cd_type_cmd_tso_mss |= (u64)I40E_TX_CTX_DESC_TSYN <<
1802 I40E_TXD_CTX_QW1_CMD_SHIFT;
1803
1804 pf->ptp_tx_start = jiffies;
1805 schedule_work(&pf->ptp_tx_work);
1806
1807 return 1;
1808}
1809
1810/**
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +00001811 * i40e_tx_enable_csum - Enable Tx checksum offloads
1812 * @skb: send buffer
1813 * @tx_flags: Tx flags currently set
1814 * @td_cmd: Tx descriptor command bits to set
1815 * @td_offset: Tx descriptor header offsets to set
1816 * @cd_tunneling: ptr to context desc bits
1817 **/
1818static void i40e_tx_enable_csum(struct sk_buff *skb, u32 tx_flags,
1819 u32 *td_cmd, u32 *td_offset,
1820 struct i40e_ring *tx_ring,
1821 u32 *cd_tunneling)
1822{
1823 struct ipv6hdr *this_ipv6_hdr;
1824 unsigned int this_tcp_hdrlen;
1825 struct iphdr *this_ip_hdr;
1826 u32 network_hdr_len;
1827 u8 l4_hdr = 0;
1828
1829 if (skb->encapsulation) {
1830 network_hdr_len = skb_inner_network_header_len(skb);
1831 this_ip_hdr = inner_ip_hdr(skb);
1832 this_ipv6_hdr = inner_ipv6_hdr(skb);
1833 this_tcp_hdrlen = inner_tcp_hdrlen(skb);
1834
1835 if (tx_flags & I40E_TX_FLAGS_IPV4) {
1836
1837 if (tx_flags & I40E_TX_FLAGS_TSO) {
1838 *cd_tunneling |= I40E_TX_CTX_EXT_IP_IPV4;
1839 ip_hdr(skb)->check = 0;
1840 } else {
1841 *cd_tunneling |=
1842 I40E_TX_CTX_EXT_IP_IPV4_NO_CSUM;
1843 }
1844 } else if (tx_flags & I40E_TX_FLAGS_IPV6) {
1845 if (tx_flags & I40E_TX_FLAGS_TSO) {
1846 *cd_tunneling |= I40E_TX_CTX_EXT_IP_IPV6;
1847 ip_hdr(skb)->check = 0;
1848 } else {
1849 *cd_tunneling |=
1850 I40E_TX_CTX_EXT_IP_IPV4_NO_CSUM;
1851 }
1852 }
1853
1854 /* Now set the ctx descriptor fields */
1855 *cd_tunneling |= (skb_network_header_len(skb) >> 2) <<
1856 I40E_TXD_CTX_QW0_EXT_IPLEN_SHIFT |
1857 I40E_TXD_CTX_UDP_TUNNELING |
1858 ((skb_inner_network_offset(skb) -
1859 skb_transport_offset(skb)) >> 1) <<
1860 I40E_TXD_CTX_QW0_NATLEN_SHIFT;
1861
1862 } else {
1863 network_hdr_len = skb_network_header_len(skb);
1864 this_ip_hdr = ip_hdr(skb);
1865 this_ipv6_hdr = ipv6_hdr(skb);
1866 this_tcp_hdrlen = tcp_hdrlen(skb);
1867 }
1868
1869 /* Enable IP checksum offloads */
1870 if (tx_flags & I40E_TX_FLAGS_IPV4) {
1871 l4_hdr = this_ip_hdr->protocol;
1872 /* the stack computes the IP header already, the only time we
1873 * need the hardware to recompute it is in the case of TSO.
1874 */
1875 if (tx_flags & I40E_TX_FLAGS_TSO) {
1876 *td_cmd |= I40E_TX_DESC_CMD_IIPT_IPV4_CSUM;
1877 this_ip_hdr->check = 0;
1878 } else {
1879 *td_cmd |= I40E_TX_DESC_CMD_IIPT_IPV4;
1880 }
1881 /* Now set the td_offset for IP header length */
1882 *td_offset = (network_hdr_len >> 2) <<
1883 I40E_TX_DESC_LENGTH_IPLEN_SHIFT;
1884 } else if (tx_flags & I40E_TX_FLAGS_IPV6) {
1885 l4_hdr = this_ipv6_hdr->nexthdr;
1886 *td_cmd |= I40E_TX_DESC_CMD_IIPT_IPV6;
1887 /* Now set the td_offset for IP header length */
1888 *td_offset = (network_hdr_len >> 2) <<
1889 I40E_TX_DESC_LENGTH_IPLEN_SHIFT;
1890 }
1891 /* words in MACLEN + dwords in IPLEN + dwords in L4Len */
1892 *td_offset |= (skb_network_offset(skb) >> 1) <<
1893 I40E_TX_DESC_LENGTH_MACLEN_SHIFT;
1894
1895 /* Enable L4 checksum offloads */
1896 switch (l4_hdr) {
1897 case IPPROTO_TCP:
1898 /* enable checksum offloads */
1899 *td_cmd |= I40E_TX_DESC_CMD_L4T_EOFT_TCP;
1900 *td_offset |= (this_tcp_hdrlen >> 2) <<
1901 I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
1902 break;
1903 case IPPROTO_SCTP:
1904 /* enable SCTP checksum offload */
1905 *td_cmd |= I40E_TX_DESC_CMD_L4T_EOFT_SCTP;
1906 *td_offset |= (sizeof(struct sctphdr) >> 2) <<
1907 I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
1908 break;
1909 case IPPROTO_UDP:
1910 /* enable UDP checksum offload */
1911 *td_cmd |= I40E_TX_DESC_CMD_L4T_EOFT_UDP;
1912 *td_offset |= (sizeof(struct udphdr) >> 2) <<
1913 I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
1914 break;
1915 default:
1916 break;
1917 }
1918}
1919
1920/**
1921 * i40e_create_tx_ctx Build the Tx context descriptor
1922 * @tx_ring: ring to create the descriptor on
1923 * @cd_type_cmd_tso_mss: Quad Word 1
1924 * @cd_tunneling: Quad Word 0 - bits 0-31
1925 * @cd_l2tag2: Quad Word 0 - bits 32-63
1926 **/
1927static void i40e_create_tx_ctx(struct i40e_ring *tx_ring,
1928 const u64 cd_type_cmd_tso_mss,
1929 const u32 cd_tunneling, const u32 cd_l2tag2)
1930{
1931 struct i40e_tx_context_desc *context_desc;
Alexander Duyckfc4ac672013-09-28 06:00:22 +00001932 int i = tx_ring->next_to_use;
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +00001933
1934 if (!cd_type_cmd_tso_mss && !cd_tunneling && !cd_l2tag2)
1935 return;
1936
1937 /* grab the next descriptor */
Alexander Duyckfc4ac672013-09-28 06:00:22 +00001938 context_desc = I40E_TX_CTXTDESC(tx_ring, i);
1939
1940 i++;
1941 tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +00001942
1943 /* cpu_to_le32 and assign to struct fields */
1944 context_desc->tunneling_params = cpu_to_le32(cd_tunneling);
1945 context_desc->l2tag2 = cpu_to_le16(cd_l2tag2);
1946 context_desc->type_cmd_tso_mss = cpu_to_le64(cd_type_cmd_tso_mss);
1947}
1948
1949/**
1950 * i40e_tx_map - Build the Tx descriptor
1951 * @tx_ring: ring to send buffer on
1952 * @skb: send buffer
1953 * @first: first buffer info buffer to use
1954 * @tx_flags: collected send information
1955 * @hdr_len: size of the packet header
1956 * @td_cmd: the command field in the descriptor
1957 * @td_offset: offset for checksum or crc
1958 **/
1959static void i40e_tx_map(struct i40e_ring *tx_ring, struct sk_buff *skb,
1960 struct i40e_tx_buffer *first, u32 tx_flags,
1961 const u8 hdr_len, u32 td_cmd, u32 td_offset)
1962{
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +00001963 unsigned int data_len = skb->data_len;
1964 unsigned int size = skb_headlen(skb);
Alexander Duycka5e9c572013-09-28 06:00:27 +00001965 struct skb_frag_struct *frag;
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +00001966 struct i40e_tx_buffer *tx_bi;
1967 struct i40e_tx_desc *tx_desc;
Alexander Duycka5e9c572013-09-28 06:00:27 +00001968 u16 i = tx_ring->next_to_use;
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +00001969 u32 td_tag = 0;
1970 dma_addr_t dma;
1971 u16 gso_segs;
1972
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +00001973 if (tx_flags & I40E_TX_FLAGS_HW_VLAN) {
1974 td_cmd |= I40E_TX_DESC_CMD_IL2TAG1;
1975 td_tag = (tx_flags & I40E_TX_FLAGS_VLAN_MASK) >>
1976 I40E_TX_FLAGS_VLAN_SHIFT;
1977 }
1978
Alexander Duycka5e9c572013-09-28 06:00:27 +00001979 if (tx_flags & (I40E_TX_FLAGS_TSO | I40E_TX_FLAGS_FSO))
1980 gso_segs = skb_shinfo(skb)->gso_segs;
1981 else
1982 gso_segs = 1;
1983
1984 /* multiply data chunks by size of headers */
1985 first->bytecount = skb->len - hdr_len + (gso_segs * hdr_len);
1986 first->gso_segs = gso_segs;
1987 first->skb = skb;
1988 first->tx_flags = tx_flags;
1989
1990 dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE);
1991
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +00001992 tx_desc = I40E_TX_DESC(tx_ring, i);
Alexander Duycka5e9c572013-09-28 06:00:27 +00001993 tx_bi = first;
1994
1995 for (frag = &skb_shinfo(skb)->frags[0];; frag++) {
1996 if (dma_mapping_error(tx_ring->dev, dma))
1997 goto dma_error;
1998
1999 /* record length, and DMA address */
2000 dma_unmap_len_set(tx_bi, len, size);
2001 dma_unmap_addr_set(tx_bi, dma, dma);
2002
2003 tx_desc->buffer_addr = cpu_to_le64(dma);
2004
2005 while (unlikely(size > I40E_MAX_DATA_PER_TXD)) {
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +00002006 tx_desc->cmd_type_offset_bsz =
2007 build_ctob(td_cmd, td_offset,
2008 I40E_MAX_DATA_PER_TXD, td_tag);
2009
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +00002010 tx_desc++;
2011 i++;
2012 if (i == tx_ring->count) {
2013 tx_desc = I40E_TX_DESC(tx_ring, 0);
2014 i = 0;
2015 }
Alexander Duycka5e9c572013-09-28 06:00:27 +00002016
2017 dma += I40E_MAX_DATA_PER_TXD;
2018 size -= I40E_MAX_DATA_PER_TXD;
2019
2020 tx_desc->buffer_addr = cpu_to_le64(dma);
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +00002021 }
2022
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +00002023 if (likely(!data_len))
2024 break;
2025
Alexander Duycka5e9c572013-09-28 06:00:27 +00002026 tx_desc->cmd_type_offset_bsz = build_ctob(td_cmd, td_offset,
2027 size, td_tag);
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +00002028
2029 tx_desc++;
2030 i++;
2031 if (i == tx_ring->count) {
2032 tx_desc = I40E_TX_DESC(tx_ring, 0);
2033 i = 0;
2034 }
2035
Alexander Duycka5e9c572013-09-28 06:00:27 +00002036 size = skb_frag_size(frag);
2037 data_len -= size;
2038
2039 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size,
2040 DMA_TO_DEVICE);
2041
2042 tx_bi = &tx_ring->tx_bi[i];
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +00002043 }
2044
Alexander Duycka5e9c572013-09-28 06:00:27 +00002045 tx_desc->cmd_type_offset_bsz =
2046 build_ctob(td_cmd, td_offset, size, td_tag) |
2047 cpu_to_le64((u64)I40E_TXD_CMD << I40E_TXD_QW1_CMD_SHIFT);
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +00002048
Alexander Duyck7070ce02013-09-28 06:00:37 +00002049 netdev_tx_sent_queue(netdev_get_tx_queue(tx_ring->netdev,
2050 tx_ring->queue_index),
2051 first->bytecount);
2052
Alexander Duycka5e9c572013-09-28 06:00:27 +00002053 /* set the timestamp */
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +00002054 first->time_stamp = jiffies;
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +00002055
2056 /* Force memory writes to complete before letting h/w
2057 * know there are new descriptors to fetch. (Only
2058 * applicable for weak-ordered memory model archs,
2059 * such as IA-64).
2060 */
2061 wmb();
2062
Alexander Duycka5e9c572013-09-28 06:00:27 +00002063 /* set next_to_watch value indicating a packet is present */
2064 first->next_to_watch = tx_desc;
2065
2066 i++;
2067 if (i == tx_ring->count)
2068 i = 0;
2069
2070 tx_ring->next_to_use = i;
2071
2072 /* notify HW of packet */
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +00002073 writel(i, tx_ring->tail);
Alexander Duycka5e9c572013-09-28 06:00:27 +00002074
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +00002075 return;
2076
2077dma_error:
Alexander Duycka5e9c572013-09-28 06:00:27 +00002078 dev_info(tx_ring->dev, "TX DMA map failed\n");
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +00002079
2080 /* clear dma mappings for failed tx_bi map */
2081 for (;;) {
2082 tx_bi = &tx_ring->tx_bi[i];
Alexander Duycka5e9c572013-09-28 06:00:27 +00002083 i40e_unmap_and_free_tx_resource(tx_ring, tx_bi);
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +00002084 if (tx_bi == first)
2085 break;
2086 if (i == 0)
2087 i = tx_ring->count;
2088 i--;
2089 }
2090
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +00002091 tx_ring->next_to_use = i;
2092}
2093
2094/**
2095 * __i40e_maybe_stop_tx - 2nd level check for tx stop conditions
2096 * @tx_ring: the ring to be checked
2097 * @size: the size buffer we want to assure is available
2098 *
2099 * Returns -EBUSY if a stop is needed, else 0
2100 **/
2101static inline int __i40e_maybe_stop_tx(struct i40e_ring *tx_ring, int size)
2102{
2103 netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index);
Greg Rose8e9dca52013-12-18 13:45:53 +00002104 /* Memory barrier before checking head and tail */
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +00002105 smp_mb();
2106
2107 /* Check again in a case another CPU has just made room available. */
2108 if (likely(I40E_DESC_UNUSED(tx_ring) < size))
2109 return -EBUSY;
2110
2111 /* A reprieve! - use start_queue because it doesn't call schedule */
2112 netif_start_subqueue(tx_ring->netdev, tx_ring->queue_index);
2113 ++tx_ring->tx_stats.restart_queue;
2114 return 0;
2115}
2116
2117/**
2118 * i40e_maybe_stop_tx - 1st level check for tx stop conditions
2119 * @tx_ring: the ring to be checked
2120 * @size: the size buffer we want to assure is available
2121 *
2122 * Returns 0 if stop is not needed
2123 **/
2124static int i40e_maybe_stop_tx(struct i40e_ring *tx_ring, int size)
2125{
2126 if (likely(I40E_DESC_UNUSED(tx_ring) >= size))
2127 return 0;
2128 return __i40e_maybe_stop_tx(tx_ring, size);
2129}
2130
2131/**
2132 * i40e_xmit_descriptor_count - calculate number of tx descriptors needed
2133 * @skb: send buffer
2134 * @tx_ring: ring to send buffer on
2135 *
2136 * Returns number of data descriptors needed for this skb. Returns 0 to indicate
2137 * there is not enough descriptors available in this ring since we need at least
2138 * one descriptor.
2139 **/
2140static int i40e_xmit_descriptor_count(struct sk_buff *skb,
2141 struct i40e_ring *tx_ring)
2142{
2143#if PAGE_SIZE > I40E_MAX_DATA_PER_TXD
2144 unsigned int f;
2145#endif
2146 int count = 0;
2147
2148 /* need: 1 descriptor per page * PAGE_SIZE/I40E_MAX_DATA_PER_TXD,
2149 * + 1 desc for skb_head_len/I40E_MAX_DATA_PER_TXD,
Jesse Brandeburgbe560522014-02-06 05:51:13 +00002150 * + 4 desc gap to avoid the cache line where head is,
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +00002151 * + 1 desc for context descriptor,
2152 * otherwise try next time
2153 */
2154#if PAGE_SIZE > I40E_MAX_DATA_PER_TXD
2155 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++)
2156 count += TXD_USE_COUNT(skb_shinfo(skb)->frags[f].size);
2157#else
2158 count += skb_shinfo(skb)->nr_frags;
2159#endif
2160 count += TXD_USE_COUNT(skb_headlen(skb));
Jesse Brandeburgbe560522014-02-06 05:51:13 +00002161 if (i40e_maybe_stop_tx(tx_ring, count + 4 + 1)) {
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +00002162 tx_ring->tx_stats.tx_busy++;
2163 return 0;
2164 }
2165 return count;
2166}
2167
2168/**
2169 * i40e_xmit_frame_ring - Sends buffer on Tx ring
2170 * @skb: send buffer
2171 * @tx_ring: ring to send buffer on
2172 *
2173 * Returns NETDEV_TX_OK if sent, else an error code
2174 **/
2175static netdev_tx_t i40e_xmit_frame_ring(struct sk_buff *skb,
2176 struct i40e_ring *tx_ring)
2177{
2178 u64 cd_type_cmd_tso_mss = I40E_TX_DESC_DTYPE_CONTEXT;
2179 u32 cd_tunneling = 0, cd_l2tag2 = 0;
2180 struct i40e_tx_buffer *first;
2181 u32 td_offset = 0;
2182 u32 tx_flags = 0;
2183 __be16 protocol;
2184 u32 td_cmd = 0;
2185 u8 hdr_len = 0;
Jacob Kellerbeb0dff2014-01-11 05:43:19 +00002186 int tsyn;
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +00002187 int tso;
2188 if (0 == i40e_xmit_descriptor_count(skb, tx_ring))
2189 return NETDEV_TX_BUSY;
2190
2191 /* prepare the xmit flags */
2192 if (i40e_tx_prepare_vlan_flags(skb, tx_ring, &tx_flags))
2193 goto out_drop;
2194
2195 /* obtain protocol of skb */
2196 protocol = skb->protocol;
2197
2198 /* record the location of the first descriptor for this packet */
2199 first = &tx_ring->tx_bi[tx_ring->next_to_use];
2200
2201 /* setup IPv4/IPv6 offloads */
Jesse Brandeburg0e2fe46c2013-11-28 06:39:29 +00002202 if (protocol == htons(ETH_P_IP))
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +00002203 tx_flags |= I40E_TX_FLAGS_IPV4;
Jesse Brandeburg0e2fe46c2013-11-28 06:39:29 +00002204 else if (protocol == htons(ETH_P_IPV6))
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +00002205 tx_flags |= I40E_TX_FLAGS_IPV6;
2206
2207 tso = i40e_tso(tx_ring, skb, tx_flags, protocol, &hdr_len,
2208 &cd_type_cmd_tso_mss, &cd_tunneling);
2209
2210 if (tso < 0)
2211 goto out_drop;
2212 else if (tso)
2213 tx_flags |= I40E_TX_FLAGS_TSO;
2214
2215 skb_tx_timestamp(skb);
2216
Jacob Kellerbeb0dff2014-01-11 05:43:19 +00002217 tsyn = i40e_tsyn(tx_ring, skb, tx_flags, &cd_type_cmd_tso_mss);
2218
2219 if (tsyn)
2220 tx_flags |= I40E_TX_FLAGS_TSYN;
2221
Alexander Duyckb1941302013-09-28 06:00:32 +00002222 /* always enable CRC insertion offload */
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +00002223 td_cmd |= I40E_TX_DESC_CMD_ICRC;
2224
Alexander Duyckb1941302013-09-28 06:00:32 +00002225 /* Always offload the checksum, since it's in the data descriptor */
2226 if (skb->ip_summed == CHECKSUM_PARTIAL) {
2227 tx_flags |= I40E_TX_FLAGS_CSUM;
2228
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +00002229 i40e_tx_enable_csum(skb, tx_flags, &td_cmd, &td_offset,
2230 tx_ring, &cd_tunneling);
Alexander Duyckb1941302013-09-28 06:00:32 +00002231 }
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +00002232
2233 i40e_create_tx_ctx(tx_ring, cd_type_cmd_tso_mss,
2234 cd_tunneling, cd_l2tag2);
2235
2236 /* Add Flow Director ATR if it's enabled.
2237 *
2238 * NOTE: this must always be directly before the data descriptor.
2239 */
2240 i40e_atr(tx_ring, skb, tx_flags, protocol);
2241
2242 i40e_tx_map(tx_ring, skb, first, tx_flags, hdr_len,
2243 td_cmd, td_offset);
2244
2245 i40e_maybe_stop_tx(tx_ring, DESC_NEEDED);
2246
2247 return NETDEV_TX_OK;
2248
2249out_drop:
2250 dev_kfree_skb_any(skb);
2251 return NETDEV_TX_OK;
2252}
2253
2254/**
2255 * i40e_lan_xmit_frame - Selects the correct VSI and Tx queue to send buffer
2256 * @skb: send buffer
2257 * @netdev: network interface device structure
2258 *
2259 * Returns NETDEV_TX_OK if sent, else an error code
2260 **/
2261netdev_tx_t i40e_lan_xmit_frame(struct sk_buff *skb, struct net_device *netdev)
2262{
2263 struct i40e_netdev_priv *np = netdev_priv(netdev);
2264 struct i40e_vsi *vsi = np->vsi;
Alexander Duyck9f65e152013-09-28 06:00:58 +00002265 struct i40e_ring *tx_ring = vsi->tx_rings[skb->queue_mapping];
Jesse Brandeburgfd0a05c2013-09-11 08:39:51 +00002266
2267 /* hardware can't handle really short frames, hardware padding works
2268 * beyond this point
2269 */
2270 if (unlikely(skb->len < I40E_MIN_TX_LEN)) {
2271 if (skb_pad(skb, I40E_MIN_TX_LEN - skb->len))
2272 return NETDEV_TX_OK;
2273 skb->len = I40E_MIN_TX_LEN;
2274 skb_set_tail_pointer(skb, I40E_MIN_TX_LEN);
2275 }
2276
2277 return i40e_xmit_frame_ring(skb, tx_ring);
2278}