blob: 58f6fc055f6a55cd1e9e3bc52af92b96cd31c9a4 [file] [log] [blame]
Christoph Lameter8199d3a2005-03-30 13:34:31 -08001/*****************************************************************************
2 * *
3 * File: sge.c *
Scott Bardone559fb512005-06-23 01:40:19 -04004 * $Revision: 1.26 $ *
5 * $Date: 2005/06/21 18:29:48 $ *
Christoph Lameter8199d3a2005-03-30 13:34:31 -08006 * Description: *
7 * DMA engine. *
8 * part of the Chelsio 10Gb Ethernet Driver. *
9 * *
10 * This program is free software; you can redistribute it and/or modify *
11 * it under the terms of the GNU General Public License, version 2, as *
12 * published by the Free Software Foundation. *
13 * *
14 * You should have received a copy of the GNU General Public License along *
15 * with this program; if not, write to the Free Software Foundation, Inc., *
16 * 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. *
17 * *
18 * THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR IMPLIED *
19 * WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF *
20 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. *
21 * *
22 * http://www.chelsio.com *
23 * *
24 * Copyright (c) 2003 - 2005 Chelsio Communications, Inc. *
25 * All rights reserved. *
26 * *
27 * Maintainers: maintainers@chelsio.com *
28 * *
29 * Authors: Dimitrios Michailidis <dm@chelsio.com> *
30 * Tina Yang <tainay@chelsio.com> *
31 * Felix Marti <felix@chelsio.com> *
32 * Scott Bardone <sbardone@chelsio.com> *
33 * Kurt Ottaway <kottaway@chelsio.com> *
34 * Frank DiMambro <frank@chelsio.com> *
35 * *
36 * History: *
37 * *
38 ****************************************************************************/
39
40#include "common.h"
41
Christoph Lameter8199d3a2005-03-30 13:34:31 -080042#include <linux/types.h>
43#include <linux/errno.h>
44#include <linux/pci.h>
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -080045#include <linux/ktime.h>
Christoph Lameter8199d3a2005-03-30 13:34:31 -080046#include <linux/netdevice.h>
47#include <linux/etherdevice.h>
48#include <linux/if_vlan.h>
49#include <linux/skbuff.h>
50#include <linux/init.h>
51#include <linux/mm.h>
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -080052#include <linux/tcp.h>
Christoph Lameter8199d3a2005-03-30 13:34:31 -080053#include <linux/ip.h>
54#include <linux/in.h>
55#include <linux/if_arp.h>
56
57#include "cpl5_cmd.h"
58#include "sge.h"
59#include "regs.h"
60#include "espi.h"
61
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -080062/* This belongs in if_ether.h */
63#define ETH_P_CPL5 0xf
Christoph Lameter8199d3a2005-03-30 13:34:31 -080064
65#define SGE_CMDQ_N 2
66#define SGE_FREELQ_N 2
Scott Bardone559fb512005-06-23 01:40:19 -040067#define SGE_CMDQ0_E_N 1024
Christoph Lameter8199d3a2005-03-30 13:34:31 -080068#define SGE_CMDQ1_E_N 128
69#define SGE_FREEL_SIZE 4096
70#define SGE_JUMBO_FREEL_SIZE 512
71#define SGE_FREEL_REFILL_THRESH 16
72#define SGE_RESPQ_E_N 1024
Scott Bardone559fb512005-06-23 01:40:19 -040073#define SGE_INTRTIMER_NRES 1000
Christoph Lameter8199d3a2005-03-30 13:34:31 -080074#define SGE_RX_SM_BUF_SIZE 1536
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -080075#define SGE_TX_DESC_MAX_PLEN 16384
Christoph Lameter8199d3a2005-03-30 13:34:31 -080076
Scott Bardone559fb512005-06-23 01:40:19 -040077#define SGE_RESPQ_REPLENISH_THRES (SGE_RESPQ_E_N / 4)
Christoph Lameter8199d3a2005-03-30 13:34:31 -080078
79/*
Scott Bardone559fb512005-06-23 01:40:19 -040080 * Period of the TX buffer reclaim timer. This timer does not need to run
81 * frequently as TX buffers are usually reclaimed by new TX packets.
82 */
83#define TX_RECLAIM_PERIOD (HZ / 4)
84
Scott Bardone559fb512005-06-23 01:40:19 -040085#define M_CMD_LEN 0x7fffffff
86#define V_CMD_LEN(v) (v)
87#define G_CMD_LEN(v) ((v) & M_CMD_LEN)
88#define V_CMD_GEN1(v) ((v) << 31)
89#define V_CMD_GEN2(v) (v)
90#define F_CMD_DATAVALID (1 << 1)
91#define F_CMD_SOP (1 << 2)
92#define V_CMD_EOP(v) ((v) << 3)
93
94/*
95 * Command queue, receive buffer list, and response queue descriptors.
Christoph Lameter8199d3a2005-03-30 13:34:31 -080096 */
97#if defined(__BIG_ENDIAN_BITFIELD)
98struct cmdQ_e {
Scott Bardone559fb512005-06-23 01:40:19 -040099 u32 addr_lo;
100 u32 len_gen;
101 u32 flags;
102 u32 addr_hi;
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800103};
104
105struct freelQ_e {
Scott Bardone559fb512005-06-23 01:40:19 -0400106 u32 addr_lo;
107 u32 len_gen;
108 u32 gen2;
109 u32 addr_hi;
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800110};
111
112struct respQ_e {
113 u32 Qsleeping : 4;
114 u32 Cmdq1CreditReturn : 5;
115 u32 Cmdq1DmaComplete : 5;
116 u32 Cmdq0CreditReturn : 5;
117 u32 Cmdq0DmaComplete : 5;
118 u32 FreelistQid : 2;
119 u32 CreditValid : 1;
120 u32 DataValid : 1;
121 u32 Offload : 1;
122 u32 Eop : 1;
123 u32 Sop : 1;
124 u32 GenerationBit : 1;
125 u32 BufferLength;
126};
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800127#elif defined(__LITTLE_ENDIAN_BITFIELD)
128struct cmdQ_e {
Scott Bardone559fb512005-06-23 01:40:19 -0400129 u32 len_gen;
130 u32 addr_lo;
131 u32 addr_hi;
132 u32 flags;
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800133};
134
135struct freelQ_e {
Scott Bardone559fb512005-06-23 01:40:19 -0400136 u32 len_gen;
137 u32 addr_lo;
138 u32 addr_hi;
139 u32 gen2;
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800140};
141
142struct respQ_e {
143 u32 BufferLength;
144 u32 GenerationBit : 1;
145 u32 Sop : 1;
146 u32 Eop : 1;
147 u32 Offload : 1;
148 u32 DataValid : 1;
149 u32 CreditValid : 1;
150 u32 FreelistQid : 2;
151 u32 Cmdq0DmaComplete : 5;
152 u32 Cmdq0CreditReturn : 5;
153 u32 Cmdq1DmaComplete : 5;
154 u32 Cmdq1CreditReturn : 5;
155 u32 Qsleeping : 4;
156} ;
157#endif
158
159/*
160 * SW Context Command and Freelist Queue Descriptors
161 */
162struct cmdQ_ce {
163 struct sk_buff *skb;
164 DECLARE_PCI_UNMAP_ADDR(dma_addr);
165 DECLARE_PCI_UNMAP_LEN(dma_len);
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800166};
167
168struct freelQ_ce {
169 struct sk_buff *skb;
170 DECLARE_PCI_UNMAP_ADDR(dma_addr);
171 DECLARE_PCI_UNMAP_LEN(dma_len);
172};
173
174/*
Scott Bardone559fb512005-06-23 01:40:19 -0400175 * SW command, freelist and response rings
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800176 */
177struct cmdQ {
Scott Bardone559fb512005-06-23 01:40:19 -0400178 unsigned long status; /* HW DMA fetch status */
179 unsigned int in_use; /* # of in-use command descriptors */
180 unsigned int size; /* # of descriptors */
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -0800181 unsigned int processed; /* total # of descs HW has processed */
182 unsigned int cleaned; /* total # of descs SW has reclaimed */
183 unsigned int stop_thres; /* SW TX queue suspend threshold */
Scott Bardone559fb512005-06-23 01:40:19 -0400184 u16 pidx; /* producer index (SW) */
185 u16 cidx; /* consumer index (HW) */
186 u8 genbit; /* current generation (=valid) bit */
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -0800187 u8 sop; /* is next entry start of packet? */
Scott Bardone559fb512005-06-23 01:40:19 -0400188 struct cmdQ_e *entries; /* HW command descriptor Q */
189 struct cmdQ_ce *centries; /* SW command context descriptor Q */
Scott Bardone559fb512005-06-23 01:40:19 -0400190 dma_addr_t dma_addr; /* DMA addr HW command descriptor Q */
Francois Romieu356bd142006-12-11 23:47:00 +0100191 spinlock_t lock; /* Lock to protect cmdQ enqueuing */
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800192};
193
194struct freelQ {
Scott Bardone559fb512005-06-23 01:40:19 -0400195 unsigned int credits; /* # of available RX buffers */
196 unsigned int size; /* free list capacity */
197 u16 pidx; /* producer index (SW) */
198 u16 cidx; /* consumer index (HW) */
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800199 u16 rx_buffer_size; /* Buffer size on this free list */
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -0800200 u16 dma_offset; /* DMA offset to align IP headers */
201 u16 recycleq_idx; /* skb recycle q to use */
Scott Bardone559fb512005-06-23 01:40:19 -0400202 u8 genbit; /* current generation (=valid) bit */
203 struct freelQ_e *entries; /* HW freelist descriptor Q */
204 struct freelQ_ce *centries; /* SW freelist context descriptor Q */
205 dma_addr_t dma_addr; /* DMA addr HW freelist descriptor Q */
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800206};
207
208struct respQ {
Scott Bardone559fb512005-06-23 01:40:19 -0400209 unsigned int credits; /* credits to be returned to SGE */
210 unsigned int size; /* # of response Q descriptors */
211 u16 cidx; /* consumer index (SW) */
212 u8 genbit; /* current generation(=valid) bit */
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800213 struct respQ_e *entries; /* HW response descriptor Q */
Scott Bardone559fb512005-06-23 01:40:19 -0400214 dma_addr_t dma_addr; /* DMA addr HW response descriptor Q */
215};
216
217/* Bit flags for cmdQ.status */
218enum {
219 CMDQ_STAT_RUNNING = 1, /* fetch engine is running */
220 CMDQ_STAT_LAST_PKT_DB = 2 /* last packet rung the doorbell */
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800221};
222
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -0800223/* T204 TX SW scheduler */
224
225/* Per T204 TX port */
226struct sched_port {
227 unsigned int avail; /* available bits - quota */
228 unsigned int drain_bits_per_1024ns; /* drain rate */
229 unsigned int speed; /* drain rate, mbps */
230 unsigned int mtu; /* mtu size */
231 struct sk_buff_head skbq; /* pending skbs */
232};
233
234/* Per T204 device */
235struct sched {
236 ktime_t last_updated; /* last time quotas were computed */
Francois Romieu356bd142006-12-11 23:47:00 +0100237 unsigned int max_avail; /* max bits to be sent to any port */
238 unsigned int port; /* port index (round robin ports) */
239 unsigned int num; /* num skbs in per port queues */
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -0800240 struct sched_port p[MAX_NPORTS];
241 struct tasklet_struct sched_tsk;/* tasklet used to run scheduler */
242};
243static void restart_sched(unsigned long);
244
245
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800246/*
247 * Main SGE data structure
248 *
249 * Interrupts are handled by a single CPU and it is likely that on a MP system
250 * the application is migrated to another CPU. In that scenario, we try to
251 * seperate the RX(in irq context) and TX state in order to decrease memory
252 * contention.
253 */
254struct sge {
Francois Romieu356bd142006-12-11 23:47:00 +0100255 struct adapter *adapter; /* adapter backpointer */
Scott Bardone559fb512005-06-23 01:40:19 -0400256 struct net_device *netdev; /* netdevice backpointer */
Francois Romieu356bd142006-12-11 23:47:00 +0100257 struct freelQ freelQ[SGE_FREELQ_N]; /* buffer free lists */
258 struct respQ respQ; /* response Q */
Scott Bardone559fb512005-06-23 01:40:19 -0400259 unsigned long stopped_tx_queues; /* bitmap of suspended Tx queues */
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800260 unsigned int rx_pkt_pad; /* RX padding for L2 packets */
261 unsigned int jumbo_fl; /* jumbo freelist Q index */
Scott Bardone559fb512005-06-23 01:40:19 -0400262 unsigned int intrtimer_nres; /* no-resource interrupt timer */
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -0800263 unsigned int fixed_intrtimer;/* non-adaptive interrupt timer */
Scott Bardone559fb512005-06-23 01:40:19 -0400264 struct timer_list tx_reclaim_timer; /* reclaims TX buffers */
265 struct timer_list espibug_timer;
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -0800266 unsigned long espibug_timeout;
267 struct sk_buff *espibug_skb[MAX_NPORTS];
Scott Bardone559fb512005-06-23 01:40:19 -0400268 u32 sge_control; /* shadow value of sge control reg */
269 struct sge_intr_counts stats;
Stephen Hemminger56f643c2006-12-01 16:36:21 -0800270 struct sge_port_stats *port_stats[MAX_NPORTS];
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -0800271 struct sched *tx_sched;
Scott Bardone559fb512005-06-23 01:40:19 -0400272 struct cmdQ cmdQ[SGE_CMDQ_N] ____cacheline_aligned_in_smp;
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800273};
274
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800275/*
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -0800276 * stop tasklet and free all pending skb's
277 */
278static void tx_sched_stop(struct sge *sge)
279{
280 struct sched *s = sge->tx_sched;
281 int i;
282
283 tasklet_kill(&s->sched_tsk);
284
285 for (i = 0; i < MAX_NPORTS; i++)
286 __skb_queue_purge(&s->p[s->port].skbq);
287}
288
289/*
290 * t1_sched_update_parms() is called when the MTU or link speed changes. It
291 * re-computes scheduler parameters to scope with the change.
292 */
293unsigned int t1_sched_update_parms(struct sge *sge, unsigned int port,
294 unsigned int mtu, unsigned int speed)
295{
296 struct sched *s = sge->tx_sched;
297 struct sched_port *p = &s->p[port];
298 unsigned int max_avail_segs;
299
300 pr_debug("t1_sched_update_params mtu=%d speed=%d\n", mtu, speed);
301 if (speed)
302 p->speed = speed;
303 if (mtu)
304 p->mtu = mtu;
305
306 if (speed || mtu) {
307 unsigned long long drain = 1024ULL * p->speed * (p->mtu - 40);
308 do_div(drain, (p->mtu + 50) * 1000);
309 p->drain_bits_per_1024ns = (unsigned int) drain;
310
311 if (p->speed < 1000)
312 p->drain_bits_per_1024ns =
313 90 * p->drain_bits_per_1024ns / 100;
314 }
315
316 if (board_info(sge->adapter)->board == CHBT_BOARD_CHT204) {
317 p->drain_bits_per_1024ns -= 16;
318 s->max_avail = max(4096U, p->mtu + 16 + 14 + 4);
319 max_avail_segs = max(1U, 4096 / (p->mtu - 40));
320 } else {
321 s->max_avail = 16384;
322 max_avail_segs = max(1U, 9000 / (p->mtu - 40));
323 }
324
325 pr_debug("t1_sched_update_parms: mtu %u speed %u max_avail %u "
326 "max_avail_segs %u drain_bits_per_1024ns %u\n", p->mtu,
327 p->speed, s->max_avail, max_avail_segs,
328 p->drain_bits_per_1024ns);
329
330 return max_avail_segs * (p->mtu - 40);
331}
332
Adrian Bunk68d579f2007-11-21 15:02:55 -0800333#if 0
334
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -0800335/*
336 * t1_sched_max_avail_bytes() tells the scheduler the maximum amount of
337 * data that can be pushed per port.
338 */
339void t1_sched_set_max_avail_bytes(struct sge *sge, unsigned int val)
340{
341 struct sched *s = sge->tx_sched;
342 unsigned int i;
343
344 s->max_avail = val;
345 for (i = 0; i < MAX_NPORTS; i++)
346 t1_sched_update_parms(sge, i, 0, 0);
347}
348
349/*
350 * t1_sched_set_drain_bits_per_us() tells the scheduler at which rate a port
351 * is draining.
352 */
353void t1_sched_set_drain_bits_per_us(struct sge *sge, unsigned int port,
354 unsigned int val)
355{
356 struct sched *s = sge->tx_sched;
357 struct sched_port *p = &s->p[port];
358 p->drain_bits_per_1024ns = val * 1024 / 1000;
359 t1_sched_update_parms(sge, port, 0, 0);
360}
361
Adrian Bunk68d579f2007-11-21 15:02:55 -0800362#endif /* 0 */
363
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -0800364
365/*
366 * get_clock() implements a ns clock (see ktime_get)
367 */
368static inline ktime_t get_clock(void)
369{
370 struct timespec ts;
371
372 ktime_get_ts(&ts);
373 return timespec_to_ktime(ts);
374}
375
376/*
377 * tx_sched_init() allocates resources and does basic initialization.
378 */
379static int tx_sched_init(struct sge *sge)
380{
381 struct sched *s;
382 int i;
383
384 s = kzalloc(sizeof (struct sched), GFP_KERNEL);
385 if (!s)
386 return -ENOMEM;
387
388 pr_debug("tx_sched_init\n");
389 tasklet_init(&s->sched_tsk, restart_sched, (unsigned long) sge);
390 sge->tx_sched = s;
391
392 for (i = 0; i < MAX_NPORTS; i++) {
393 skb_queue_head_init(&s->p[i].skbq);
394 t1_sched_update_parms(sge, i, 1500, 1000);
395 }
396
397 return 0;
398}
399
400/*
401 * sched_update_avail() computes the delta since the last time it was called
402 * and updates the per port quota (number of bits that can be sent to the any
403 * port).
404 */
405static inline int sched_update_avail(struct sge *sge)
406{
407 struct sched *s = sge->tx_sched;
408 ktime_t now = get_clock();
409 unsigned int i;
410 long long delta_time_ns;
411
412 delta_time_ns = ktime_to_ns(ktime_sub(now, s->last_updated));
413
414 pr_debug("sched_update_avail delta=%lld\n", delta_time_ns);
415 if (delta_time_ns < 15000)
416 return 0;
417
418 for (i = 0; i < MAX_NPORTS; i++) {
419 struct sched_port *p = &s->p[i];
420 unsigned int delta_avail;
421
422 delta_avail = (p->drain_bits_per_1024ns * delta_time_ns) >> 13;
423 p->avail = min(p->avail + delta_avail, s->max_avail);
424 }
425
426 s->last_updated = now;
427
428 return 1;
429}
430
431/*
432 * sched_skb() is called from two different places. In the tx path, any
433 * packet generating load on an output port will call sched_skb()
434 * (skb != NULL). In addition, sched_skb() is called from the irq/soft irq
435 * context (skb == NULL).
436 * The scheduler only returns a skb (which will then be sent) if the
437 * length of the skb is <= the current quota of the output port.
438 */
439static struct sk_buff *sched_skb(struct sge *sge, struct sk_buff *skb,
440 unsigned int credits)
441{
442 struct sched *s = sge->tx_sched;
443 struct sk_buff_head *skbq;
444 unsigned int i, len, update = 1;
445
446 pr_debug("sched_skb %p\n", skb);
447 if (!skb) {
448 if (!s->num)
449 return NULL;
450 } else {
451 skbq = &s->p[skb->dev->if_port].skbq;
452 __skb_queue_tail(skbq, skb);
453 s->num++;
454 skb = NULL;
455 }
456
457 if (credits < MAX_SKB_FRAGS + 1)
458 goto out;
459
Francois Romieu356bd142006-12-11 23:47:00 +0100460again:
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -0800461 for (i = 0; i < MAX_NPORTS; i++) {
462 s->port = ++s->port & (MAX_NPORTS - 1);
463 skbq = &s->p[s->port].skbq;
464
465 skb = skb_peek(skbq);
466
467 if (!skb)
468 continue;
469
470 len = skb->len;
471 if (len <= s->p[s->port].avail) {
472 s->p[s->port].avail -= len;
473 s->num--;
474 __skb_unlink(skb, skbq);
475 goto out;
476 }
477 skb = NULL;
478 }
479
480 if (update-- && sched_update_avail(sge))
481 goto again;
482
Francois Romieu356bd142006-12-11 23:47:00 +0100483out:
484 /* If there are more pending skbs, we use the hardware to schedule us
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -0800485 * again.
486 */
487 if (s->num && !skb) {
488 struct cmdQ *q = &sge->cmdQ[0];
489 clear_bit(CMDQ_STAT_LAST_PKT_DB, &q->status);
490 if (test_and_set_bit(CMDQ_STAT_RUNNING, &q->status) == 0) {
491 set_bit(CMDQ_STAT_LAST_PKT_DB, &q->status);
492 writel(F_CMDQ0_ENABLE, sge->adapter->regs + A_SG_DOORBELL);
493 }
494 }
495 pr_debug("sched_skb ret %p\n", skb);
496
497 return skb;
498}
499
500/*
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800501 * PIO to indicate that memory mapped Q contains valid descriptor(s).
502 */
Scott Bardone559fb512005-06-23 01:40:19 -0400503static inline void doorbell_pio(struct adapter *adapter, u32 val)
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800504{
505 wmb();
Scott Bardone559fb512005-06-23 01:40:19 -0400506 writel(val, adapter->regs + A_SG_DOORBELL);
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800507}
508
509/*
510 * Frees all RX buffers on the freelist Q. The caller must make sure that
511 * the SGE is turned off before calling this function.
512 */
Scott Bardone559fb512005-06-23 01:40:19 -0400513static void free_freelQ_buffers(struct pci_dev *pdev, struct freelQ *q)
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800514{
Scott Bardone559fb512005-06-23 01:40:19 -0400515 unsigned int cidx = q->cidx;
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800516
Scott Bardone559fb512005-06-23 01:40:19 -0400517 while (q->credits--) {
518 struct freelQ_ce *ce = &q->centries[cidx];
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800519
520 pci_unmap_single(pdev, pci_unmap_addr(ce, dma_addr),
521 pci_unmap_len(ce, dma_len),
522 PCI_DMA_FROMDEVICE);
523 dev_kfree_skb(ce->skb);
524 ce->skb = NULL;
Scott Bardone559fb512005-06-23 01:40:19 -0400525 if (++cidx == q->size)
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800526 cidx = 0;
527 }
528}
529
530/*
531 * Free RX free list and response queue resources.
532 */
533static void free_rx_resources(struct sge *sge)
534{
535 struct pci_dev *pdev = sge->adapter->pdev;
536 unsigned int size, i;
537
538 if (sge->respQ.entries) {
Scott Bardone559fb512005-06-23 01:40:19 -0400539 size = sizeof(struct respQ_e) * sge->respQ.size;
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800540 pci_free_consistent(pdev, size, sge->respQ.entries,
541 sge->respQ.dma_addr);
542 }
543
544 for (i = 0; i < SGE_FREELQ_N; i++) {
Scott Bardone559fb512005-06-23 01:40:19 -0400545 struct freelQ *q = &sge->freelQ[i];
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800546
Scott Bardone559fb512005-06-23 01:40:19 -0400547 if (q->centries) {
548 free_freelQ_buffers(pdev, q);
549 kfree(q->centries);
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800550 }
Scott Bardone559fb512005-06-23 01:40:19 -0400551 if (q->entries) {
552 size = sizeof(struct freelQ_e) * q->size;
553 pci_free_consistent(pdev, size, q->entries,
554 q->dma_addr);
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800555 }
556 }
557}
558
559/*
560 * Allocates basic RX resources, consisting of memory mapped freelist Qs and a
Scott Bardone559fb512005-06-23 01:40:19 -0400561 * response queue.
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800562 */
563static int alloc_rx_resources(struct sge *sge, struct sge_params *p)
564{
565 struct pci_dev *pdev = sge->adapter->pdev;
566 unsigned int size, i;
567
568 for (i = 0; i < SGE_FREELQ_N; i++) {
Scott Bardone559fb512005-06-23 01:40:19 -0400569 struct freelQ *q = &sge->freelQ[i];
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800570
Scott Bardone559fb512005-06-23 01:40:19 -0400571 q->genbit = 1;
572 q->size = p->freelQ_size[i];
573 q->dma_offset = sge->rx_pkt_pad ? 0 : NET_IP_ALIGN;
574 size = sizeof(struct freelQ_e) * q->size;
Francois Romieu3e0f75b2006-12-05 23:57:41 +0100575 q->entries = pci_alloc_consistent(pdev, size, &q->dma_addr);
Scott Bardone559fb512005-06-23 01:40:19 -0400576 if (!q->entries)
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800577 goto err_no_mem;
Francois Romieu3e0f75b2006-12-05 23:57:41 +0100578
Scott Bardone559fb512005-06-23 01:40:19 -0400579 size = sizeof(struct freelQ_ce) * q->size;
Stephen Hemmingercbee9f92006-11-17 17:01:52 -0800580 q->centries = kzalloc(size, GFP_KERNEL);
Scott Bardone559fb512005-06-23 01:40:19 -0400581 if (!q->centries)
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800582 goto err_no_mem;
583 }
584
585 /*
586 * Calculate the buffer sizes for the two free lists. FL0 accommodates
587 * regular sized Ethernet frames, FL1 is sized not to exceed 16K,
588 * including all the sk_buff overhead.
589 *
590 * Note: For T2 FL0 and FL1 are reversed.
591 */
592 sge->freelQ[!sge->jumbo_fl].rx_buffer_size = SGE_RX_SM_BUF_SIZE +
593 sizeof(struct cpl_rx_data) +
594 sge->freelQ[!sge->jumbo_fl].dma_offset;
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -0800595
596 size = (16 * 1024) -
597 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
598
599 sge->freelQ[sge->jumbo_fl].rx_buffer_size = size;
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800600
Scott Bardone559fb512005-06-23 01:40:19 -0400601 /*
602 * Setup which skb recycle Q should be used when recycling buffers from
603 * each free list.
604 */
605 sge->freelQ[!sge->jumbo_fl].recycleq_idx = 0;
606 sge->freelQ[sge->jumbo_fl].recycleq_idx = 1;
607
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800608 sge->respQ.genbit = 1;
Scott Bardone559fb512005-06-23 01:40:19 -0400609 sge->respQ.size = SGE_RESPQ_E_N;
610 sge->respQ.credits = 0;
611 size = sizeof(struct respQ_e) * sge->respQ.size;
Francois Romieu3e0f75b2006-12-05 23:57:41 +0100612 sge->respQ.entries =
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800613 pci_alloc_consistent(pdev, size, &sge->respQ.dma_addr);
614 if (!sge->respQ.entries)
615 goto err_no_mem;
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800616 return 0;
617
618err_no_mem:
619 free_rx_resources(sge);
620 return -ENOMEM;
621}
622
623/*
Scott Bardone559fb512005-06-23 01:40:19 -0400624 * Reclaims n TX descriptors and frees the buffers associated with them.
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800625 */
Scott Bardone559fb512005-06-23 01:40:19 -0400626static void free_cmdQ_buffers(struct sge *sge, struct cmdQ *q, unsigned int n)
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800627{
Scott Bardone559fb512005-06-23 01:40:19 -0400628 struct cmdQ_ce *ce;
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800629 struct pci_dev *pdev = sge->adapter->pdev;
Scott Bardone559fb512005-06-23 01:40:19 -0400630 unsigned int cidx = q->cidx;
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800631
Scott Bardone559fb512005-06-23 01:40:19 -0400632 q->in_use -= n;
633 ce = &q->centries[cidx];
634 while (n--) {
Francois Romieu3e0f75b2006-12-05 23:57:41 +0100635 if (likely(pci_unmap_len(ce, dma_len))) {
636 pci_unmap_single(pdev, pci_unmap_addr(ce, dma_addr),
637 pci_unmap_len(ce, dma_len),
638 PCI_DMA_TODEVICE);
639 if (q->sop)
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -0800640 q->sop = 0;
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -0800641 }
Scott Bardone559fb512005-06-23 01:40:19 -0400642 if (ce->skb) {
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -0800643 dev_kfree_skb_any(ce->skb);
Scott Bardone559fb512005-06-23 01:40:19 -0400644 q->sop = 1;
645 }
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800646 ce++;
Scott Bardone559fb512005-06-23 01:40:19 -0400647 if (++cidx == q->size) {
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800648 cidx = 0;
Scott Bardone559fb512005-06-23 01:40:19 -0400649 ce = q->centries;
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800650 }
651 }
Scott Bardone559fb512005-06-23 01:40:19 -0400652 q->cidx = cidx;
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800653}
654
655/*
656 * Free TX resources.
657 *
658 * Assumes that SGE is stopped and all interrupts are disabled.
659 */
660static void free_tx_resources(struct sge *sge)
661{
662 struct pci_dev *pdev = sge->adapter->pdev;
663 unsigned int size, i;
664
665 for (i = 0; i < SGE_CMDQ_N; i++) {
Scott Bardone559fb512005-06-23 01:40:19 -0400666 struct cmdQ *q = &sge->cmdQ[i];
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800667
Scott Bardone559fb512005-06-23 01:40:19 -0400668 if (q->centries) {
669 if (q->in_use)
670 free_cmdQ_buffers(sge, q, q->in_use);
671 kfree(q->centries);
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800672 }
Scott Bardone559fb512005-06-23 01:40:19 -0400673 if (q->entries) {
674 size = sizeof(struct cmdQ_e) * q->size;
675 pci_free_consistent(pdev, size, q->entries,
676 q->dma_addr);
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800677 }
678 }
679}
680
681/*
682 * Allocates basic TX resources, consisting of memory mapped command Qs.
683 */
684static int alloc_tx_resources(struct sge *sge, struct sge_params *p)
685{
686 struct pci_dev *pdev = sge->adapter->pdev;
687 unsigned int size, i;
688
689 for (i = 0; i < SGE_CMDQ_N; i++) {
Scott Bardone559fb512005-06-23 01:40:19 -0400690 struct cmdQ *q = &sge->cmdQ[i];
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800691
Scott Bardone559fb512005-06-23 01:40:19 -0400692 q->genbit = 1;
693 q->sop = 1;
694 q->size = p->cmdQ_size[i];
695 q->in_use = 0;
696 q->status = 0;
697 q->processed = q->cleaned = 0;
698 q->stop_thres = 0;
699 spin_lock_init(&q->lock);
700 size = sizeof(struct cmdQ_e) * q->size;
Francois Romieu3e0f75b2006-12-05 23:57:41 +0100701 q->entries = pci_alloc_consistent(pdev, size, &q->dma_addr);
Scott Bardone559fb512005-06-23 01:40:19 -0400702 if (!q->entries)
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800703 goto err_no_mem;
Francois Romieu3e0f75b2006-12-05 23:57:41 +0100704
Scott Bardone559fb512005-06-23 01:40:19 -0400705 size = sizeof(struct cmdQ_ce) * q->size;
Stephen Hemmingercbee9f92006-11-17 17:01:52 -0800706 q->centries = kzalloc(size, GFP_KERNEL);
Scott Bardone559fb512005-06-23 01:40:19 -0400707 if (!q->centries)
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800708 goto err_no_mem;
709 }
710
Scott Bardone559fb512005-06-23 01:40:19 -0400711 /*
712 * CommandQ 0 handles Ethernet and TOE packets, while queue 1 is TOE
713 * only. For queue 0 set the stop threshold so we can handle one more
714 * packet from each port, plus reserve an additional 24 entries for
715 * Ethernet packets only. Queue 1 never suspends nor do we reserve
716 * space for Ethernet packets.
717 */
718 sge->cmdQ[0].stop_thres = sge->adapter->params.nports *
719 (MAX_SKB_FRAGS + 1);
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800720 return 0;
721
722err_no_mem:
723 free_tx_resources(sge);
724 return -ENOMEM;
725}
726
727static inline void setup_ring_params(struct adapter *adapter, u64 addr,
728 u32 size, int base_reg_lo,
729 int base_reg_hi, int size_reg)
730{
Scott Bardone559fb512005-06-23 01:40:19 -0400731 writel((u32)addr, adapter->regs + base_reg_lo);
732 writel(addr >> 32, adapter->regs + base_reg_hi);
733 writel(size, adapter->regs + size_reg);
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800734}
735
736/*
737 * Enable/disable VLAN acceleration.
738 */
739void t1_set_vlan_accel(struct adapter *adapter, int on_off)
740{
741 struct sge *sge = adapter->sge;
742
743 sge->sge_control &= ~F_VLAN_XTRACT;
744 if (on_off)
745 sge->sge_control |= F_VLAN_XTRACT;
746 if (adapter->open_device_map) {
Scott Bardone559fb512005-06-23 01:40:19 -0400747 writel(sge->sge_control, adapter->regs + A_SG_CONTROL);
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -0800748 readl(adapter->regs + A_SG_CONTROL); /* flush */
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800749 }
750}
751
752/*
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800753 * Programs the various SGE registers. However, the engine is not yet enabled,
754 * but sge->sge_control is setup and ready to go.
755 */
756static void configure_sge(struct sge *sge, struct sge_params *p)
757{
758 struct adapter *ap = sge->adapter;
Francois Romieu356bd142006-12-11 23:47:00 +0100759
Scott Bardone559fb512005-06-23 01:40:19 -0400760 writel(0, ap->regs + A_SG_CONTROL);
761 setup_ring_params(ap, sge->cmdQ[0].dma_addr, sge->cmdQ[0].size,
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800762 A_SG_CMD0BASELWR, A_SG_CMD0BASEUPR, A_SG_CMD0SIZE);
Scott Bardone559fb512005-06-23 01:40:19 -0400763 setup_ring_params(ap, sge->cmdQ[1].dma_addr, sge->cmdQ[1].size,
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800764 A_SG_CMD1BASELWR, A_SG_CMD1BASEUPR, A_SG_CMD1SIZE);
765 setup_ring_params(ap, sge->freelQ[0].dma_addr,
Scott Bardone559fb512005-06-23 01:40:19 -0400766 sge->freelQ[0].size, A_SG_FL0BASELWR,
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800767 A_SG_FL0BASEUPR, A_SG_FL0SIZE);
768 setup_ring_params(ap, sge->freelQ[1].dma_addr,
Scott Bardone559fb512005-06-23 01:40:19 -0400769 sge->freelQ[1].size, A_SG_FL1BASELWR,
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800770 A_SG_FL1BASEUPR, A_SG_FL1SIZE);
771
772 /* The threshold comparison uses <. */
Scott Bardone559fb512005-06-23 01:40:19 -0400773 writel(SGE_RX_SM_BUF_SIZE + 1, ap->regs + A_SG_FLTHRESHOLD);
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800774
Scott Bardone559fb512005-06-23 01:40:19 -0400775 setup_ring_params(ap, sge->respQ.dma_addr, sge->respQ.size,
776 A_SG_RSPBASELWR, A_SG_RSPBASEUPR, A_SG_RSPSIZE);
777 writel((u32)sge->respQ.size - 1, ap->regs + A_SG_RSPQUEUECREDIT);
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800778
779 sge->sge_control = F_CMDQ0_ENABLE | F_CMDQ1_ENABLE | F_FL0_ENABLE |
780 F_FL1_ENABLE | F_CPL_ENABLE | F_RESPONSE_QUEUE_ENABLE |
781 V_CMDQ_PRIORITY(2) | F_DISABLE_CMDQ1_GTS | F_ISCSI_COALESCE |
782 V_RX_PKT_OFFSET(sge->rx_pkt_pad);
783
784#if defined(__BIG_ENDIAN_BITFIELD)
785 sge->sge_control |= F_ENABLE_BIG_ENDIAN;
786#endif
787
Scott Bardone559fb512005-06-23 01:40:19 -0400788 /* Initialize no-resource timer */
789 sge->intrtimer_nres = SGE_INTRTIMER_NRES * core_ticks_per_usec(ap);
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800790
Scott Bardone559fb512005-06-23 01:40:19 -0400791 t1_sge_set_coalesce_params(sge, p);
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800792}
793
794/*
795 * Return the payload capacity of the jumbo free-list buffers.
796 */
797static inline unsigned int jumbo_payload_capacity(const struct sge *sge)
798{
799 return sge->freelQ[sge->jumbo_fl].rx_buffer_size -
Scott Bardone559fb512005-06-23 01:40:19 -0400800 sge->freelQ[sge->jumbo_fl].dma_offset -
801 sizeof(struct cpl_rx_data);
802}
803
804/*
805 * Frees all SGE related resources and the sge structure itself
806 */
807void t1_sge_destroy(struct sge *sge)
808{
Stephen Hemminger56f643c2006-12-01 16:36:21 -0800809 int i;
810
811 for_each_port(sge->adapter, i)
812 free_percpu(sge->port_stats[i]);
813
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -0800814 kfree(sge->tx_sched);
Scott Bardone559fb512005-06-23 01:40:19 -0400815 free_tx_resources(sge);
816 free_rx_resources(sge);
817 kfree(sge);
818}
819
820/*
821 * Allocates new RX buffers on the freelist Q (and tracks them on the freelist
822 * context Q) until the Q is full or alloc_skb fails.
823 *
824 * It is possible that the generation bits already match, indicating that the
825 * buffer is already valid and nothing needs to be done. This happens when we
826 * copied a received buffer into a new sk_buff during the interrupt processing.
827 *
828 * If the SGE doesn't automatically align packets properly (!sge->rx_pkt_pad),
829 * we specify a RX_OFFSET in order to make sure that the IP header is 4B
830 * aligned.
831 */
832static void refill_free_list(struct sge *sge, struct freelQ *q)
833{
834 struct pci_dev *pdev = sge->adapter->pdev;
835 struct freelQ_ce *ce = &q->centries[q->pidx];
836 struct freelQ_e *e = &q->entries[q->pidx];
837 unsigned int dma_len = q->rx_buffer_size - q->dma_offset;
838
Scott Bardone559fb512005-06-23 01:40:19 -0400839 while (q->credits < q->size) {
840 struct sk_buff *skb;
841 dma_addr_t mapping;
842
843 skb = alloc_skb(q->rx_buffer_size, GFP_ATOMIC);
844 if (!skb)
845 break;
846
847 skb_reserve(skb, q->dma_offset);
848 mapping = pci_map_single(pdev, skb->data, dma_len,
849 PCI_DMA_FROMDEVICE);
Stephen Hemminger24a427c2007-01-08 11:26:12 -0800850 skb_reserve(skb, sge->rx_pkt_pad);
851
Scott Bardone559fb512005-06-23 01:40:19 -0400852 ce->skb = skb;
853 pci_unmap_addr_set(ce, dma_addr, mapping);
854 pci_unmap_len_set(ce, dma_len, dma_len);
855 e->addr_lo = (u32)mapping;
856 e->addr_hi = (u64)mapping >> 32;
857 e->len_gen = V_CMD_LEN(dma_len) | V_CMD_GEN1(q->genbit);
858 wmb();
859 e->gen2 = V_CMD_GEN2(q->genbit);
860
861 e++;
862 ce++;
863 if (++q->pidx == q->size) {
864 q->pidx = 0;
865 q->genbit ^= 1;
866 ce = q->centries;
867 e = q->entries;
868 }
869 q->credits++;
870 }
Scott Bardone559fb512005-06-23 01:40:19 -0400871}
872
873/*
874 * Calls refill_free_list for both free lists. If we cannot fill at least 1/4
875 * of both rings, we go into 'few interrupt mode' in order to give the system
876 * time to free up resources.
877 */
878static void freelQs_empty(struct sge *sge)
879{
880 struct adapter *adapter = sge->adapter;
881 u32 irq_reg = readl(adapter->regs + A_SG_INT_ENABLE);
882 u32 irqholdoff_reg;
883
884 refill_free_list(sge, &sge->freelQ[0]);
885 refill_free_list(sge, &sge->freelQ[1]);
886
887 if (sge->freelQ[0].credits > (sge->freelQ[0].size >> 2) &&
888 sge->freelQ[1].credits > (sge->freelQ[1].size >> 2)) {
889 irq_reg |= F_FL_EXHAUSTED;
890 irqholdoff_reg = sge->fixed_intrtimer;
891 } else {
892 /* Clear the F_FL_EXHAUSTED interrupts for now */
893 irq_reg &= ~F_FL_EXHAUSTED;
894 irqholdoff_reg = sge->intrtimer_nres;
895 }
896 writel(irqholdoff_reg, adapter->regs + A_SG_INTRTIMER);
897 writel(irq_reg, adapter->regs + A_SG_INT_ENABLE);
898
899 /* We reenable the Qs to force a freelist GTS interrupt later */
900 doorbell_pio(adapter, F_FL0_ENABLE | F_FL1_ENABLE);
901}
902
903#define SGE_PL_INTR_MASK (F_PL_INTR_SGE_ERR | F_PL_INTR_SGE_DATA)
904#define SGE_INT_FATAL (F_RESPQ_OVERFLOW | F_PACKET_TOO_BIG | F_PACKET_MISMATCH)
905#define SGE_INT_ENABLE (F_RESPQ_EXHAUSTED | F_RESPQ_OVERFLOW | \
906 F_FL_EXHAUSTED | F_PACKET_TOO_BIG | F_PACKET_MISMATCH)
907
908/*
909 * Disable SGE Interrupts
910 */
911void t1_sge_intr_disable(struct sge *sge)
912{
913 u32 val = readl(sge->adapter->regs + A_PL_ENABLE);
914
915 writel(val & ~SGE_PL_INTR_MASK, sge->adapter->regs + A_PL_ENABLE);
916 writel(0, sge->adapter->regs + A_SG_INT_ENABLE);
917}
918
919/*
920 * Enable SGE interrupts.
921 */
922void t1_sge_intr_enable(struct sge *sge)
923{
924 u32 en = SGE_INT_ENABLE;
925 u32 val = readl(sge->adapter->regs + A_PL_ENABLE);
926
927 if (sge->adapter->flags & TSO_CAPABLE)
928 en &= ~F_PACKET_TOO_BIG;
929 writel(en, sge->adapter->regs + A_SG_INT_ENABLE);
930 writel(val | SGE_PL_INTR_MASK, sge->adapter->regs + A_PL_ENABLE);
931}
932
933/*
934 * Clear SGE interrupts.
935 */
936void t1_sge_intr_clear(struct sge *sge)
937{
938 writel(SGE_PL_INTR_MASK, sge->adapter->regs + A_PL_CAUSE);
939 writel(0xffffffff, sge->adapter->regs + A_SG_INT_CAUSE);
940}
941
942/*
943 * SGE 'Error' interrupt handler
944 */
945int t1_sge_intr_error_handler(struct sge *sge)
946{
947 struct adapter *adapter = sge->adapter;
948 u32 cause = readl(adapter->regs + A_SG_INT_CAUSE);
949
950 if (adapter->flags & TSO_CAPABLE)
951 cause &= ~F_PACKET_TOO_BIG;
952 if (cause & F_RESPQ_EXHAUSTED)
953 sge->stats.respQ_empty++;
954 if (cause & F_RESPQ_OVERFLOW) {
955 sge->stats.respQ_overflow++;
956 CH_ALERT("%s: SGE response queue overflow\n",
957 adapter->name);
958 }
959 if (cause & F_FL_EXHAUSTED) {
960 sge->stats.freelistQ_empty++;
961 freelQs_empty(sge);
962 }
963 if (cause & F_PACKET_TOO_BIG) {
964 sge->stats.pkt_too_big++;
965 CH_ALERT("%s: SGE max packet size exceeded\n",
966 adapter->name);
967 }
968 if (cause & F_PACKET_MISMATCH) {
969 sge->stats.pkt_mismatch++;
970 CH_ALERT("%s: SGE packet mismatch\n", adapter->name);
971 }
972 if (cause & SGE_INT_FATAL)
973 t1_fatal_err(adapter);
974
975 writel(cause, adapter->regs + A_SG_INT_CAUSE);
976 return 0;
977}
978
Stephen Hemminger56f643c2006-12-01 16:36:21 -0800979const struct sge_intr_counts *t1_sge_get_intr_counts(const struct sge *sge)
Scott Bardone559fb512005-06-23 01:40:19 -0400980{
981 return &sge->stats;
982}
983
Stephen Hemminger56f643c2006-12-01 16:36:21 -0800984void t1_sge_get_port_stats(const struct sge *sge, int port,
985 struct sge_port_stats *ss)
Scott Bardone559fb512005-06-23 01:40:19 -0400986{
Stephen Hemminger56f643c2006-12-01 16:36:21 -0800987 int cpu;
988
989 memset(ss, 0, sizeof(*ss));
990 for_each_possible_cpu(cpu) {
991 struct sge_port_stats *st = per_cpu_ptr(sge->port_stats[port], cpu);
992
Stephen Hemminger56f643c2006-12-01 16:36:21 -0800993 ss->rx_cso_good += st->rx_cso_good;
Stephen Hemminger56f643c2006-12-01 16:36:21 -0800994 ss->tx_cso += st->tx_cso;
995 ss->tx_tso += st->tx_tso;
Divy Le Ray7832ee02007-11-27 13:30:09 -0800996 ss->tx_need_hdrroom += st->tx_need_hdrroom;
Stephen Hemminger56f643c2006-12-01 16:36:21 -0800997 ss->vlan_xtract += st->vlan_xtract;
998 ss->vlan_insert += st->vlan_insert;
999 }
Scott Bardone559fb512005-06-23 01:40:19 -04001000}
1001
1002/**
1003 * recycle_fl_buf - recycle a free list buffer
1004 * @fl: the free list
1005 * @idx: index of buffer to recycle
1006 *
1007 * Recycles the specified buffer on the given free list by adding it at
1008 * the next available slot on the list.
1009 */
1010static void recycle_fl_buf(struct freelQ *fl, int idx)
1011{
1012 struct freelQ_e *from = &fl->entries[idx];
1013 struct freelQ_e *to = &fl->entries[fl->pidx];
1014
1015 fl->centries[fl->pidx] = fl->centries[idx];
1016 to->addr_lo = from->addr_lo;
1017 to->addr_hi = from->addr_hi;
1018 to->len_gen = G_CMD_LEN(from->len_gen) | V_CMD_GEN1(fl->genbit);
1019 wmb();
1020 to->gen2 = V_CMD_GEN2(fl->genbit);
1021 fl->credits++;
1022
1023 if (++fl->pidx == fl->size) {
1024 fl->pidx = 0;
1025 fl->genbit ^= 1;
1026 }
1027}
1028
Stephen Hemminger24a427c2007-01-08 11:26:12 -08001029static int copybreak __read_mostly = 256;
1030module_param(copybreak, int, 0);
1031MODULE_PARM_DESC(copybreak, "Receive copy threshold");
1032
Scott Bardone559fb512005-06-23 01:40:19 -04001033/**
1034 * get_packet - return the next ingress packet buffer
1035 * @pdev: the PCI device that received the packet
1036 * @fl: the SGE free list holding the packet
1037 * @len: the actual packet length, excluding any SGE padding
Scott Bardone559fb512005-06-23 01:40:19 -04001038 *
1039 * Get the next packet from a free list and complete setup of the
1040 * sk_buff. If the packet is small we make a copy and recycle the
1041 * original buffer, otherwise we use the original buffer itself. If a
1042 * positive drop threshold is supplied packets are dropped and their
1043 * buffers recycled if (a) the number of remaining buffers is under the
1044 * threshold and the packet is too big to copy, or (b) the packet should
1045 * be copied but there is no memory for the copy.
1046 */
1047static inline struct sk_buff *get_packet(struct pci_dev *pdev,
Stephen Hemminger24a427c2007-01-08 11:26:12 -08001048 struct freelQ *fl, unsigned int len)
Scott Bardone559fb512005-06-23 01:40:19 -04001049{
1050 struct sk_buff *skb;
Stephen Hemminger24a427c2007-01-08 11:26:12 -08001051 const struct freelQ_ce *ce = &fl->centries[fl->cidx];
Scott Bardone559fb512005-06-23 01:40:19 -04001052
Stephen Hemminger24a427c2007-01-08 11:26:12 -08001053 if (len < copybreak) {
1054 skb = alloc_skb(len + 2, GFP_ATOMIC);
1055 if (!skb)
Scott Bardone559fb512005-06-23 01:40:19 -04001056 goto use_orig_buf;
1057
Stephen Hemminger24a427c2007-01-08 11:26:12 -08001058 skb_reserve(skb, 2); /* align IP header */
1059 skb_put(skb, len);
1060 pci_dma_sync_single_for_cpu(pdev,
1061 pci_unmap_addr(ce, dma_addr),
1062 pci_unmap_len(ce, dma_len),
1063 PCI_DMA_FROMDEVICE);
Arnaldo Carvalho de Melod626f622007-03-27 18:55:52 -03001064 skb_copy_from_linear_data(ce->skb, skb->data, len);
Stephen Hemminger24a427c2007-01-08 11:26:12 -08001065 pci_dma_sync_single_for_device(pdev,
1066 pci_unmap_addr(ce, dma_addr),
1067 pci_unmap_len(ce, dma_len),
1068 PCI_DMA_FROMDEVICE);
Scott Bardone559fb512005-06-23 01:40:19 -04001069 recycle_fl_buf(fl, fl->cidx);
1070 return skb;
1071 }
1072
Stephen Hemminger24a427c2007-01-08 11:26:12 -08001073use_orig_buf:
1074 if (fl->credits < 2) {
Scott Bardone559fb512005-06-23 01:40:19 -04001075 recycle_fl_buf(fl, fl->cidx);
1076 return NULL;
1077 }
1078
Scott Bardone559fb512005-06-23 01:40:19 -04001079 pci_unmap_single(pdev, pci_unmap_addr(ce, dma_addr),
1080 pci_unmap_len(ce, dma_len), PCI_DMA_FROMDEVICE);
1081 skb = ce->skb;
Stephen Hemminger24a427c2007-01-08 11:26:12 -08001082 prefetch(skb->data);
1083
Scott Bardone559fb512005-06-23 01:40:19 -04001084 skb_put(skb, len);
1085 return skb;
1086}
1087
1088/**
1089 * unexpected_offload - handle an unexpected offload packet
1090 * @adapter: the adapter
1091 * @fl: the free list that received the packet
1092 *
1093 * Called when we receive an unexpected offload packet (e.g., the TOE
1094 * function is disabled or the card is a NIC). Prints a message and
1095 * recycles the buffer.
1096 */
1097static void unexpected_offload(struct adapter *adapter, struct freelQ *fl)
1098{
1099 struct freelQ_ce *ce = &fl->centries[fl->cidx];
1100 struct sk_buff *skb = ce->skb;
1101
1102 pci_dma_sync_single_for_cpu(adapter->pdev, pci_unmap_addr(ce, dma_addr),
1103 pci_unmap_len(ce, dma_len), PCI_DMA_FROMDEVICE);
1104 CH_ERR("%s: unexpected offload packet, cmd %u\n",
1105 adapter->name, *skb->data);
1106 recycle_fl_buf(fl, fl->cidx);
1107}
1108
1109/*
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001110 * T1/T2 SGE limits the maximum DMA size per TX descriptor to
1111 * SGE_TX_DESC_MAX_PLEN (16KB). If the PAGE_SIZE is larger than 16KB, the
1112 * stack might send more than SGE_TX_DESC_MAX_PLEN in a contiguous manner.
1113 * Note that the *_large_page_tx_descs stuff will be optimized out when
1114 * PAGE_SIZE <= SGE_TX_DESC_MAX_PLEN.
1115 *
1116 * compute_large_page_descs() computes how many additional descriptors are
1117 * required to break down the stack's request.
1118 */
1119static inline unsigned int compute_large_page_tx_descs(struct sk_buff *skb)
1120{
1121 unsigned int count = 0;
Francois Romieu356bd142006-12-11 23:47:00 +01001122
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001123 if (PAGE_SIZE > SGE_TX_DESC_MAX_PLEN) {
1124 unsigned int nfrags = skb_shinfo(skb)->nr_frags;
1125 unsigned int i, len = skb->len - skb->data_len;
1126 while (len > SGE_TX_DESC_MAX_PLEN) {
1127 count++;
1128 len -= SGE_TX_DESC_MAX_PLEN;
1129 }
1130 for (i = 0; nfrags--; i++) {
1131 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
1132 len = frag->size;
1133 while (len > SGE_TX_DESC_MAX_PLEN) {
1134 count++;
1135 len -= SGE_TX_DESC_MAX_PLEN;
1136 }
1137 }
1138 }
1139 return count;
1140}
1141
1142/*
1143 * Write a cmdQ entry.
1144 *
1145 * Since this function writes the 'flags' field, it must not be used to
1146 * write the first cmdQ entry.
1147 */
1148static inline void write_tx_desc(struct cmdQ_e *e, dma_addr_t mapping,
1149 unsigned int len, unsigned int gen,
1150 unsigned int eop)
1151{
1152 if (unlikely(len > SGE_TX_DESC_MAX_PLEN))
1153 BUG();
1154 e->addr_lo = (u32)mapping;
1155 e->addr_hi = (u64)mapping >> 32;
1156 e->len_gen = V_CMD_LEN(len) | V_CMD_GEN1(gen);
1157 e->flags = F_CMD_DATAVALID | V_CMD_EOP(eop) | V_CMD_GEN2(gen);
1158}
1159
1160/*
1161 * See comment for previous function.
1162 *
1163 * write_tx_descs_large_page() writes additional SGE tx descriptors if
1164 * *desc_len exceeds HW's capability.
1165 */
1166static inline unsigned int write_large_page_tx_descs(unsigned int pidx,
1167 struct cmdQ_e **e,
1168 struct cmdQ_ce **ce,
1169 unsigned int *gen,
1170 dma_addr_t *desc_mapping,
1171 unsigned int *desc_len,
1172 unsigned int nfrags,
1173 struct cmdQ *q)
1174{
1175 if (PAGE_SIZE > SGE_TX_DESC_MAX_PLEN) {
1176 struct cmdQ_e *e1 = *e;
1177 struct cmdQ_ce *ce1 = *ce;
1178
1179 while (*desc_len > SGE_TX_DESC_MAX_PLEN) {
1180 *desc_len -= SGE_TX_DESC_MAX_PLEN;
1181 write_tx_desc(e1, *desc_mapping, SGE_TX_DESC_MAX_PLEN,
1182 *gen, nfrags == 0 && *desc_len == 0);
1183 ce1->skb = NULL;
1184 pci_unmap_len_set(ce1, dma_len, 0);
1185 *desc_mapping += SGE_TX_DESC_MAX_PLEN;
1186 if (*desc_len) {
1187 ce1++;
1188 e1++;
1189 if (++pidx == q->size) {
1190 pidx = 0;
1191 *gen ^= 1;
1192 ce1 = q->centries;
1193 e1 = q->entries;
1194 }
1195 }
1196 }
1197 *e = e1;
1198 *ce = ce1;
1199 }
1200 return pidx;
1201}
1202
1203/*
Scott Bardone559fb512005-06-23 01:40:19 -04001204 * Write the command descriptors to transmit the given skb starting at
1205 * descriptor pidx with the given generation.
1206 */
1207static inline void write_tx_descs(struct adapter *adapter, struct sk_buff *skb,
1208 unsigned int pidx, unsigned int gen,
1209 struct cmdQ *q)
1210{
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001211 dma_addr_t mapping, desc_mapping;
Scott Bardone559fb512005-06-23 01:40:19 -04001212 struct cmdQ_e *e, *e1;
1213 struct cmdQ_ce *ce;
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001214 unsigned int i, flags, first_desc_len, desc_len,
1215 nfrags = skb_shinfo(skb)->nr_frags;
1216
1217 e = e1 = &q->entries[pidx];
1218 ce = &q->centries[pidx];
Scott Bardone559fb512005-06-23 01:40:19 -04001219
1220 mapping = pci_map_single(adapter->pdev, skb->data,
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001221 skb->len - skb->data_len, PCI_DMA_TODEVICE);
1222
1223 desc_mapping = mapping;
1224 desc_len = skb->len - skb->data_len;
1225
1226 flags = F_CMD_DATAVALID | F_CMD_SOP |
1227 V_CMD_EOP(nfrags == 0 && desc_len <= SGE_TX_DESC_MAX_PLEN) |
1228 V_CMD_GEN2(gen);
1229 first_desc_len = (desc_len <= SGE_TX_DESC_MAX_PLEN) ?
1230 desc_len : SGE_TX_DESC_MAX_PLEN;
1231 e->addr_lo = (u32)desc_mapping;
1232 e->addr_hi = (u64)desc_mapping >> 32;
1233 e->len_gen = V_CMD_LEN(first_desc_len) | V_CMD_GEN1(gen);
1234 ce->skb = NULL;
1235 pci_unmap_len_set(ce, dma_len, 0);
1236
1237 if (PAGE_SIZE > SGE_TX_DESC_MAX_PLEN &&
1238 desc_len > SGE_TX_DESC_MAX_PLEN) {
1239 desc_mapping += first_desc_len;
1240 desc_len -= first_desc_len;
1241 e1++;
1242 ce++;
1243 if (++pidx == q->size) {
1244 pidx = 0;
1245 gen ^= 1;
1246 e1 = q->entries;
1247 ce = q->centries;
1248 }
1249 pidx = write_large_page_tx_descs(pidx, &e1, &ce, &gen,
1250 &desc_mapping, &desc_len,
1251 nfrags, q);
1252
1253 if (likely(desc_len))
1254 write_tx_desc(e1, desc_mapping, desc_len, gen,
1255 nfrags == 0);
1256 }
1257
Scott Bardone559fb512005-06-23 01:40:19 -04001258 ce->skb = NULL;
1259 pci_unmap_addr_set(ce, dma_addr, mapping);
1260 pci_unmap_len_set(ce, dma_len, skb->len - skb->data_len);
1261
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001262 for (i = 0; nfrags--; i++) {
Scott Bardone559fb512005-06-23 01:40:19 -04001263 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
Scott Bardone559fb512005-06-23 01:40:19 -04001264 e1++;
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001265 ce++;
Scott Bardone559fb512005-06-23 01:40:19 -04001266 if (++pidx == q->size) {
1267 pidx = 0;
1268 gen ^= 1;
Scott Bardone559fb512005-06-23 01:40:19 -04001269 e1 = q->entries;
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001270 ce = q->centries;
Scott Bardone559fb512005-06-23 01:40:19 -04001271 }
1272
1273 mapping = pci_map_page(adapter->pdev, frag->page,
1274 frag->page_offset, frag->size,
1275 PCI_DMA_TODEVICE);
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001276 desc_mapping = mapping;
1277 desc_len = frag->size;
1278
1279 pidx = write_large_page_tx_descs(pidx, &e1, &ce, &gen,
1280 &desc_mapping, &desc_len,
1281 nfrags, q);
1282 if (likely(desc_len))
1283 write_tx_desc(e1, desc_mapping, desc_len, gen,
1284 nfrags == 0);
Scott Bardone559fb512005-06-23 01:40:19 -04001285 ce->skb = NULL;
1286 pci_unmap_addr_set(ce, dma_addr, mapping);
1287 pci_unmap_len_set(ce, dma_len, frag->size);
Scott Bardone559fb512005-06-23 01:40:19 -04001288 }
Scott Bardone559fb512005-06-23 01:40:19 -04001289 ce->skb = skb;
1290 wmb();
1291 e->flags = flags;
1292}
1293
1294/*
1295 * Clean up completed Tx buffers.
1296 */
1297static inline void reclaim_completed_tx(struct sge *sge, struct cmdQ *q)
1298{
1299 unsigned int reclaim = q->processed - q->cleaned;
1300
1301 if (reclaim) {
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001302 pr_debug("reclaim_completed_tx processed:%d cleaned:%d\n",
1303 q->processed, q->cleaned);
Scott Bardone559fb512005-06-23 01:40:19 -04001304 free_cmdQ_buffers(sge, q, reclaim);
1305 q->cleaned += reclaim;
1306 }
1307}
1308
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001309/*
1310 * Called from tasklet. Checks the scheduler for any
1311 * pending skbs that can be sent.
1312 */
1313static void restart_sched(unsigned long arg)
1314{
1315 struct sge *sge = (struct sge *) arg;
1316 struct adapter *adapter = sge->adapter;
1317 struct cmdQ *q = &sge->cmdQ[0];
1318 struct sk_buff *skb;
1319 unsigned int credits, queued_skb = 0;
1320
1321 spin_lock(&q->lock);
1322 reclaim_completed_tx(sge, q);
1323
1324 credits = q->size - q->in_use;
1325 pr_debug("restart_sched credits=%d\n", credits);
1326 while ((skb = sched_skb(sge, NULL, credits)) != NULL) {
1327 unsigned int genbit, pidx, count;
1328 count = 1 + skb_shinfo(skb)->nr_frags;
Francois Romieu356bd142006-12-11 23:47:00 +01001329 count += compute_large_page_tx_descs(skb);
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001330 q->in_use += count;
1331 genbit = q->genbit;
1332 pidx = q->pidx;
1333 q->pidx += count;
1334 if (q->pidx >= q->size) {
1335 q->pidx -= q->size;
1336 q->genbit ^= 1;
1337 }
1338 write_tx_descs(adapter, skb, pidx, genbit, q);
1339 credits = q->size - q->in_use;
1340 queued_skb = 1;
1341 }
1342
1343 if (queued_skb) {
1344 clear_bit(CMDQ_STAT_LAST_PKT_DB, &q->status);
1345 if (test_and_set_bit(CMDQ_STAT_RUNNING, &q->status) == 0) {
1346 set_bit(CMDQ_STAT_LAST_PKT_DB, &q->status);
1347 writel(F_CMDQ0_ENABLE, adapter->regs + A_SG_DOORBELL);
1348 }
1349 }
1350 spin_unlock(&q->lock);
1351}
Scott Bardone559fb512005-06-23 01:40:19 -04001352
Scott Bardone559fb512005-06-23 01:40:19 -04001353/**
1354 * sge_rx - process an ingress ethernet packet
1355 * @sge: the sge structure
1356 * @fl: the free list that contains the packet buffer
1357 * @len: the packet length
1358 *
1359 * Process an ingress ethernet pakcet and deliver it to the stack.
1360 */
Stephen Hemminger24a427c2007-01-08 11:26:12 -08001361static void sge_rx(struct sge *sge, struct freelQ *fl, unsigned int len)
Scott Bardone559fb512005-06-23 01:40:19 -04001362{
1363 struct sk_buff *skb;
Stephen Hemminger24a427c2007-01-08 11:26:12 -08001364 const struct cpl_rx_pkt *p;
Scott Bardone559fb512005-06-23 01:40:19 -04001365 struct adapter *adapter = sge->adapter;
Stephen Hemminger56f643c2006-12-01 16:36:21 -08001366 struct sge_port_stats *st;
Scott Bardone559fb512005-06-23 01:40:19 -04001367
Stephen Hemminger24a427c2007-01-08 11:26:12 -08001368 skb = get_packet(adapter->pdev, fl, len - sge->rx_pkt_pad);
Stephen Hemminger56f643c2006-12-01 16:36:21 -08001369 if (unlikely(!skb)) {
1370 sge->stats.rx_drops++;
Stephen Hemminger24a427c2007-01-08 11:26:12 -08001371 return;
Scott Bardone559fb512005-06-23 01:40:19 -04001372 }
1373
Stephen Hemminger24a427c2007-01-08 11:26:12 -08001374 p = (const struct cpl_rx_pkt *) skb->data;
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001375 if (p->iff >= adapter->params.nports) {
1376 kfree_skb(skb);
Stephen Hemminger24a427c2007-01-08 11:26:12 -08001377 return;
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001378 }
Stephen Hemminger24a427c2007-01-08 11:26:12 -08001379 __skb_pull(skb, sizeof(*p));
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001380
Stephen Hemminger56f643c2006-12-01 16:36:21 -08001381 st = per_cpu_ptr(sge->port_stats[p->iff], smp_processor_id());
Stephen Hemminger56f643c2006-12-01 16:36:21 -08001382
Arnaldo Carvalho de Melo4c13eb62007-04-25 17:40:23 -07001383 skb->protocol = eth_type_trans(skb, adapter->port[p->iff].dev);
Scott Bardone559fb512005-06-23 01:40:19 -04001384 if ((adapter->flags & RX_CSUM_ENABLED) && p->csum == 0xffff &&
1385 skb->protocol == htons(ETH_P_IP) &&
1386 (skb->data[9] == IPPROTO_TCP || skb->data[9] == IPPROTO_UDP)) {
Stephen Hemminger56f643c2006-12-01 16:36:21 -08001387 ++st->rx_cso_good;
Scott Bardone559fb512005-06-23 01:40:19 -04001388 skb->ip_summed = CHECKSUM_UNNECESSARY;
1389 } else
1390 skb->ip_summed = CHECKSUM_NONE;
1391
1392 if (unlikely(adapter->vlan_grp && p->vlan_valid)) {
Stephen Hemminger56f643c2006-12-01 16:36:21 -08001393 st->vlan_xtract++;
Francois Romieu4422b002008-07-11 00:29:19 +02001394 vlan_hwaccel_receive_skb(skb, adapter->vlan_grp,
1395 ntohs(p->vlan));
1396 } else
Scott Bardone559fb512005-06-23 01:40:19 -04001397 netif_receive_skb(skb);
Scott Bardone559fb512005-06-23 01:40:19 -04001398}
1399
1400/*
1401 * Returns true if a command queue has enough available descriptors that
1402 * we can resume Tx operation after temporarily disabling its packet queue.
1403 */
1404static inline int enough_free_Tx_descs(const struct cmdQ *q)
1405{
1406 unsigned int r = q->processed - q->cleaned;
1407
1408 return q->in_use - r < (q->size >> 1);
1409}
1410
1411/*
1412 * Called when sufficient space has become available in the SGE command queues
1413 * after the Tx packet schedulers have been suspended to restart the Tx path.
1414 */
1415static void restart_tx_queues(struct sge *sge)
1416{
1417 struct adapter *adap = sge->adapter;
Francois Romieu3e0f75b2006-12-05 23:57:41 +01001418 int i;
Scott Bardone559fb512005-06-23 01:40:19 -04001419
Francois Romieu3e0f75b2006-12-05 23:57:41 +01001420 if (!enough_free_Tx_descs(&sge->cmdQ[0]))
1421 return;
Scott Bardone559fb512005-06-23 01:40:19 -04001422
Francois Romieu3e0f75b2006-12-05 23:57:41 +01001423 for_each_port(adap, i) {
1424 struct net_device *nd = adap->port[i].dev;
Scott Bardone559fb512005-06-23 01:40:19 -04001425
Francois Romieu3e0f75b2006-12-05 23:57:41 +01001426 if (test_and_clear_bit(nd->if_port, &sge->stopped_tx_queues) &&
1427 netif_running(nd)) {
1428 sge->stats.cmdQ_restarted[2]++;
1429 netif_wake_queue(nd);
Scott Bardone559fb512005-06-23 01:40:19 -04001430 }
1431 }
1432}
1433
1434/*
Francois Romieu356bd142006-12-11 23:47:00 +01001435 * update_tx_info is called from the interrupt handler/NAPI to return cmdQ0
Scott Bardone559fb512005-06-23 01:40:19 -04001436 * information.
1437 */
Francois Romieu356bd142006-12-11 23:47:00 +01001438static unsigned int update_tx_info(struct adapter *adapter,
1439 unsigned int flags,
Scott Bardone559fb512005-06-23 01:40:19 -04001440 unsigned int pr0)
1441{
1442 struct sge *sge = adapter->sge;
1443 struct cmdQ *cmdq = &sge->cmdQ[0];
1444
1445 cmdq->processed += pr0;
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001446 if (flags & (F_FL0_ENABLE | F_FL1_ENABLE)) {
1447 freelQs_empty(sge);
1448 flags &= ~(F_FL0_ENABLE | F_FL1_ENABLE);
1449 }
Scott Bardone559fb512005-06-23 01:40:19 -04001450 if (flags & F_CMDQ0_ENABLE) {
1451 clear_bit(CMDQ_STAT_RUNNING, &cmdq->status);
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001452
Scott Bardone559fb512005-06-23 01:40:19 -04001453 if (cmdq->cleaned + cmdq->in_use != cmdq->processed &&
1454 !test_and_set_bit(CMDQ_STAT_LAST_PKT_DB, &cmdq->status)) {
1455 set_bit(CMDQ_STAT_RUNNING, &cmdq->status);
1456 writel(F_CMDQ0_ENABLE, adapter->regs + A_SG_DOORBELL);
1457 }
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001458 if (sge->tx_sched)
1459 tasklet_hi_schedule(&sge->tx_sched->sched_tsk);
1460
1461 flags &= ~F_CMDQ0_ENABLE;
Scott Bardone559fb512005-06-23 01:40:19 -04001462 }
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001463
Scott Bardone559fb512005-06-23 01:40:19 -04001464 if (unlikely(sge->stopped_tx_queues != 0))
1465 restart_tx_queues(sge);
1466
1467 return flags;
1468}
1469
1470/*
1471 * Process SGE responses, up to the supplied budget. Returns the number of
1472 * responses processed. A negative budget is effectively unlimited.
1473 */
1474static int process_responses(struct adapter *adapter, int budget)
1475{
1476 struct sge *sge = adapter->sge;
1477 struct respQ *q = &sge->respQ;
1478 struct respQ_e *e = &q->entries[q->cidx];
Stephen Hemminger24a427c2007-01-08 11:26:12 -08001479 int done = 0;
Scott Bardone559fb512005-06-23 01:40:19 -04001480 unsigned int flags = 0;
1481 unsigned int cmdq_processed[SGE_CMDQ_N] = {0, 0};
Francois Romieu356bd142006-12-11 23:47:00 +01001482
Stephen Hemminger24a427c2007-01-08 11:26:12 -08001483 while (done < budget && e->GenerationBit == q->genbit) {
Scott Bardone559fb512005-06-23 01:40:19 -04001484 flags |= e->Qsleeping;
Francois Romieu356bd142006-12-11 23:47:00 +01001485
Scott Bardone559fb512005-06-23 01:40:19 -04001486 cmdq_processed[0] += e->Cmdq0CreditReturn;
1487 cmdq_processed[1] += e->Cmdq1CreditReturn;
Francois Romieu356bd142006-12-11 23:47:00 +01001488
Scott Bardone559fb512005-06-23 01:40:19 -04001489 /* We batch updates to the TX side to avoid cacheline
1490 * ping-pong of TX state information on MP where the sender
1491 * might run on a different CPU than this function...
1492 */
Stephen Hemminger24a427c2007-01-08 11:26:12 -08001493 if (unlikely((flags & F_CMDQ0_ENABLE) || cmdq_processed[0] > 64)) {
Scott Bardone559fb512005-06-23 01:40:19 -04001494 flags = update_tx_info(adapter, flags, cmdq_processed[0]);
1495 cmdq_processed[0] = 0;
1496 }
Stephen Hemminger24a427c2007-01-08 11:26:12 -08001497
Scott Bardone559fb512005-06-23 01:40:19 -04001498 if (unlikely(cmdq_processed[1] > 16)) {
1499 sge->cmdQ[1].processed += cmdq_processed[1];
1500 cmdq_processed[1] = 0;
1501 }
Stephen Hemminger24a427c2007-01-08 11:26:12 -08001502
Scott Bardone559fb512005-06-23 01:40:19 -04001503 if (likely(e->DataValid)) {
1504 struct freelQ *fl = &sge->freelQ[e->FreelistQid];
1505
Eric Sesterhenn5d9428d2006-04-02 13:52:48 +02001506 BUG_ON(!e->Sop || !e->Eop);
Scott Bardone559fb512005-06-23 01:40:19 -04001507 if (unlikely(e->Offload))
1508 unexpected_offload(adapter, fl);
1509 else
1510 sge_rx(sge, fl, e->BufferLength);
1511
Stephen Hemminger24a427c2007-01-08 11:26:12 -08001512 ++done;
1513
Scott Bardone559fb512005-06-23 01:40:19 -04001514 /*
1515 * Note: this depends on each packet consuming a
1516 * single free-list buffer; cf. the BUG above.
1517 */
1518 if (++fl->cidx == fl->size)
1519 fl->cidx = 0;
Stephen Hemminger24a427c2007-01-08 11:26:12 -08001520 prefetch(fl->centries[fl->cidx].skb);
1521
Scott Bardone559fb512005-06-23 01:40:19 -04001522 if (unlikely(--fl->credits <
1523 fl->size - SGE_FREEL_REFILL_THRESH))
1524 refill_free_list(sge, fl);
1525 } else
1526 sge->stats.pure_rsps++;
1527
1528 e++;
1529 if (unlikely(++q->cidx == q->size)) {
1530 q->cidx = 0;
1531 q->genbit ^= 1;
1532 e = q->entries;
1533 }
1534 prefetch(e);
1535
1536 if (++q->credits > SGE_RESPQ_REPLENISH_THRES) {
1537 writel(q->credits, adapter->regs + A_SG_RSPQUEUECREDIT);
1538 q->credits = 0;
1539 }
Scott Bardone559fb512005-06-23 01:40:19 -04001540 }
1541
Francois Romieu356bd142006-12-11 23:47:00 +01001542 flags = update_tx_info(adapter, flags, cmdq_processed[0]);
Scott Bardone559fb512005-06-23 01:40:19 -04001543 sge->cmdQ[1].processed += cmdq_processed[1];
1544
Stephen Hemminger24a427c2007-01-08 11:26:12 -08001545 return done;
Scott Bardone559fb512005-06-23 01:40:19 -04001546}
1547
Stephen Hemminger3de00b82007-01-08 11:26:30 -08001548static inline int responses_pending(const struct adapter *adapter)
1549{
1550 const struct respQ *Q = &adapter->sge->respQ;
1551 const struct respQ_e *e = &Q->entries[Q->cidx];
1552
1553 return (e->GenerationBit == Q->genbit);
1554}
1555
Scott Bardone559fb512005-06-23 01:40:19 -04001556/*
1557 * A simpler version of process_responses() that handles only pure (i.e.,
1558 * non data-carrying) responses. Such respones are too light-weight to justify
1559 * calling a softirq when using NAPI, so we handle them specially in hard
1560 * interrupt context. The function is called with a pointer to a response,
1561 * which the caller must ensure is a valid pure response. Returns 1 if it
1562 * encounters a valid data-carrying response, 0 otherwise.
1563 */
Stephen Hemminger3de00b82007-01-08 11:26:30 -08001564static int process_pure_responses(struct adapter *adapter)
Scott Bardone559fb512005-06-23 01:40:19 -04001565{
1566 struct sge *sge = adapter->sge;
1567 struct respQ *q = &sge->respQ;
Stephen Hemminger3de00b82007-01-08 11:26:30 -08001568 struct respQ_e *e = &q->entries[q->cidx];
Stephen Hemminger24a427c2007-01-08 11:26:12 -08001569 const struct freelQ *fl = &sge->freelQ[e->FreelistQid];
Scott Bardone559fb512005-06-23 01:40:19 -04001570 unsigned int flags = 0;
1571 unsigned int cmdq_processed[SGE_CMDQ_N] = {0, 0};
1572
Stephen Hemminger24a427c2007-01-08 11:26:12 -08001573 prefetch(fl->centries[fl->cidx].skb);
Stephen Hemminger3de00b82007-01-08 11:26:30 -08001574 if (e->DataValid)
1575 return 1;
Stephen Hemminger24a427c2007-01-08 11:26:12 -08001576
Scott Bardone559fb512005-06-23 01:40:19 -04001577 do {
1578 flags |= e->Qsleeping;
1579
1580 cmdq_processed[0] += e->Cmdq0CreditReturn;
1581 cmdq_processed[1] += e->Cmdq1CreditReturn;
Francois Romieu356bd142006-12-11 23:47:00 +01001582
Scott Bardone559fb512005-06-23 01:40:19 -04001583 e++;
1584 if (unlikely(++q->cidx == q->size)) {
1585 q->cidx = 0;
1586 q->genbit ^= 1;
1587 e = q->entries;
1588 }
1589 prefetch(e);
1590
1591 if (++q->credits > SGE_RESPQ_REPLENISH_THRES) {
1592 writel(q->credits, adapter->regs + A_SG_RSPQUEUECREDIT);
1593 q->credits = 0;
1594 }
1595 sge->stats.pure_rsps++;
1596 } while (e->GenerationBit == q->genbit && !e->DataValid);
1597
Francois Romieu356bd142006-12-11 23:47:00 +01001598 flags = update_tx_info(adapter, flags, cmdq_processed[0]);
Scott Bardone559fb512005-06-23 01:40:19 -04001599 sge->cmdQ[1].processed += cmdq_processed[1];
1600
1601 return e->GenerationBit == q->genbit;
1602}
1603
1604/*
1605 * Handler for new data events when using NAPI. This does not need any locking
1606 * or protection from interrupts as data interrupts are off at this point and
1607 * other adapter interrupts do not interfere.
1608 */
Stephen Hemmingerbea33482007-10-03 16:41:36 -07001609int t1_poll(struct napi_struct *napi, int budget)
Scott Bardone559fb512005-06-23 01:40:19 -04001610{
Stephen Hemmingerbea33482007-10-03 16:41:36 -07001611 struct adapter *adapter = container_of(napi, struct adapter, napi);
Divy Le Ray445cf802007-11-27 13:30:15 -08001612 int work_done = process_responses(adapter, budget);
Stephen Hemminger7fe26a62006-12-08 11:08:33 -08001613
Divy Le Ray445cf802007-11-27 13:30:15 -08001614 if (likely(work_done < budget)) {
Ben Hutchings288379f2009-01-19 16:43:59 -08001615 napi_complete(napi);
Stephen Hemmingerbea33482007-10-03 16:41:36 -07001616 writel(adapter->sge->respQ.cidx,
1617 adapter->regs + A_SG_SLEEPING);
1618 }
1619 return work_done;
Scott Bardone559fb512005-06-23 01:40:19 -04001620}
1621
Stephen Hemminger7fe26a62006-12-08 11:08:33 -08001622irqreturn_t t1_interrupt(int irq, void *data)
Scott Bardone559fb512005-06-23 01:40:19 -04001623{
Scott Bardone559fb512005-06-23 01:40:19 -04001624 struct adapter *adapter = data;
1625 struct sge *sge = adapter->sge;
Stephen Hemminger3de00b82007-01-08 11:26:30 -08001626 int handled;
Scott Bardone559fb512005-06-23 01:40:19 -04001627
Stephen Hemminger3de00b82007-01-08 11:26:30 -08001628 if (likely(responses_pending(adapter))) {
Francois Romieu356bd142006-12-11 23:47:00 +01001629 writel(F_PL_INTR_SGE_DATA, adapter->regs + A_PL_CAUSE);
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001630
Stephen Hemmingerbea33482007-10-03 16:41:36 -07001631 if (napi_schedule_prep(&adapter->napi)) {
Stephen Hemminger3de00b82007-01-08 11:26:30 -08001632 if (process_pure_responses(adapter))
Ben Hutchings288379f2009-01-19 16:43:59 -08001633 __napi_schedule(&adapter->napi);
Stephen Hemminger3de00b82007-01-08 11:26:30 -08001634 else {
1635 /* no data, no NAPI needed */
1636 writel(sge->respQ.cidx, adapter->regs + A_SG_SLEEPING);
Francois Romieu4422b002008-07-11 00:29:19 +02001637 /* undo schedule_prep */
1638 napi_enable(&adapter->napi);
Stephen Hemminger7fe26a62006-12-08 11:08:33 -08001639 }
Stephen Hemminger7fe26a62006-12-08 11:08:33 -08001640 }
Stephen Hemminger3de00b82007-01-08 11:26:30 -08001641 return IRQ_HANDLED;
1642 }
1643
1644 spin_lock(&adapter->async_lock);
1645 handled = t1_slow_intr_handler(adapter);
1646 spin_unlock(&adapter->async_lock);
Stephen Hemminger7fe26a62006-12-08 11:08:33 -08001647
Scott Bardone559fb512005-06-23 01:40:19 -04001648 if (!handled)
1649 sge->stats.unhandled_irqs++;
Stephen Hemminger3de00b82007-01-08 11:26:30 -08001650
Scott Bardone559fb512005-06-23 01:40:19 -04001651 return IRQ_RETVAL(handled != 0);
1652}
1653
Scott Bardone559fb512005-06-23 01:40:19 -04001654/*
1655 * Enqueues the sk_buff onto the cmdQ[qid] and has hardware fetch it.
1656 *
1657 * The code figures out how many entries the sk_buff will require in the
1658 * cmdQ and updates the cmdQ data structure with the state once the enqueue
1659 * has complete. Then, it doesn't access the global structure anymore, but
1660 * uses the corresponding fields on the stack. In conjuction with a spinlock
1661 * around that code, we can make the function reentrant without holding the
1662 * lock when we actually enqueue (which might be expensive, especially on
1663 * architectures with IO MMUs).
1664 *
1665 * This runs with softirqs disabled.
1666 */
Stephen Hemmingeraa845052005-12-14 14:38:44 -08001667static int t1_sge_tx(struct sk_buff *skb, struct adapter *adapter,
1668 unsigned int qid, struct net_device *dev)
Scott Bardone559fb512005-06-23 01:40:19 -04001669{
1670 struct sge *sge = adapter->sge;
1671 struct cmdQ *q = &sge->cmdQ[qid];
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001672 unsigned int credits, pidx, genbit, count, use_sched_skb = 0;
Scott Bardone559fb512005-06-23 01:40:19 -04001673
Stephen Hemmingercabdfb32006-12-01 16:36:22 -08001674 if (!spin_trylock(&q->lock))
1675 return NETDEV_TX_LOCKED;
1676
Scott Bardone559fb512005-06-23 01:40:19 -04001677 reclaim_completed_tx(sge, q);
1678
1679 pidx = q->pidx;
1680 credits = q->size - q->in_use;
1681 count = 1 + skb_shinfo(skb)->nr_frags;
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001682 count += compute_large_page_tx_descs(skb);
Scott Bardone559fb512005-06-23 01:40:19 -04001683
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001684 /* Ethernet packet */
1685 if (unlikely(credits < count)) {
1686 if (!netif_queue_stopped(dev)) {
Scott Bardone559fb512005-06-23 01:40:19 -04001687 netif_stop_queue(dev);
1688 set_bit(dev->if_port, &sge->stopped_tx_queues);
Scott Bardone232a3472006-03-16 19:20:40 -05001689 sge->stats.cmdQ_full[2]++;
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001690 CH_ERR("%s: Tx ring full while queue awake!\n",
1691 adapter->name);
Scott Bardone559fb512005-06-23 01:40:19 -04001692 }
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001693 spin_unlock(&q->lock);
1694 return NETDEV_TX_BUSY;
Scott Bardone559fb512005-06-23 01:40:19 -04001695 }
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001696
1697 if (unlikely(credits - count < q->stop_thres)) {
1698 netif_stop_queue(dev);
1699 set_bit(dev->if_port, &sge->stopped_tx_queues);
1700 sge->stats.cmdQ_full[2]++;
1701 }
1702
1703 /* T204 cmdQ0 skbs that are destined for a certain port have to go
1704 * through the scheduler.
1705 */
1706 if (sge->tx_sched && !qid && skb->dev) {
Francois Romieu356bd142006-12-11 23:47:00 +01001707use_sched:
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001708 use_sched_skb = 1;
1709 /* Note that the scheduler might return a different skb than
1710 * the one passed in.
1711 */
1712 skb = sched_skb(sge, skb, credits);
1713 if (!skb) {
1714 spin_unlock(&q->lock);
1715 return NETDEV_TX_OK;
1716 }
1717 pidx = q->pidx;
1718 count = 1 + skb_shinfo(skb)->nr_frags;
1719 count += compute_large_page_tx_descs(skb);
1720 }
1721
Scott Bardone559fb512005-06-23 01:40:19 -04001722 q->in_use += count;
1723 genbit = q->genbit;
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001724 pidx = q->pidx;
Scott Bardone559fb512005-06-23 01:40:19 -04001725 q->pidx += count;
1726 if (q->pidx >= q->size) {
1727 q->pidx -= q->size;
1728 q->genbit ^= 1;
1729 }
1730 spin_unlock(&q->lock);
1731
1732 write_tx_descs(adapter, skb, pidx, genbit, q);
1733
1734 /*
1735 * We always ring the doorbell for cmdQ1. For cmdQ0, we only ring
1736 * the doorbell if the Q is asleep. There is a natural race, where
1737 * the hardware is going to sleep just after we checked, however,
1738 * then the interrupt handler will detect the outstanding TX packet
1739 * and ring the doorbell for us.
1740 */
1741 if (qid)
1742 doorbell_pio(adapter, F_CMDQ1_ENABLE);
1743 else {
1744 clear_bit(CMDQ_STAT_LAST_PKT_DB, &q->status);
1745 if (test_and_set_bit(CMDQ_STAT_RUNNING, &q->status) == 0) {
1746 set_bit(CMDQ_STAT_LAST_PKT_DB, &q->status);
1747 writel(F_CMDQ0_ENABLE, adapter->regs + A_SG_DOORBELL);
1748 }
1749 }
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001750
1751 if (use_sched_skb) {
1752 if (spin_trylock(&q->lock)) {
1753 credits = q->size - q->in_use;
1754 skb = NULL;
1755 goto use_sched;
1756 }
1757 }
Stephen Hemmingeraa845052005-12-14 14:38:44 -08001758 return NETDEV_TX_OK;
Scott Bardone559fb512005-06-23 01:40:19 -04001759}
1760
1761#define MK_ETH_TYPE_MSS(type, mss) (((mss) & 0x3FFF) | ((type) << 14))
1762
1763/*
1764 * eth_hdr_len - return the length of an Ethernet header
1765 * @data: pointer to the start of the Ethernet header
1766 *
1767 * Returns the length of an Ethernet header, including optional VLAN tag.
1768 */
1769static inline int eth_hdr_len(const void *data)
1770{
1771 const struct ethhdr *e = data;
1772
1773 return e->h_proto == htons(ETH_P_8021Q) ? VLAN_ETH_HLEN : ETH_HLEN;
1774}
1775
1776/*
1777 * Adds the CPL header to the sk_buff and passes it to t1_sge_tx.
1778 */
1779int t1_start_xmit(struct sk_buff *skb, struct net_device *dev)
1780{
Wang Chenc3ccc122008-11-16 23:06:39 -08001781 struct adapter *adapter = dev->ml_priv;
Scott Bardone559fb512005-06-23 01:40:19 -04001782 struct sge *sge = adapter->sge;
Divy Le Ray7832ee02007-11-27 13:30:09 -08001783 struct sge_port_stats *st = per_cpu_ptr(sge->port_stats[dev->if_port],
1784 smp_processor_id());
Scott Bardone559fb512005-06-23 01:40:19 -04001785 struct cpl_tx_pkt *cpl;
Stephen Hemmingercabdfb32006-12-01 16:36:22 -08001786 struct sk_buff *orig_skb = skb;
1787 int ret;
Scott Bardone559fb512005-06-23 01:40:19 -04001788
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001789 if (skb->protocol == htons(ETH_P_CPL5))
1790 goto send;
1791
Divy Le Ray7832ee02007-11-27 13:30:09 -08001792 /*
1793 * We are using a non-standard hard_header_len.
1794 * Allocate more header room in the rare cases it is not big enough.
1795 */
1796 if (unlikely(skb_headroom(skb) < dev->hard_header_len - ETH_HLEN)) {
1797 skb = skb_realloc_headroom(skb, sizeof(struct cpl_tx_pkt_lso));
1798 ++st->tx_need_hdrroom;
1799 dev_kfree_skb_any(orig_skb);
1800 if (!skb)
1801 return NETDEV_TX_OK;
1802 }
1803
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001804 if (skb_shinfo(skb)->gso_size) {
Scott Bardone559fb512005-06-23 01:40:19 -04001805 int eth_type;
1806 struct cpl_tx_pkt_lso *hdr;
1807
Stephen Hemminger56f643c2006-12-01 16:36:21 -08001808 ++st->tx_tso;
Scott Bardone559fb512005-06-23 01:40:19 -04001809
Arnaldo Carvalho de Melobbe735e2007-03-10 22:16:10 -03001810 eth_type = skb_network_offset(skb) == ETH_HLEN ?
Scott Bardone559fb512005-06-23 01:40:19 -04001811 CPL_ETH_II : CPL_ETH_II_VLAN;
1812
1813 hdr = (struct cpl_tx_pkt_lso *)skb_push(skb, sizeof(*hdr));
1814 hdr->opcode = CPL_TX_PKT_LSO;
1815 hdr->ip_csum_dis = hdr->l4_csum_dis = 0;
Arnaldo Carvalho de Meloeddc9ec2007-04-20 22:47:35 -07001816 hdr->ip_hdr_words = ip_hdr(skb)->ihl;
Arnaldo Carvalho de Meloaa8223c2007-04-10 21:04:22 -07001817 hdr->tcp_hdr_words = tcp_hdr(skb)->doff;
Scott Bardone559fb512005-06-23 01:40:19 -04001818 hdr->eth_type_mss = htons(MK_ETH_TYPE_MSS(eth_type,
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001819 skb_shinfo(skb)->gso_size));
Scott Bardone559fb512005-06-23 01:40:19 -04001820 hdr->len = htonl(skb->len - sizeof(*hdr));
1821 cpl = (struct cpl_tx_pkt *)hdr;
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001822 } else {
Scott Bardone559fb512005-06-23 01:40:19 -04001823 /*
Francois Romieu356bd142006-12-11 23:47:00 +01001824 * Packets shorter than ETH_HLEN can break the MAC, drop them
Scott Bardone559fb512005-06-23 01:40:19 -04001825 * early. Also, we may get oversized packets because some
1826 * parts of the kernel don't handle our unusual hard_header_len
1827 * right, drop those too.
1828 */
1829 if (unlikely(skb->len < ETH_HLEN ||
1830 skb->len > dev->mtu + eth_hdr_len(skb->data))) {
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001831 pr_debug("%s: packet size %d hdr %d mtu%d\n", dev->name,
1832 skb->len, eth_hdr_len(skb->data), dev->mtu);
Scott Bardone559fb512005-06-23 01:40:19 -04001833 dev_kfree_skb_any(skb);
Stephen Hemmingeraa845052005-12-14 14:38:44 -08001834 return NETDEV_TX_OK;
Scott Bardone559fb512005-06-23 01:40:19 -04001835 }
1836
Scott Bardone559fb512005-06-23 01:40:19 -04001837 if (!(adapter->flags & UDP_CSUM_CAPABLE) &&
Patrick McHardy84fa7932006-08-29 16:44:56 -07001838 skb->ip_summed == CHECKSUM_PARTIAL &&
Arnaldo Carvalho de Meloeddc9ec2007-04-20 22:47:35 -07001839 ip_hdr(skb)->protocol == IPPROTO_UDP) {
Patrick McHardy84fa7932006-08-29 16:44:56 -07001840 if (unlikely(skb_checksum_help(skb))) {
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001841 pr_debug("%s: unable to do udp checksum\n", dev->name);
Scott Bardone559fb512005-06-23 01:40:19 -04001842 dev_kfree_skb_any(skb);
Stephen Hemmingeraa845052005-12-14 14:38:44 -08001843 return NETDEV_TX_OK;
Scott Bardone559fb512005-06-23 01:40:19 -04001844 }
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001845 }
Scott Bardone559fb512005-06-23 01:40:19 -04001846
1847 /* Hmmm, assuming to catch the gratious arp... and we'll use
1848 * it to flush out stuck espi packets...
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001849 */
1850 if ((unlikely(!adapter->sge->espibug_skb[dev->if_port]))) {
Scott Bardone559fb512005-06-23 01:40:19 -04001851 if (skb->protocol == htons(ETH_P_ARP) &&
Arnaldo Carvalho de Melod0a92be2007-03-12 20:56:31 -03001852 arp_hdr(skb)->ar_op == htons(ARPOP_REQUEST)) {
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001853 adapter->sge->espibug_skb[dev->if_port] = skb;
Scott Bardone559fb512005-06-23 01:40:19 -04001854 /* We want to re-use this skb later. We
1855 * simply bump the reference count and it
1856 * will not be freed...
1857 */
1858 skb = skb_get(skb);
1859 }
1860 }
1861
1862 cpl = (struct cpl_tx_pkt *)__skb_push(skb, sizeof(*cpl));
1863 cpl->opcode = CPL_TX_PKT;
1864 cpl->ip_csum_dis = 1; /* SW calculates IP csum */
Patrick McHardy84fa7932006-08-29 16:44:56 -07001865 cpl->l4_csum_dis = skb->ip_summed == CHECKSUM_PARTIAL ? 0 : 1;
Scott Bardone559fb512005-06-23 01:40:19 -04001866 /* the length field isn't used so don't bother setting it */
1867
Patrick McHardy84fa7932006-08-29 16:44:56 -07001868 st->tx_cso += (skb->ip_summed == CHECKSUM_PARTIAL);
Scott Bardone559fb512005-06-23 01:40:19 -04001869 }
1870 cpl->iff = dev->if_port;
1871
1872#if defined(CONFIG_VLAN_8021Q) || defined(CONFIG_VLAN_8021Q_MODULE)
1873 if (adapter->vlan_grp && vlan_tx_tag_present(skb)) {
1874 cpl->vlan_valid = 1;
1875 cpl->vlan = htons(vlan_tx_tag_get(skb));
1876 st->vlan_insert++;
1877 } else
1878#endif
1879 cpl->vlan_valid = 0;
1880
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001881send:
Scott Bardone559fb512005-06-23 01:40:19 -04001882 dev->trans_start = jiffies;
Stephen Hemmingercabdfb32006-12-01 16:36:22 -08001883 ret = t1_sge_tx(skb, adapter, 0, dev);
1884
1885 /* If transmit busy, and we reallocated skb's due to headroom limit,
1886 * then silently discard to avoid leak.
1887 */
1888 if (unlikely(ret != NETDEV_TX_OK && skb != orig_skb)) {
Francois Romieu356bd142006-12-11 23:47:00 +01001889 dev_kfree_skb_any(skb);
Stephen Hemmingercabdfb32006-12-01 16:36:22 -08001890 ret = NETDEV_TX_OK;
Francois Romieu356bd142006-12-11 23:47:00 +01001891 }
Stephen Hemmingercabdfb32006-12-01 16:36:22 -08001892 return ret;
Scott Bardone559fb512005-06-23 01:40:19 -04001893}
1894
1895/*
1896 * Callback for the Tx buffer reclaim timer. Runs with softirqs disabled.
1897 */
1898static void sge_tx_reclaim_cb(unsigned long data)
1899{
1900 int i;
1901 struct sge *sge = (struct sge *)data;
1902
1903 for (i = 0; i < SGE_CMDQ_N; ++i) {
1904 struct cmdQ *q = &sge->cmdQ[i];
1905
1906 if (!spin_trylock(&q->lock))
1907 continue;
1908
1909 reclaim_completed_tx(sge, q);
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001910 if (i == 0 && q->in_use) { /* flush pending credits */
1911 writel(F_CMDQ0_ENABLE, sge->adapter->regs + A_SG_DOORBELL);
1912 }
Scott Bardone559fb512005-06-23 01:40:19 -04001913 spin_unlock(&q->lock);
1914 }
1915 mod_timer(&sge->tx_reclaim_timer, jiffies + TX_RECLAIM_PERIOD);
1916}
1917
1918/*
1919 * Propagate changes of the SGE coalescing parameters to the HW.
1920 */
1921int t1_sge_set_coalesce_params(struct sge *sge, struct sge_params *p)
1922{
Scott Bardone559fb512005-06-23 01:40:19 -04001923 sge->fixed_intrtimer = p->rx_coalesce_usecs *
1924 core_ticks_per_usec(sge->adapter);
1925 writel(sge->fixed_intrtimer, sge->adapter->regs + A_SG_INTRTIMER);
1926 return 0;
Christoph Lameter8199d3a2005-03-30 13:34:31 -08001927}
1928
1929/*
1930 * Allocates both RX and TX resources and configures the SGE. However,
1931 * the hardware is not enabled yet.
1932 */
1933int t1_sge_configure(struct sge *sge, struct sge_params *p)
1934{
1935 if (alloc_rx_resources(sge, p))
1936 return -ENOMEM;
1937 if (alloc_tx_resources(sge, p)) {
1938 free_rx_resources(sge);
1939 return -ENOMEM;
1940 }
1941 configure_sge(sge, p);
1942
1943 /*
1944 * Now that we have sized the free lists calculate the payload
1945 * capacity of the large buffers. Other parts of the driver use
1946 * this to set the max offload coalescing size so that RX packets
1947 * do not overflow our large buffers.
1948 */
1949 p->large_buf_capacity = jumbo_payload_capacity(sge);
1950 return 0;
1951}
1952
1953/*
Scott Bardone559fb512005-06-23 01:40:19 -04001954 * Disables the DMA engine.
Christoph Lameter8199d3a2005-03-30 13:34:31 -08001955 */
Scott Bardone559fb512005-06-23 01:40:19 -04001956void t1_sge_stop(struct sge *sge)
Christoph Lameter8199d3a2005-03-30 13:34:31 -08001957{
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001958 int i;
Scott Bardone559fb512005-06-23 01:40:19 -04001959 writel(0, sge->adapter->regs + A_SG_CONTROL);
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001960 readl(sge->adapter->regs + A_SG_CONTROL); /* flush */
1961
Scott Bardone559fb512005-06-23 01:40:19 -04001962 if (is_T2(sge->adapter))
1963 del_timer_sync(&sge->espibug_timer);
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001964
Scott Bardone559fb512005-06-23 01:40:19 -04001965 del_timer_sync(&sge->tx_reclaim_timer);
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001966 if (sge->tx_sched)
1967 tx_sched_stop(sge);
1968
1969 for (i = 0; i < MAX_NPORTS; i++)
Wei Yongjunf4fe5a92009-02-25 00:45:09 +00001970 kfree_skb(sge->espibug_skb[i]);
Christoph Lameter8199d3a2005-03-30 13:34:31 -08001971}
1972
1973/*
Scott Bardone559fb512005-06-23 01:40:19 -04001974 * Enables the DMA engine.
Christoph Lameter8199d3a2005-03-30 13:34:31 -08001975 */
Scott Bardone559fb512005-06-23 01:40:19 -04001976void t1_sge_start(struct sge *sge)
Christoph Lameter8199d3a2005-03-30 13:34:31 -08001977{
Christoph Lameter8199d3a2005-03-30 13:34:31 -08001978 refill_free_list(sge, &sge->freelQ[0]);
1979 refill_free_list(sge, &sge->freelQ[1]);
1980
Scott Bardone559fb512005-06-23 01:40:19 -04001981 writel(sge->sge_control, sge->adapter->regs + A_SG_CONTROL);
1982 doorbell_pio(sge->adapter, F_FL0_ENABLE | F_FL1_ENABLE);
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001983 readl(sge->adapter->regs + A_SG_CONTROL); /* flush */
Christoph Lameter8199d3a2005-03-30 13:34:31 -08001984
Scott Bardone559fb512005-06-23 01:40:19 -04001985 mod_timer(&sge->tx_reclaim_timer, jiffies + TX_RECLAIM_PERIOD);
Christoph Lameter8199d3a2005-03-30 13:34:31 -08001986
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001987 if (is_T2(sge->adapter))
Scott Bardone559fb512005-06-23 01:40:19 -04001988 mod_timer(&sge->espibug_timer, jiffies + sge->espibug_timeout);
Christoph Lameter8199d3a2005-03-30 13:34:31 -08001989}
1990
1991/*
Scott Bardone559fb512005-06-23 01:40:19 -04001992 * Callback for the T2 ESPI 'stuck packet feature' workaorund
Christoph Lameter8199d3a2005-03-30 13:34:31 -08001993 */
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001994static void espibug_workaround_t204(unsigned long data)
1995{
1996 struct adapter *adapter = (struct adapter *)data;
1997 struct sge *sge = adapter->sge;
1998 unsigned int nports = adapter->params.nports;
1999 u32 seop[MAX_NPORTS];
2000
2001 if (adapter->open_device_map & PORT_MASK) {
2002 int i;
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08002003
Francois Romieu356bd142006-12-11 23:47:00 +01002004 if (t1_espi_get_mon_t204(adapter, &(seop[0]), 0) < 0)
2005 return;
2006
2007 for (i = 0; i < nports; i++) {
2008 struct sk_buff *skb = sge->espibug_skb[i];
2009
2010 if (!netif_running(adapter->port[i].dev) ||
2011 netif_queue_stopped(adapter->port[i].dev) ||
2012 !seop[i] || ((seop[i] & 0xfff) != 0) || !skb)
2013 continue;
2014
2015 if (!skb->cb[0]) {
2016 u8 ch_mac_addr[ETH_ALEN] = {
2017 0x0, 0x7, 0x43, 0x0, 0x0, 0x0
2018 };
2019
Arnaldo Carvalho de Melo27d7ff42007-03-31 11:55:19 -03002020 skb_copy_to_linear_data_offset(skb,
2021 sizeof(struct cpl_tx_pkt),
2022 ch_mac_addr,
2023 ETH_ALEN);
2024 skb_copy_to_linear_data_offset(skb,
2025 skb->len - 10,
2026 ch_mac_addr,
2027 ETH_ALEN);
Francois Romieu356bd142006-12-11 23:47:00 +01002028 skb->cb[0] = 0xff;
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08002029 }
Francois Romieu356bd142006-12-11 23:47:00 +01002030
2031 /* bump the reference count to avoid freeing of
2032 * the skb once the DMA has completed.
2033 */
2034 skb = skb_get(skb);
2035 t1_sge_tx(skb, adapter, 0, adapter->port[i].dev);
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08002036 }
2037 }
2038 mod_timer(&sge->espibug_timer, jiffies + sge->espibug_timeout);
2039}
2040
2041static void espibug_workaround(unsigned long data)
Christoph Lameter8199d3a2005-03-30 13:34:31 -08002042{
Scott Bardone559fb512005-06-23 01:40:19 -04002043 struct adapter *adapter = (struct adapter *)data;
Christoph Lameter8199d3a2005-03-30 13:34:31 -08002044 struct sge *sge = adapter->sge;
Christoph Lameter8199d3a2005-03-30 13:34:31 -08002045
Scott Bardone559fb512005-06-23 01:40:19 -04002046 if (netif_running(adapter->port[0].dev)) {
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08002047 struct sk_buff *skb = sge->espibug_skb[0];
2048 u32 seop = t1_espi_get_mon(adapter, 0x930, 0);
Christoph Lameter8199d3a2005-03-30 13:34:31 -08002049
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08002050 if ((seop & 0xfff0fff) == 0xfff && skb) {
2051 if (!skb->cb[0]) {
2052 u8 ch_mac_addr[ETH_ALEN] =
2053 {0x0, 0x7, 0x43, 0x0, 0x0, 0x0};
Arnaldo Carvalho de Melo27d7ff42007-03-31 11:55:19 -03002054 skb_copy_to_linear_data_offset(skb,
2055 sizeof(struct cpl_tx_pkt),
2056 ch_mac_addr,
2057 ETH_ALEN);
2058 skb_copy_to_linear_data_offset(skb,
2059 skb->len - 10,
2060 ch_mac_addr,
2061 ETH_ALEN);
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08002062 skb->cb[0] = 0xff;
2063 }
Christoph Lameter8199d3a2005-03-30 13:34:31 -08002064
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08002065 /* bump the reference count to avoid freeing of the
2066 * skb once the DMA has completed.
2067 */
2068 skb = skb_get(skb);
2069 t1_sge_tx(skb, adapter, 0, adapter->port[0].dev);
2070 }
Christoph Lameter8199d3a2005-03-30 13:34:31 -08002071 }
Scott Bardone559fb512005-06-23 01:40:19 -04002072 mod_timer(&sge->espibug_timer, jiffies + sge->espibug_timeout);
Christoph Lameter8199d3a2005-03-30 13:34:31 -08002073}
2074
Christoph Lameter8199d3a2005-03-30 13:34:31 -08002075/*
Scott Bardone559fb512005-06-23 01:40:19 -04002076 * Creates a t1_sge structure and returns suggested resource parameters.
Christoph Lameter8199d3a2005-03-30 13:34:31 -08002077 */
Scott Bardone559fb512005-06-23 01:40:19 -04002078struct sge * __devinit t1_sge_create(struct adapter *adapter,
2079 struct sge_params *p)
Christoph Lameter8199d3a2005-03-30 13:34:31 -08002080{
Stephen Hemmingercbee9f92006-11-17 17:01:52 -08002081 struct sge *sge = kzalloc(sizeof(*sge), GFP_KERNEL);
Stephen Hemminger56f643c2006-12-01 16:36:21 -08002082 int i;
Christoph Lameter8199d3a2005-03-30 13:34:31 -08002083
Scott Bardone559fb512005-06-23 01:40:19 -04002084 if (!sge)
2085 return NULL;
Christoph Lameter8199d3a2005-03-30 13:34:31 -08002086
Scott Bardone559fb512005-06-23 01:40:19 -04002087 sge->adapter = adapter;
2088 sge->netdev = adapter->port[0].dev;
2089 sge->rx_pkt_pad = t1_is_T1B(adapter) ? 0 : 2;
2090 sge->jumbo_fl = t1_is_T1B(adapter) ? 1 : 0;
2091
Stephen Hemminger56f643c2006-12-01 16:36:21 -08002092 for_each_port(adapter, i) {
2093 sge->port_stats[i] = alloc_percpu(struct sge_port_stats);
2094 if (!sge->port_stats[i])
2095 goto nomem_port;
2096 }
2097
Scott Bardone559fb512005-06-23 01:40:19 -04002098 init_timer(&sge->tx_reclaim_timer);
2099 sge->tx_reclaim_timer.data = (unsigned long)sge;
2100 sge->tx_reclaim_timer.function = sge_tx_reclaim_cb;
2101
2102 if (is_T2(sge->adapter)) {
2103 init_timer(&sge->espibug_timer);
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08002104
2105 if (adapter->params.nports > 1) {
2106 tx_sched_init(sge);
2107 sge->espibug_timer.function = espibug_workaround_t204;
Francois Romieud7487422006-12-11 23:49:13 +01002108 } else
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08002109 sge->espibug_timer.function = espibug_workaround;
Scott Bardone559fb512005-06-23 01:40:19 -04002110 sge->espibug_timer.data = (unsigned long)sge->adapter;
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08002111
Scott Bardone559fb512005-06-23 01:40:19 -04002112 sge->espibug_timeout = 1;
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08002113 /* for T204, every 10ms */
2114 if (adapter->params.nports > 1)
2115 sge->espibug_timeout = HZ/100;
Christoph Lameter8199d3a2005-03-30 13:34:31 -08002116 }
Francois Romieu356bd142006-12-11 23:47:00 +01002117
Christoph Lameter8199d3a2005-03-30 13:34:31 -08002118
Scott Bardone559fb512005-06-23 01:40:19 -04002119 p->cmdQ_size[0] = SGE_CMDQ0_E_N;
2120 p->cmdQ_size[1] = SGE_CMDQ1_E_N;
2121 p->freelQ_size[!sge->jumbo_fl] = SGE_FREEL_SIZE;
2122 p->freelQ_size[sge->jumbo_fl] = SGE_JUMBO_FREEL_SIZE;
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08002123 if (sge->tx_sched) {
2124 if (board_info(sge->adapter)->board == CHBT_BOARD_CHT204)
2125 p->rx_coalesce_usecs = 15;
2126 else
2127 p->rx_coalesce_usecs = 50;
2128 } else
2129 p->rx_coalesce_usecs = 50;
2130
Scott Bardone559fb512005-06-23 01:40:19 -04002131 p->coalesce_enable = 0;
2132 p->sample_interval_usecs = 0;
Christoph Lameter8199d3a2005-03-30 13:34:31 -08002133
Scott Bardone559fb512005-06-23 01:40:19 -04002134 return sge;
Stephen Hemminger56f643c2006-12-01 16:36:21 -08002135nomem_port:
2136 while (i >= 0) {
2137 free_percpu(sge->port_stats[i]);
2138 --i;
2139 }
2140 kfree(sge);
2141 return NULL;
2142
Christoph Lameter8199d3a2005-03-30 13:34:31 -08002143}