blob: fa0a1f129d37d12cdbc97b2633e829e408c6e192 [file] [log] [blame]
Christoph Lameter8199d3a2005-03-30 13:34:31 -08001/*****************************************************************************
2 * *
3 * File: sge.c *
Scott Bardone559fb512005-06-23 01:40:19 -04004 * $Revision: 1.26 $ *
5 * $Date: 2005/06/21 18:29:48 $ *
Christoph Lameter8199d3a2005-03-30 13:34:31 -08006 * Description: *
7 * DMA engine. *
8 * part of the Chelsio 10Gb Ethernet Driver. *
9 * *
10 * This program is free software; you can redistribute it and/or modify *
11 * it under the terms of the GNU General Public License, version 2, as *
12 * published by the Free Software Foundation. *
13 * *
14 * You should have received a copy of the GNU General Public License along *
15 * with this program; if not, write to the Free Software Foundation, Inc., *
16 * 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. *
17 * *
18 * THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR IMPLIED *
19 * WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF *
20 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. *
21 * *
22 * http://www.chelsio.com *
23 * *
24 * Copyright (c) 2003 - 2005 Chelsio Communications, Inc. *
25 * All rights reserved. *
26 * *
27 * Maintainers: maintainers@chelsio.com *
28 * *
29 * Authors: Dimitrios Michailidis <dm@chelsio.com> *
30 * Tina Yang <tainay@chelsio.com> *
31 * Felix Marti <felix@chelsio.com> *
32 * Scott Bardone <sbardone@chelsio.com> *
33 * Kurt Ottaway <kottaway@chelsio.com> *
34 * Frank DiMambro <frank@chelsio.com> *
35 * *
36 * History: *
37 * *
38 ****************************************************************************/
39
40#include "common.h"
41
Christoph Lameter8199d3a2005-03-30 13:34:31 -080042#include <linux/types.h>
43#include <linux/errno.h>
44#include <linux/pci.h>
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -080045#include <linux/ktime.h>
Christoph Lameter8199d3a2005-03-30 13:34:31 -080046#include <linux/netdevice.h>
47#include <linux/etherdevice.h>
48#include <linux/if_vlan.h>
49#include <linux/skbuff.h>
50#include <linux/init.h>
51#include <linux/mm.h>
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -080052#include <linux/tcp.h>
Christoph Lameter8199d3a2005-03-30 13:34:31 -080053#include <linux/ip.h>
54#include <linux/in.h>
55#include <linux/if_arp.h>
56
57#include "cpl5_cmd.h"
58#include "sge.h"
59#include "regs.h"
60#include "espi.h"
61
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -080062/* This belongs in if_ether.h */
63#define ETH_P_CPL5 0xf
Christoph Lameter8199d3a2005-03-30 13:34:31 -080064
65#define SGE_CMDQ_N 2
66#define SGE_FREELQ_N 2
Scott Bardone559fb512005-06-23 01:40:19 -040067#define SGE_CMDQ0_E_N 1024
Christoph Lameter8199d3a2005-03-30 13:34:31 -080068#define SGE_CMDQ1_E_N 128
69#define SGE_FREEL_SIZE 4096
70#define SGE_JUMBO_FREEL_SIZE 512
71#define SGE_FREEL_REFILL_THRESH 16
72#define SGE_RESPQ_E_N 1024
Scott Bardone559fb512005-06-23 01:40:19 -040073#define SGE_INTRTIMER_NRES 1000
Christoph Lameter8199d3a2005-03-30 13:34:31 -080074#define SGE_RX_SM_BUF_SIZE 1536
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -080075#define SGE_TX_DESC_MAX_PLEN 16384
Christoph Lameter8199d3a2005-03-30 13:34:31 -080076
Scott Bardone559fb512005-06-23 01:40:19 -040077#define SGE_RESPQ_REPLENISH_THRES (SGE_RESPQ_E_N / 4)
Christoph Lameter8199d3a2005-03-30 13:34:31 -080078
79/*
Scott Bardone559fb512005-06-23 01:40:19 -040080 * Period of the TX buffer reclaim timer. This timer does not need to run
81 * frequently as TX buffers are usually reclaimed by new TX packets.
82 */
83#define TX_RECLAIM_PERIOD (HZ / 4)
84
Scott Bardone559fb512005-06-23 01:40:19 -040085#define M_CMD_LEN 0x7fffffff
86#define V_CMD_LEN(v) (v)
87#define G_CMD_LEN(v) ((v) & M_CMD_LEN)
88#define V_CMD_GEN1(v) ((v) << 31)
89#define V_CMD_GEN2(v) (v)
90#define F_CMD_DATAVALID (1 << 1)
91#define F_CMD_SOP (1 << 2)
92#define V_CMD_EOP(v) ((v) << 3)
93
94/*
95 * Command queue, receive buffer list, and response queue descriptors.
Christoph Lameter8199d3a2005-03-30 13:34:31 -080096 */
97#if defined(__BIG_ENDIAN_BITFIELD)
98struct cmdQ_e {
Scott Bardone559fb512005-06-23 01:40:19 -040099 u32 addr_lo;
100 u32 len_gen;
101 u32 flags;
102 u32 addr_hi;
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800103};
104
105struct freelQ_e {
Scott Bardone559fb512005-06-23 01:40:19 -0400106 u32 addr_lo;
107 u32 len_gen;
108 u32 gen2;
109 u32 addr_hi;
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800110};
111
112struct respQ_e {
113 u32 Qsleeping : 4;
114 u32 Cmdq1CreditReturn : 5;
115 u32 Cmdq1DmaComplete : 5;
116 u32 Cmdq0CreditReturn : 5;
117 u32 Cmdq0DmaComplete : 5;
118 u32 FreelistQid : 2;
119 u32 CreditValid : 1;
120 u32 DataValid : 1;
121 u32 Offload : 1;
122 u32 Eop : 1;
123 u32 Sop : 1;
124 u32 GenerationBit : 1;
125 u32 BufferLength;
126};
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800127#elif defined(__LITTLE_ENDIAN_BITFIELD)
128struct cmdQ_e {
Scott Bardone559fb512005-06-23 01:40:19 -0400129 u32 len_gen;
130 u32 addr_lo;
131 u32 addr_hi;
132 u32 flags;
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800133};
134
135struct freelQ_e {
Scott Bardone559fb512005-06-23 01:40:19 -0400136 u32 len_gen;
137 u32 addr_lo;
138 u32 addr_hi;
139 u32 gen2;
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800140};
141
142struct respQ_e {
143 u32 BufferLength;
144 u32 GenerationBit : 1;
145 u32 Sop : 1;
146 u32 Eop : 1;
147 u32 Offload : 1;
148 u32 DataValid : 1;
149 u32 CreditValid : 1;
150 u32 FreelistQid : 2;
151 u32 Cmdq0DmaComplete : 5;
152 u32 Cmdq0CreditReturn : 5;
153 u32 Cmdq1DmaComplete : 5;
154 u32 Cmdq1CreditReturn : 5;
155 u32 Qsleeping : 4;
156} ;
157#endif
158
159/*
160 * SW Context Command and Freelist Queue Descriptors
161 */
162struct cmdQ_ce {
163 struct sk_buff *skb;
164 DECLARE_PCI_UNMAP_ADDR(dma_addr);
165 DECLARE_PCI_UNMAP_LEN(dma_len);
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800166};
167
168struct freelQ_ce {
169 struct sk_buff *skb;
170 DECLARE_PCI_UNMAP_ADDR(dma_addr);
171 DECLARE_PCI_UNMAP_LEN(dma_len);
172};
173
174/*
Scott Bardone559fb512005-06-23 01:40:19 -0400175 * SW command, freelist and response rings
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800176 */
177struct cmdQ {
Scott Bardone559fb512005-06-23 01:40:19 -0400178 unsigned long status; /* HW DMA fetch status */
179 unsigned int in_use; /* # of in-use command descriptors */
180 unsigned int size; /* # of descriptors */
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -0800181 unsigned int processed; /* total # of descs HW has processed */
182 unsigned int cleaned; /* total # of descs SW has reclaimed */
183 unsigned int stop_thres; /* SW TX queue suspend threshold */
Scott Bardone559fb512005-06-23 01:40:19 -0400184 u16 pidx; /* producer index (SW) */
185 u16 cidx; /* consumer index (HW) */
186 u8 genbit; /* current generation (=valid) bit */
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -0800187 u8 sop; /* is next entry start of packet? */
Scott Bardone559fb512005-06-23 01:40:19 -0400188 struct cmdQ_e *entries; /* HW command descriptor Q */
189 struct cmdQ_ce *centries; /* SW command context descriptor Q */
Scott Bardone559fb512005-06-23 01:40:19 -0400190 dma_addr_t dma_addr; /* DMA addr HW command descriptor Q */
Francois Romieu356bd142006-12-11 23:47:00 +0100191 spinlock_t lock; /* Lock to protect cmdQ enqueuing */
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800192};
193
194struct freelQ {
Scott Bardone559fb512005-06-23 01:40:19 -0400195 unsigned int credits; /* # of available RX buffers */
196 unsigned int size; /* free list capacity */
197 u16 pidx; /* producer index (SW) */
198 u16 cidx; /* consumer index (HW) */
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800199 u16 rx_buffer_size; /* Buffer size on this free list */
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -0800200 u16 dma_offset; /* DMA offset to align IP headers */
201 u16 recycleq_idx; /* skb recycle q to use */
Scott Bardone559fb512005-06-23 01:40:19 -0400202 u8 genbit; /* current generation (=valid) bit */
203 struct freelQ_e *entries; /* HW freelist descriptor Q */
204 struct freelQ_ce *centries; /* SW freelist context descriptor Q */
205 dma_addr_t dma_addr; /* DMA addr HW freelist descriptor Q */
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800206};
207
208struct respQ {
Scott Bardone559fb512005-06-23 01:40:19 -0400209 unsigned int credits; /* credits to be returned to SGE */
210 unsigned int size; /* # of response Q descriptors */
211 u16 cidx; /* consumer index (SW) */
212 u8 genbit; /* current generation(=valid) bit */
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800213 struct respQ_e *entries; /* HW response descriptor Q */
Scott Bardone559fb512005-06-23 01:40:19 -0400214 dma_addr_t dma_addr; /* DMA addr HW response descriptor Q */
215};
216
217/* Bit flags for cmdQ.status */
218enum {
219 CMDQ_STAT_RUNNING = 1, /* fetch engine is running */
220 CMDQ_STAT_LAST_PKT_DB = 2 /* last packet rung the doorbell */
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800221};
222
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -0800223/* T204 TX SW scheduler */
224
225/* Per T204 TX port */
226struct sched_port {
227 unsigned int avail; /* available bits - quota */
228 unsigned int drain_bits_per_1024ns; /* drain rate */
229 unsigned int speed; /* drain rate, mbps */
230 unsigned int mtu; /* mtu size */
231 struct sk_buff_head skbq; /* pending skbs */
232};
233
234/* Per T204 device */
235struct sched {
236 ktime_t last_updated; /* last time quotas were computed */
Francois Romieu356bd142006-12-11 23:47:00 +0100237 unsigned int max_avail; /* max bits to be sent to any port */
238 unsigned int port; /* port index (round robin ports) */
239 unsigned int num; /* num skbs in per port queues */
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -0800240 struct sched_port p[MAX_NPORTS];
241 struct tasklet_struct sched_tsk;/* tasklet used to run scheduler */
242};
243static void restart_sched(unsigned long);
244
245
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800246/*
247 * Main SGE data structure
248 *
249 * Interrupts are handled by a single CPU and it is likely that on a MP system
250 * the application is migrated to another CPU. In that scenario, we try to
251 * seperate the RX(in irq context) and TX state in order to decrease memory
252 * contention.
253 */
254struct sge {
Francois Romieu356bd142006-12-11 23:47:00 +0100255 struct adapter *adapter; /* adapter backpointer */
Scott Bardone559fb512005-06-23 01:40:19 -0400256 struct net_device *netdev; /* netdevice backpointer */
Francois Romieu356bd142006-12-11 23:47:00 +0100257 struct freelQ freelQ[SGE_FREELQ_N]; /* buffer free lists */
258 struct respQ respQ; /* response Q */
Scott Bardone559fb512005-06-23 01:40:19 -0400259 unsigned long stopped_tx_queues; /* bitmap of suspended Tx queues */
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800260 unsigned int rx_pkt_pad; /* RX padding for L2 packets */
261 unsigned int jumbo_fl; /* jumbo freelist Q index */
Scott Bardone559fb512005-06-23 01:40:19 -0400262 unsigned int intrtimer_nres; /* no-resource interrupt timer */
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -0800263 unsigned int fixed_intrtimer;/* non-adaptive interrupt timer */
Scott Bardone559fb512005-06-23 01:40:19 -0400264 struct timer_list tx_reclaim_timer; /* reclaims TX buffers */
265 struct timer_list espibug_timer;
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -0800266 unsigned long espibug_timeout;
267 struct sk_buff *espibug_skb[MAX_NPORTS];
Scott Bardone559fb512005-06-23 01:40:19 -0400268 u32 sge_control; /* shadow value of sge control reg */
269 struct sge_intr_counts stats;
Stephen Hemminger56f643c2006-12-01 16:36:21 -0800270 struct sge_port_stats *port_stats[MAX_NPORTS];
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -0800271 struct sched *tx_sched;
Scott Bardone559fb512005-06-23 01:40:19 -0400272 struct cmdQ cmdQ[SGE_CMDQ_N] ____cacheline_aligned_in_smp;
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800273};
274
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800275/*
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -0800276 * stop tasklet and free all pending skb's
277 */
278static void tx_sched_stop(struct sge *sge)
279{
280 struct sched *s = sge->tx_sched;
281 int i;
282
283 tasklet_kill(&s->sched_tsk);
284
285 for (i = 0; i < MAX_NPORTS; i++)
286 __skb_queue_purge(&s->p[s->port].skbq);
287}
288
289/*
290 * t1_sched_update_parms() is called when the MTU or link speed changes. It
291 * re-computes scheduler parameters to scope with the change.
292 */
293unsigned int t1_sched_update_parms(struct sge *sge, unsigned int port,
294 unsigned int mtu, unsigned int speed)
295{
296 struct sched *s = sge->tx_sched;
297 struct sched_port *p = &s->p[port];
298 unsigned int max_avail_segs;
299
300 pr_debug("t1_sched_update_params mtu=%d speed=%d\n", mtu, speed);
301 if (speed)
302 p->speed = speed;
303 if (mtu)
304 p->mtu = mtu;
305
306 if (speed || mtu) {
307 unsigned long long drain = 1024ULL * p->speed * (p->mtu - 40);
308 do_div(drain, (p->mtu + 50) * 1000);
309 p->drain_bits_per_1024ns = (unsigned int) drain;
310
311 if (p->speed < 1000)
312 p->drain_bits_per_1024ns =
313 90 * p->drain_bits_per_1024ns / 100;
314 }
315
316 if (board_info(sge->adapter)->board == CHBT_BOARD_CHT204) {
317 p->drain_bits_per_1024ns -= 16;
318 s->max_avail = max(4096U, p->mtu + 16 + 14 + 4);
319 max_avail_segs = max(1U, 4096 / (p->mtu - 40));
320 } else {
321 s->max_avail = 16384;
322 max_avail_segs = max(1U, 9000 / (p->mtu - 40));
323 }
324
325 pr_debug("t1_sched_update_parms: mtu %u speed %u max_avail %u "
326 "max_avail_segs %u drain_bits_per_1024ns %u\n", p->mtu,
327 p->speed, s->max_avail, max_avail_segs,
328 p->drain_bits_per_1024ns);
329
330 return max_avail_segs * (p->mtu - 40);
331}
332
Adrian Bunk68d579f2007-11-21 15:02:55 -0800333#if 0
334
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -0800335/*
336 * t1_sched_max_avail_bytes() tells the scheduler the maximum amount of
337 * data that can be pushed per port.
338 */
339void t1_sched_set_max_avail_bytes(struct sge *sge, unsigned int val)
340{
341 struct sched *s = sge->tx_sched;
342 unsigned int i;
343
344 s->max_avail = val;
345 for (i = 0; i < MAX_NPORTS; i++)
346 t1_sched_update_parms(sge, i, 0, 0);
347}
348
349/*
350 * t1_sched_set_drain_bits_per_us() tells the scheduler at which rate a port
351 * is draining.
352 */
353void t1_sched_set_drain_bits_per_us(struct sge *sge, unsigned int port,
354 unsigned int val)
355{
356 struct sched *s = sge->tx_sched;
357 struct sched_port *p = &s->p[port];
358 p->drain_bits_per_1024ns = val * 1024 / 1000;
359 t1_sched_update_parms(sge, port, 0, 0);
360}
361
Adrian Bunk68d579f2007-11-21 15:02:55 -0800362#endif /* 0 */
363
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -0800364
365/*
366 * get_clock() implements a ns clock (see ktime_get)
367 */
368static inline ktime_t get_clock(void)
369{
370 struct timespec ts;
371
372 ktime_get_ts(&ts);
373 return timespec_to_ktime(ts);
374}
375
376/*
377 * tx_sched_init() allocates resources and does basic initialization.
378 */
379static int tx_sched_init(struct sge *sge)
380{
381 struct sched *s;
382 int i;
383
384 s = kzalloc(sizeof (struct sched), GFP_KERNEL);
385 if (!s)
386 return -ENOMEM;
387
388 pr_debug("tx_sched_init\n");
389 tasklet_init(&s->sched_tsk, restart_sched, (unsigned long) sge);
390 sge->tx_sched = s;
391
392 for (i = 0; i < MAX_NPORTS; i++) {
393 skb_queue_head_init(&s->p[i].skbq);
394 t1_sched_update_parms(sge, i, 1500, 1000);
395 }
396
397 return 0;
398}
399
400/*
401 * sched_update_avail() computes the delta since the last time it was called
402 * and updates the per port quota (number of bits that can be sent to the any
403 * port).
404 */
405static inline int sched_update_avail(struct sge *sge)
406{
407 struct sched *s = sge->tx_sched;
408 ktime_t now = get_clock();
409 unsigned int i;
410 long long delta_time_ns;
411
412 delta_time_ns = ktime_to_ns(ktime_sub(now, s->last_updated));
413
414 pr_debug("sched_update_avail delta=%lld\n", delta_time_ns);
415 if (delta_time_ns < 15000)
416 return 0;
417
418 for (i = 0; i < MAX_NPORTS; i++) {
419 struct sched_port *p = &s->p[i];
420 unsigned int delta_avail;
421
422 delta_avail = (p->drain_bits_per_1024ns * delta_time_ns) >> 13;
423 p->avail = min(p->avail + delta_avail, s->max_avail);
424 }
425
426 s->last_updated = now;
427
428 return 1;
429}
430
431/*
432 * sched_skb() is called from two different places. In the tx path, any
433 * packet generating load on an output port will call sched_skb()
434 * (skb != NULL). In addition, sched_skb() is called from the irq/soft irq
435 * context (skb == NULL).
436 * The scheduler only returns a skb (which will then be sent) if the
437 * length of the skb is <= the current quota of the output port.
438 */
439static struct sk_buff *sched_skb(struct sge *sge, struct sk_buff *skb,
440 unsigned int credits)
441{
442 struct sched *s = sge->tx_sched;
443 struct sk_buff_head *skbq;
444 unsigned int i, len, update = 1;
445
446 pr_debug("sched_skb %p\n", skb);
447 if (!skb) {
448 if (!s->num)
449 return NULL;
450 } else {
451 skbq = &s->p[skb->dev->if_port].skbq;
452 __skb_queue_tail(skbq, skb);
453 s->num++;
454 skb = NULL;
455 }
456
457 if (credits < MAX_SKB_FRAGS + 1)
458 goto out;
459
Francois Romieu356bd142006-12-11 23:47:00 +0100460again:
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -0800461 for (i = 0; i < MAX_NPORTS; i++) {
462 s->port = ++s->port & (MAX_NPORTS - 1);
463 skbq = &s->p[s->port].skbq;
464
465 skb = skb_peek(skbq);
466
467 if (!skb)
468 continue;
469
470 len = skb->len;
471 if (len <= s->p[s->port].avail) {
472 s->p[s->port].avail -= len;
473 s->num--;
474 __skb_unlink(skb, skbq);
475 goto out;
476 }
477 skb = NULL;
478 }
479
480 if (update-- && sched_update_avail(sge))
481 goto again;
482
Francois Romieu356bd142006-12-11 23:47:00 +0100483out:
484 /* If there are more pending skbs, we use the hardware to schedule us
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -0800485 * again.
486 */
487 if (s->num && !skb) {
488 struct cmdQ *q = &sge->cmdQ[0];
489 clear_bit(CMDQ_STAT_LAST_PKT_DB, &q->status);
490 if (test_and_set_bit(CMDQ_STAT_RUNNING, &q->status) == 0) {
491 set_bit(CMDQ_STAT_LAST_PKT_DB, &q->status);
492 writel(F_CMDQ0_ENABLE, sge->adapter->regs + A_SG_DOORBELL);
493 }
494 }
495 pr_debug("sched_skb ret %p\n", skb);
496
497 return skb;
498}
499
500/*
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800501 * PIO to indicate that memory mapped Q contains valid descriptor(s).
502 */
Scott Bardone559fb512005-06-23 01:40:19 -0400503static inline void doorbell_pio(struct adapter *adapter, u32 val)
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800504{
505 wmb();
Scott Bardone559fb512005-06-23 01:40:19 -0400506 writel(val, adapter->regs + A_SG_DOORBELL);
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800507}
508
509/*
510 * Frees all RX buffers on the freelist Q. The caller must make sure that
511 * the SGE is turned off before calling this function.
512 */
Scott Bardone559fb512005-06-23 01:40:19 -0400513static void free_freelQ_buffers(struct pci_dev *pdev, struct freelQ *q)
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800514{
Scott Bardone559fb512005-06-23 01:40:19 -0400515 unsigned int cidx = q->cidx;
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800516
Scott Bardone559fb512005-06-23 01:40:19 -0400517 while (q->credits--) {
518 struct freelQ_ce *ce = &q->centries[cidx];
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800519
520 pci_unmap_single(pdev, pci_unmap_addr(ce, dma_addr),
521 pci_unmap_len(ce, dma_len),
522 PCI_DMA_FROMDEVICE);
523 dev_kfree_skb(ce->skb);
524 ce->skb = NULL;
Scott Bardone559fb512005-06-23 01:40:19 -0400525 if (++cidx == q->size)
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800526 cidx = 0;
527 }
528}
529
530/*
531 * Free RX free list and response queue resources.
532 */
533static void free_rx_resources(struct sge *sge)
534{
535 struct pci_dev *pdev = sge->adapter->pdev;
536 unsigned int size, i;
537
538 if (sge->respQ.entries) {
Scott Bardone559fb512005-06-23 01:40:19 -0400539 size = sizeof(struct respQ_e) * sge->respQ.size;
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800540 pci_free_consistent(pdev, size, sge->respQ.entries,
541 sge->respQ.dma_addr);
542 }
543
544 for (i = 0; i < SGE_FREELQ_N; i++) {
Scott Bardone559fb512005-06-23 01:40:19 -0400545 struct freelQ *q = &sge->freelQ[i];
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800546
Scott Bardone559fb512005-06-23 01:40:19 -0400547 if (q->centries) {
548 free_freelQ_buffers(pdev, q);
549 kfree(q->centries);
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800550 }
Scott Bardone559fb512005-06-23 01:40:19 -0400551 if (q->entries) {
552 size = sizeof(struct freelQ_e) * q->size;
553 pci_free_consistent(pdev, size, q->entries,
554 q->dma_addr);
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800555 }
556 }
557}
558
559/*
560 * Allocates basic RX resources, consisting of memory mapped freelist Qs and a
Scott Bardone559fb512005-06-23 01:40:19 -0400561 * response queue.
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800562 */
563static int alloc_rx_resources(struct sge *sge, struct sge_params *p)
564{
565 struct pci_dev *pdev = sge->adapter->pdev;
566 unsigned int size, i;
567
568 for (i = 0; i < SGE_FREELQ_N; i++) {
Scott Bardone559fb512005-06-23 01:40:19 -0400569 struct freelQ *q = &sge->freelQ[i];
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800570
Scott Bardone559fb512005-06-23 01:40:19 -0400571 q->genbit = 1;
572 q->size = p->freelQ_size[i];
573 q->dma_offset = sge->rx_pkt_pad ? 0 : NET_IP_ALIGN;
574 size = sizeof(struct freelQ_e) * q->size;
Francois Romieu3e0f75b2006-12-05 23:57:41 +0100575 q->entries = pci_alloc_consistent(pdev, size, &q->dma_addr);
Scott Bardone559fb512005-06-23 01:40:19 -0400576 if (!q->entries)
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800577 goto err_no_mem;
Francois Romieu3e0f75b2006-12-05 23:57:41 +0100578
Scott Bardone559fb512005-06-23 01:40:19 -0400579 size = sizeof(struct freelQ_ce) * q->size;
Stephen Hemmingercbee9f92006-11-17 17:01:52 -0800580 q->centries = kzalloc(size, GFP_KERNEL);
Scott Bardone559fb512005-06-23 01:40:19 -0400581 if (!q->centries)
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800582 goto err_no_mem;
583 }
584
585 /*
586 * Calculate the buffer sizes for the two free lists. FL0 accommodates
587 * regular sized Ethernet frames, FL1 is sized not to exceed 16K,
588 * including all the sk_buff overhead.
589 *
590 * Note: For T2 FL0 and FL1 are reversed.
591 */
592 sge->freelQ[!sge->jumbo_fl].rx_buffer_size = SGE_RX_SM_BUF_SIZE +
593 sizeof(struct cpl_rx_data) +
594 sge->freelQ[!sge->jumbo_fl].dma_offset;
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -0800595
596 size = (16 * 1024) -
597 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
598
599 sge->freelQ[sge->jumbo_fl].rx_buffer_size = size;
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800600
Scott Bardone559fb512005-06-23 01:40:19 -0400601 /*
602 * Setup which skb recycle Q should be used when recycling buffers from
603 * each free list.
604 */
605 sge->freelQ[!sge->jumbo_fl].recycleq_idx = 0;
606 sge->freelQ[sge->jumbo_fl].recycleq_idx = 1;
607
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800608 sge->respQ.genbit = 1;
Scott Bardone559fb512005-06-23 01:40:19 -0400609 sge->respQ.size = SGE_RESPQ_E_N;
610 sge->respQ.credits = 0;
611 size = sizeof(struct respQ_e) * sge->respQ.size;
Francois Romieu3e0f75b2006-12-05 23:57:41 +0100612 sge->respQ.entries =
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800613 pci_alloc_consistent(pdev, size, &sge->respQ.dma_addr);
614 if (!sge->respQ.entries)
615 goto err_no_mem;
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800616 return 0;
617
618err_no_mem:
619 free_rx_resources(sge);
620 return -ENOMEM;
621}
622
623/*
Scott Bardone559fb512005-06-23 01:40:19 -0400624 * Reclaims n TX descriptors and frees the buffers associated with them.
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800625 */
Scott Bardone559fb512005-06-23 01:40:19 -0400626static void free_cmdQ_buffers(struct sge *sge, struct cmdQ *q, unsigned int n)
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800627{
Scott Bardone559fb512005-06-23 01:40:19 -0400628 struct cmdQ_ce *ce;
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800629 struct pci_dev *pdev = sge->adapter->pdev;
Scott Bardone559fb512005-06-23 01:40:19 -0400630 unsigned int cidx = q->cidx;
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800631
Scott Bardone559fb512005-06-23 01:40:19 -0400632 q->in_use -= n;
633 ce = &q->centries[cidx];
634 while (n--) {
Francois Romieu3e0f75b2006-12-05 23:57:41 +0100635 if (likely(pci_unmap_len(ce, dma_len))) {
636 pci_unmap_single(pdev, pci_unmap_addr(ce, dma_addr),
637 pci_unmap_len(ce, dma_len),
638 PCI_DMA_TODEVICE);
639 if (q->sop)
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -0800640 q->sop = 0;
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -0800641 }
Scott Bardone559fb512005-06-23 01:40:19 -0400642 if (ce->skb) {
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -0800643 dev_kfree_skb_any(ce->skb);
Scott Bardone559fb512005-06-23 01:40:19 -0400644 q->sop = 1;
645 }
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800646 ce++;
Scott Bardone559fb512005-06-23 01:40:19 -0400647 if (++cidx == q->size) {
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800648 cidx = 0;
Scott Bardone559fb512005-06-23 01:40:19 -0400649 ce = q->centries;
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800650 }
651 }
Scott Bardone559fb512005-06-23 01:40:19 -0400652 q->cidx = cidx;
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800653}
654
655/*
656 * Free TX resources.
657 *
658 * Assumes that SGE is stopped and all interrupts are disabled.
659 */
660static void free_tx_resources(struct sge *sge)
661{
662 struct pci_dev *pdev = sge->adapter->pdev;
663 unsigned int size, i;
664
665 for (i = 0; i < SGE_CMDQ_N; i++) {
Scott Bardone559fb512005-06-23 01:40:19 -0400666 struct cmdQ *q = &sge->cmdQ[i];
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800667
Scott Bardone559fb512005-06-23 01:40:19 -0400668 if (q->centries) {
669 if (q->in_use)
670 free_cmdQ_buffers(sge, q, q->in_use);
671 kfree(q->centries);
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800672 }
Scott Bardone559fb512005-06-23 01:40:19 -0400673 if (q->entries) {
674 size = sizeof(struct cmdQ_e) * q->size;
675 pci_free_consistent(pdev, size, q->entries,
676 q->dma_addr);
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800677 }
678 }
679}
680
681/*
682 * Allocates basic TX resources, consisting of memory mapped command Qs.
683 */
684static int alloc_tx_resources(struct sge *sge, struct sge_params *p)
685{
686 struct pci_dev *pdev = sge->adapter->pdev;
687 unsigned int size, i;
688
689 for (i = 0; i < SGE_CMDQ_N; i++) {
Scott Bardone559fb512005-06-23 01:40:19 -0400690 struct cmdQ *q = &sge->cmdQ[i];
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800691
Scott Bardone559fb512005-06-23 01:40:19 -0400692 q->genbit = 1;
693 q->sop = 1;
694 q->size = p->cmdQ_size[i];
695 q->in_use = 0;
696 q->status = 0;
697 q->processed = q->cleaned = 0;
698 q->stop_thres = 0;
699 spin_lock_init(&q->lock);
700 size = sizeof(struct cmdQ_e) * q->size;
Francois Romieu3e0f75b2006-12-05 23:57:41 +0100701 q->entries = pci_alloc_consistent(pdev, size, &q->dma_addr);
Scott Bardone559fb512005-06-23 01:40:19 -0400702 if (!q->entries)
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800703 goto err_no_mem;
Francois Romieu3e0f75b2006-12-05 23:57:41 +0100704
Scott Bardone559fb512005-06-23 01:40:19 -0400705 size = sizeof(struct cmdQ_ce) * q->size;
Stephen Hemmingercbee9f92006-11-17 17:01:52 -0800706 q->centries = kzalloc(size, GFP_KERNEL);
Scott Bardone559fb512005-06-23 01:40:19 -0400707 if (!q->centries)
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800708 goto err_no_mem;
709 }
710
Scott Bardone559fb512005-06-23 01:40:19 -0400711 /*
712 * CommandQ 0 handles Ethernet and TOE packets, while queue 1 is TOE
713 * only. For queue 0 set the stop threshold so we can handle one more
714 * packet from each port, plus reserve an additional 24 entries for
715 * Ethernet packets only. Queue 1 never suspends nor do we reserve
716 * space for Ethernet packets.
717 */
718 sge->cmdQ[0].stop_thres = sge->adapter->params.nports *
719 (MAX_SKB_FRAGS + 1);
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800720 return 0;
721
722err_no_mem:
723 free_tx_resources(sge);
724 return -ENOMEM;
725}
726
727static inline void setup_ring_params(struct adapter *adapter, u64 addr,
728 u32 size, int base_reg_lo,
729 int base_reg_hi, int size_reg)
730{
Scott Bardone559fb512005-06-23 01:40:19 -0400731 writel((u32)addr, adapter->regs + base_reg_lo);
732 writel(addr >> 32, adapter->regs + base_reg_hi);
733 writel(size, adapter->regs + size_reg);
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800734}
735
736/*
737 * Enable/disable VLAN acceleration.
738 */
739void t1_set_vlan_accel(struct adapter *adapter, int on_off)
740{
741 struct sge *sge = adapter->sge;
742
743 sge->sge_control &= ~F_VLAN_XTRACT;
744 if (on_off)
745 sge->sge_control |= F_VLAN_XTRACT;
746 if (adapter->open_device_map) {
Scott Bardone559fb512005-06-23 01:40:19 -0400747 writel(sge->sge_control, adapter->regs + A_SG_CONTROL);
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -0800748 readl(adapter->regs + A_SG_CONTROL); /* flush */
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800749 }
750}
751
752/*
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800753 * Programs the various SGE registers. However, the engine is not yet enabled,
754 * but sge->sge_control is setup and ready to go.
755 */
756static void configure_sge(struct sge *sge, struct sge_params *p)
757{
758 struct adapter *ap = sge->adapter;
Francois Romieu356bd142006-12-11 23:47:00 +0100759
Scott Bardone559fb512005-06-23 01:40:19 -0400760 writel(0, ap->regs + A_SG_CONTROL);
761 setup_ring_params(ap, sge->cmdQ[0].dma_addr, sge->cmdQ[0].size,
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800762 A_SG_CMD0BASELWR, A_SG_CMD0BASEUPR, A_SG_CMD0SIZE);
Scott Bardone559fb512005-06-23 01:40:19 -0400763 setup_ring_params(ap, sge->cmdQ[1].dma_addr, sge->cmdQ[1].size,
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800764 A_SG_CMD1BASELWR, A_SG_CMD1BASEUPR, A_SG_CMD1SIZE);
765 setup_ring_params(ap, sge->freelQ[0].dma_addr,
Scott Bardone559fb512005-06-23 01:40:19 -0400766 sge->freelQ[0].size, A_SG_FL0BASELWR,
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800767 A_SG_FL0BASEUPR, A_SG_FL0SIZE);
768 setup_ring_params(ap, sge->freelQ[1].dma_addr,
Scott Bardone559fb512005-06-23 01:40:19 -0400769 sge->freelQ[1].size, A_SG_FL1BASELWR,
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800770 A_SG_FL1BASEUPR, A_SG_FL1SIZE);
771
772 /* The threshold comparison uses <. */
Scott Bardone559fb512005-06-23 01:40:19 -0400773 writel(SGE_RX_SM_BUF_SIZE + 1, ap->regs + A_SG_FLTHRESHOLD);
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800774
Scott Bardone559fb512005-06-23 01:40:19 -0400775 setup_ring_params(ap, sge->respQ.dma_addr, sge->respQ.size,
776 A_SG_RSPBASELWR, A_SG_RSPBASEUPR, A_SG_RSPSIZE);
777 writel((u32)sge->respQ.size - 1, ap->regs + A_SG_RSPQUEUECREDIT);
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800778
779 sge->sge_control = F_CMDQ0_ENABLE | F_CMDQ1_ENABLE | F_FL0_ENABLE |
780 F_FL1_ENABLE | F_CPL_ENABLE | F_RESPONSE_QUEUE_ENABLE |
781 V_CMDQ_PRIORITY(2) | F_DISABLE_CMDQ1_GTS | F_ISCSI_COALESCE |
782 V_RX_PKT_OFFSET(sge->rx_pkt_pad);
783
784#if defined(__BIG_ENDIAN_BITFIELD)
785 sge->sge_control |= F_ENABLE_BIG_ENDIAN;
786#endif
787
Scott Bardone559fb512005-06-23 01:40:19 -0400788 /* Initialize no-resource timer */
789 sge->intrtimer_nres = SGE_INTRTIMER_NRES * core_ticks_per_usec(ap);
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800790
Scott Bardone559fb512005-06-23 01:40:19 -0400791 t1_sge_set_coalesce_params(sge, p);
Christoph Lameter8199d3a2005-03-30 13:34:31 -0800792}
793
794/*
795 * Return the payload capacity of the jumbo free-list buffers.
796 */
797static inline unsigned int jumbo_payload_capacity(const struct sge *sge)
798{
799 return sge->freelQ[sge->jumbo_fl].rx_buffer_size -
Scott Bardone559fb512005-06-23 01:40:19 -0400800 sge->freelQ[sge->jumbo_fl].dma_offset -
801 sizeof(struct cpl_rx_data);
802}
803
804/*
805 * Frees all SGE related resources and the sge structure itself
806 */
807void t1_sge_destroy(struct sge *sge)
808{
Stephen Hemminger56f643c2006-12-01 16:36:21 -0800809 int i;
810
811 for_each_port(sge->adapter, i)
812 free_percpu(sge->port_stats[i]);
813
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -0800814 kfree(sge->tx_sched);
Scott Bardone559fb512005-06-23 01:40:19 -0400815 free_tx_resources(sge);
816 free_rx_resources(sge);
817 kfree(sge);
818}
819
820/*
821 * Allocates new RX buffers on the freelist Q (and tracks them on the freelist
822 * context Q) until the Q is full or alloc_skb fails.
823 *
824 * It is possible that the generation bits already match, indicating that the
825 * buffer is already valid and nothing needs to be done. This happens when we
826 * copied a received buffer into a new sk_buff during the interrupt processing.
827 *
828 * If the SGE doesn't automatically align packets properly (!sge->rx_pkt_pad),
829 * we specify a RX_OFFSET in order to make sure that the IP header is 4B
830 * aligned.
831 */
832static void refill_free_list(struct sge *sge, struct freelQ *q)
833{
834 struct pci_dev *pdev = sge->adapter->pdev;
835 struct freelQ_ce *ce = &q->centries[q->pidx];
836 struct freelQ_e *e = &q->entries[q->pidx];
837 unsigned int dma_len = q->rx_buffer_size - q->dma_offset;
838
Scott Bardone559fb512005-06-23 01:40:19 -0400839 while (q->credits < q->size) {
840 struct sk_buff *skb;
841 dma_addr_t mapping;
842
843 skb = alloc_skb(q->rx_buffer_size, GFP_ATOMIC);
844 if (!skb)
845 break;
846
847 skb_reserve(skb, q->dma_offset);
848 mapping = pci_map_single(pdev, skb->data, dma_len,
849 PCI_DMA_FROMDEVICE);
Stephen Hemminger24a427c2007-01-08 11:26:12 -0800850 skb_reserve(skb, sge->rx_pkt_pad);
851
Scott Bardone559fb512005-06-23 01:40:19 -0400852 ce->skb = skb;
853 pci_unmap_addr_set(ce, dma_addr, mapping);
854 pci_unmap_len_set(ce, dma_len, dma_len);
855 e->addr_lo = (u32)mapping;
856 e->addr_hi = (u64)mapping >> 32;
857 e->len_gen = V_CMD_LEN(dma_len) | V_CMD_GEN1(q->genbit);
858 wmb();
859 e->gen2 = V_CMD_GEN2(q->genbit);
860
861 e++;
862 ce++;
863 if (++q->pidx == q->size) {
864 q->pidx = 0;
865 q->genbit ^= 1;
866 ce = q->centries;
867 e = q->entries;
868 }
869 q->credits++;
870 }
Scott Bardone559fb512005-06-23 01:40:19 -0400871}
872
873/*
874 * Calls refill_free_list for both free lists. If we cannot fill at least 1/4
875 * of both rings, we go into 'few interrupt mode' in order to give the system
876 * time to free up resources.
877 */
878static void freelQs_empty(struct sge *sge)
879{
880 struct adapter *adapter = sge->adapter;
881 u32 irq_reg = readl(adapter->regs + A_SG_INT_ENABLE);
882 u32 irqholdoff_reg;
883
884 refill_free_list(sge, &sge->freelQ[0]);
885 refill_free_list(sge, &sge->freelQ[1]);
886
887 if (sge->freelQ[0].credits > (sge->freelQ[0].size >> 2) &&
888 sge->freelQ[1].credits > (sge->freelQ[1].size >> 2)) {
889 irq_reg |= F_FL_EXHAUSTED;
890 irqholdoff_reg = sge->fixed_intrtimer;
891 } else {
892 /* Clear the F_FL_EXHAUSTED interrupts for now */
893 irq_reg &= ~F_FL_EXHAUSTED;
894 irqholdoff_reg = sge->intrtimer_nres;
895 }
896 writel(irqholdoff_reg, adapter->regs + A_SG_INTRTIMER);
897 writel(irq_reg, adapter->regs + A_SG_INT_ENABLE);
898
899 /* We reenable the Qs to force a freelist GTS interrupt later */
900 doorbell_pio(adapter, F_FL0_ENABLE | F_FL1_ENABLE);
901}
902
903#define SGE_PL_INTR_MASK (F_PL_INTR_SGE_ERR | F_PL_INTR_SGE_DATA)
904#define SGE_INT_FATAL (F_RESPQ_OVERFLOW | F_PACKET_TOO_BIG | F_PACKET_MISMATCH)
905#define SGE_INT_ENABLE (F_RESPQ_EXHAUSTED | F_RESPQ_OVERFLOW | \
906 F_FL_EXHAUSTED | F_PACKET_TOO_BIG | F_PACKET_MISMATCH)
907
908/*
909 * Disable SGE Interrupts
910 */
911void t1_sge_intr_disable(struct sge *sge)
912{
913 u32 val = readl(sge->adapter->regs + A_PL_ENABLE);
914
915 writel(val & ~SGE_PL_INTR_MASK, sge->adapter->regs + A_PL_ENABLE);
916 writel(0, sge->adapter->regs + A_SG_INT_ENABLE);
917}
918
919/*
920 * Enable SGE interrupts.
921 */
922void t1_sge_intr_enable(struct sge *sge)
923{
924 u32 en = SGE_INT_ENABLE;
925 u32 val = readl(sge->adapter->regs + A_PL_ENABLE);
926
927 if (sge->adapter->flags & TSO_CAPABLE)
928 en &= ~F_PACKET_TOO_BIG;
929 writel(en, sge->adapter->regs + A_SG_INT_ENABLE);
930 writel(val | SGE_PL_INTR_MASK, sge->adapter->regs + A_PL_ENABLE);
931}
932
933/*
934 * Clear SGE interrupts.
935 */
936void t1_sge_intr_clear(struct sge *sge)
937{
938 writel(SGE_PL_INTR_MASK, sge->adapter->regs + A_PL_CAUSE);
939 writel(0xffffffff, sge->adapter->regs + A_SG_INT_CAUSE);
940}
941
942/*
943 * SGE 'Error' interrupt handler
944 */
945int t1_sge_intr_error_handler(struct sge *sge)
946{
947 struct adapter *adapter = sge->adapter;
948 u32 cause = readl(adapter->regs + A_SG_INT_CAUSE);
949
950 if (adapter->flags & TSO_CAPABLE)
951 cause &= ~F_PACKET_TOO_BIG;
952 if (cause & F_RESPQ_EXHAUSTED)
953 sge->stats.respQ_empty++;
954 if (cause & F_RESPQ_OVERFLOW) {
955 sge->stats.respQ_overflow++;
956 CH_ALERT("%s: SGE response queue overflow\n",
957 adapter->name);
958 }
959 if (cause & F_FL_EXHAUSTED) {
960 sge->stats.freelistQ_empty++;
961 freelQs_empty(sge);
962 }
963 if (cause & F_PACKET_TOO_BIG) {
964 sge->stats.pkt_too_big++;
965 CH_ALERT("%s: SGE max packet size exceeded\n",
966 adapter->name);
967 }
968 if (cause & F_PACKET_MISMATCH) {
969 sge->stats.pkt_mismatch++;
970 CH_ALERT("%s: SGE packet mismatch\n", adapter->name);
971 }
972 if (cause & SGE_INT_FATAL)
973 t1_fatal_err(adapter);
974
975 writel(cause, adapter->regs + A_SG_INT_CAUSE);
976 return 0;
977}
978
Stephen Hemminger56f643c2006-12-01 16:36:21 -0800979const struct sge_intr_counts *t1_sge_get_intr_counts(const struct sge *sge)
Scott Bardone559fb512005-06-23 01:40:19 -0400980{
981 return &sge->stats;
982}
983
Stephen Hemminger56f643c2006-12-01 16:36:21 -0800984void t1_sge_get_port_stats(const struct sge *sge, int port,
985 struct sge_port_stats *ss)
Scott Bardone559fb512005-06-23 01:40:19 -0400986{
Stephen Hemminger56f643c2006-12-01 16:36:21 -0800987 int cpu;
988
989 memset(ss, 0, sizeof(*ss));
990 for_each_possible_cpu(cpu) {
991 struct sge_port_stats *st = per_cpu_ptr(sge->port_stats[port], cpu);
992
Stephen Hemminger56f643c2006-12-01 16:36:21 -0800993 ss->rx_cso_good += st->rx_cso_good;
Stephen Hemminger56f643c2006-12-01 16:36:21 -0800994 ss->tx_cso += st->tx_cso;
995 ss->tx_tso += st->tx_tso;
Divy Le Ray7832ee02007-11-27 13:30:09 -0800996 ss->tx_need_hdrroom += st->tx_need_hdrroom;
Stephen Hemminger56f643c2006-12-01 16:36:21 -0800997 ss->vlan_xtract += st->vlan_xtract;
998 ss->vlan_insert += st->vlan_insert;
999 }
Scott Bardone559fb512005-06-23 01:40:19 -04001000}
1001
1002/**
1003 * recycle_fl_buf - recycle a free list buffer
1004 * @fl: the free list
1005 * @idx: index of buffer to recycle
1006 *
1007 * Recycles the specified buffer on the given free list by adding it at
1008 * the next available slot on the list.
1009 */
1010static void recycle_fl_buf(struct freelQ *fl, int idx)
1011{
1012 struct freelQ_e *from = &fl->entries[idx];
1013 struct freelQ_e *to = &fl->entries[fl->pidx];
1014
1015 fl->centries[fl->pidx] = fl->centries[idx];
1016 to->addr_lo = from->addr_lo;
1017 to->addr_hi = from->addr_hi;
1018 to->len_gen = G_CMD_LEN(from->len_gen) | V_CMD_GEN1(fl->genbit);
1019 wmb();
1020 to->gen2 = V_CMD_GEN2(fl->genbit);
1021 fl->credits++;
1022
1023 if (++fl->pidx == fl->size) {
1024 fl->pidx = 0;
1025 fl->genbit ^= 1;
1026 }
1027}
1028
Stephen Hemminger24a427c2007-01-08 11:26:12 -08001029static int copybreak __read_mostly = 256;
1030module_param(copybreak, int, 0);
1031MODULE_PARM_DESC(copybreak, "Receive copy threshold");
1032
Scott Bardone559fb512005-06-23 01:40:19 -04001033/**
1034 * get_packet - return the next ingress packet buffer
1035 * @pdev: the PCI device that received the packet
1036 * @fl: the SGE free list holding the packet
1037 * @len: the actual packet length, excluding any SGE padding
1038 * @dma_pad: padding at beginning of buffer left by SGE DMA
1039 * @skb_pad: padding to be used if the packet is copied
1040 * @copy_thres: length threshold under which a packet should be copied
1041 * @drop_thres: # of remaining buffers before we start dropping packets
1042 *
1043 * Get the next packet from a free list and complete setup of the
1044 * sk_buff. If the packet is small we make a copy and recycle the
1045 * original buffer, otherwise we use the original buffer itself. If a
1046 * positive drop threshold is supplied packets are dropped and their
1047 * buffers recycled if (a) the number of remaining buffers is under the
1048 * threshold and the packet is too big to copy, or (b) the packet should
1049 * be copied but there is no memory for the copy.
1050 */
1051static inline struct sk_buff *get_packet(struct pci_dev *pdev,
Stephen Hemminger24a427c2007-01-08 11:26:12 -08001052 struct freelQ *fl, unsigned int len)
Scott Bardone559fb512005-06-23 01:40:19 -04001053{
1054 struct sk_buff *skb;
Stephen Hemminger24a427c2007-01-08 11:26:12 -08001055 const struct freelQ_ce *ce = &fl->centries[fl->cidx];
Scott Bardone559fb512005-06-23 01:40:19 -04001056
Stephen Hemminger24a427c2007-01-08 11:26:12 -08001057 if (len < copybreak) {
1058 skb = alloc_skb(len + 2, GFP_ATOMIC);
1059 if (!skb)
Scott Bardone559fb512005-06-23 01:40:19 -04001060 goto use_orig_buf;
1061
Stephen Hemminger24a427c2007-01-08 11:26:12 -08001062 skb_reserve(skb, 2); /* align IP header */
1063 skb_put(skb, len);
1064 pci_dma_sync_single_for_cpu(pdev,
1065 pci_unmap_addr(ce, dma_addr),
1066 pci_unmap_len(ce, dma_len),
1067 PCI_DMA_FROMDEVICE);
Arnaldo Carvalho de Melod626f622007-03-27 18:55:52 -03001068 skb_copy_from_linear_data(ce->skb, skb->data, len);
Stephen Hemminger24a427c2007-01-08 11:26:12 -08001069 pci_dma_sync_single_for_device(pdev,
1070 pci_unmap_addr(ce, dma_addr),
1071 pci_unmap_len(ce, dma_len),
1072 PCI_DMA_FROMDEVICE);
Scott Bardone559fb512005-06-23 01:40:19 -04001073 recycle_fl_buf(fl, fl->cidx);
1074 return skb;
1075 }
1076
Stephen Hemminger24a427c2007-01-08 11:26:12 -08001077use_orig_buf:
1078 if (fl->credits < 2) {
Scott Bardone559fb512005-06-23 01:40:19 -04001079 recycle_fl_buf(fl, fl->cidx);
1080 return NULL;
1081 }
1082
Scott Bardone559fb512005-06-23 01:40:19 -04001083 pci_unmap_single(pdev, pci_unmap_addr(ce, dma_addr),
1084 pci_unmap_len(ce, dma_len), PCI_DMA_FROMDEVICE);
1085 skb = ce->skb;
Stephen Hemminger24a427c2007-01-08 11:26:12 -08001086 prefetch(skb->data);
1087
Scott Bardone559fb512005-06-23 01:40:19 -04001088 skb_put(skb, len);
1089 return skb;
1090}
1091
1092/**
1093 * unexpected_offload - handle an unexpected offload packet
1094 * @adapter: the adapter
1095 * @fl: the free list that received the packet
1096 *
1097 * Called when we receive an unexpected offload packet (e.g., the TOE
1098 * function is disabled or the card is a NIC). Prints a message and
1099 * recycles the buffer.
1100 */
1101static void unexpected_offload(struct adapter *adapter, struct freelQ *fl)
1102{
1103 struct freelQ_ce *ce = &fl->centries[fl->cidx];
1104 struct sk_buff *skb = ce->skb;
1105
1106 pci_dma_sync_single_for_cpu(adapter->pdev, pci_unmap_addr(ce, dma_addr),
1107 pci_unmap_len(ce, dma_len), PCI_DMA_FROMDEVICE);
1108 CH_ERR("%s: unexpected offload packet, cmd %u\n",
1109 adapter->name, *skb->data);
1110 recycle_fl_buf(fl, fl->cidx);
1111}
1112
1113/*
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001114 * T1/T2 SGE limits the maximum DMA size per TX descriptor to
1115 * SGE_TX_DESC_MAX_PLEN (16KB). If the PAGE_SIZE is larger than 16KB, the
1116 * stack might send more than SGE_TX_DESC_MAX_PLEN in a contiguous manner.
1117 * Note that the *_large_page_tx_descs stuff will be optimized out when
1118 * PAGE_SIZE <= SGE_TX_DESC_MAX_PLEN.
1119 *
1120 * compute_large_page_descs() computes how many additional descriptors are
1121 * required to break down the stack's request.
1122 */
1123static inline unsigned int compute_large_page_tx_descs(struct sk_buff *skb)
1124{
1125 unsigned int count = 0;
Francois Romieu356bd142006-12-11 23:47:00 +01001126
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001127 if (PAGE_SIZE > SGE_TX_DESC_MAX_PLEN) {
1128 unsigned int nfrags = skb_shinfo(skb)->nr_frags;
1129 unsigned int i, len = skb->len - skb->data_len;
1130 while (len > SGE_TX_DESC_MAX_PLEN) {
1131 count++;
1132 len -= SGE_TX_DESC_MAX_PLEN;
1133 }
1134 for (i = 0; nfrags--; i++) {
1135 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
1136 len = frag->size;
1137 while (len > SGE_TX_DESC_MAX_PLEN) {
1138 count++;
1139 len -= SGE_TX_DESC_MAX_PLEN;
1140 }
1141 }
1142 }
1143 return count;
1144}
1145
1146/*
1147 * Write a cmdQ entry.
1148 *
1149 * Since this function writes the 'flags' field, it must not be used to
1150 * write the first cmdQ entry.
1151 */
1152static inline void write_tx_desc(struct cmdQ_e *e, dma_addr_t mapping,
1153 unsigned int len, unsigned int gen,
1154 unsigned int eop)
1155{
1156 if (unlikely(len > SGE_TX_DESC_MAX_PLEN))
1157 BUG();
1158 e->addr_lo = (u32)mapping;
1159 e->addr_hi = (u64)mapping >> 32;
1160 e->len_gen = V_CMD_LEN(len) | V_CMD_GEN1(gen);
1161 e->flags = F_CMD_DATAVALID | V_CMD_EOP(eop) | V_CMD_GEN2(gen);
1162}
1163
1164/*
1165 * See comment for previous function.
1166 *
1167 * write_tx_descs_large_page() writes additional SGE tx descriptors if
1168 * *desc_len exceeds HW's capability.
1169 */
1170static inline unsigned int write_large_page_tx_descs(unsigned int pidx,
1171 struct cmdQ_e **e,
1172 struct cmdQ_ce **ce,
1173 unsigned int *gen,
1174 dma_addr_t *desc_mapping,
1175 unsigned int *desc_len,
1176 unsigned int nfrags,
1177 struct cmdQ *q)
1178{
1179 if (PAGE_SIZE > SGE_TX_DESC_MAX_PLEN) {
1180 struct cmdQ_e *e1 = *e;
1181 struct cmdQ_ce *ce1 = *ce;
1182
1183 while (*desc_len > SGE_TX_DESC_MAX_PLEN) {
1184 *desc_len -= SGE_TX_DESC_MAX_PLEN;
1185 write_tx_desc(e1, *desc_mapping, SGE_TX_DESC_MAX_PLEN,
1186 *gen, nfrags == 0 && *desc_len == 0);
1187 ce1->skb = NULL;
1188 pci_unmap_len_set(ce1, dma_len, 0);
1189 *desc_mapping += SGE_TX_DESC_MAX_PLEN;
1190 if (*desc_len) {
1191 ce1++;
1192 e1++;
1193 if (++pidx == q->size) {
1194 pidx = 0;
1195 *gen ^= 1;
1196 ce1 = q->centries;
1197 e1 = q->entries;
1198 }
1199 }
1200 }
1201 *e = e1;
1202 *ce = ce1;
1203 }
1204 return pidx;
1205}
1206
1207/*
Scott Bardone559fb512005-06-23 01:40:19 -04001208 * Write the command descriptors to transmit the given skb starting at
1209 * descriptor pidx with the given generation.
1210 */
1211static inline void write_tx_descs(struct adapter *adapter, struct sk_buff *skb,
1212 unsigned int pidx, unsigned int gen,
1213 struct cmdQ *q)
1214{
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001215 dma_addr_t mapping, desc_mapping;
Scott Bardone559fb512005-06-23 01:40:19 -04001216 struct cmdQ_e *e, *e1;
1217 struct cmdQ_ce *ce;
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001218 unsigned int i, flags, first_desc_len, desc_len,
1219 nfrags = skb_shinfo(skb)->nr_frags;
1220
1221 e = e1 = &q->entries[pidx];
1222 ce = &q->centries[pidx];
Scott Bardone559fb512005-06-23 01:40:19 -04001223
1224 mapping = pci_map_single(adapter->pdev, skb->data,
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001225 skb->len - skb->data_len, PCI_DMA_TODEVICE);
1226
1227 desc_mapping = mapping;
1228 desc_len = skb->len - skb->data_len;
1229
1230 flags = F_CMD_DATAVALID | F_CMD_SOP |
1231 V_CMD_EOP(nfrags == 0 && desc_len <= SGE_TX_DESC_MAX_PLEN) |
1232 V_CMD_GEN2(gen);
1233 first_desc_len = (desc_len <= SGE_TX_DESC_MAX_PLEN) ?
1234 desc_len : SGE_TX_DESC_MAX_PLEN;
1235 e->addr_lo = (u32)desc_mapping;
1236 e->addr_hi = (u64)desc_mapping >> 32;
1237 e->len_gen = V_CMD_LEN(first_desc_len) | V_CMD_GEN1(gen);
1238 ce->skb = NULL;
1239 pci_unmap_len_set(ce, dma_len, 0);
1240
1241 if (PAGE_SIZE > SGE_TX_DESC_MAX_PLEN &&
1242 desc_len > SGE_TX_DESC_MAX_PLEN) {
1243 desc_mapping += first_desc_len;
1244 desc_len -= first_desc_len;
1245 e1++;
1246 ce++;
1247 if (++pidx == q->size) {
1248 pidx = 0;
1249 gen ^= 1;
1250 e1 = q->entries;
1251 ce = q->centries;
1252 }
1253 pidx = write_large_page_tx_descs(pidx, &e1, &ce, &gen,
1254 &desc_mapping, &desc_len,
1255 nfrags, q);
1256
1257 if (likely(desc_len))
1258 write_tx_desc(e1, desc_mapping, desc_len, gen,
1259 nfrags == 0);
1260 }
1261
Scott Bardone559fb512005-06-23 01:40:19 -04001262 ce->skb = NULL;
1263 pci_unmap_addr_set(ce, dma_addr, mapping);
1264 pci_unmap_len_set(ce, dma_len, skb->len - skb->data_len);
1265
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001266 for (i = 0; nfrags--; i++) {
Scott Bardone559fb512005-06-23 01:40:19 -04001267 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
Scott Bardone559fb512005-06-23 01:40:19 -04001268 e1++;
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001269 ce++;
Scott Bardone559fb512005-06-23 01:40:19 -04001270 if (++pidx == q->size) {
1271 pidx = 0;
1272 gen ^= 1;
Scott Bardone559fb512005-06-23 01:40:19 -04001273 e1 = q->entries;
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001274 ce = q->centries;
Scott Bardone559fb512005-06-23 01:40:19 -04001275 }
1276
1277 mapping = pci_map_page(adapter->pdev, frag->page,
1278 frag->page_offset, frag->size,
1279 PCI_DMA_TODEVICE);
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001280 desc_mapping = mapping;
1281 desc_len = frag->size;
1282
1283 pidx = write_large_page_tx_descs(pidx, &e1, &ce, &gen,
1284 &desc_mapping, &desc_len,
1285 nfrags, q);
1286 if (likely(desc_len))
1287 write_tx_desc(e1, desc_mapping, desc_len, gen,
1288 nfrags == 0);
Scott Bardone559fb512005-06-23 01:40:19 -04001289 ce->skb = NULL;
1290 pci_unmap_addr_set(ce, dma_addr, mapping);
1291 pci_unmap_len_set(ce, dma_len, frag->size);
Scott Bardone559fb512005-06-23 01:40:19 -04001292 }
Scott Bardone559fb512005-06-23 01:40:19 -04001293 ce->skb = skb;
1294 wmb();
1295 e->flags = flags;
1296}
1297
1298/*
1299 * Clean up completed Tx buffers.
1300 */
1301static inline void reclaim_completed_tx(struct sge *sge, struct cmdQ *q)
1302{
1303 unsigned int reclaim = q->processed - q->cleaned;
1304
1305 if (reclaim) {
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001306 pr_debug("reclaim_completed_tx processed:%d cleaned:%d\n",
1307 q->processed, q->cleaned);
Scott Bardone559fb512005-06-23 01:40:19 -04001308 free_cmdQ_buffers(sge, q, reclaim);
1309 q->cleaned += reclaim;
1310 }
1311}
1312
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001313/*
1314 * Called from tasklet. Checks the scheduler for any
1315 * pending skbs that can be sent.
1316 */
1317static void restart_sched(unsigned long arg)
1318{
1319 struct sge *sge = (struct sge *) arg;
1320 struct adapter *adapter = sge->adapter;
1321 struct cmdQ *q = &sge->cmdQ[0];
1322 struct sk_buff *skb;
1323 unsigned int credits, queued_skb = 0;
1324
1325 spin_lock(&q->lock);
1326 reclaim_completed_tx(sge, q);
1327
1328 credits = q->size - q->in_use;
1329 pr_debug("restart_sched credits=%d\n", credits);
1330 while ((skb = sched_skb(sge, NULL, credits)) != NULL) {
1331 unsigned int genbit, pidx, count;
1332 count = 1 + skb_shinfo(skb)->nr_frags;
Francois Romieu356bd142006-12-11 23:47:00 +01001333 count += compute_large_page_tx_descs(skb);
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001334 q->in_use += count;
1335 genbit = q->genbit;
1336 pidx = q->pidx;
1337 q->pidx += count;
1338 if (q->pidx >= q->size) {
1339 q->pidx -= q->size;
1340 q->genbit ^= 1;
1341 }
1342 write_tx_descs(adapter, skb, pidx, genbit, q);
1343 credits = q->size - q->in_use;
1344 queued_skb = 1;
1345 }
1346
1347 if (queued_skb) {
1348 clear_bit(CMDQ_STAT_LAST_PKT_DB, &q->status);
1349 if (test_and_set_bit(CMDQ_STAT_RUNNING, &q->status) == 0) {
1350 set_bit(CMDQ_STAT_LAST_PKT_DB, &q->status);
1351 writel(F_CMDQ0_ENABLE, adapter->regs + A_SG_DOORBELL);
1352 }
1353 }
1354 spin_unlock(&q->lock);
1355}
Scott Bardone559fb512005-06-23 01:40:19 -04001356
Scott Bardone559fb512005-06-23 01:40:19 -04001357/**
1358 * sge_rx - process an ingress ethernet packet
1359 * @sge: the sge structure
1360 * @fl: the free list that contains the packet buffer
1361 * @len: the packet length
1362 *
1363 * Process an ingress ethernet pakcet and deliver it to the stack.
1364 */
Stephen Hemminger24a427c2007-01-08 11:26:12 -08001365static void sge_rx(struct sge *sge, struct freelQ *fl, unsigned int len)
Scott Bardone559fb512005-06-23 01:40:19 -04001366{
1367 struct sk_buff *skb;
Stephen Hemminger24a427c2007-01-08 11:26:12 -08001368 const struct cpl_rx_pkt *p;
Scott Bardone559fb512005-06-23 01:40:19 -04001369 struct adapter *adapter = sge->adapter;
Stephen Hemminger56f643c2006-12-01 16:36:21 -08001370 struct sge_port_stats *st;
Scott Bardone559fb512005-06-23 01:40:19 -04001371
Stephen Hemminger24a427c2007-01-08 11:26:12 -08001372 skb = get_packet(adapter->pdev, fl, len - sge->rx_pkt_pad);
Stephen Hemminger56f643c2006-12-01 16:36:21 -08001373 if (unlikely(!skb)) {
1374 sge->stats.rx_drops++;
Stephen Hemminger24a427c2007-01-08 11:26:12 -08001375 return;
Scott Bardone559fb512005-06-23 01:40:19 -04001376 }
1377
Stephen Hemminger24a427c2007-01-08 11:26:12 -08001378 p = (const struct cpl_rx_pkt *) skb->data;
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001379 if (p->iff >= adapter->params.nports) {
1380 kfree_skb(skb);
Stephen Hemminger24a427c2007-01-08 11:26:12 -08001381 return;
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001382 }
Stephen Hemminger24a427c2007-01-08 11:26:12 -08001383 __skb_pull(skb, sizeof(*p));
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001384
Stephen Hemminger56f643c2006-12-01 16:36:21 -08001385 st = per_cpu_ptr(sge->port_stats[p->iff], smp_processor_id());
Stephen Hemminger56f643c2006-12-01 16:36:21 -08001386
Arnaldo Carvalho de Melo4c13eb62007-04-25 17:40:23 -07001387 skb->protocol = eth_type_trans(skb, adapter->port[p->iff].dev);
Scott Bardone559fb512005-06-23 01:40:19 -04001388 if ((adapter->flags & RX_CSUM_ENABLED) && p->csum == 0xffff &&
1389 skb->protocol == htons(ETH_P_IP) &&
1390 (skb->data[9] == IPPROTO_TCP || skb->data[9] == IPPROTO_UDP)) {
Stephen Hemminger56f643c2006-12-01 16:36:21 -08001391 ++st->rx_cso_good;
Scott Bardone559fb512005-06-23 01:40:19 -04001392 skb->ip_summed = CHECKSUM_UNNECESSARY;
1393 } else
1394 skb->ip_summed = CHECKSUM_NONE;
1395
1396 if (unlikely(adapter->vlan_grp && p->vlan_valid)) {
Stephen Hemminger56f643c2006-12-01 16:36:21 -08001397 st->vlan_xtract++;
Francois Romieu4422b002008-07-11 00:29:19 +02001398 vlan_hwaccel_receive_skb(skb, adapter->vlan_grp,
1399 ntohs(p->vlan));
1400 } else
Scott Bardone559fb512005-06-23 01:40:19 -04001401 netif_receive_skb(skb);
Scott Bardone559fb512005-06-23 01:40:19 -04001402}
1403
1404/*
1405 * Returns true if a command queue has enough available descriptors that
1406 * we can resume Tx operation after temporarily disabling its packet queue.
1407 */
1408static inline int enough_free_Tx_descs(const struct cmdQ *q)
1409{
1410 unsigned int r = q->processed - q->cleaned;
1411
1412 return q->in_use - r < (q->size >> 1);
1413}
1414
1415/*
1416 * Called when sufficient space has become available in the SGE command queues
1417 * after the Tx packet schedulers have been suspended to restart the Tx path.
1418 */
1419static void restart_tx_queues(struct sge *sge)
1420{
1421 struct adapter *adap = sge->adapter;
Francois Romieu3e0f75b2006-12-05 23:57:41 +01001422 int i;
Scott Bardone559fb512005-06-23 01:40:19 -04001423
Francois Romieu3e0f75b2006-12-05 23:57:41 +01001424 if (!enough_free_Tx_descs(&sge->cmdQ[0]))
1425 return;
Scott Bardone559fb512005-06-23 01:40:19 -04001426
Francois Romieu3e0f75b2006-12-05 23:57:41 +01001427 for_each_port(adap, i) {
1428 struct net_device *nd = adap->port[i].dev;
Scott Bardone559fb512005-06-23 01:40:19 -04001429
Francois Romieu3e0f75b2006-12-05 23:57:41 +01001430 if (test_and_clear_bit(nd->if_port, &sge->stopped_tx_queues) &&
1431 netif_running(nd)) {
1432 sge->stats.cmdQ_restarted[2]++;
1433 netif_wake_queue(nd);
Scott Bardone559fb512005-06-23 01:40:19 -04001434 }
1435 }
1436}
1437
1438/*
Francois Romieu356bd142006-12-11 23:47:00 +01001439 * update_tx_info is called from the interrupt handler/NAPI to return cmdQ0
Scott Bardone559fb512005-06-23 01:40:19 -04001440 * information.
1441 */
Francois Romieu356bd142006-12-11 23:47:00 +01001442static unsigned int update_tx_info(struct adapter *adapter,
1443 unsigned int flags,
Scott Bardone559fb512005-06-23 01:40:19 -04001444 unsigned int pr0)
1445{
1446 struct sge *sge = adapter->sge;
1447 struct cmdQ *cmdq = &sge->cmdQ[0];
1448
1449 cmdq->processed += pr0;
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001450 if (flags & (F_FL0_ENABLE | F_FL1_ENABLE)) {
1451 freelQs_empty(sge);
1452 flags &= ~(F_FL0_ENABLE | F_FL1_ENABLE);
1453 }
Scott Bardone559fb512005-06-23 01:40:19 -04001454 if (flags & F_CMDQ0_ENABLE) {
1455 clear_bit(CMDQ_STAT_RUNNING, &cmdq->status);
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001456
Scott Bardone559fb512005-06-23 01:40:19 -04001457 if (cmdq->cleaned + cmdq->in_use != cmdq->processed &&
1458 !test_and_set_bit(CMDQ_STAT_LAST_PKT_DB, &cmdq->status)) {
1459 set_bit(CMDQ_STAT_RUNNING, &cmdq->status);
1460 writel(F_CMDQ0_ENABLE, adapter->regs + A_SG_DOORBELL);
1461 }
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001462 if (sge->tx_sched)
1463 tasklet_hi_schedule(&sge->tx_sched->sched_tsk);
1464
1465 flags &= ~F_CMDQ0_ENABLE;
Scott Bardone559fb512005-06-23 01:40:19 -04001466 }
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001467
Scott Bardone559fb512005-06-23 01:40:19 -04001468 if (unlikely(sge->stopped_tx_queues != 0))
1469 restart_tx_queues(sge);
1470
1471 return flags;
1472}
1473
1474/*
1475 * Process SGE responses, up to the supplied budget. Returns the number of
1476 * responses processed. A negative budget is effectively unlimited.
1477 */
1478static int process_responses(struct adapter *adapter, int budget)
1479{
1480 struct sge *sge = adapter->sge;
1481 struct respQ *q = &sge->respQ;
1482 struct respQ_e *e = &q->entries[q->cidx];
Stephen Hemminger24a427c2007-01-08 11:26:12 -08001483 int done = 0;
Scott Bardone559fb512005-06-23 01:40:19 -04001484 unsigned int flags = 0;
1485 unsigned int cmdq_processed[SGE_CMDQ_N] = {0, 0};
Francois Romieu356bd142006-12-11 23:47:00 +01001486
Stephen Hemminger24a427c2007-01-08 11:26:12 -08001487 while (done < budget && e->GenerationBit == q->genbit) {
Scott Bardone559fb512005-06-23 01:40:19 -04001488 flags |= e->Qsleeping;
Francois Romieu356bd142006-12-11 23:47:00 +01001489
Scott Bardone559fb512005-06-23 01:40:19 -04001490 cmdq_processed[0] += e->Cmdq0CreditReturn;
1491 cmdq_processed[1] += e->Cmdq1CreditReturn;
Francois Romieu356bd142006-12-11 23:47:00 +01001492
Scott Bardone559fb512005-06-23 01:40:19 -04001493 /* We batch updates to the TX side to avoid cacheline
1494 * ping-pong of TX state information on MP where the sender
1495 * might run on a different CPU than this function...
1496 */
Stephen Hemminger24a427c2007-01-08 11:26:12 -08001497 if (unlikely((flags & F_CMDQ0_ENABLE) || cmdq_processed[0] > 64)) {
Scott Bardone559fb512005-06-23 01:40:19 -04001498 flags = update_tx_info(adapter, flags, cmdq_processed[0]);
1499 cmdq_processed[0] = 0;
1500 }
Stephen Hemminger24a427c2007-01-08 11:26:12 -08001501
Scott Bardone559fb512005-06-23 01:40:19 -04001502 if (unlikely(cmdq_processed[1] > 16)) {
1503 sge->cmdQ[1].processed += cmdq_processed[1];
1504 cmdq_processed[1] = 0;
1505 }
Stephen Hemminger24a427c2007-01-08 11:26:12 -08001506
Scott Bardone559fb512005-06-23 01:40:19 -04001507 if (likely(e->DataValid)) {
1508 struct freelQ *fl = &sge->freelQ[e->FreelistQid];
1509
Eric Sesterhenn5d9428d2006-04-02 13:52:48 +02001510 BUG_ON(!e->Sop || !e->Eop);
Scott Bardone559fb512005-06-23 01:40:19 -04001511 if (unlikely(e->Offload))
1512 unexpected_offload(adapter, fl);
1513 else
1514 sge_rx(sge, fl, e->BufferLength);
1515
Stephen Hemminger24a427c2007-01-08 11:26:12 -08001516 ++done;
1517
Scott Bardone559fb512005-06-23 01:40:19 -04001518 /*
1519 * Note: this depends on each packet consuming a
1520 * single free-list buffer; cf. the BUG above.
1521 */
1522 if (++fl->cidx == fl->size)
1523 fl->cidx = 0;
Stephen Hemminger24a427c2007-01-08 11:26:12 -08001524 prefetch(fl->centries[fl->cidx].skb);
1525
Scott Bardone559fb512005-06-23 01:40:19 -04001526 if (unlikely(--fl->credits <
1527 fl->size - SGE_FREEL_REFILL_THRESH))
1528 refill_free_list(sge, fl);
1529 } else
1530 sge->stats.pure_rsps++;
1531
1532 e++;
1533 if (unlikely(++q->cidx == q->size)) {
1534 q->cidx = 0;
1535 q->genbit ^= 1;
1536 e = q->entries;
1537 }
1538 prefetch(e);
1539
1540 if (++q->credits > SGE_RESPQ_REPLENISH_THRES) {
1541 writel(q->credits, adapter->regs + A_SG_RSPQUEUECREDIT);
1542 q->credits = 0;
1543 }
Scott Bardone559fb512005-06-23 01:40:19 -04001544 }
1545
Francois Romieu356bd142006-12-11 23:47:00 +01001546 flags = update_tx_info(adapter, flags, cmdq_processed[0]);
Scott Bardone559fb512005-06-23 01:40:19 -04001547 sge->cmdQ[1].processed += cmdq_processed[1];
1548
Stephen Hemminger24a427c2007-01-08 11:26:12 -08001549 return done;
Scott Bardone559fb512005-06-23 01:40:19 -04001550}
1551
Stephen Hemminger3de00b82007-01-08 11:26:30 -08001552static inline int responses_pending(const struct adapter *adapter)
1553{
1554 const struct respQ *Q = &adapter->sge->respQ;
1555 const struct respQ_e *e = &Q->entries[Q->cidx];
1556
1557 return (e->GenerationBit == Q->genbit);
1558}
1559
Scott Bardone559fb512005-06-23 01:40:19 -04001560/*
1561 * A simpler version of process_responses() that handles only pure (i.e.,
1562 * non data-carrying) responses. Such respones are too light-weight to justify
1563 * calling a softirq when using NAPI, so we handle them specially in hard
1564 * interrupt context. The function is called with a pointer to a response,
1565 * which the caller must ensure is a valid pure response. Returns 1 if it
1566 * encounters a valid data-carrying response, 0 otherwise.
1567 */
Stephen Hemminger3de00b82007-01-08 11:26:30 -08001568static int process_pure_responses(struct adapter *adapter)
Scott Bardone559fb512005-06-23 01:40:19 -04001569{
1570 struct sge *sge = adapter->sge;
1571 struct respQ *q = &sge->respQ;
Stephen Hemminger3de00b82007-01-08 11:26:30 -08001572 struct respQ_e *e = &q->entries[q->cidx];
Stephen Hemminger24a427c2007-01-08 11:26:12 -08001573 const struct freelQ *fl = &sge->freelQ[e->FreelistQid];
Scott Bardone559fb512005-06-23 01:40:19 -04001574 unsigned int flags = 0;
1575 unsigned int cmdq_processed[SGE_CMDQ_N] = {0, 0};
1576
Stephen Hemminger24a427c2007-01-08 11:26:12 -08001577 prefetch(fl->centries[fl->cidx].skb);
Stephen Hemminger3de00b82007-01-08 11:26:30 -08001578 if (e->DataValid)
1579 return 1;
Stephen Hemminger24a427c2007-01-08 11:26:12 -08001580
Scott Bardone559fb512005-06-23 01:40:19 -04001581 do {
1582 flags |= e->Qsleeping;
1583
1584 cmdq_processed[0] += e->Cmdq0CreditReturn;
1585 cmdq_processed[1] += e->Cmdq1CreditReturn;
Francois Romieu356bd142006-12-11 23:47:00 +01001586
Scott Bardone559fb512005-06-23 01:40:19 -04001587 e++;
1588 if (unlikely(++q->cidx == q->size)) {
1589 q->cidx = 0;
1590 q->genbit ^= 1;
1591 e = q->entries;
1592 }
1593 prefetch(e);
1594
1595 if (++q->credits > SGE_RESPQ_REPLENISH_THRES) {
1596 writel(q->credits, adapter->regs + A_SG_RSPQUEUECREDIT);
1597 q->credits = 0;
1598 }
1599 sge->stats.pure_rsps++;
1600 } while (e->GenerationBit == q->genbit && !e->DataValid);
1601
Francois Romieu356bd142006-12-11 23:47:00 +01001602 flags = update_tx_info(adapter, flags, cmdq_processed[0]);
Scott Bardone559fb512005-06-23 01:40:19 -04001603 sge->cmdQ[1].processed += cmdq_processed[1];
1604
1605 return e->GenerationBit == q->genbit;
1606}
1607
1608/*
1609 * Handler for new data events when using NAPI. This does not need any locking
1610 * or protection from interrupts as data interrupts are off at this point and
1611 * other adapter interrupts do not interfere.
1612 */
Stephen Hemmingerbea33482007-10-03 16:41:36 -07001613int t1_poll(struct napi_struct *napi, int budget)
Scott Bardone559fb512005-06-23 01:40:19 -04001614{
Stephen Hemmingerbea33482007-10-03 16:41:36 -07001615 struct adapter *adapter = container_of(napi, struct adapter, napi);
1616 struct net_device *dev = adapter->port[0].dev;
Divy Le Ray445cf802007-11-27 13:30:15 -08001617 int work_done = process_responses(adapter, budget);
Stephen Hemminger7fe26a62006-12-08 11:08:33 -08001618
Divy Le Ray445cf802007-11-27 13:30:15 -08001619 if (likely(work_done < budget)) {
Stephen Hemmingerbea33482007-10-03 16:41:36 -07001620 netif_rx_complete(dev, napi);
1621 writel(adapter->sge->respQ.cidx,
1622 adapter->regs + A_SG_SLEEPING);
1623 }
1624 return work_done;
Scott Bardone559fb512005-06-23 01:40:19 -04001625}
1626
Stephen Hemminger7fe26a62006-12-08 11:08:33 -08001627irqreturn_t t1_interrupt(int irq, void *data)
Scott Bardone559fb512005-06-23 01:40:19 -04001628{
Scott Bardone559fb512005-06-23 01:40:19 -04001629 struct adapter *adapter = data;
1630 struct sge *sge = adapter->sge;
Stephen Hemminger3de00b82007-01-08 11:26:30 -08001631 int handled;
Scott Bardone559fb512005-06-23 01:40:19 -04001632
Stephen Hemminger3de00b82007-01-08 11:26:30 -08001633 if (likely(responses_pending(adapter))) {
1634 struct net_device *dev = sge->netdev;
Scott Bardone559fb512005-06-23 01:40:19 -04001635
Francois Romieu356bd142006-12-11 23:47:00 +01001636 writel(F_PL_INTR_SGE_DATA, adapter->regs + A_PL_CAUSE);
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001637
Stephen Hemmingerbea33482007-10-03 16:41:36 -07001638 if (napi_schedule_prep(&adapter->napi)) {
Stephen Hemminger3de00b82007-01-08 11:26:30 -08001639 if (process_pure_responses(adapter))
Stephen Hemmingerbea33482007-10-03 16:41:36 -07001640 __netif_rx_schedule(dev, &adapter->napi);
Stephen Hemminger3de00b82007-01-08 11:26:30 -08001641 else {
1642 /* no data, no NAPI needed */
1643 writel(sge->respQ.cidx, adapter->regs + A_SG_SLEEPING);
Francois Romieu4422b002008-07-11 00:29:19 +02001644 /* undo schedule_prep */
1645 napi_enable(&adapter->napi);
Stephen Hemminger7fe26a62006-12-08 11:08:33 -08001646 }
Stephen Hemminger7fe26a62006-12-08 11:08:33 -08001647 }
Stephen Hemminger3de00b82007-01-08 11:26:30 -08001648 return IRQ_HANDLED;
1649 }
1650
1651 spin_lock(&adapter->async_lock);
1652 handled = t1_slow_intr_handler(adapter);
1653 spin_unlock(&adapter->async_lock);
Stephen Hemminger7fe26a62006-12-08 11:08:33 -08001654
Scott Bardone559fb512005-06-23 01:40:19 -04001655 if (!handled)
1656 sge->stats.unhandled_irqs++;
Stephen Hemminger3de00b82007-01-08 11:26:30 -08001657
Scott Bardone559fb512005-06-23 01:40:19 -04001658 return IRQ_RETVAL(handled != 0);
1659}
1660
Scott Bardone559fb512005-06-23 01:40:19 -04001661/*
1662 * Enqueues the sk_buff onto the cmdQ[qid] and has hardware fetch it.
1663 *
1664 * The code figures out how many entries the sk_buff will require in the
1665 * cmdQ and updates the cmdQ data structure with the state once the enqueue
1666 * has complete. Then, it doesn't access the global structure anymore, but
1667 * uses the corresponding fields on the stack. In conjuction with a spinlock
1668 * around that code, we can make the function reentrant without holding the
1669 * lock when we actually enqueue (which might be expensive, especially on
1670 * architectures with IO MMUs).
1671 *
1672 * This runs with softirqs disabled.
1673 */
Stephen Hemmingeraa845052005-12-14 14:38:44 -08001674static int t1_sge_tx(struct sk_buff *skb, struct adapter *adapter,
1675 unsigned int qid, struct net_device *dev)
Scott Bardone559fb512005-06-23 01:40:19 -04001676{
1677 struct sge *sge = adapter->sge;
1678 struct cmdQ *q = &sge->cmdQ[qid];
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001679 unsigned int credits, pidx, genbit, count, use_sched_skb = 0;
Scott Bardone559fb512005-06-23 01:40:19 -04001680
Stephen Hemmingercabdfb32006-12-01 16:36:22 -08001681 if (!spin_trylock(&q->lock))
1682 return NETDEV_TX_LOCKED;
1683
Scott Bardone559fb512005-06-23 01:40:19 -04001684 reclaim_completed_tx(sge, q);
1685
1686 pidx = q->pidx;
1687 credits = q->size - q->in_use;
1688 count = 1 + skb_shinfo(skb)->nr_frags;
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001689 count += compute_large_page_tx_descs(skb);
Scott Bardone559fb512005-06-23 01:40:19 -04001690
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001691 /* Ethernet packet */
1692 if (unlikely(credits < count)) {
1693 if (!netif_queue_stopped(dev)) {
Scott Bardone559fb512005-06-23 01:40:19 -04001694 netif_stop_queue(dev);
1695 set_bit(dev->if_port, &sge->stopped_tx_queues);
Scott Bardone232a3472006-03-16 19:20:40 -05001696 sge->stats.cmdQ_full[2]++;
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001697 CH_ERR("%s: Tx ring full while queue awake!\n",
1698 adapter->name);
Scott Bardone559fb512005-06-23 01:40:19 -04001699 }
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001700 spin_unlock(&q->lock);
1701 return NETDEV_TX_BUSY;
Scott Bardone559fb512005-06-23 01:40:19 -04001702 }
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001703
1704 if (unlikely(credits - count < q->stop_thres)) {
1705 netif_stop_queue(dev);
1706 set_bit(dev->if_port, &sge->stopped_tx_queues);
1707 sge->stats.cmdQ_full[2]++;
1708 }
1709
1710 /* T204 cmdQ0 skbs that are destined for a certain port have to go
1711 * through the scheduler.
1712 */
1713 if (sge->tx_sched && !qid && skb->dev) {
Francois Romieu356bd142006-12-11 23:47:00 +01001714use_sched:
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001715 use_sched_skb = 1;
1716 /* Note that the scheduler might return a different skb than
1717 * the one passed in.
1718 */
1719 skb = sched_skb(sge, skb, credits);
1720 if (!skb) {
1721 spin_unlock(&q->lock);
1722 return NETDEV_TX_OK;
1723 }
1724 pidx = q->pidx;
1725 count = 1 + skb_shinfo(skb)->nr_frags;
1726 count += compute_large_page_tx_descs(skb);
1727 }
1728
Scott Bardone559fb512005-06-23 01:40:19 -04001729 q->in_use += count;
1730 genbit = q->genbit;
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001731 pidx = q->pidx;
Scott Bardone559fb512005-06-23 01:40:19 -04001732 q->pidx += count;
1733 if (q->pidx >= q->size) {
1734 q->pidx -= q->size;
1735 q->genbit ^= 1;
1736 }
1737 spin_unlock(&q->lock);
1738
1739 write_tx_descs(adapter, skb, pidx, genbit, q);
1740
1741 /*
1742 * We always ring the doorbell for cmdQ1. For cmdQ0, we only ring
1743 * the doorbell if the Q is asleep. There is a natural race, where
1744 * the hardware is going to sleep just after we checked, however,
1745 * then the interrupt handler will detect the outstanding TX packet
1746 * and ring the doorbell for us.
1747 */
1748 if (qid)
1749 doorbell_pio(adapter, F_CMDQ1_ENABLE);
1750 else {
1751 clear_bit(CMDQ_STAT_LAST_PKT_DB, &q->status);
1752 if (test_and_set_bit(CMDQ_STAT_RUNNING, &q->status) == 0) {
1753 set_bit(CMDQ_STAT_LAST_PKT_DB, &q->status);
1754 writel(F_CMDQ0_ENABLE, adapter->regs + A_SG_DOORBELL);
1755 }
1756 }
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001757
1758 if (use_sched_skb) {
1759 if (spin_trylock(&q->lock)) {
1760 credits = q->size - q->in_use;
1761 skb = NULL;
1762 goto use_sched;
1763 }
1764 }
Stephen Hemmingeraa845052005-12-14 14:38:44 -08001765 return NETDEV_TX_OK;
Scott Bardone559fb512005-06-23 01:40:19 -04001766}
1767
1768#define MK_ETH_TYPE_MSS(type, mss) (((mss) & 0x3FFF) | ((type) << 14))
1769
1770/*
1771 * eth_hdr_len - return the length of an Ethernet header
1772 * @data: pointer to the start of the Ethernet header
1773 *
1774 * Returns the length of an Ethernet header, including optional VLAN tag.
1775 */
1776static inline int eth_hdr_len(const void *data)
1777{
1778 const struct ethhdr *e = data;
1779
1780 return e->h_proto == htons(ETH_P_8021Q) ? VLAN_ETH_HLEN : ETH_HLEN;
1781}
1782
1783/*
1784 * Adds the CPL header to the sk_buff and passes it to t1_sge_tx.
1785 */
1786int t1_start_xmit(struct sk_buff *skb, struct net_device *dev)
1787{
1788 struct adapter *adapter = dev->priv;
Scott Bardone559fb512005-06-23 01:40:19 -04001789 struct sge *sge = adapter->sge;
Divy Le Ray7832ee02007-11-27 13:30:09 -08001790 struct sge_port_stats *st = per_cpu_ptr(sge->port_stats[dev->if_port],
1791 smp_processor_id());
Scott Bardone559fb512005-06-23 01:40:19 -04001792 struct cpl_tx_pkt *cpl;
Stephen Hemmingercabdfb32006-12-01 16:36:22 -08001793 struct sk_buff *orig_skb = skb;
1794 int ret;
Scott Bardone559fb512005-06-23 01:40:19 -04001795
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001796 if (skb->protocol == htons(ETH_P_CPL5))
1797 goto send;
1798
Divy Le Ray7832ee02007-11-27 13:30:09 -08001799 /*
1800 * We are using a non-standard hard_header_len.
1801 * Allocate more header room in the rare cases it is not big enough.
1802 */
1803 if (unlikely(skb_headroom(skb) < dev->hard_header_len - ETH_HLEN)) {
1804 skb = skb_realloc_headroom(skb, sizeof(struct cpl_tx_pkt_lso));
1805 ++st->tx_need_hdrroom;
1806 dev_kfree_skb_any(orig_skb);
1807 if (!skb)
1808 return NETDEV_TX_OK;
1809 }
1810
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001811 if (skb_shinfo(skb)->gso_size) {
Scott Bardone559fb512005-06-23 01:40:19 -04001812 int eth_type;
1813 struct cpl_tx_pkt_lso *hdr;
1814
Stephen Hemminger56f643c2006-12-01 16:36:21 -08001815 ++st->tx_tso;
Scott Bardone559fb512005-06-23 01:40:19 -04001816
Arnaldo Carvalho de Melobbe735e2007-03-10 22:16:10 -03001817 eth_type = skb_network_offset(skb) == ETH_HLEN ?
Scott Bardone559fb512005-06-23 01:40:19 -04001818 CPL_ETH_II : CPL_ETH_II_VLAN;
1819
1820 hdr = (struct cpl_tx_pkt_lso *)skb_push(skb, sizeof(*hdr));
1821 hdr->opcode = CPL_TX_PKT_LSO;
1822 hdr->ip_csum_dis = hdr->l4_csum_dis = 0;
Arnaldo Carvalho de Meloeddc9ec2007-04-20 22:47:35 -07001823 hdr->ip_hdr_words = ip_hdr(skb)->ihl;
Arnaldo Carvalho de Meloaa8223c2007-04-10 21:04:22 -07001824 hdr->tcp_hdr_words = tcp_hdr(skb)->doff;
Scott Bardone559fb512005-06-23 01:40:19 -04001825 hdr->eth_type_mss = htons(MK_ETH_TYPE_MSS(eth_type,
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001826 skb_shinfo(skb)->gso_size));
Scott Bardone559fb512005-06-23 01:40:19 -04001827 hdr->len = htonl(skb->len - sizeof(*hdr));
1828 cpl = (struct cpl_tx_pkt *)hdr;
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001829 } else {
Scott Bardone559fb512005-06-23 01:40:19 -04001830 /*
Francois Romieu356bd142006-12-11 23:47:00 +01001831 * Packets shorter than ETH_HLEN can break the MAC, drop them
Scott Bardone559fb512005-06-23 01:40:19 -04001832 * early. Also, we may get oversized packets because some
1833 * parts of the kernel don't handle our unusual hard_header_len
1834 * right, drop those too.
1835 */
1836 if (unlikely(skb->len < ETH_HLEN ||
1837 skb->len > dev->mtu + eth_hdr_len(skb->data))) {
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001838 pr_debug("%s: packet size %d hdr %d mtu%d\n", dev->name,
1839 skb->len, eth_hdr_len(skb->data), dev->mtu);
Scott Bardone559fb512005-06-23 01:40:19 -04001840 dev_kfree_skb_any(skb);
Stephen Hemmingeraa845052005-12-14 14:38:44 -08001841 return NETDEV_TX_OK;
Scott Bardone559fb512005-06-23 01:40:19 -04001842 }
1843
Scott Bardone559fb512005-06-23 01:40:19 -04001844 if (!(adapter->flags & UDP_CSUM_CAPABLE) &&
Patrick McHardy84fa7932006-08-29 16:44:56 -07001845 skb->ip_summed == CHECKSUM_PARTIAL &&
Arnaldo Carvalho de Meloeddc9ec2007-04-20 22:47:35 -07001846 ip_hdr(skb)->protocol == IPPROTO_UDP) {
Patrick McHardy84fa7932006-08-29 16:44:56 -07001847 if (unlikely(skb_checksum_help(skb))) {
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001848 pr_debug("%s: unable to do udp checksum\n", dev->name);
Scott Bardone559fb512005-06-23 01:40:19 -04001849 dev_kfree_skb_any(skb);
Stephen Hemmingeraa845052005-12-14 14:38:44 -08001850 return NETDEV_TX_OK;
Scott Bardone559fb512005-06-23 01:40:19 -04001851 }
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001852 }
Scott Bardone559fb512005-06-23 01:40:19 -04001853
1854 /* Hmmm, assuming to catch the gratious arp... and we'll use
1855 * it to flush out stuck espi packets...
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001856 */
1857 if ((unlikely(!adapter->sge->espibug_skb[dev->if_port]))) {
Scott Bardone559fb512005-06-23 01:40:19 -04001858 if (skb->protocol == htons(ETH_P_ARP) &&
Arnaldo Carvalho de Melod0a92be2007-03-12 20:56:31 -03001859 arp_hdr(skb)->ar_op == htons(ARPOP_REQUEST)) {
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001860 adapter->sge->espibug_skb[dev->if_port] = skb;
Scott Bardone559fb512005-06-23 01:40:19 -04001861 /* We want to re-use this skb later. We
1862 * simply bump the reference count and it
1863 * will not be freed...
1864 */
1865 skb = skb_get(skb);
1866 }
1867 }
1868
1869 cpl = (struct cpl_tx_pkt *)__skb_push(skb, sizeof(*cpl));
1870 cpl->opcode = CPL_TX_PKT;
1871 cpl->ip_csum_dis = 1; /* SW calculates IP csum */
Patrick McHardy84fa7932006-08-29 16:44:56 -07001872 cpl->l4_csum_dis = skb->ip_summed == CHECKSUM_PARTIAL ? 0 : 1;
Scott Bardone559fb512005-06-23 01:40:19 -04001873 /* the length field isn't used so don't bother setting it */
1874
Patrick McHardy84fa7932006-08-29 16:44:56 -07001875 st->tx_cso += (skb->ip_summed == CHECKSUM_PARTIAL);
Scott Bardone559fb512005-06-23 01:40:19 -04001876 }
1877 cpl->iff = dev->if_port;
1878
1879#if defined(CONFIG_VLAN_8021Q) || defined(CONFIG_VLAN_8021Q_MODULE)
1880 if (adapter->vlan_grp && vlan_tx_tag_present(skb)) {
1881 cpl->vlan_valid = 1;
1882 cpl->vlan = htons(vlan_tx_tag_get(skb));
1883 st->vlan_insert++;
1884 } else
1885#endif
1886 cpl->vlan_valid = 0;
1887
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001888send:
Scott Bardone559fb512005-06-23 01:40:19 -04001889 dev->trans_start = jiffies;
Stephen Hemmingercabdfb32006-12-01 16:36:22 -08001890 ret = t1_sge_tx(skb, adapter, 0, dev);
1891
1892 /* If transmit busy, and we reallocated skb's due to headroom limit,
1893 * then silently discard to avoid leak.
1894 */
1895 if (unlikely(ret != NETDEV_TX_OK && skb != orig_skb)) {
Francois Romieu356bd142006-12-11 23:47:00 +01001896 dev_kfree_skb_any(skb);
Stephen Hemmingercabdfb32006-12-01 16:36:22 -08001897 ret = NETDEV_TX_OK;
Francois Romieu356bd142006-12-11 23:47:00 +01001898 }
Stephen Hemmingercabdfb32006-12-01 16:36:22 -08001899 return ret;
Scott Bardone559fb512005-06-23 01:40:19 -04001900}
1901
1902/*
1903 * Callback for the Tx buffer reclaim timer. Runs with softirqs disabled.
1904 */
1905static void sge_tx_reclaim_cb(unsigned long data)
1906{
1907 int i;
1908 struct sge *sge = (struct sge *)data;
1909
1910 for (i = 0; i < SGE_CMDQ_N; ++i) {
1911 struct cmdQ *q = &sge->cmdQ[i];
1912
1913 if (!spin_trylock(&q->lock))
1914 continue;
1915
1916 reclaim_completed_tx(sge, q);
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001917 if (i == 0 && q->in_use) { /* flush pending credits */
1918 writel(F_CMDQ0_ENABLE, sge->adapter->regs + A_SG_DOORBELL);
1919 }
Scott Bardone559fb512005-06-23 01:40:19 -04001920 spin_unlock(&q->lock);
1921 }
1922 mod_timer(&sge->tx_reclaim_timer, jiffies + TX_RECLAIM_PERIOD);
1923}
1924
1925/*
1926 * Propagate changes of the SGE coalescing parameters to the HW.
1927 */
1928int t1_sge_set_coalesce_params(struct sge *sge, struct sge_params *p)
1929{
Scott Bardone559fb512005-06-23 01:40:19 -04001930 sge->fixed_intrtimer = p->rx_coalesce_usecs *
1931 core_ticks_per_usec(sge->adapter);
1932 writel(sge->fixed_intrtimer, sge->adapter->regs + A_SG_INTRTIMER);
1933 return 0;
Christoph Lameter8199d3a2005-03-30 13:34:31 -08001934}
1935
1936/*
1937 * Allocates both RX and TX resources and configures the SGE. However,
1938 * the hardware is not enabled yet.
1939 */
1940int t1_sge_configure(struct sge *sge, struct sge_params *p)
1941{
1942 if (alloc_rx_resources(sge, p))
1943 return -ENOMEM;
1944 if (alloc_tx_resources(sge, p)) {
1945 free_rx_resources(sge);
1946 return -ENOMEM;
1947 }
1948 configure_sge(sge, p);
1949
1950 /*
1951 * Now that we have sized the free lists calculate the payload
1952 * capacity of the large buffers. Other parts of the driver use
1953 * this to set the max offload coalescing size so that RX packets
1954 * do not overflow our large buffers.
1955 */
1956 p->large_buf_capacity = jumbo_payload_capacity(sge);
1957 return 0;
1958}
1959
1960/*
Scott Bardone559fb512005-06-23 01:40:19 -04001961 * Disables the DMA engine.
Christoph Lameter8199d3a2005-03-30 13:34:31 -08001962 */
Scott Bardone559fb512005-06-23 01:40:19 -04001963void t1_sge_stop(struct sge *sge)
Christoph Lameter8199d3a2005-03-30 13:34:31 -08001964{
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001965 int i;
Scott Bardone559fb512005-06-23 01:40:19 -04001966 writel(0, sge->adapter->regs + A_SG_CONTROL);
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001967 readl(sge->adapter->regs + A_SG_CONTROL); /* flush */
1968
Scott Bardone559fb512005-06-23 01:40:19 -04001969 if (is_T2(sge->adapter))
1970 del_timer_sync(&sge->espibug_timer);
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001971
Scott Bardone559fb512005-06-23 01:40:19 -04001972 del_timer_sync(&sge->tx_reclaim_timer);
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001973 if (sge->tx_sched)
1974 tx_sched_stop(sge);
1975
1976 for (i = 0; i < MAX_NPORTS; i++)
1977 if (sge->espibug_skb[i])
1978 kfree_skb(sge->espibug_skb[i]);
Christoph Lameter8199d3a2005-03-30 13:34:31 -08001979}
1980
1981/*
Scott Bardone559fb512005-06-23 01:40:19 -04001982 * Enables the DMA engine.
Christoph Lameter8199d3a2005-03-30 13:34:31 -08001983 */
Scott Bardone559fb512005-06-23 01:40:19 -04001984void t1_sge_start(struct sge *sge)
Christoph Lameter8199d3a2005-03-30 13:34:31 -08001985{
Christoph Lameter8199d3a2005-03-30 13:34:31 -08001986 refill_free_list(sge, &sge->freelQ[0]);
1987 refill_free_list(sge, &sge->freelQ[1]);
1988
Scott Bardone559fb512005-06-23 01:40:19 -04001989 writel(sge->sge_control, sge->adapter->regs + A_SG_CONTROL);
1990 doorbell_pio(sge->adapter, F_FL0_ENABLE | F_FL1_ENABLE);
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001991 readl(sge->adapter->regs + A_SG_CONTROL); /* flush */
Christoph Lameter8199d3a2005-03-30 13:34:31 -08001992
Scott Bardone559fb512005-06-23 01:40:19 -04001993 mod_timer(&sge->tx_reclaim_timer, jiffies + TX_RECLAIM_PERIOD);
Christoph Lameter8199d3a2005-03-30 13:34:31 -08001994
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08001995 if (is_T2(sge->adapter))
Scott Bardone559fb512005-06-23 01:40:19 -04001996 mod_timer(&sge->espibug_timer, jiffies + sge->espibug_timeout);
Christoph Lameter8199d3a2005-03-30 13:34:31 -08001997}
1998
1999/*
Scott Bardone559fb512005-06-23 01:40:19 -04002000 * Callback for the T2 ESPI 'stuck packet feature' workaorund
Christoph Lameter8199d3a2005-03-30 13:34:31 -08002001 */
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08002002static void espibug_workaround_t204(unsigned long data)
2003{
2004 struct adapter *adapter = (struct adapter *)data;
2005 struct sge *sge = adapter->sge;
2006 unsigned int nports = adapter->params.nports;
2007 u32 seop[MAX_NPORTS];
2008
2009 if (adapter->open_device_map & PORT_MASK) {
2010 int i;
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08002011
Francois Romieu356bd142006-12-11 23:47:00 +01002012 if (t1_espi_get_mon_t204(adapter, &(seop[0]), 0) < 0)
2013 return;
2014
2015 for (i = 0; i < nports; i++) {
2016 struct sk_buff *skb = sge->espibug_skb[i];
2017
2018 if (!netif_running(adapter->port[i].dev) ||
2019 netif_queue_stopped(adapter->port[i].dev) ||
2020 !seop[i] || ((seop[i] & 0xfff) != 0) || !skb)
2021 continue;
2022
2023 if (!skb->cb[0]) {
2024 u8 ch_mac_addr[ETH_ALEN] = {
2025 0x0, 0x7, 0x43, 0x0, 0x0, 0x0
2026 };
2027
Arnaldo Carvalho de Melo27d7ff42007-03-31 11:55:19 -03002028 skb_copy_to_linear_data_offset(skb,
2029 sizeof(struct cpl_tx_pkt),
2030 ch_mac_addr,
2031 ETH_ALEN);
2032 skb_copy_to_linear_data_offset(skb,
2033 skb->len - 10,
2034 ch_mac_addr,
2035 ETH_ALEN);
Francois Romieu356bd142006-12-11 23:47:00 +01002036 skb->cb[0] = 0xff;
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08002037 }
Francois Romieu356bd142006-12-11 23:47:00 +01002038
2039 /* bump the reference count to avoid freeing of
2040 * the skb once the DMA has completed.
2041 */
2042 skb = skb_get(skb);
2043 t1_sge_tx(skb, adapter, 0, adapter->port[i].dev);
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08002044 }
2045 }
2046 mod_timer(&sge->espibug_timer, jiffies + sge->espibug_timeout);
2047}
2048
2049static void espibug_workaround(unsigned long data)
Christoph Lameter8199d3a2005-03-30 13:34:31 -08002050{
Scott Bardone559fb512005-06-23 01:40:19 -04002051 struct adapter *adapter = (struct adapter *)data;
Christoph Lameter8199d3a2005-03-30 13:34:31 -08002052 struct sge *sge = adapter->sge;
Christoph Lameter8199d3a2005-03-30 13:34:31 -08002053
Scott Bardone559fb512005-06-23 01:40:19 -04002054 if (netif_running(adapter->port[0].dev)) {
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08002055 struct sk_buff *skb = sge->espibug_skb[0];
2056 u32 seop = t1_espi_get_mon(adapter, 0x930, 0);
Christoph Lameter8199d3a2005-03-30 13:34:31 -08002057
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08002058 if ((seop & 0xfff0fff) == 0xfff && skb) {
2059 if (!skb->cb[0]) {
2060 u8 ch_mac_addr[ETH_ALEN] =
2061 {0x0, 0x7, 0x43, 0x0, 0x0, 0x0};
Arnaldo Carvalho de Melo27d7ff42007-03-31 11:55:19 -03002062 skb_copy_to_linear_data_offset(skb,
2063 sizeof(struct cpl_tx_pkt),
2064 ch_mac_addr,
2065 ETH_ALEN);
2066 skb_copy_to_linear_data_offset(skb,
2067 skb->len - 10,
2068 ch_mac_addr,
2069 ETH_ALEN);
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08002070 skb->cb[0] = 0xff;
2071 }
Christoph Lameter8199d3a2005-03-30 13:34:31 -08002072
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08002073 /* bump the reference count to avoid freeing of the
2074 * skb once the DMA has completed.
2075 */
2076 skb = skb_get(skb);
2077 t1_sge_tx(skb, adapter, 0, adapter->port[0].dev);
2078 }
Christoph Lameter8199d3a2005-03-30 13:34:31 -08002079 }
Scott Bardone559fb512005-06-23 01:40:19 -04002080 mod_timer(&sge->espibug_timer, jiffies + sge->espibug_timeout);
Christoph Lameter8199d3a2005-03-30 13:34:31 -08002081}
2082
Christoph Lameter8199d3a2005-03-30 13:34:31 -08002083/*
Scott Bardone559fb512005-06-23 01:40:19 -04002084 * Creates a t1_sge structure and returns suggested resource parameters.
Christoph Lameter8199d3a2005-03-30 13:34:31 -08002085 */
Scott Bardone559fb512005-06-23 01:40:19 -04002086struct sge * __devinit t1_sge_create(struct adapter *adapter,
2087 struct sge_params *p)
Christoph Lameter8199d3a2005-03-30 13:34:31 -08002088{
Stephen Hemmingercbee9f92006-11-17 17:01:52 -08002089 struct sge *sge = kzalloc(sizeof(*sge), GFP_KERNEL);
Stephen Hemminger56f643c2006-12-01 16:36:21 -08002090 int i;
Christoph Lameter8199d3a2005-03-30 13:34:31 -08002091
Scott Bardone559fb512005-06-23 01:40:19 -04002092 if (!sge)
2093 return NULL;
Christoph Lameter8199d3a2005-03-30 13:34:31 -08002094
Scott Bardone559fb512005-06-23 01:40:19 -04002095 sge->adapter = adapter;
2096 sge->netdev = adapter->port[0].dev;
2097 sge->rx_pkt_pad = t1_is_T1B(adapter) ? 0 : 2;
2098 sge->jumbo_fl = t1_is_T1B(adapter) ? 1 : 0;
2099
Stephen Hemminger56f643c2006-12-01 16:36:21 -08002100 for_each_port(adapter, i) {
2101 sge->port_stats[i] = alloc_percpu(struct sge_port_stats);
2102 if (!sge->port_stats[i])
2103 goto nomem_port;
2104 }
2105
Scott Bardone559fb512005-06-23 01:40:19 -04002106 init_timer(&sge->tx_reclaim_timer);
2107 sge->tx_reclaim_timer.data = (unsigned long)sge;
2108 sge->tx_reclaim_timer.function = sge_tx_reclaim_cb;
2109
2110 if (is_T2(sge->adapter)) {
2111 init_timer(&sge->espibug_timer);
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08002112
2113 if (adapter->params.nports > 1) {
2114 tx_sched_init(sge);
2115 sge->espibug_timer.function = espibug_workaround_t204;
Francois Romieud7487422006-12-11 23:49:13 +01002116 } else
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08002117 sge->espibug_timer.function = espibug_workaround;
Scott Bardone559fb512005-06-23 01:40:19 -04002118 sge->espibug_timer.data = (unsigned long)sge->adapter;
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08002119
Scott Bardone559fb512005-06-23 01:40:19 -04002120 sge->espibug_timeout = 1;
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08002121 /* for T204, every 10ms */
2122 if (adapter->params.nports > 1)
2123 sge->espibug_timeout = HZ/100;
Christoph Lameter8199d3a2005-03-30 13:34:31 -08002124 }
Francois Romieu356bd142006-12-11 23:47:00 +01002125
Christoph Lameter8199d3a2005-03-30 13:34:31 -08002126
Scott Bardone559fb512005-06-23 01:40:19 -04002127 p->cmdQ_size[0] = SGE_CMDQ0_E_N;
2128 p->cmdQ_size[1] = SGE_CMDQ1_E_N;
2129 p->freelQ_size[!sge->jumbo_fl] = SGE_FREEL_SIZE;
2130 p->freelQ_size[sge->jumbo_fl] = SGE_JUMBO_FREEL_SIZE;
Stephen Hemmingerf1d3d382006-12-01 16:36:16 -08002131 if (sge->tx_sched) {
2132 if (board_info(sge->adapter)->board == CHBT_BOARD_CHT204)
2133 p->rx_coalesce_usecs = 15;
2134 else
2135 p->rx_coalesce_usecs = 50;
2136 } else
2137 p->rx_coalesce_usecs = 50;
2138
Scott Bardone559fb512005-06-23 01:40:19 -04002139 p->coalesce_enable = 0;
2140 p->sample_interval_usecs = 0;
Christoph Lameter8199d3a2005-03-30 13:34:31 -08002141
Scott Bardone559fb512005-06-23 01:40:19 -04002142 return sge;
Stephen Hemminger56f643c2006-12-01 16:36:21 -08002143nomem_port:
2144 while (i >= 0) {
2145 free_percpu(sge->port_stats[i]);
2146 --i;
2147 }
2148 kfree(sge);
2149 return NULL;
2150
Christoph Lameter8199d3a2005-03-30 13:34:31 -08002151}