| Eric Dumazet | c9e6bc6 | 2012-09-27 19:29:05 +0000 | [diff] [blame^] | 1 | #ifndef _NET_GRO_CELLS_H |
| 2 | #define _NET_GRO_CELLS_H |
| 3 | |
| 4 | #include <linux/skbuff.h> |
| 5 | #include <linux/slab.h> |
| 6 | #include <linux/netdevice.h> |
| 7 | |
| 8 | struct gro_cell { |
| 9 | struct sk_buff_head napi_skbs; |
| 10 | struct napi_struct napi; |
| 11 | } ____cacheline_aligned_in_smp; |
| 12 | |
| 13 | struct gro_cells { |
| 14 | unsigned int gro_cells_mask; |
| 15 | struct gro_cell *cells; |
| 16 | }; |
| 17 | |
| 18 | static inline void gro_cells_receive(struct gro_cells *gcells, struct sk_buff *skb) |
| 19 | { |
| 20 | unsigned long flags; |
| 21 | struct gro_cell *cell = gcells->cells; |
| 22 | struct net_device *dev = skb->dev; |
| 23 | |
| 24 | if (!cell || skb_cloned(skb) || !(dev->features & NETIF_F_GRO)) { |
| 25 | netif_rx(skb); |
| 26 | return; |
| 27 | } |
| 28 | |
| 29 | if (skb_rx_queue_recorded(skb)) |
| 30 | cell += skb_get_rx_queue(skb) & gcells->gro_cells_mask; |
| 31 | |
| 32 | if (skb_queue_len(&cell->napi_skbs) > netdev_max_backlog) { |
| 33 | atomic_long_inc(&dev->rx_dropped); |
| 34 | kfree_skb(skb); |
| 35 | return; |
| 36 | } |
| 37 | |
| 38 | spin_lock_irqsave(&cell->napi_skbs.lock, flags); |
| 39 | |
| 40 | __skb_queue_tail(&cell->napi_skbs, skb); |
| 41 | if (skb_queue_len(&cell->napi_skbs) == 1) |
| 42 | napi_schedule(&cell->napi); |
| 43 | |
| 44 | spin_unlock_irqrestore(&cell->napi_skbs.lock, flags); |
| 45 | } |
| 46 | |
| 47 | static inline int gro_cell_poll(struct napi_struct *napi, int budget) |
| 48 | { |
| 49 | struct gro_cell *cell = container_of(napi, struct gro_cell, napi); |
| 50 | struct sk_buff *skb; |
| 51 | int work_done = 0; |
| 52 | |
| 53 | while (work_done < budget) { |
| 54 | skb = skb_dequeue(&cell->napi_skbs); |
| 55 | if (!skb) |
| 56 | break; |
| 57 | |
| 58 | napi_gro_receive(napi, skb); |
| 59 | work_done++; |
| 60 | } |
| 61 | |
| 62 | if (work_done < budget) |
| 63 | napi_complete(napi); |
| 64 | return work_done; |
| 65 | } |
| 66 | |
| 67 | static inline int gro_cells_init(struct gro_cells *gcells, struct net_device *dev) |
| 68 | { |
| 69 | int i; |
| 70 | |
| 71 | gcells->gro_cells_mask = roundup_pow_of_two(netif_get_num_default_rss_queues()) - 1; |
| 72 | gcells->cells = kcalloc(sizeof(struct gro_cell), |
| 73 | gcells->gro_cells_mask + 1, |
| 74 | GFP_KERNEL); |
| 75 | if (!gcells->cells) |
| 76 | return -ENOMEM; |
| 77 | |
| 78 | for (i = 0; i <= gcells->gro_cells_mask; i++) { |
| 79 | struct gro_cell *cell = gcells->cells + i; |
| 80 | |
| 81 | skb_queue_head_init(&cell->napi_skbs); |
| 82 | netif_napi_add(dev, &cell->napi, gro_cell_poll, 64); |
| 83 | napi_enable(&cell->napi); |
| 84 | } |
| 85 | return 0; |
| 86 | } |
| 87 | |
| 88 | static inline void gro_cells_destroy(struct gro_cells *gcells) |
| 89 | { |
| 90 | struct gro_cell *cell = gcells->cells; |
| 91 | int i; |
| 92 | |
| 93 | if (!cell) |
| 94 | return; |
| 95 | for (i = 0; i <= gcells->gro_cells_mask; i++,cell++) { |
| 96 | netif_napi_del(&cell->napi); |
| 97 | skb_queue_purge(&cell->napi_skbs); |
| 98 | } |
| 99 | kfree(gcells->cells); |
| 100 | gcells->cells = NULL; |
| 101 | } |
| 102 | |
| 103 | #endif |