[PATCH] e1000: Added RX buffer enhancements
Align the prefetches to a dword to help speed them up.
Recycle skb's and early replenish.
Force memory writes to complete before fetching more descriptors.
Signed-off-by: Jeff Kirsher <jeffrey.t.kirsher@intel.com>
Signed-off-by: Jeff Kirsher <jeffrey.t.kirsher@intel.com>
Signed-off-by: John Ronciak <john.ronciak@intel.com>
Signed-off-by: Jeff Garzik <jgarzik@pobox.com>
diff --git a/drivers/net/e1000/e1000_main.c b/drivers/net/e1000/e1000_main.c
index 73b2a7b..53a9cca 100644
--- a/drivers/net/e1000/e1000_main.c
+++ b/drivers/net/e1000/e1000_main.c
@@ -1653,23 +1653,8 @@
rctl |= adapter->rx_buffer_len << 0x11;
} else {
rctl &= ~E1000_RCTL_SZ_4096;
- rctl |= E1000_RCTL_BSEX;
- switch (adapter->rx_buffer_len) {
- case E1000_RXBUFFER_2048:
- default:
- rctl |= E1000_RCTL_SZ_2048;
- rctl &= ~E1000_RCTL_BSEX;
- break;
- case E1000_RXBUFFER_4096:
- rctl |= E1000_RCTL_SZ_4096;
- break;
- case E1000_RXBUFFER_8192:
- rctl |= E1000_RCTL_SZ_8192;
- break;
- case E1000_RXBUFFER_16384:
- rctl |= E1000_RCTL_SZ_16384;
- break;
- }
+ rctl &= ~E1000_RCTL_BSEX;
+ rctl |= E1000_RCTL_SZ_2048;
}
#ifndef CONFIG_E1000_DISABLE_PACKET_SPLIT
@@ -3571,7 +3556,6 @@
struct pci_dev *pdev = adapter->pdev;
struct e1000_rx_desc *rx_desc;
struct e1000_buffer *buffer_info;
- struct sk_buff *skb;
unsigned long flags;
uint32_t length;
uint8_t last_byte;
@@ -3581,9 +3565,10 @@
i = rx_ring->next_to_clean;
rx_desc = E1000_RX_DESC(*rx_ring, i);
+ buffer_info = &rx_ring->buffer_info[i];
- while(rx_desc->status & E1000_RXD_STAT_DD) {
- buffer_info = &rx_ring->buffer_info[i];
+ while (rx_desc->status & E1000_RXD_STAT_DD) {
+ struct sk_buff *skb;
u8 status;
#ifdef CONFIG_E1000_NAPI
if(*work_done >= work_to_do)
@@ -3591,6 +3576,7 @@
(*work_done)++;
#endif
status = rx_desc->status;
+ skb = buffer_info->skb;
cleaned = TRUE;
cleaned_count++;
pci_unmap_single(pdev,
@@ -3598,20 +3584,50 @@
buffer_info->length,
PCI_DMA_FROMDEVICE);
- skb = buffer_info->skb;
length = le16_to_cpu(rx_desc->length);
- if(unlikely(!(rx_desc->status & E1000_RXD_STAT_EOP))) {
- /* All receives must fit into a single buffer */
- E1000_DBG("%s: Receive packet consumed multiple"
- " buffers\n", netdev->name);
- dev_kfree_skb_irq(skb);
+ skb_put(skb, length);
+
+ if (!(status & E1000_RXD_STAT_EOP)) {
+ if (!rx_ring->rx_skb_top) {
+ rx_ring->rx_skb_top = skb;
+ rx_ring->rx_skb_top->len = length;
+ rx_ring->rx_skb_prev = skb;
+ } else {
+ if (skb_shinfo(rx_ring->rx_skb_top)->frag_list) {
+ rx_ring->rx_skb_prev->next = skb;
+ skb->prev = rx_ring->rx_skb_prev;
+ } else {
+ skb_shinfo(rx_ring->rx_skb_top)->frag_list = skb;
+ }
+ rx_ring->rx_skb_prev = skb;
+ rx_ring->rx_skb_top->data_len += length;
+ }
goto next_desc;
+ } else {
+ if (rx_ring->rx_skb_top) {
+ if (skb_shinfo(rx_ring->rx_skb_top)
+ ->frag_list) {
+ rx_ring->rx_skb_prev->next = skb;
+ skb->prev = rx_ring->rx_skb_prev;
+ } else
+ skb_shinfo(rx_ring->rx_skb_top)
+ ->frag_list = skb;
+
+ rx_ring->rx_skb_top->data_len += length;
+ rx_ring->rx_skb_top->len +=
+ rx_ring->rx_skb_top->data_len;
+
+ skb = rx_ring->rx_skb_top;
+ multi_descriptor = TRUE;
+ rx_ring->rx_skb_top = NULL;
+ rx_ring->rx_skb_prev = NULL;
+ }
}
if(unlikely(rx_desc->errors & E1000_RXD_ERR_FRAME_ERR_MASK)) {
last_byte = *(skb->data + length - 1);
- if(TBI_ACCEPT(&adapter->hw, rx_desc->status,
+ if (TBI_ACCEPT(&adapter->hw, status,
rx_desc->errors, length, last_byte)) {
spin_lock_irqsave(&adapter->stats_lock, flags);
e1000_tbi_adjust_stats(&adapter->hw,
@@ -3668,7 +3684,7 @@
}
#else /* CONFIG_E1000_NAPI */
if(unlikely(adapter->vlgrp &&
- (rx_desc->status & E1000_RXD_STAT_VP))) {
+ (status & E1000_RXD_STAT_VP))) {
vlan_hwaccel_rx(skb, adapter->vlgrp,
le16_to_cpu(rx_desc->special) &
E1000_RXD_SPC_VLAN_MASK);
@@ -3795,12 +3811,8 @@
skb->protocol = eth_type_trans(skb, netdev);
if(likely(rx_desc->wb.upper.header_status &
- E1000_RXDPS_HDRSTAT_HDRSP)) {
+ E1000_RXDPS_HDRSTAT_HDRSP))
adapter->rx_hdr_split++;
-#ifdef HAVE_RX_ZERO_COPY
- skb_shinfo(skb)->zero_copy = TRUE;
-#endif
- }
#ifdef CONFIG_E1000_NAPI
if(unlikely(adapter->vlgrp && (staterr & E1000_RXD_STAT_VP))) {
vlan_hwaccel_receive_skb(skb, adapter->vlgrp,
@@ -3940,20 +3952,22 @@
rx_desc = E1000_RX_DESC(*rx_ring, i);
rx_desc->buffer_addr = cpu_to_le64(buffer_info->dma);
- if(unlikely((i & ~(E1000_RX_BUFFER_WRITE - 1)) == i)) {
- /* Force memory writes to complete before letting h/w
- * know there are new descriptors to fetch. (Only
- * applicable for weak-ordered memory model archs,
- * such as IA-64). */
- wmb();
- writel(i, adapter->hw.hw_addr + rx_ring->rdt);
- }
-
if(unlikely(++i == rx_ring->count)) i = 0;
buffer_info = &rx_ring->buffer_info[i];
}
- rx_ring->next_to_use = i;
+ if (likely(rx_ring->next_to_use != i)) {
+ rx_ring->next_to_use = i;
+ if (unlikely(i-- == 0))
+ i = (rx_ring->count - 1);
+
+ /* Force memory writes to complete before letting h/w
+ * know there are new descriptors to fetch. (Only
+ * applicable for weak-ordered memory model archs,
+ * such as IA-64). */
+ wmb();
+ writel(i, adapter->hw.hw_addr + rx_ring->rdt);
+ }
}
/**
@@ -3988,8 +4002,10 @@
if (likely(!ps_page->ps_page[j])) {
ps_page->ps_page[j] =
alloc_page(GFP_ATOMIC);
- if (unlikely(!ps_page->ps_page[j]))
+ if (unlikely(!ps_page->ps_page[j])) {
+ adapter->alloc_rx_buff_failed++;
goto no_buffers;
+ }
ps_page_dma->ps_page_dma[j] =
pci_map_page(pdev,
ps_page->ps_page[j],
@@ -4008,8 +4024,10 @@
skb = dev_alloc_skb(adapter->rx_ps_bsize0 + NET_IP_ALIGN);
- if(unlikely(!skb))
+ if (unlikely(!skb)) {
+ adapter->alloc_rx_buff_failed++;
break;
+ }
/* Make buffer alignment 2 beyond a 16 byte boundary
* this will result in a 16 byte aligned IP header after
@@ -4027,19 +4045,6 @@
rx_desc->read.buffer_addr[0] = cpu_to_le64(buffer_info->dma);
- if(unlikely((i & ~(E1000_RX_BUFFER_WRITE - 1)) == i)) {
- /* Force memory writes to complete before letting h/w
- * know there are new descriptors to fetch. (Only
- * applicable for weak-ordered memory model archs,
- * such as IA-64). */
- wmb();
- /* Hardware increments by 16 bytes, but packet split
- * descriptors are 32 bytes...so we increment tail
- * twice as much.
- */
- writel(i<<1, adapter->hw.hw_addr + rx_ring->rdt);
- }
-
if(unlikely(++i == rx_ring->count)) i = 0;
buffer_info = &rx_ring->buffer_info[i];
ps_page = &rx_ring->ps_page[i];
@@ -4047,7 +4052,21 @@
}
no_buffers:
- rx_ring->next_to_use = i;
+ if (likely(rx_ring->next_to_use != i)) {
+ rx_ring->next_to_use = i;
+ if (unlikely(i-- == 0)) i = (rx_ring->count - 1);
+
+ /* Force memory writes to complete before letting h/w
+ * know there are new descriptors to fetch. (Only
+ * applicable for weak-ordered memory model archs,
+ * such as IA-64). */
+ wmb();
+ /* Hardware increments by 16 bytes, but packet split
+ * descriptors are 32 bytes...so we increment tail
+ * twice as much.
+ */
+ writel(i<<1, adapter->hw.hw_addr + rx_ring->rdt);
+ }
}
/**