net: skb_shared_info optimization
skb_dma_unmap() is quite expensive for small packets,
because we use two different cache lines from skb_shared_info.
One to access nr_frags, one to access dma_maps[0]
Instead of dma_maps being an array of MAX_SKB_FRAGS + 1 elements,
let dma_head alone in a new dma_head field, close to nr_frags,
to reduce cache lines misses.
Tested on my dev machine (bnx2 & tg3 adapters), nice speedup !
Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
diff --git a/drivers/net/bnx2.c b/drivers/net/bnx2.c
index f530172..391d2d4 100644
--- a/drivers/net/bnx2.c
+++ b/drivers/net/bnx2.c
@@ -5487,7 +5487,7 @@
dev_kfree_skb(skb);
return -EIO;
}
- map = skb_shinfo(skb)->dma_maps[0];
+ map = skb_shinfo(skb)->dma_head;
REG_WR(bp, BNX2_HC_COMMAND,
bp->hc_cmd | BNX2_HC_COMMAND_COAL_NOW_WO_INT);
@@ -6167,7 +6167,7 @@
}
sp = skb_shinfo(skb);
- mapping = sp->dma_maps[0];
+ mapping = sp->dma_head;
tx_buf = &txr->tx_buf_ring[ring_prod];
tx_buf->skb = skb;
@@ -6191,7 +6191,7 @@
txbd = &txr->tx_desc_ring[ring_prod];
len = frag->size;
- mapping = sp->dma_maps[i + 1];
+ mapping = sp->dma_maps[i];
txbd->tx_bd_haddr_hi = (u64) mapping >> 32;
txbd->tx_bd_haddr_lo = (u64) mapping & 0xffffffff;
diff --git a/drivers/net/e1000/e1000_main.c b/drivers/net/e1000/e1000_main.c
index 05e87a5..8d36743 100644
--- a/drivers/net/e1000/e1000_main.c
+++ b/drivers/net/e1000/e1000_main.c
@@ -2998,7 +2998,7 @@
size -= 4;
buffer_info->length = size;
- buffer_info->dma = map[0] + offset;
+ buffer_info->dma = skb_shinfo(skb)->dma_head + offset;
buffer_info->time_stamp = jiffies;
buffer_info->next_to_watch = i;
@@ -3039,7 +3039,7 @@
size -= 4;
buffer_info->length = size;
- buffer_info->dma = map[f + 1] + offset;
+ buffer_info->dma = map[f] + offset;
buffer_info->time_stamp = jiffies;
buffer_info->next_to_watch = i;
diff --git a/drivers/net/e1000e/netdev.c b/drivers/net/e1000e/netdev.c
index 38694c7..9043f1b 100644
--- a/drivers/net/e1000e/netdev.c
+++ b/drivers/net/e1000e/netdev.c
@@ -3916,7 +3916,7 @@
buffer_info->length = size;
buffer_info->time_stamp = jiffies;
buffer_info->next_to_watch = i;
- buffer_info->dma = map[0] + offset;
+ buffer_info->dma = skb_shinfo(skb)->dma_head + offset;
count++;
len -= size;
@@ -3947,7 +3947,7 @@
buffer_info->length = size;
buffer_info->time_stamp = jiffies;
buffer_info->next_to_watch = i;
- buffer_info->dma = map[f + 1] + offset;
+ buffer_info->dma = map[f] + offset;
len -= size;
offset += size;
diff --git a/drivers/net/igb/igb_main.c b/drivers/net/igb/igb_main.c
index 958b287..ea17319 100644
--- a/drivers/net/igb/igb_main.c
+++ b/drivers/net/igb/igb_main.c
@@ -3139,8 +3139,7 @@
/* set time_stamp *before* dma to help avoid a possible race */
buffer_info->time_stamp = jiffies;
buffer_info->next_to_watch = i;
- buffer_info->dma = map[count];
- count++;
+ buffer_info->dma = skb_shinfo(skb)->dma_head;
for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) {
struct skb_frag_struct *frag;
@@ -3164,7 +3163,7 @@
tx_ring->buffer_info[i].skb = skb;
tx_ring->buffer_info[first].next_to_watch = i;
- return count;
+ return count + 1;
}
static inline void igb_tx_queue_adv(struct igb_adapter *adapter,
diff --git a/drivers/net/igbvf/netdev.c b/drivers/net/igbvf/netdev.c
index 5f7ba1a4..22aadb7 100644
--- a/drivers/net/igbvf/netdev.c
+++ b/drivers/net/igbvf/netdev.c
@@ -2119,8 +2119,7 @@
/* set time_stamp *before* dma to help avoid a possible race */
buffer_info->time_stamp = jiffies;
buffer_info->next_to_watch = i;
- buffer_info->dma = map[count];
- count++;
+ buffer_info->dma = skb_shinfo(skb)->dma_head;
for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) {
struct skb_frag_struct *frag;
@@ -2144,7 +2143,7 @@
tx_ring->buffer_info[i].skb = skb;
tx_ring->buffer_info[first].next_to_watch = i;
- return count;
+ return count + 1;
}
static inline void igbvf_tx_queue_adv(struct igbvf_adapter *adapter,
diff --git a/drivers/net/ixgb/ixgb_main.c b/drivers/net/ixgb/ixgb_main.c
index 6eb7f37..9c897cf 100644
--- a/drivers/net/ixgb/ixgb_main.c
+++ b/drivers/net/ixgb/ixgb_main.c
@@ -1300,7 +1300,7 @@
buffer_info->length = size;
WARN_ON(buffer_info->dma != 0);
buffer_info->time_stamp = jiffies;
- buffer_info->dma = map[0] + offset;
+ buffer_info->dma = skb_shinfo(skb)->dma_head + offset;
pci_map_single(adapter->pdev,
skb->data + offset,
size,
@@ -1340,7 +1340,7 @@
buffer_info->length = size;
buffer_info->time_stamp = jiffies;
- buffer_info->dma = map[f + 1] + offset;
+ buffer_info->dma = map[f] + offset;
buffer_info->next_to_watch = 0;
len -= size;
diff --git a/drivers/net/ixgbe/ixgbe_main.c b/drivers/net/ixgbe/ixgbe_main.c
index d36003c..09994e9 100644
--- a/drivers/net/ixgbe/ixgbe_main.c
+++ b/drivers/net/ixgbe/ixgbe_main.c
@@ -4837,7 +4837,7 @@
size = min(len, (uint)IXGBE_MAX_DATA_PER_TXD);
tx_buffer_info->length = size;
- tx_buffer_info->dma = map[0] + offset;
+ tx_buffer_info->dma = skb_shinfo(skb)->dma_head + offset;
tx_buffer_info->time_stamp = jiffies;
tx_buffer_info->next_to_watch = i;
@@ -4869,7 +4869,7 @@
size = min(len, (uint)IXGBE_MAX_DATA_PER_TXD);
tx_buffer_info->length = size;
- tx_buffer_info->dma = map[f + 1] + offset;
+ tx_buffer_info->dma = map[f] + offset;
tx_buffer_info->time_stamp = jiffies;
tx_buffer_info->next_to_watch = i;
diff --git a/drivers/net/tg3.c b/drivers/net/tg3.c
index a39b534..46a3f86 100644
--- a/drivers/net/tg3.c
+++ b/drivers/net/tg3.c
@@ -5021,7 +5021,7 @@
/* New SKB is guaranteed to be linear. */
entry = *start;
ret = skb_dma_map(&tp->pdev->dev, new_skb, DMA_TO_DEVICE);
- new_addr = skb_shinfo(new_skb)->dma_maps[0];
+ new_addr = skb_shinfo(new_skb)->dma_head;
/* Make sure new skb does not cross any 4G boundaries.
* Drop the packet if it does.
@@ -5155,7 +5155,7 @@
sp = skb_shinfo(skb);
- mapping = sp->dma_maps[0];
+ mapping = sp->dma_head;
tp->tx_buffers[entry].skb = skb;
@@ -5173,7 +5173,7 @@
skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
len = frag->size;
- mapping = sp->dma_maps[i + 1];
+ mapping = sp->dma_maps[i];
tp->tx_buffers[entry].skb = NULL;
tg3_set_txd(tp, entry, mapping, len,
@@ -5331,7 +5331,7 @@
sp = skb_shinfo(skb);
- mapping = sp->dma_maps[0];
+ mapping = sp->dma_head;
tp->tx_buffers[entry].skb = skb;
@@ -5356,7 +5356,7 @@
skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
len = frag->size;
- mapping = sp->dma_maps[i + 1];
+ mapping = sp->dma_maps[i];
tp->tx_buffers[entry].skb = NULL;