net: fs_enet: Add NAPI TX
When using a MPC8xx as a router, 'perf' shows a significant time spent in
fs_enet_interrupt() and fs_enet_start_xmit().
'perf annotate' shows that the time spent in fs_enet_start_xmit is indeed spent
between spin_unlock_irqrestore() and the following instruction, hence in
interrupt handling. This is due to the TX complete interrupt that fires after
each transmitted packet.
This patch modifies the handling of TX complete to use NAPI.
With this patch, my NAT router offers a throughput improved by 21%
Original performance:
[root@localhost tmp]# scp toto pgs:/tmp
toto 100% 256MB 2.8MB/s 01:31
Performance with the patch:
[root@localhost tmp]# scp toto pgs:/tmp
toto 100% 256MB 3.4MB/s 01:16
Signed-off-by: Christophe Leroy <christophe.leroy@c-s.fr>
Signed-off-by: David S. Miller <davem@davemloft.net>
diff --git a/drivers/net/ethernet/freescale/fs_enet/fs_enet-main.c b/drivers/net/ethernet/freescale/fs_enet/fs_enet-main.c
index 71a25b4..c92c3b7 100644
--- a/drivers/net/ethernet/freescale/fs_enet/fs_enet-main.c
+++ b/drivers/net/ethernet/freescale/fs_enet/fs_enet-main.c
@@ -215,17 +215,23 @@
return received;
}
-static void fs_enet_tx(struct net_device *dev)
+static int fs_enet_tx_napi(struct napi_struct *napi, int budget)
{
- struct fs_enet_private *fep = netdev_priv(dev);
+ struct fs_enet_private *fep = container_of(napi, struct fs_enet_private,
+ napi_tx);
+ struct net_device *dev = fep->ndev;
cbd_t __iomem *bdp;
struct sk_buff *skb;
int dirtyidx, do_wake, do_restart;
u16 sc;
+ int has_tx_work = 0;
spin_lock(&fep->tx_lock);
bdp = fep->dirty_tx;
+ /* clear TX status bits for napi*/
+ (*fep->ops->napi_clear_tx_event)(dev);
+
do_wake = do_restart = 0;
while (((sc = CBDR_SC(bdp)) & BD_ENET_TX_READY) == 0) {
dirtyidx = bdp - fep->tx_bd_base;
@@ -278,7 +284,7 @@
/*
* Free the sk buffer associated with this last transmit.
*/
- dev_kfree_skb_irq(skb);
+ dev_kfree_skb(skb);
fep->tx_skbuff[dirtyidx] = NULL;
/*
@@ -295,6 +301,7 @@
*/
if (!fep->tx_free++)
do_wake = 1;
+ has_tx_work = 1;
}
fep->dirty_tx = bdp;
@@ -302,10 +309,19 @@
if (do_restart)
(*fep->ops->tx_restart)(dev);
+ if (!has_tx_work) {
+ napi_complete(napi);
+ (*fep->ops->napi_enable_tx)(dev);
+ }
+
spin_unlock(&fep->tx_lock);
if (do_wake)
netif_wake_queue(dev);
+
+ if (has_tx_work)
+ return budget;
+ return 0;
}
/*
@@ -350,8 +366,17 @@
__napi_schedule(&fep->napi);
}
- if (int_events & fep->ev_tx)
- fs_enet_tx(dev);
+ if (int_events & fep->ev_tx) {
+ napi_ok = napi_schedule_prep(&fep->napi_tx);
+
+ (*fep->ops->napi_disable_tx)(dev);
+ (*fep->ops->clear_int_events)(dev, fep->ev_napi_tx);
+
+ /* NOTE: it is possible for FCCs in NAPI mode */
+ /* to submit a spurious interrupt while in poll */
+ if (napi_ok)
+ __napi_schedule(&fep->napi_tx);
+ }
}
handled = nr > 0;
@@ -484,7 +509,6 @@
cbd_t __iomem *bdp;
int curidx;
u16 sc;
- unsigned long flags;
#ifdef CONFIG_FS_ENET_MPC5121_FEC
if (((unsigned long)skb->data) & 0x3) {
@@ -499,7 +523,7 @@
}
}
#endif
- spin_lock_irqsave(&fep->tx_lock, flags);
+ spin_lock(&fep->tx_lock);
/*
* Fill in a Tx ring entry
@@ -508,7 +532,7 @@
if (!fep->tx_free || (CBDR_SC(bdp) & BD_ENET_TX_READY)) {
netif_stop_queue(dev);
- spin_unlock_irqrestore(&fep->tx_lock, flags);
+ spin_unlock(&fep->tx_lock);
/*
* Ooops. All transmit buffers are full. Bail out.
@@ -564,7 +588,7 @@
(*fep->ops->tx_kickstart)(dev);
- spin_unlock_irqrestore(&fep->tx_lock, flags);
+ spin_unlock(&fep->tx_lock);
return NETDEV_TX_OK;
}
@@ -685,6 +709,7 @@
fs_init_bds(fep->ndev);
napi_enable(&fep->napi);
+ napi_enable(&fep->napi_tx);
/* Install our interrupt handler. */
r = request_irq(fep->interrupt, fs_enet_interrupt, IRQF_SHARED,
@@ -692,6 +717,7 @@
if (r != 0) {
dev_err(fep->dev, "Could not allocate FS_ENET IRQ!");
napi_disable(&fep->napi);
+ napi_disable(&fep->napi_tx);
return -EINVAL;
}
@@ -699,6 +725,7 @@
if (err) {
free_irq(fep->interrupt, dev);
napi_disable(&fep->napi);
+ napi_disable(&fep->napi_tx);
return err;
}
phy_start(fep->phydev);
@@ -716,6 +743,7 @@
netif_stop_queue(dev);
netif_carrier_off(dev);
napi_disable(&fep->napi);
+ napi_disable(&fep->napi_tx);
phy_stop(fep->phydev);
spin_lock_irqsave(&fep->lock, flags);
@@ -971,6 +999,7 @@
ndev->netdev_ops = &fs_enet_netdev_ops;
ndev->watchdog_timeo = 2 * HZ;
netif_napi_add(ndev, &fep->napi, fs_enet_rx_napi, fpi->napi_weight);
+ netif_napi_add(ndev, &fep->napi_tx, fs_enet_tx_napi, 2);
ndev->ethtool_ops = &fs_ethtool_ops;
diff --git a/drivers/net/ethernet/freescale/fs_enet/fs_enet.h b/drivers/net/ethernet/freescale/fs_enet/fs_enet.h
index 1ece4b1..3a4b49e 100644
--- a/drivers/net/ethernet/freescale/fs_enet/fs_enet.h
+++ b/drivers/net/ethernet/freescale/fs_enet/fs_enet.h
@@ -84,6 +84,9 @@
void (*napi_clear_rx_event)(struct net_device *dev);
void (*napi_enable_rx)(struct net_device *dev);
void (*napi_disable_rx)(struct net_device *dev);
+ void (*napi_clear_tx_event)(struct net_device *dev);
+ void (*napi_enable_tx)(struct net_device *dev);
+ void (*napi_disable_tx)(struct net_device *dev);
void (*rx_bd_done)(struct net_device *dev);
void (*tx_kickstart)(struct net_device *dev);
u32 (*get_int_events)(struct net_device *dev);
@@ -119,6 +122,7 @@
struct fs_enet_private {
struct napi_struct napi;
+ struct napi_struct napi_tx;
struct device *dev; /* pointer back to the device (must be initialized first) */
struct net_device *ndev;
spinlock_t lock; /* during all ops except TX pckt processing */
@@ -149,6 +153,7 @@
/* event masks */
u32 ev_napi_rx; /* mask of NAPI rx events */
+ u32 ev_napi_tx; /* mask of NAPI rx events */
u32 ev_rx; /* rx event mask */
u32 ev_tx; /* tx event mask */
u32 ev_err; /* error event mask */
@@ -191,8 +196,8 @@
#define DRV_MODULE_NAME "fs_enet"
#define PFX DRV_MODULE_NAME ": "
-#define DRV_MODULE_VERSION "1.0"
-#define DRV_MODULE_RELDATE "Aug 8, 2005"
+#define DRV_MODULE_VERSION "1.1"
+#define DRV_MODULE_RELDATE "Sep 22, 2014"
/***************************************************************************/
diff --git a/drivers/net/ethernet/freescale/fs_enet/mac-fcc.c b/drivers/net/ethernet/freescale/fs_enet/mac-fcc.c
index f5383ab..2c578db 100644
--- a/drivers/net/ethernet/freescale/fs_enet/mac-fcc.c
+++ b/drivers/net/ethernet/freescale/fs_enet/mac-fcc.c
@@ -125,6 +125,7 @@
}
#define FCC_NAPI_RX_EVENT_MSK (FCC_ENET_RXF | FCC_ENET_RXB)
+#define FCC_NAPI_TX_EVENT_MSK (FCC_ENET_TXF | FCC_ENET_TXB)
#define FCC_RX_EVENT (FCC_ENET_RXF)
#define FCC_TX_EVENT (FCC_ENET_TXB)
#define FCC_ERR_EVENT_MSK (FCC_ENET_TXE)
@@ -137,6 +138,7 @@
return -EINVAL;
fep->ev_napi_rx = FCC_NAPI_RX_EVENT_MSK;
+ fep->ev_napi_tx = FCC_NAPI_TX_EVENT_MSK;
fep->ev_rx = FCC_RX_EVENT;
fep->ev_tx = FCC_TX_EVENT;
fep->ev_err = FCC_ERR_EVENT_MSK;
@@ -446,6 +448,30 @@
C16(fccp, fcc_fccm, FCC_NAPI_RX_EVENT_MSK);
}
+static void napi_clear_tx_event(struct net_device *dev)
+{
+ struct fs_enet_private *fep = netdev_priv(dev);
+ fcc_t __iomem *fccp = fep->fcc.fccp;
+
+ W16(fccp, fcc_fcce, FCC_NAPI_TX_EVENT_MSK);
+}
+
+static void napi_enable_tx(struct net_device *dev)
+{
+ struct fs_enet_private *fep = netdev_priv(dev);
+ fcc_t __iomem *fccp = fep->fcc.fccp;
+
+ S16(fccp, fcc_fccm, FCC_NAPI_TX_EVENT_MSK);
+}
+
+static void napi_disable_tx(struct net_device *dev)
+{
+ struct fs_enet_private *fep = netdev_priv(dev);
+ fcc_t __iomem *fccp = fep->fcc.fccp;
+
+ C16(fccp, fcc_fccm, FCC_NAPI_TX_EVENT_MSK);
+}
+
static void rx_bd_done(struct net_device *dev)
{
/* nothing */
@@ -572,6 +598,9 @@
.napi_clear_rx_event = napi_clear_rx_event,
.napi_enable_rx = napi_enable_rx,
.napi_disable_rx = napi_disable_rx,
+ .napi_clear_tx_event = napi_clear_tx_event,
+ .napi_enable_tx = napi_enable_tx,
+ .napi_disable_tx = napi_disable_tx,
.rx_bd_done = rx_bd_done,
.tx_kickstart = tx_kickstart,
.get_int_events = get_int_events,
diff --git a/drivers/net/ethernet/freescale/fs_enet/mac-fec.c b/drivers/net/ethernet/freescale/fs_enet/mac-fec.c
index 1eedfba..3d4e08b 100644
--- a/drivers/net/ethernet/freescale/fs_enet/mac-fec.c
+++ b/drivers/net/ethernet/freescale/fs_enet/mac-fec.c
@@ -110,6 +110,7 @@
}
#define FEC_NAPI_RX_EVENT_MSK (FEC_ENET_RXF | FEC_ENET_RXB)
+#define FEC_NAPI_TX_EVENT_MSK (FEC_ENET_TXF | FEC_ENET_TXB)
#define FEC_RX_EVENT (FEC_ENET_RXF)
#define FEC_TX_EVENT (FEC_ENET_TXF)
#define FEC_ERR_EVENT_MSK (FEC_ENET_HBERR | FEC_ENET_BABR | \
@@ -126,6 +127,7 @@
fep->fec.htlo = 0;
fep->ev_napi_rx = FEC_NAPI_RX_EVENT_MSK;
+ fep->ev_napi_tx = FEC_NAPI_TX_EVENT_MSK;
fep->ev_rx = FEC_RX_EVENT;
fep->ev_tx = FEC_TX_EVENT;
fep->ev_err = FEC_ERR_EVENT_MSK;
@@ -415,6 +417,30 @@
FC(fecp, imask, FEC_NAPI_RX_EVENT_MSK);
}
+static void napi_clear_tx_event(struct net_device *dev)
+{
+ struct fs_enet_private *fep = netdev_priv(dev);
+ struct fec __iomem *fecp = fep->fec.fecp;
+
+ FW(fecp, ievent, FEC_NAPI_TX_EVENT_MSK);
+}
+
+static void napi_enable_tx(struct net_device *dev)
+{
+ struct fs_enet_private *fep = netdev_priv(dev);
+ struct fec __iomem *fecp = fep->fec.fecp;
+
+ FS(fecp, imask, FEC_NAPI_TX_EVENT_MSK);
+}
+
+static void napi_disable_tx(struct net_device *dev)
+{
+ struct fs_enet_private *fep = netdev_priv(dev);
+ struct fec __iomem *fecp = fep->fec.fecp;
+
+ FC(fecp, imask, FEC_NAPI_TX_EVENT_MSK);
+}
+
static void rx_bd_done(struct net_device *dev)
{
struct fs_enet_private *fep = netdev_priv(dev);
@@ -487,6 +513,9 @@
.napi_clear_rx_event = napi_clear_rx_event,
.napi_enable_rx = napi_enable_rx,
.napi_disable_rx = napi_disable_rx,
+ .napi_clear_tx_event = napi_clear_tx_event,
+ .napi_enable_tx = napi_enable_tx,
+ .napi_disable_tx = napi_disable_tx,
.rx_bd_done = rx_bd_done,
.tx_kickstart = tx_kickstart,
.get_int_events = get_int_events,
diff --git a/drivers/net/ethernet/freescale/fs_enet/mac-scc.c b/drivers/net/ethernet/freescale/fs_enet/mac-scc.c
index 90b3b19..41aa0b4 100644
--- a/drivers/net/ethernet/freescale/fs_enet/mac-scc.c
+++ b/drivers/net/ethernet/freescale/fs_enet/mac-scc.c
@@ -116,6 +116,7 @@
}
#define SCC_NAPI_RX_EVENT_MSK (SCCE_ENET_RXF | SCCE_ENET_RXB)
+#define SCC_NAPI_TX_EVENT_MSK (SCCE_ENET_TXF | SCCE_ENET_TXB)
#define SCC_RX_EVENT (SCCE_ENET_RXF)
#define SCC_TX_EVENT (SCCE_ENET_TXB)
#define SCC_ERR_EVENT_MSK (SCCE_ENET_TXE | SCCE_ENET_BSY)
@@ -130,6 +131,7 @@
fep->scc.htlo = 0;
fep->ev_napi_rx = SCC_NAPI_RX_EVENT_MSK;
+ fep->ev_napi_tx = SCC_NAPI_TX_EVENT_MSK;
fep->ev_rx = SCC_RX_EVENT;
fep->ev_tx = SCC_TX_EVENT | SCCE_ENET_TXE;
fep->ev_err = SCC_ERR_EVENT_MSK;
@@ -398,6 +400,30 @@
C16(sccp, scc_sccm, SCC_NAPI_RX_EVENT_MSK);
}
+static void napi_clear_tx_event(struct net_device *dev)
+{
+ struct fs_enet_private *fep = netdev_priv(dev);
+ scc_t __iomem *sccp = fep->scc.sccp;
+
+ W16(sccp, scc_scce, SCC_NAPI_TX_EVENT_MSK);
+}
+
+static void napi_enable_tx(struct net_device *dev)
+{
+ struct fs_enet_private *fep = netdev_priv(dev);
+ scc_t __iomem *sccp = fep->scc.sccp;
+
+ S16(sccp, scc_sccm, SCC_NAPI_TX_EVENT_MSK);
+}
+
+static void napi_disable_tx(struct net_device *dev)
+{
+ struct fs_enet_private *fep = netdev_priv(dev);
+ scc_t __iomem *sccp = fep->scc.sccp;
+
+ C16(sccp, scc_sccm, SCC_NAPI_TX_EVENT_MSK);
+}
+
static void rx_bd_done(struct net_device *dev)
{
/* nothing */
@@ -471,6 +497,9 @@
.napi_clear_rx_event = napi_clear_rx_event,
.napi_enable_rx = napi_enable_rx,
.napi_disable_rx = napi_disable_rx,
+ .napi_clear_tx_event = napi_clear_tx_event,
+ .napi_enable_tx = napi_enable_tx,
+ .napi_disable_tx = napi_disable_tx,
.rx_bd_done = rx_bd_done,
.tx_kickstart = tx_kickstart,
.get_int_events = get_int_events,