igb: add DCA support
Add DCA support in the similar method that it was added to the ixgbe
driver recently. DCA allows the network device to put data in the
CPU cache and notify the chipset of that event. This reduces cache
misses during receives.
Signed-off-by: Jeb Cramer <cramerj@intel.com>
Signed-off-by: Mitch Williams <mitch.a.williams@intel.com>
Signed-off-by: Auke Kok <auke-jan.h.kok@intel.com>
Signed-off-by: Shannon Nelson <shannon.nelson@intel.com>
Signed-off-by: Jeff Garzik <jgarzik@redhat.com>
diff --git a/drivers/net/igb/e1000_82575.h b/drivers/net/igb/e1000_82575.h
index d78ad33..02e57a8 100644
--- a/drivers/net/igb/e1000_82575.h
+++ b/drivers/net/igb/e1000_82575.h
@@ -144,9 +144,20 @@
#define E1000_RXDCTL_QUEUE_ENABLE 0x02000000 /* Enable specific Rx Queue */
/* Direct Cache Access (DCA) definitions */
+#define E1000_DCA_CTRL_DCA_ENABLE 0x00000000 /* DCA Enable */
+#define E1000_DCA_CTRL_DCA_DISABLE 0x00000001 /* DCA Disable */
+#define E1000_DCA_CTRL_DCA_MODE_CB1 0x00 /* DCA Mode CB1 */
+#define E1000_DCA_CTRL_DCA_MODE_CB2 0x02 /* DCA Mode CB2 */
+#define E1000_DCA_RXCTRL_CPUID_MASK 0x0000001F /* Rx CPUID Mask */
+#define E1000_DCA_RXCTRL_DESC_DCA_EN (1 << 5) /* DCA Rx Desc enable */
+#define E1000_DCA_RXCTRL_HEAD_DCA_EN (1 << 6) /* DCA Rx Desc header enable */
+#define E1000_DCA_RXCTRL_DATA_DCA_EN (1 << 7) /* DCA Rx Desc payload enable */
+#define E1000_DCA_TXCTRL_CPUID_MASK 0x0000001F /* Tx CPUID Mask */
+#define E1000_DCA_TXCTRL_DESC_DCA_EN (1 << 5) /* DCA Tx Desc enable */
#define E1000_DCA_TXCTRL_TX_WB_RO_EN (1 << 11) /* Tx Desc writeback RO bit */
+
#endif
diff --git a/drivers/net/igb/e1000_regs.h b/drivers/net/igb/e1000_regs.h
index ff187b7..d25e914 100644
--- a/drivers/net/igb/e1000_regs.h
+++ b/drivers/net/igb/e1000_regs.h
@@ -235,6 +235,8 @@
#define E1000_FACTPS 0x05B30 /* Function Active and Power State to MNG */
#define E1000_SWSM 0x05B50 /* SW Semaphore */
#define E1000_FWSM 0x05B54 /* FW Semaphore */
+#define E1000_DCA_ID 0x05B70 /* DCA Requester ID Information - RO */
+#define E1000_DCA_CTRL 0x05B74 /* DCA Control - RW */
#define E1000_HICR 0x08F00 /* Host Inteface Control */
/* RSS registers */
diff --git a/drivers/net/igb/igb.h b/drivers/net/igb/igb.h
index 5915efc..d4a0423 100644
--- a/drivers/net/igb/igb.h
+++ b/drivers/net/igb/igb.h
@@ -271,7 +271,9 @@
/* to not mess up cache alignment, always add to the bottom */
unsigned long state;
unsigned int msi_enabled;
-
+#ifdef CONFIG_DCA
+ unsigned int dca_enabled;
+#endif
u32 eeprom_wol;
/* for ioport free */
diff --git a/drivers/net/igb/igb_main.c b/drivers/net/igb/igb_main.c
index f975bfe..e8ef541 100644
--- a/drivers/net/igb/igb_main.c
+++ b/drivers/net/igb/igb_main.c
@@ -41,7 +41,9 @@
#include <linux/delay.h>
#include <linux/interrupt.h>
#include <linux/if_ether.h>
-
+#ifdef CONFIG_DCA
+#include <linux/dca.h>
+#endif
#include "igb.h"
#define DRV_VERSION "1.0.8-k2"
@@ -102,6 +104,11 @@
static irqreturn_t igb_msix_rx(int irq, void *);
static irqreturn_t igb_msix_tx(int irq, void *);
static int igb_clean_rx_ring_msix(struct napi_struct *, int);
+#ifdef CONFIG_DCA
+static void igb_update_rx_dca(struct igb_ring *);
+static void igb_update_tx_dca(struct igb_ring *);
+static void igb_setup_dca(struct igb_adapter *);
+#endif /* CONFIG_DCA */
static bool igb_clean_tx_irq(struct igb_ring *);
static int igb_poll(struct napi_struct *, int);
static bool igb_clean_rx_irq_adv(struct igb_ring *, int *, int);
@@ -119,6 +126,14 @@
static int igb_resume(struct pci_dev *);
#endif
static void igb_shutdown(struct pci_dev *);
+#ifdef CONFIG_DCA
+static int igb_notify_dca(struct notifier_block *, unsigned long, void *);
+static struct notifier_block dca_notifier = {
+ .notifier_call = igb_notify_dca,
+ .next = NULL,
+ .priority = 0
+};
+#endif
#ifdef CONFIG_NET_POLL_CONTROLLER
/* for netdump / net console */
@@ -183,6 +198,9 @@
printk(KERN_INFO "%s\n", igb_copyright);
ret = pci_register_driver(&igb_driver);
+#ifdef CONFIG_DCA
+ dca_register_notify(&dca_notifier);
+#endif
return ret;
}
@@ -196,6 +214,9 @@
**/
static void __exit igb_exit_module(void)
{
+#ifdef CONFIG_DCA
+ dca_unregister_notify(&dca_notifier);
+#endif
pci_unregister_driver(&igb_driver);
}
@@ -1130,6 +1151,17 @@
if (err)
goto err_register;
+#ifdef CONFIG_DCA
+ if (dca_add_requester(&pdev->dev) == 0) {
+ adapter->dca_enabled = true;
+ dev_info(&pdev->dev, "DCA enabled\n");
+ /* Always use CB2 mode, difference is masked
+ * in the CB driver. */
+ wr32(E1000_DCA_CTRL, 2);
+ igb_setup_dca(adapter);
+ }
+#endif
+
dev_info(&pdev->dev, "Intel(R) Gigabit Ethernet Network Connection\n");
/* print bus type/speed/width info */
dev_info(&pdev->dev,
@@ -1193,6 +1225,7 @@
{
struct net_device *netdev = pci_get_drvdata(pdev);
struct igb_adapter *adapter = netdev_priv(netdev);
+ struct e1000_hw *hw = &adapter->hw;
/* flush_scheduled work may reschedule our watchdog task, so
* explicitly disable watchdog tasks from being rescheduled */
@@ -1202,6 +1235,15 @@
flush_scheduled_work();
+#ifdef CONFIG_DCA
+ if (adapter->dca_enabled) {
+ dev_info(&pdev->dev, "DCA disabled\n");
+ dca_remove_requester(&pdev->dev);
+ adapter->dca_enabled = false;
+ wr32(E1000_DCA_CTRL, 1);
+ }
+#endif
+
/* Release control of h/w to f/w. If f/w is AMT enabled, this
* would have already happened in close and is redundant. */
igb_release_hw_control(adapter);
@@ -3112,7 +3154,10 @@
if (!tx_ring->itr_val)
wr32(E1000_EIMC, tx_ring->eims_value);
-
+#ifdef CONFIG_DCA
+ if (adapter->dca_enabled)
+ igb_update_tx_dca(tx_ring);
+#endif
tx_ring->total_bytes = 0;
tx_ring->total_packets = 0;
@@ -3146,9 +3191,119 @@
if (netif_rx_schedule_prep(adapter->netdev, &rx_ring->napi))
__netif_rx_schedule(adapter->netdev, &rx_ring->napi);
- return IRQ_HANDLED;
+#ifdef CONFIG_DCA
+ if (adapter->dca_enabled)
+ igb_update_rx_dca(rx_ring);
+#endif
+ return IRQ_HANDLED;
}
+#ifdef CONFIG_DCA
+static void igb_update_rx_dca(struct igb_ring *rx_ring)
+{
+ u32 dca_rxctrl;
+ struct igb_adapter *adapter = rx_ring->adapter;
+ struct e1000_hw *hw = &adapter->hw;
+ int cpu = get_cpu();
+ int q = rx_ring - adapter->rx_ring;
+
+ if (rx_ring->cpu != cpu) {
+ dca_rxctrl = rd32(E1000_DCA_RXCTRL(q));
+ dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK;
+ dca_rxctrl |= dca_get_tag(cpu);
+ dca_rxctrl |= E1000_DCA_RXCTRL_DESC_DCA_EN;
+ dca_rxctrl |= E1000_DCA_RXCTRL_HEAD_DCA_EN;
+ dca_rxctrl |= E1000_DCA_RXCTRL_DATA_DCA_EN;
+ wr32(E1000_DCA_RXCTRL(q), dca_rxctrl);
+ rx_ring->cpu = cpu;
+ }
+ put_cpu();
+}
+
+static void igb_update_tx_dca(struct igb_ring *tx_ring)
+{
+ u32 dca_txctrl;
+ struct igb_adapter *adapter = tx_ring->adapter;
+ struct e1000_hw *hw = &adapter->hw;
+ int cpu = get_cpu();
+ int q = tx_ring - adapter->tx_ring;
+
+ if (tx_ring->cpu != cpu) {
+ dca_txctrl = rd32(E1000_DCA_TXCTRL(q));
+ dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK;
+ dca_txctrl |= dca_get_tag(cpu);
+ dca_txctrl |= E1000_DCA_TXCTRL_DESC_DCA_EN;
+ wr32(E1000_DCA_TXCTRL(q), dca_txctrl);
+ tx_ring->cpu = cpu;
+ }
+ put_cpu();
+}
+
+static void igb_setup_dca(struct igb_adapter *adapter)
+{
+ int i;
+
+ if (!(adapter->dca_enabled))
+ return;
+
+ for (i = 0; i < adapter->num_tx_queues; i++) {
+ adapter->tx_ring[i].cpu = -1;
+ igb_update_tx_dca(&adapter->tx_ring[i]);
+ }
+ for (i = 0; i < adapter->num_rx_queues; i++) {
+ adapter->rx_ring[i].cpu = -1;
+ igb_update_rx_dca(&adapter->rx_ring[i]);
+ }
+}
+
+static int __igb_notify_dca(struct device *dev, void *data)
+{
+ struct net_device *netdev = dev_get_drvdata(dev);
+ struct igb_adapter *adapter = netdev_priv(netdev);
+ struct e1000_hw *hw = &adapter->hw;
+ unsigned long event = *(unsigned long *)data;
+
+ switch (event) {
+ case DCA_PROVIDER_ADD:
+ /* if already enabled, don't do it again */
+ if (adapter->dca_enabled)
+ break;
+ adapter->dca_enabled = true;
+ /* Always use CB2 mode, difference is masked
+ * in the CB driver. */
+ wr32(E1000_DCA_CTRL, 2);
+ if (dca_add_requester(dev) == 0) {
+ dev_info(&adapter->pdev->dev, "DCA enabled\n");
+ igb_setup_dca(adapter);
+ break;
+ }
+ /* Fall Through since DCA is disabled. */
+ case DCA_PROVIDER_REMOVE:
+ if (adapter->dca_enabled) {
+ /* without this a class_device is left
+ * hanging around in the sysfs model */
+ dca_remove_requester(dev);
+ dev_info(&adapter->pdev->dev, "DCA disabled\n");
+ adapter->dca_enabled = false;
+ wr32(E1000_DCA_CTRL, 1);
+ }
+ break;
+ }
+
+ return 0;
+}
+
+static int igb_notify_dca(struct notifier_block *nb, unsigned long event,
+ void *p)
+{
+ int ret_val;
+
+ ret_val = driver_for_each_device(&igb_driver.driver, NULL, &event,
+ __igb_notify_dca);
+
+ return ret_val ? NOTIFY_BAD : NOTIFY_DONE;
+}
+#endif /* CONFIG_DCA */
/**
* igb_intr_msi - Interrupt Handler
@@ -3239,7 +3394,16 @@
int tx_clean_complete, work_done = 0;
/* this poll routine only supports one tx and one rx queue */
+#ifdef CONFIG_DCA
+ if (adapter->dca_enabled)
+ igb_update_tx_dca(&adapter->tx_ring[0]);
+#endif
tx_clean_complete = igb_clean_tx_irq(&adapter->tx_ring[0]);
+
+#ifdef CONFIG_DCA
+ if (adapter->dca_enabled)
+ igb_update_rx_dca(&adapter->rx_ring[0]);
+#endif
igb_clean_rx_irq_adv(&adapter->rx_ring[0], &work_done, budget);
/* If no Tx and not enough Rx work done, exit the polling mode */
@@ -3268,6 +3432,10 @@
if (!netif_carrier_ok(netdev))
goto quit_polling;
+#ifdef CONFIG_DCA
+ if (adapter->dca_enabled)
+ igb_update_rx_dca(rx_ring);
+#endif
igb_clean_rx_irq_adv(rx_ring, &work_done, budget);