hif/src/hif_irq_affinity.c

/*
 * Copyright (c) 2015-2017 The Linux Foundation. All rights reserved.
 *
 * Permission to use, copy, modify, and/or distribute this software for
 * any purpose with or without fee is hereby granted, provided that the
 * above copyright notice and this permission notice appear in all
 * copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL
 * WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE
 * AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL
 * DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR
 * PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
 * TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
 * PERFORMANCE OF THIS SOFTWARE.
 */

/**
 * DOC: hif_irq_afinity.c
 *
 * This irq afinity implementation is os dependent, so this can be treated as
 * an abstraction layer...  Should this be moved into a /linux folder?
 */

#include <linux/string.h> /* memset */

/* Linux headers */
#include <linux/cpumask.h>
#include <linux/cpufreq.h>
#include <linux/cpu.h>
#include <linux/topology.h>
#include <linux/interrupt.h>
#include <linux/irq.h>
#ifdef CONFIG_SCHED_CORE_CTL
#include <linux/sched/core_ctl.h>
#endif
#include <linux/pm.h>
#include <hif_napi.h>
#include <hif_irq_affinity.h>
#include <hif_exec.h>
#include <hif_main.h>

#if defined(FEATURE_NAPI_DEBUG) && defined(HIF_IRQ_AFFINITY)
/*
 * Local functions
 * - no argument checks, all internal/trusted callers
 */
static void hnc_dump_cpus(struct qca_napi_data *napid)
{
	hif_napi_stats(napid);
}
#else
static void hnc_dump_cpus(struct qca_napi_data *napid) { /* no-op */ };
#endif /* FEATURE_NAPI_DEBUG */

#ifdef HIF_IRQ_AFFINITY
/**
 *
 * hif_exec_event() - reacts to events that impact irq affinity
 * @hif : pointer to hif context
 * @evnt: event that has been detected
 * @data: more data regarding the event
 *
 * Description:
 *   This function handles two types of events:
 *   1- Events that change the state of NAPI (enabled/disabled):
 *      {NAPI_EVT_INI_FILE, NAPI_EVT_CMD_STATE}
 *      The state is retrievable by "hdd_napi_enabled(-1)"
 *    - NAPI will be on if either INI file is on and it has not been disabled
 *                                by a subsequent vendor CMD,
 *                         or     it has been enabled by a vendor CMD.
 *   2- Events that change the CPU affinity of a NAPI instance/IRQ:
 *      {NAPI_EVT_TPUT_STATE, NAPI_EVT_CPU_STATE}
 *    - NAPI will support a throughput mode (HI/LO), kept at napid->napi_mode
 *    - NAPI will switch throughput mode based on hdd_napi_throughput_policy()
 *    - In LO tput mode, NAPI will yield control if its interrupts to the system
 *      management functions. However in HI throughput mode, NAPI will actively
 *      manage its interrupts/instances (by trying to disperse them out to
 *      separate performance cores).
 *    - CPU eligibility is kept up-to-date by NAPI_EVT_CPU_STATE events.
 *
 *    + In some cases (roaming peer management is the only case so far), a
 *      a client can trigger a "SERIALIZE" event. Basically, this means that the
 *      users is asking NAPI to go into a truly single execution context state.
 *      So, NAPI indicates to msm-irqbalancer that it wants to be blacklisted,
 *      (if called for the first time) and then moves all IRQs (for NAPI
 *      instances) to be collapsed to a single core. If called multiple times,
 *      it will just re-collapse the CPUs. This is because blacklist-on() API
 *      is reference-counted, and because the API has already been called.
 *
 *      Such a user, should call "DESERIALIZE" (NORMAL) event, to set NAPI to go
 *      to its "normal" operation. Optionally, they can give a timeout value (in
 *      multiples of BusBandwidthCheckPeriod -- 100 msecs by default). In this
 *      case, NAPI will just set the current throughput state to uninitialized
 *      and set the delay period. Once policy handler is called, it would skip
 *      applying the policy delay period times, and otherwise apply the policy.
 *
 * Return:
 *  < 0: some error
 *  = 0: event handled successfully
 */
int hif_exec_event(struct hif_opaque_softc *hif_ctx, enum qca_napi_event event,
		   void *data)
{
	int      rc = 0;
	uint32_t prev_state;
	struct hif_softc *hif = HIF_GET_SOFTC(hif_ctx);
	struct qca_napi_data *napid = &(hif->napi_data);
	enum qca_napi_tput_state tput_mode = QCA_NAPI_TPUT_UNINITIALIZED;
	enum {
		BLACKLIST_NOT_PENDING,
		BLACKLIST_ON_PENDING,
		BLACKLIST_OFF_PENDING
	     } blacklist_pending = BLACKLIST_NOT_PENDING;

	NAPI_DEBUG("%s: -->(event=%d, aux=%pK)", __func__, event, data);

	qdf_spin_lock_bh(&(napid->lock));
	prev_state = napid->state;
	switch (event) {
	case NAPI_EVT_INI_FILE:
	case NAPI_EVT_CMD_STATE:
	case NAPI_EVT_INT_STATE:
		/* deprecated */
		break;

	case NAPI_EVT_CPU_STATE: {
		int cpu = ((unsigned long int)data >> 16);
		int val = ((unsigned long int)data & 0x0ff);

		NAPI_DEBUG("%s: evt=CPU_STATE on CPU %d value=%d",
			   __func__, cpu, val);

		/* state has already been set by hnc_cpu_notify_cb */
		if ((val == QCA_NAPI_CPU_DOWN) &&
		    (napid->napi_mode == QCA_NAPI_TPUT_HI) && /* we manage */
		    (napid->napi_cpu[cpu].napis != 0)) {
			NAPI_DEBUG("%s: Migrating NAPIs out of cpu %d",
				   __func__, cpu);
			rc = hif_exec_cpu_migrate(napid,
						  cpu,
						  HNC_ACT_RELOCATE);
			napid->napi_cpu[cpu].napis = 0;
		}
		/* in QCA_NAPI_TPUT_LO case, napis MUST == 0 */
		break;
	}

	case NAPI_EVT_TPUT_STATE: {
		tput_mode = (enum qca_napi_tput_state)data;
		if (tput_mode == QCA_NAPI_TPUT_LO) {
			/* from TPUT_HI -> TPUT_LO */
			NAPI_DEBUG("%s: Moving to napi_tput_LO state",
				   __func__);
			blacklist_pending = BLACKLIST_OFF_PENDING;
			/*
			 * Ideally we should "collapse" interrupts here, since
			 * we are "dispersing" interrupts in the "else" case.
			 * This allows the possibility that our interrupts may
			 * still be on the perf cluster the next time we enter
			 * high tput mode. However, the irq_balancer is free
			 * to move our interrupts to power cluster once
			 * blacklisting has been turned off in the "else" case.
			 */
		} else {
			/* from TPUT_LO -> TPUT->HI */
			NAPI_DEBUG("%s: Moving to napi_tput_HI state",
				   __func__);
			rc = hif_exec_cpu_migrate(napid,
						  HNC_ANY_CPU,
						  HNC_ACT_DISPERSE);

			blacklist_pending = BLACKLIST_ON_PENDING;
		}
		napid->napi_mode = tput_mode;
		break;
	}

	case NAPI_EVT_USR_SERIAL: {
		unsigned long users = (unsigned long)data;

		NAPI_DEBUG("%s: User forced SERIALIZATION; users=%ld",
			   __func__, users);

		rc = hif_exec_cpu_migrate(napid,
					  HNC_ANY_CPU,
					  HNC_ACT_COLLAPSE);
		if ((users == 0) && (rc == 0))
			blacklist_pending = BLACKLIST_ON_PENDING;
		break;
	}
	case NAPI_EVT_USR_NORMAL: {
		NAPI_DEBUG("%s: User forced DE-SERIALIZATION", __func__);
		/*
		 * Deserialization timeout is handled at hdd layer;
		 * just mark current mode to uninitialized to ensure
		 * it will be set when the delay is over
		 */
		napid->napi_mode = QCA_NAPI_TPUT_UNINITIALIZED;
		break;
	}
	default: {
		HIF_ERROR("%s: unknown event: %d (data=0x%0lx)",
			  __func__, event, (unsigned long) data);
		break;
	} /* default */
	}; /* switch */


	switch (blacklist_pending) {
	case BLACKLIST_ON_PENDING:
		/* assume the control of WLAN IRQs */
		hif_napi_cpu_blacklist(napid, BLACKLIST_ON);
		break;
	case BLACKLIST_OFF_PENDING:
		/* yield the control of WLAN IRQs */
		hif_napi_cpu_blacklist(napid, BLACKLIST_OFF);
		break;
	default: /* nothing to do */
		break;
	} /* switch blacklist_pending */

	qdf_spin_unlock_bh(&(napid->lock));

	NAPI_DEBUG("<--[rc=%d]", rc);
	return rc;
}
#endif

/**
 * hif_napi_correct_cpu() - correct the interrupt affinity for napi if needed
 * @napi_info: pointer to qca_napi_info for the napi instance
 *
 * Return: true  => interrupt already on correct cpu, no correction needed
 *         false => interrupt on wrong cpu, correction done for cpu affinity
 *                   of the interrupt
 */
static inline
bool hif_exec_correct_cpu(struct hif_exec_context *exec_ctx)
{
	bool right_cpu = true;
	int rc = 0;
	cpumask_t cpumask;
	int cpu;
	struct hif_softc *hif_softc = HIF_GET_SOFTC(exec_ctx->hif);
	struct qca_napi_data *napid;
	int ind;

	napid = &hif_softc->napi_data;

	if (!(napid->flags & QCA_NAPI_FEATURE_CPU_CORRECTION))
		goto done;

	cpu = qdf_get_cpu();
	if (likely((cpu == exec_ctx->cpu) ||
		    hif_exec_cpu_blacklist(napid, BLACKLIST_QUERY) == 0))
		goto done;

	right_cpu = false;

	NAPI_DEBUG("interrupt on wrong CPU, correcting");
	cpumask.bits[0] = (0x01 << exec_ctx->cpu);

	for (ind = 0; ind < exec_ctx->numirq; ind++) {
		if (exec_ctx->os_irq[ind]) {
			irq_modify_status(exec_ctx->os_irq[ind],
					  IRQ_NO_BALANCING, 0);
			rc = irq_set_affinity_hint(exec_ctx->os_irq[ind],
						   &cpumask);
			irq_modify_status(exec_ctx->os_irq[ind], 0,
					  IRQ_NO_BALANCING);

			if (rc)
				HIF_ERROR("error setting irq affinity hint: %d",
					  rc);
			else
				exec_ctx->stats[cpu].cpu_corrected++;
		}
	}
done:
	return right_cpu;
}

/**
 * hncm_migrate_to() - migrates a NAPI to a CPU
 * @napid: pointer to NAPI block
 * @ce_id: CE_id of the NAPI instance
 * @didx : index in the CPU topology table for the CPU to migrate to
 *
 * Migrates NAPI (identified by the CE_id) to the destination core
 * Updates the napi_map of the destination entry
 *
 * Return:
 *  =0 : success
 *  <0 : error
 */
static int hncm_exec_migrate_to(struct qca_napi_data *napid, uint8_t ctx_id,
				int didx)
{
	struct hif_exec_context *exec_ctx;
	int rc = 0;
	int status = 0;
	int ind;
	cpumask_t cpumask;


	NAPI_DEBUG("-->%s(napi_cd=%d, didx=%d)", __func__, napi_ce, didx);

	cpumask.bits[0] = (1 << didx);
	exec_ctx = hif_exec_get_ctx(&napid->hif_softc->osc, ctx_id);
	if (exec_ctx == NULL)
		return -EINVAL;

	for (ind = 0; ind < exec_ctx->numirq; ind++) {
		if (exec_ctx->os_irq[ind]) {
			irq_modify_status(exec_ctx->os_irq[ind],
					  IRQ_NO_BALANCING, 0);
			rc = irq_set_affinity_hint(exec_ctx->os_irq[ind],
						   &cpumask);
			if (rc)
				status = rc;
		}
	}

	/* unmark the napis bitmap in the cpu table */
	napid->napi_cpu[exec_ctx->cpu].napis &= ~(0x01 << ctx_id);
	/* mark the napis bitmap for the new designated cpu */
	napid->napi_cpu[didx].napis |= (0x01 << ctx_id);
	exec_ctx->cpu = didx;

	NAPI_DEBUG("<--%s[%d]", __func__, rc);
	return status;
}

/**
 * hncm_dest_cpu() - finds a destination CPU for NAPI
 * @napid: pointer to NAPI block
 * @act  : RELOCATE | COLLAPSE | DISPERSE
 *
 * Finds the designated destionation for the next IRQ.
 * RELOCATE: translated to either COLLAPSE or DISPERSE based
 *           on napid->napi_mode (throughput state)
 * COLLAPSE: All have the same destination: the first online CPU in lilcl
 * DISPERSE: One of the CPU in bigcl, which has the smallest number of
 *           NAPIs on it
 *
 * Return: >=0 : index in the cpu topology table
 *       : < 0 : error
 */
static int hncm_dest_cpu(struct qca_napi_data *napid, int act)
{
	int destidx = -1;
	int head, i;

	NAPI_DEBUG("-->%s(act=%d)", __func__, act);
	if (act == HNC_ACT_RELOCATE) {
		if (napid->napi_mode == QCA_NAPI_TPUT_LO)
			act = HNC_ACT_COLLAPSE;
		else
			act = HNC_ACT_DISPERSE;
		NAPI_DEBUG("%s: act changed from HNC_ACT_RELOCATE to %d",
			   __func__, act);
	}
	if (act == HNC_ACT_COLLAPSE) {
		head = i = napid->lilcl_head;
retry_collapse:
		while (i >= 0) {
			if (napid->napi_cpu[i].state == QCA_NAPI_CPU_UP) {
				destidx = i;
				break;
			}
			i = napid->napi_cpu[i].cluster_nxt;
		}
		if ((destidx < 0) && (head == napid->lilcl_head)) {
			NAPI_DEBUG("%s: COLLAPSE: no lilcl dest, try bigcl",
				__func__);
			head = i = napid->bigcl_head;
			goto retry_collapse;
		}
	} else { /* HNC_ACT_DISPERSE */
		int smallest = 99; /* all 32 bits full */
		int smallidx = -1;

		head = i = napid->bigcl_head;
retry_disperse:
		while (i >= 0) {
			if ((napid->napi_cpu[i].state == QCA_NAPI_CPU_UP) &&
			    (hweight32(napid->napi_cpu[i].napis) <= smallest)) {
				smallest = napid->napi_cpu[i].napis;
				smallidx = i;
			}
			i = napid->napi_cpu[i].cluster_nxt;
		}
		destidx = smallidx;
		if ((destidx < 0) && (head == napid->bigcl_head)) {
			NAPI_DEBUG("%s: DISPERSE: no bigcl dest, try lilcl",
				__func__);
			head = i = napid->lilcl_head;
			goto retry_disperse;
		}
	}
	NAPI_DEBUG("<--%s[dest=%d]", __func__, destidx);
	return destidx;
}
/**
 * hif_napi_cpu_migrate() - migrate IRQs away
 * @cpu: -1: all CPUs <n> specific CPU
 * @act: COLLAPSE | DISPERSE
 *
 * Moves IRQs/NAPIs from specific or all CPUs (specified by @cpu) to eligible
 * cores. Eligible cores are:
 * act=COLLAPSE -> the first online core of the little cluster
 * act=DISPERSE -> separate cores of the big cluster, so that each core will
 *                 host minimum number of NAPIs/IRQs (napid->cpus[cpu].napis)
 *
 * Note that this function is called with a spinlock acquired already.
 *
 * Return: =0: success
 *         <0: error
 */
int hif_exec_cpu_migrate(struct qca_napi_data *napid, int cpu, int action)
{
	int      rc = 0;
	struct qca_napi_cpu *cpup;
	int      i, dind;
	uint32_t napis;


	NAPI_DEBUG("-->%s(.., cpu=%d, act=%d)",
		   __func__, cpu, action);

	if (napid->exec_map == 0) {
		NAPI_DEBUG("%s: datapath contexts to disperse", __func__);
		goto hncm_return;
	}
	cpup = napid->napi_cpu;

	switch (action) {
	case HNC_ACT_RELOCATE:
	case HNC_ACT_DISPERSE:
	case HNC_ACT_COLLAPSE: {
		/* first find the src napi set */
		if (cpu == HNC_ANY_CPU)
			napis = napid->exec_map;
		else
			napis = cpup[cpu].napis;
		/* then clear the napi bitmap on each CPU */
		for (i = 0; i < NR_CPUS; i++)
			cpup[i].napis = 0;
		/* then for each of the NAPIs to disperse: */
		for (i = 0; i < HIF_MAX_GROUP; i++)
			if (napis & (1 << i)) {
				/* find a destination CPU */
				dind = hncm_dest_cpu(napid, action);
				if (dind >= 0) {
					NAPI_DEBUG("Migrating NAPI ce%d to %d",
						   i, dind);
					rc = hncm_exec_migrate_to(napid, i,
								  dind);
				} else {
					NAPI_DEBUG("No dest for NAPI ce%d", i);
					hnc_dump_cpus(napid);
					rc = -1;
				}
			}
		break;
	}
	default: {
		NAPI_DEBUG("%s: bad action: %d\n", __func__, action);
		QDF_BUG(0);
		break;
	}
	} /* switch action */

hncm_return:
	hnc_dump_cpus(napid);
	return rc;
}


/**
 * hif_exec_bl_irq() - calls irq_modify_status to enable/disable blacklisting
 * @napid: pointer to qca_napi_data structure
 * @bl_flag: blacklist flag to enable/disable blacklisting
 *
 * The function enables/disables blacklisting for all the copy engine
 * interrupts on which NAPI is enabled.
 *
 * Return: None
 */
static inline void hif_exec_bl_irq(struct qca_napi_data *napid, bool bl_flag)
{
	int i, j;
	struct hif_exec_context *exec_ctx;

	for (i = 0; i < HIF_MAX_GROUP; i++) {
		/* check if NAPI is enabled on the CE */
		if (!(napid->exec_map & (0x01 << i)))
			continue;

		/*double check that NAPI is allocated for the CE */
		exec_ctx = hif_exec_get_ctx(&napid->hif_softc->osc, i);
		if (!(exec_ctx))
			continue;

		if (bl_flag == true)
			for (j = 0; j < exec_ctx->numirq; j++)
				irq_modify_status(exec_ctx->os_irq[j],
						  0, IRQ_NO_BALANCING);
		else
			for (j = 0; j < exec_ctx->numirq; j++)
				irq_modify_status(exec_ctx->os_irq[j],
						  IRQ_NO_BALANCING, 0);
		HIF_DBG("%s: bl_flag %d CE %d", __func__, bl_flag, i);
	}
}

#ifdef CONFIG_SCHED_CORE_CTL
/* Enable this API only if kernel feature - CONFIG_SCHED_CORE_CTL is defined */
static inline int hif_napi_core_ctl_set_boost(bool boost)
{
	return core_ctl_set_boost(boost);
}
#else
static inline int hif_napi_core_ctl_set_boost(bool boost)
{
	return 0;
}
#endif

/**
 * hif_napi_cpu_blacklist() - en(dis)ables blacklisting for NAPI RX interrupts.
 * @napid: pointer to qca_napi_data structure
 * @op: blacklist operation to perform
 *
 * The function enables/disables/queries blacklisting for all CE RX
 * interrupts with NAPI enabled. Besides blacklisting, it also enables/disables
 * core_ctl_set_boost.
 * Once blacklisting is enabled, the interrupts will not be managed by the IRQ
 * balancer.
 *
 * Return: -EINVAL, in case IRQ_BLACKLISTING and CORE_CTL_BOOST is not enabled
 *         for BLACKLIST_QUERY op - blacklist refcount
 *         for BLACKLIST_ON op    - return value from core_ctl_set_boost API
 *         for BLACKLIST_OFF op   - return value from core_ctl_set_boost API
 */
int hif_exec_cpu_blacklist(struct qca_napi_data *napid,
			   enum qca_blacklist_op op)
{
	int rc = 0;
	static int ref_count; /* = 0 by the compiler */
	uint8_t flags = napid->flags;
	bool bl_en = flags & QCA_NAPI_FEATURE_IRQ_BLACKLISTING;
	bool ccb_en = flags & QCA_NAPI_FEATURE_CORE_CTL_BOOST;

	NAPI_DEBUG("-->%s(%d %d)", __func__, flags, op);

	if (!(bl_en && ccb_en)) {
		rc = -EINVAL;
		goto out;
	}

	switch (op) {
	case BLACKLIST_QUERY:
		rc = ref_count;
		break;
	case BLACKLIST_ON:
		ref_count++;
		rc = 0;
		if (ref_count == 1) {
			rc = hif_napi_core_ctl_set_boost(true);
			NAPI_DEBUG("boost_on() returns %d - refcnt=%d",
				rc, ref_count);
			hif_exec_bl_irq(napid, true);
		}
		break;
	case BLACKLIST_OFF:
		if (ref_count)
			ref_count--;
		rc = 0;
		if (ref_count == 0) {
			rc = hif_napi_core_ctl_set_boost(false);
			NAPI_DEBUG("boost_off() returns %d - refcnt=%d",
				   rc, ref_count);
			hif_exec_bl_irq(napid, false);
		}
		break;
	default:
		NAPI_DEBUG("Invalid blacklist op: %d", op);
		rc = -EINVAL;
	} /* switch */
out:
	NAPI_DEBUG("<--%s[%d]", __func__, rc);
	return rc;
}