softirq: defer softirq processing to ksoftirqd if CPU is busy with RT
Defer the softirq processing to ksoftirqd if a RT task is running
or queued on the current CPU. This complements the RT task placement
algorithm which tries to find a CPU that is not currently busy with
softirqs.
Currently NET_TX, NET_RX, BLOCK and TASKLET softirqs are only deferred
as they can potentially run for long time.
Change-Id: Id7665244af6bbd5a96d9e591cf26154e9eaa860c
Signed-off-by: Pavankumar Kondeti <pkondeti@codeaurora.org>
diff --git a/include/linux/sched.h b/include/linux/sched.h
index 779c5c4..9fa32c1 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -2621,6 +2621,7 @@
extern int set_cpus_allowed_ptr(struct task_struct *p,
const struct cpumask *new_mask);
+extern bool cpupri_check_rt(void);
#else
static inline void do_set_cpus_allowed(struct task_struct *p,
const struct cpumask *new_mask)
@@ -2633,6 +2634,10 @@
return -EINVAL;
return 0;
}
+static inline bool cpupri_check_rt(void)
+{
+ return false;
+}
#endif
struct sched_load {
diff --git a/kernel/sched/cpupri.c b/kernel/sched/cpupri.c
index ba5e3e2..87bea1e 100644
--- a/kernel/sched/cpupri.c
+++ b/kernel/sched/cpupri.c
@@ -279,3 +279,14 @@
for (i = 0; i < CPUPRI_NR_PRIORITIES; i++)
free_cpumask_var(cp->pri_to_cpu[i].mask);
}
+
+/*
+ * cpupri_check_rt - check if CPU has a RT task
+ * should be called from rcu-sched read section.
+ */
+bool cpupri_check_rt(void)
+{
+ int cpu = raw_smp_processor_id();
+
+ return cpu_rq(cpu)->rd->cpupri.cpu_to_pri[cpu] > CPUPRI_NORMAL;
+}
diff --git a/kernel/softirq.c b/kernel/softirq.c
index bde8e33..6833ffa 100644
--- a/kernel/softirq.c
+++ b/kernel/softirq.c
@@ -245,6 +245,8 @@
static inline void lockdep_softirq_end(bool in_hardirq) { }
#endif
+#define long_softirq_pending() (local_softirq_pending() & LONG_SOFTIRQ_MASK)
+#define defer_for_rt() (long_softirq_pending() && cpupri_check_rt())
asmlinkage __visible void __softirq_entry __do_softirq(void)
{
unsigned long end = jiffies + MAX_SOFTIRQ_TIME;
@@ -308,6 +310,7 @@
pending = local_softirq_pending();
if (pending) {
if (time_before(jiffies, end) && !need_resched() &&
+ !defer_for_rt() &&
--max_restart)
goto restart;
@@ -363,7 +366,7 @@
if (ksoftirqd_running())
return;
- if (!force_irqthreads) {
+ if (!force_irqthreads && !defer_for_rt()) {
#ifdef CONFIG_HAVE_IRQ_EXIT_ON_IRQ_STACK
/*
* We can safely execute softirq on the current stack if