cpu/hotplug: Let upcoming cpu bring itself fully up
Let the upcoming cpu kick the hotplug thread and let itself complete the
bringup. That way the controll side can just wait for the completion or later
when we made the hotplug machinery async not care at all.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-arch@vger.kernel.org
Cc: Rik van Riel <riel@redhat.com>
Cc: Rafael Wysocki <rafael.j.wysocki@intel.com>
Cc: "Srivatsa S. Bhat" <srivatsa@mit.edu>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Arjan van de Ven <arjan@linux.intel.com>
Cc: Sebastian Siewior <bigeasy@linutronix.de>
Cc: Rusty Russell <rusty@rustcorp.com.au>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Tejun Heo <tj@kernel.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Paul McKenney <paulmck@linux.vnet.ibm.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Paul Turner <pjt@google.com>
Link: http://lkml.kernel.org/r/20160226182341.697655464@linutronix.de
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h
index 4aa263a..ad5d7fc 100644
--- a/include/linux/cpuhotplug.h
+++ b/include/linux/cpuhotplug.h
@@ -10,9 +10,6 @@
CPUHP_AP_NOTIFY_STARTING,
CPUHP_AP_ONLINE,
CPUHP_TEARDOWN_CPU,
- CPUHP_CPU_SET_ACTIVE,
- CPUHP_KICK_AP_THREAD,
- CPUHP_BP_ONLINE,
CPUHP_AP_ONLINE_IDLE,
CPUHP_AP_SMPBOOT_THREADS,
CPUHP_AP_NOTIFY_ONLINE,
@@ -86,4 +83,10 @@
__cpuhp_remove_state(state, false);
}
+#ifdef CONFIG_SMP
+void cpuhp_online_idle(enum cpuhp_state state);
+#else
+static inline void cpuhp_online_idle(enum cpuhp_state state) { }
+#endif
+
#endif
diff --git a/kernel/cpu.c b/kernel/cpu.c
index e220e56..f1f880f 100644
--- a/kernel/cpu.c
+++ b/kernel/cpu.c
@@ -329,6 +329,14 @@
return 0;
}
+static int bringup_wait_for_ap(unsigned int cpu)
+{
+ struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
+
+ wait_for_completion(&st->done);
+ return st->result;
+}
+
static int bringup_cpu(unsigned int cpu)
{
struct task_struct *idle = idle_thread_get(cpu);
@@ -340,8 +348,9 @@
cpu_notify(CPU_UP_CANCELED, cpu);
return ret;
}
+ ret = bringup_wait_for_ap(cpu);
BUG_ON(!cpu_online(cpu));
- return 0;
+ return ret;
}
/*
@@ -470,7 +479,7 @@
}
} else {
/* Cannot happen .... */
- BUG_ON(st->state < CPUHP_KICK_AP_THREAD);
+ BUG_ON(st->state < CPUHP_AP_ONLINE_IDLE);
/* Regular hotplug work */
if (st->state < st->target)
@@ -780,7 +789,7 @@
* If the current CPU state is in the range of the AP hotplug thread,
* then we need to kick the thread.
*/
- if (st->state >= CPUHP_KICK_AP_THREAD) {
+ if (st->state > CPUHP_TEARDOWN_CPU) {
ret = cpuhp_kick_ap_work(cpu);
/*
* The AP side has done the error rollback already. Just
@@ -793,11 +802,11 @@
* We might have stopped still in the range of the AP hotplug
* thread. Nothing to do anymore.
*/
- if (st->state >= CPUHP_KICK_AP_THREAD)
+ if (st->state > CPUHP_TEARDOWN_CPU)
goto out;
}
/*
- * The AP brought itself down below CPUHP_KICK_AP_THREAD. So we need
+ * The AP brought itself down to CPUHP_TEARDOWN_CPU. So we need
* to do the further cleanups.
*/
ret = cpuhp_down_callbacks(cpu, st, cpuhp_bp_states, target);
@@ -859,18 +868,32 @@
/*
* Called from the idle task. We need to set active here, so we can kick off
- * the stopper thread.
+ * the stopper thread and unpark the smpboot threads. If the target state is
+ * beyond CPUHP_AP_ONLINE_IDLE we kick cpuhp thread and let it bring up the
+ * cpu further.
*/
-static int cpuhp_set_cpu_active(unsigned int cpu)
+void cpuhp_online_idle(enum cpuhp_state state)
{
- struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
+ struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
+ unsigned int cpu = smp_processor_id();
+
+ /* Happens for the boot cpu */
+ if (state != CPUHP_AP_ONLINE_IDLE)
+ return;
+
+ st->state = CPUHP_AP_ONLINE_IDLE;
/* The cpu is marked online, set it active now */
set_cpu_active(cpu, true);
- /* Unpark the stopper thread and the hotplug thread */
+ /* Unpark the stopper thread and the hotplug thread of this cpu */
stop_machine_unpark(cpu);
kthread_unpark(st->thread);
- return 0;
+
+ /* Should we go further up ? */
+ if (st->target > CPUHP_AP_ONLINE_IDLE)
+ __cpuhp_kick_ap_work(st);
+ else
+ complete(&st->done);
}
/* Requires cpu_add_remove_lock to be held */
@@ -910,7 +933,7 @@
* If the current CPU state is in the range of the AP hotplug thread,
* then we need to kick the thread once more.
*/
- if (st->state >= CPUHP_KICK_AP_THREAD) {
+ if (st->state > CPUHP_BRINGUP_CPU) {
ret = cpuhp_kick_ap_work(cpu);
/*
* The AP side has done the error rollback already. Just
@@ -922,10 +945,10 @@
/*
* Try to reach the target state. We max out on the BP at
- * CPUHP_KICK_AP_THREAD. After that the AP hotplug thread is
+ * CPUHP_BRINGUP_CPU. After that the AP hotplug thread is
* responsible for bringing it up to the target state.
*/
- target = min((int)target, CPUHP_KICK_AP_THREAD);
+ target = min((int)target, CPUHP_BRINGUP_CPU);
ret = cpuhp_up_callbacks(cpu, st, cpuhp_bp_states, target);
out:
cpu_hotplug_done();
@@ -1146,22 +1169,7 @@
.teardown = takedown_cpu,
.cant_stop = true,
},
- [CPUHP_CPU_SET_ACTIVE] = {
- .name = "cpu:active",
- .startup = cpuhp_set_cpu_active,
- .teardown = NULL,
- },
- [CPUHP_KICK_AP_THREAD] = {
- .name = "cpuhp:kickthread",
- .startup = cpuhp_kick_ap_work,
- .teardown = cpuhp_kick_ap_work,
- },
#endif
- [CPUHP_BP_ONLINE] = {
- .name = "online",
- .startup = NULL,
- .teardown = NULL,
- },
};
/* Application processor state steps */
@@ -1204,7 +1212,7 @@
{
if (state >= CPUHP_AP_OFFLINE && state <= CPUHP_AP_ONLINE)
return true;
- return state > CPUHP_BP_ONLINE;
+ return state > CPUHP_BRINGUP_CPU;
}
static struct cpuhp_step *cpuhp_get_step(enum cpuhp_state state)
diff --git a/kernel/sched/idle.c b/kernel/sched/idle.c
index 544a713..a4b9813 100644
--- a/kernel/sched/idle.c
+++ b/kernel/sched/idle.c
@@ -4,6 +4,7 @@
#include <linux/sched.h>
#include <linux/cpu.h>
#include <linux/cpuidle.h>
+#include <linux/cpuhotplug.h>
#include <linux/tick.h>
#include <linux/mm.h>
#include <linux/stackprotector.h>
@@ -291,5 +292,6 @@
boot_init_stack_canary();
#endif
arch_cpu_idle_prepare();
+ cpuhp_online_idle(state);
cpu_idle_loop();
}