blob: 12bc8c3663add73a4402074d2fa2919e01d3aa09 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
Paul Mackerras69a80d32005-10-10 22:03:41 +10002 * SMP support for pSeries machines.
Linus Torvalds1da177e2005-04-16 15:20:36 -07003 *
4 * Dave Engebretsen, Peter Bergner, and
5 * Mike Corrigan {engebret|bergner|mikec}@us.ibm.com
6 *
7 * Plus various changes from other IBM teams...
8 *
9 * This program is free software; you can redistribute it and/or
10 * modify it under the terms of the GNU General Public License
11 * as published by the Free Software Foundation; either version
12 * 2 of the License, or (at your option) any later version.
13 */
14
Linus Torvalds1da177e2005-04-16 15:20:36 -070015
Linus Torvalds1da177e2005-04-16 15:20:36 -070016#include <linux/kernel.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070017#include <linux/sched.h>
18#include <linux/smp.h>
19#include <linux/interrupt.h>
20#include <linux/delay.h>
21#include <linux/init.h>
22#include <linux/spinlock.h>
23#include <linux/cache.h>
24#include <linux/err.h>
Kay Sieversedbaa602011-12-21 16:26:03 -080025#include <linux/device.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070026#include <linux/cpu.h>
27
28#include <asm/ptrace.h>
Arun Sharma600634972011-07-26 16:09:06 -070029#include <linux/atomic.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070030#include <asm/irq.h>
31#include <asm/page.h>
32#include <asm/pgtable.h>
33#include <asm/io.h>
34#include <asm/prom.h>
35#include <asm/smp.h>
36#include <asm/paca.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070037#include <asm/machdep.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070038#include <asm/cputable.h>
Stephen Rothwell1ababe12005-08-03 14:35:25 +100039#include <asm/firmware.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070040#include <asm/rtas.h>
Stephen Rothwellbbeb3f42005-09-27 13:51:59 +100041#include <asm/mpic.h>
Paul Mackerras271c3f32005-11-11 23:04:40 +110042#include <asm/vdso_datapage.h>
Benjamin Herrenschmidt8d089082007-10-25 15:27:44 +100043#include <asm/cputhreads.h>
Benjamin Herrenschmidt0b05ac62011-04-04 13:46:58 +100044#include <asm/xics.h>
Ian Munsiee5e84f02012-11-14 18:49:50 +000045#include <asm/dbell.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070046
Michael Ellermana1218722005-11-03 15:33:31 +110047#include "plpar_wrappers.h"
Michael Ellerman8feaeca2007-02-08 18:33:55 +110048#include "pseries.h"
Gautham R Shenoy3aa565f2009-10-29 19:22:53 +000049#include "offline_states.h"
Michael Ellermana1218722005-11-03 15:33:31 +110050
Linus Torvalds1da177e2005-04-16 15:20:36 -070051
52/*
Milton Miller6a75a6b2008-10-20 15:37:03 +000053 * The Primary thread of each non-boot processor was started from the OF client
54 * interface by prom_hold_cpus and is spinning on secondary_hold_spinloop.
Linus Torvalds1da177e2005-04-16 15:20:36 -070055 */
Anton Blanchardaf831e12010-04-26 15:32:37 +000056static cpumask_var_t of_spin_mask;
Linus Torvalds1da177e2005-04-16 15:20:36 -070057
Ian Munsiee5e84f02012-11-14 18:49:50 +000058/*
59 * If we multiplex IPI mechanisms, store the appropriate XICS IPI mechanism here
60 */
61static void (*xics_cause_ipi)(int cpu, unsigned long data);
62
Michael Neulingf8b67692010-04-28 13:39:41 +000063/* Query where a cpu is now. Return codes #defined in plpar_wrappers.h */
64int smp_query_cpu_stopped(unsigned int pcpu)
65{
66 int cpu_status, status;
67 int qcss_tok = rtas_token("query-cpu-stopped-state");
68
69 if (qcss_tok == RTAS_UNKNOWN_SERVICE) {
Milton Miller2d869382011-03-21 08:12:13 +000070 printk_once(KERN_INFO
71 "Firmware doesn't support query-cpu-stopped-state\n");
Michael Neulingf8b67692010-04-28 13:39:41 +000072 return QCSS_HARDWARE_ERROR;
73 }
74
75 status = rtas_call(qcss_tok, 1, 2, &cpu_status, pcpu);
76 if (status != 0) {
77 printk(KERN_ERR
78 "RTAS query-cpu-stopped-state failed: %i\n", status);
79 return status;
80 }
81
82 return cpu_status;
83}
84
Linus Torvalds1da177e2005-04-16 15:20:36 -070085/**
86 * smp_startup_cpu() - start the given cpu
87 *
88 * At boot time, there is nothing to do for primary threads which were
89 * started from Open Firmware. For anything else, call RTAS with the
90 * appropriate start location.
91 *
92 * Returns:
93 * 0 - failure
94 * 1 - success
95 */
Greg Kroah-Hartmancad5cef2012-12-21 14:04:10 -080096static inline int smp_startup_cpu(unsigned int lcpu)
Linus Torvalds1da177e2005-04-16 15:20:36 -070097{
98 int status;
99 unsigned long start_here = __pa((u32)*((unsigned long *)
Olof Johanssonf39b7a52006-08-11 00:07:08 -0500100 generic_secondary_smp_init));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700101 unsigned int pcpu;
Anton Blanchard1ed2fd22005-09-12 13:17:27 +1000102 int start_cpu;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700103
Anton Blanchardaf831e12010-04-26 15:32:37 +0000104 if (cpumask_test_cpu(lcpu, of_spin_mask))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700105 /* Already started by OF and sitting in spin loop */
106 return 1;
107
108 pcpu = get_hard_smp_processor_id(lcpu);
109
Michael Neulingaef40e82010-04-28 13:39:41 +0000110 /* Check to see if the CPU out of FW already for kexec */
111 if (smp_query_cpu_stopped(pcpu) == QCSS_NOT_STOPPED){
Anton Blanchardaf831e12010-04-26 15:32:37 +0000112 cpumask_set_cpu(lcpu, of_spin_mask);
Michael Neulingaef40e82010-04-28 13:39:41 +0000113 return 1;
114 }
115
Linus Torvalds1da177e2005-04-16 15:20:36 -0700116 /* Fixup atomic count: it exited inside IRQ handler. */
Al Virob5e2fc12006-01-12 01:06:01 -0800117 task_thread_info(paca[lcpu].__current)->preempt_count = 0;
Matt Evansc60e65d2011-03-31 19:33:08 +0000118#ifdef CONFIG_HOTPLUG_CPU
Gautham R Shenoy3aa565f2009-10-29 19:22:53 +0000119 if (get_cpu_current_state(lcpu) == CPU_STATE_INACTIVE)
120 goto out;
Matt Evansc60e65d2011-03-31 19:33:08 +0000121#endif
Anton Blanchard1ed2fd22005-09-12 13:17:27 +1000122 /*
123 * If the RTAS start-cpu token does not exist then presume the
124 * cpu is already spinning.
125 */
126 start_cpu = rtas_token("start-cpu");
127 if (start_cpu == RTAS_UNKNOWN_SERVICE)
128 return 1;
129
Michael Ellerman496b7a52006-02-16 14:13:53 +1100130 status = rtas_call(start_cpu, 3, 1, NULL, pcpu, start_here, pcpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700131 if (status != 0) {
132 printk(KERN_ERR "start-cpu failed: %i\n", status);
133 return 0;
134 }
Anton Blanchard1ed2fd22005-09-12 13:17:27 +1000135
Matt Evansc60e65d2011-03-31 19:33:08 +0000136#ifdef CONFIG_HOTPLUG_CPU
Gautham R Shenoy3aa565f2009-10-29 19:22:53 +0000137out:
Matt Evansc60e65d2011-03-31 19:33:08 +0000138#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700139 return 1;
140}
141
Greg Kroah-Hartmancad5cef2012-12-21 14:04:10 -0800142static void smp_xics_setup_cpu(int cpu)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700143{
144 if (cpu != boot_cpuid)
145 xics_setup_cpu();
Ian Munsiee5e84f02012-11-14 18:49:50 +0000146 if (cpu_has_feature(CPU_FTR_DBELL))
147 doorbell_setup_this_cpu();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700148
Stephen Rothwell1ababe12005-08-03 14:35:25 +1000149 if (firmware_has_feature(FW_FEATURE_SPLPAR))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700150 vpa_init(cpu);
151
Anton Blanchardaf831e12010-04-26 15:32:37 +0000152 cpumask_clear_cpu(cpu, of_spin_mask);
Matt Evansc60e65d2011-03-31 19:33:08 +0000153#ifdef CONFIG_HOTPLUG_CPU
Gautham R Shenoy3aa565f2009-10-29 19:22:53 +0000154 set_cpu_current_state(cpu, CPU_STATE_ONLINE);
155 set_default_offline_state(cpu);
Matt Evansc60e65d2011-03-31 19:33:08 +0000156#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700157}
158
Greg Kroah-Hartmancad5cef2012-12-21 14:04:10 -0800159static int smp_pSeries_kick_cpu(int nr)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700160{
161 BUG_ON(nr < 0 || nr >= NR_CPUS);
162
163 if (!smp_startup_cpu(nr))
Michael Ellermande300972011-04-11 21:46:19 +0000164 return -ENOENT;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700165
166 /*
167 * The processor is currently spinning, waiting for the
168 * cpu_start field to become non-zero After we set cpu_start,
169 * the processor will continue on to secondary_start
170 */
171 paca[nr].cpu_start = 1;
Matt Evansc60e65d2011-03-31 19:33:08 +0000172#ifdef CONFIG_HOTPLUG_CPU
Gautham R Shenoy3aa565f2009-10-29 19:22:53 +0000173 set_preferred_offline_state(nr, CPU_STATE_ONLINE);
174
175 if (get_cpu_current_state(nr) == CPU_STATE_INACTIVE) {
Matt Evansc60e65d2011-03-31 19:33:08 +0000176 long rc;
177 unsigned long hcpuid;
178
Gautham R Shenoy3aa565f2009-10-29 19:22:53 +0000179 hcpuid = get_hard_smp_processor_id(nr);
180 rc = plpar_hcall_norets(H_PROD, hcpuid);
181 if (rc != H_SUCCESS)
Joe Perches5a2ad982010-01-31 10:02:03 +0000182 printk(KERN_ERR "Error: Prod to wake up processor %d "
183 "Ret= %ld\n", nr, rc);
Gautham R Shenoy3aa565f2009-10-29 19:22:53 +0000184 }
Matt Evansc60e65d2011-03-31 19:33:08 +0000185#endif
Michael Ellermande300972011-04-11 21:46:19 +0000186
187 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700188}
189
190static int smp_pSeries_cpu_bootable(unsigned int nr)
191{
192 /* Special case - we inhibit secondary thread startup
Milton Miller6a75a6b2008-10-20 15:37:03 +0000193 * during boot if the user requests it.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700194 */
Nathan Fontenot954e6da2010-08-05 07:42:11 +0000195 if (system_state < SYSTEM_RUNNING && cpu_has_feature(CPU_FTR_SMT)) {
196 if (!smt_enabled_at_boot && cpu_thread_in_core(nr) != 0)
197 return 0;
198 if (smt_enabled_at_boot
199 && cpu_thread_in_core(nr) >= smt_enabled_at_boot)
200 return 0;
201 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700202
203 return 1;
204}
Benjamin Herrenschmidt0b05ac62011-04-04 13:46:58 +1000205
Ian Munsiee5e84f02012-11-14 18:49:50 +0000206/* Only used on systems that support multiple IPI mechanisms */
207static void pSeries_cause_ipi_mux(int cpu, unsigned long data)
208{
209 if (cpumask_test_cpu(cpu, cpu_sibling_mask(smp_processor_id())))
210 doorbell_cause_ipi(cpu, data);
211 else
212 xics_cause_ipi(cpu, data);
213}
214
215static __init int pSeries_smp_probe(void)
216{
217 int ret = xics_smp_probe();
218
219 if (cpu_has_feature(CPU_FTR_DBELL)) {
220 xics_cause_ipi = smp_ops->cause_ipi;
221 smp_ops->cause_ipi = pSeries_cause_ipi_mux;
222 }
223
224 return ret;
225}
226
Linus Torvalds1da177e2005-04-16 15:20:36 -0700227static struct smp_ops_t pSeries_mpic_smp_ops = {
228 .message_pass = smp_mpic_message_pass,
229 .probe = smp_mpic_probe,
230 .kick_cpu = smp_pSeries_kick_cpu,
231 .setup_cpu = smp_mpic_setup_cpu,
232};
Benjamin Herrenschmidt0b05ac62011-04-04 13:46:58 +1000233
Linus Torvalds1da177e2005-04-16 15:20:36 -0700234static struct smp_ops_t pSeries_xics_smp_ops = {
Paul Mackerras9ca980d2011-05-25 23:34:12 +0000235 .message_pass = NULL, /* Use smp_muxed_ipi_message_pass */
Ian Munsiee5e84f02012-11-14 18:49:50 +0000236 .cause_ipi = NULL, /* Filled at runtime by pSeries_smp_probe() */
237 .probe = pSeries_smp_probe,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700238 .kick_cpu = smp_pSeries_kick_cpu,
239 .setup_cpu = smp_xics_setup_cpu,
240 .cpu_bootable = smp_pSeries_cpu_bootable,
241};
242
243/* This is called very early */
Benjamin Herrenschmidt0ebfff12006-07-03 21:36:01 +1000244static void __init smp_init_pseries(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700245{
246 int i;
247
Michael Ellermanf7ebf352008-04-24 15:13:19 +1000248 pr_debug(" -> smp_init_pSeries()\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700249
Anton Blanchardaf831e12010-04-26 15:32:37 +0000250 alloc_bootmem_cpumask_var(&of_spin_mask);
251
Linus Torvalds1da177e2005-04-16 15:20:36 -0700252 /* Mark threads which are still spinning in hold loops. */
Anton Blanchard0231c292005-06-20 21:43:15 +1000253 if (cpu_has_feature(CPU_FTR_SMT)) {
254 for_each_present_cpu(i) {
Milton Miller6a75a6b2008-10-20 15:37:03 +0000255 if (cpu_thread_in_core(i) == 0)
Anton Blanchardaf831e12010-04-26 15:32:37 +0000256 cpumask_set_cpu(i, of_spin_mask);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700257 }
Anton Blanchard0231c292005-06-20 21:43:15 +1000258 } else {
Anton Blanchardaf831e12010-04-26 15:32:37 +0000259 cpumask_copy(of_spin_mask, cpu_present_mask);
Anton Blanchard0231c292005-06-20 21:43:15 +1000260 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700261
Anton Blanchardaf831e12010-04-26 15:32:37 +0000262 cpumask_clear_cpu(boot_cpuid, of_spin_mask);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700263
264 /* Non-lpar has additional take/give timebase */
265 if (rtas_token("freeze-time-base") != RTAS_UNKNOWN_SERVICE) {
Benjamin Herrenschmidtc4007a22009-06-16 16:42:50 +0000266 smp_ops->give_timebase = rtas_give_timebase;
267 smp_ops->take_timebase = rtas_take_timebase;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700268 }
269
Michael Ellermanf7ebf352008-04-24 15:13:19 +1000270 pr_debug(" <- smp_init_pSeries()\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700271}
272
Benjamin Herrenschmidt0ebfff12006-07-03 21:36:01 +1000273void __init smp_init_pseries_mpic(void)
274{
275 smp_ops = &pSeries_mpic_smp_ops;
276
277 smp_init_pseries();
278}
Benjamin Herrenschmidt0ebfff12006-07-03 21:36:01 +1000279
280void __init smp_init_pseries_xics(void)
281{
282 smp_ops = &pSeries_xics_smp_ops;
283
284 smp_init_pseries();
285}