blob: 36ecfc870e5a6cb86698cb218ac00cccc491e299 [file] [log] [blame]
Marc Zyngier021f6532014-06-30 16:01:31 +01001/*
2 * Copyright (C) 2013, 2014 ARM Limited, All Rights Reserved.
3 * Author: Marc Zyngier <marc.zyngier@arm.com>
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License version 2 as
7 * published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program. If not, see <http://www.gnu.org/licenses/>.
16 */
17
18#include <linux/cpu.h>
Sudeep Holla3708d522014-08-26 16:03:35 +010019#include <linux/cpu_pm.h>
Marc Zyngier021f6532014-06-30 16:01:31 +010020#include <linux/delay.h>
21#include <linux/interrupt.h>
22#include <linux/of.h>
23#include <linux/of_address.h>
24#include <linux/of_irq.h>
25#include <linux/percpu.h>
26#include <linux/slab.h>
27
Joel Porquet41a83e02015-07-07 17:11:46 -040028#include <linux/irqchip.h>
Marc Zyngier021f6532014-06-30 16:01:31 +010029#include <linux/irqchip/arm-gic-v3.h>
30
31#include <asm/cputype.h>
32#include <asm/exception.h>
33#include <asm/smp_plat.h>
Marc Zyngier0b6a3da2015-08-26 17:00:42 +010034#include <asm/virt.h>
Marc Zyngier021f6532014-06-30 16:01:31 +010035
36#include "irq-gic-common.h"
Marc Zyngier021f6532014-06-30 16:01:31 +010037
Marc Zyngierf5c14342014-11-24 14:35:10 +000038struct redist_region {
39 void __iomem *redist_base;
40 phys_addr_t phys_base;
41};
42
Marc Zyngier021f6532014-06-30 16:01:31 +010043struct gic_chip_data {
44 void __iomem *dist_base;
Marc Zyngierf5c14342014-11-24 14:35:10 +000045 struct redist_region *redist_regions;
46 struct rdists rdists;
Marc Zyngier021f6532014-06-30 16:01:31 +010047 struct irq_domain *domain;
48 u64 redist_stride;
Marc Zyngierf5c14342014-11-24 14:35:10 +000049 u32 nr_redist_regions;
Marc Zyngier021f6532014-06-30 16:01:31 +010050 unsigned int irq_nr;
51};
52
53static struct gic_chip_data gic_data __read_mostly;
Marc Zyngier0b6a3da2015-08-26 17:00:42 +010054static struct static_key supports_deactivate = STATIC_KEY_INIT_TRUE;
Marc Zyngier021f6532014-06-30 16:01:31 +010055
Marc Zyngierf5c14342014-11-24 14:35:10 +000056#define gic_data_rdist() (this_cpu_ptr(gic_data.rdists.rdist))
57#define gic_data_rdist_rd_base() (gic_data_rdist()->rd_base)
Marc Zyngier021f6532014-06-30 16:01:31 +010058#define gic_data_rdist_sgi_base() (gic_data_rdist_rd_base() + SZ_64K)
59
60/* Our default, arbitrary priority value. Linux only uses one anyway. */
61#define DEFAULT_PMR_VALUE 0xf0
62
63static inline unsigned int gic_irq(struct irq_data *d)
64{
65 return d->hwirq;
66}
67
68static inline int gic_irq_in_rdist(struct irq_data *d)
69{
70 return gic_irq(d) < 32;
71}
72
73static inline void __iomem *gic_dist_base(struct irq_data *d)
74{
75 if (gic_irq_in_rdist(d)) /* SGI+PPI -> SGI_base for this CPU */
76 return gic_data_rdist_sgi_base();
77
78 if (d->hwirq <= 1023) /* SPI -> dist_base */
79 return gic_data.dist_base;
80
Marc Zyngier021f6532014-06-30 16:01:31 +010081 return NULL;
82}
83
84static void gic_do_wait_for_rwp(void __iomem *base)
85{
86 u32 count = 1000000; /* 1s! */
87
88 while (readl_relaxed(base + GICD_CTLR) & GICD_CTLR_RWP) {
89 count--;
90 if (!count) {
91 pr_err_ratelimited("RWP timeout, gone fishing\n");
92 return;
93 }
94 cpu_relax();
95 udelay(1);
96 };
97}
98
99/* Wait for completion of a distributor change */
100static void gic_dist_wait_for_rwp(void)
101{
102 gic_do_wait_for_rwp(gic_data.dist_base);
103}
104
105/* Wait for completion of a redistributor change */
106static void gic_redist_wait_for_rwp(void)
107{
108 gic_do_wait_for_rwp(gic_data_rdist_rd_base());
109}
110
111/* Low level accessors */
Mark Brownc44e9d72014-07-30 20:23:14 +0100112static u64 __maybe_unused gic_read_iar(void)
Marc Zyngier021f6532014-06-30 16:01:31 +0100113{
114 u64 irqstat;
115
Catalin Marinas72c58392014-07-24 14:14:42 +0100116 asm volatile("mrs_s %0, " __stringify(ICC_IAR1_EL1) : "=r" (irqstat));
Marc Zyngier021f6532014-06-30 16:01:31 +0100117 return irqstat;
118}
119
Mark Brownc44e9d72014-07-30 20:23:14 +0100120static void __maybe_unused gic_write_pmr(u64 val)
Marc Zyngier021f6532014-06-30 16:01:31 +0100121{
Catalin Marinas72c58392014-07-24 14:14:42 +0100122 asm volatile("msr_s " __stringify(ICC_PMR_EL1) ", %0" : : "r" (val));
Marc Zyngier021f6532014-06-30 16:01:31 +0100123}
124
Mark Brownc44e9d72014-07-30 20:23:14 +0100125static void __maybe_unused gic_write_ctlr(u64 val)
Marc Zyngier021f6532014-06-30 16:01:31 +0100126{
Catalin Marinas72c58392014-07-24 14:14:42 +0100127 asm volatile("msr_s " __stringify(ICC_CTLR_EL1) ", %0" : : "r" (val));
Marc Zyngier021f6532014-06-30 16:01:31 +0100128 isb();
129}
130
Mark Brownc44e9d72014-07-30 20:23:14 +0100131static void __maybe_unused gic_write_grpen1(u64 val)
Marc Zyngier021f6532014-06-30 16:01:31 +0100132{
Catalin Marinas72c58392014-07-24 14:14:42 +0100133 asm volatile("msr_s " __stringify(ICC_GRPEN1_EL1) ", %0" : : "r" (val));
Marc Zyngier021f6532014-06-30 16:01:31 +0100134 isb();
135}
136
Mark Brownc44e9d72014-07-30 20:23:14 +0100137static void __maybe_unused gic_write_sgi1r(u64 val)
Marc Zyngier021f6532014-06-30 16:01:31 +0100138{
Catalin Marinas72c58392014-07-24 14:14:42 +0100139 asm volatile("msr_s " __stringify(ICC_SGI1R_EL1) ", %0" : : "r" (val));
Marc Zyngier021f6532014-06-30 16:01:31 +0100140}
141
142static void gic_enable_sre(void)
143{
144 u64 val;
145
Catalin Marinas72c58392014-07-24 14:14:42 +0100146 asm volatile("mrs_s %0, " __stringify(ICC_SRE_EL1) : "=r" (val));
Marc Zyngier021f6532014-06-30 16:01:31 +0100147 val |= ICC_SRE_EL1_SRE;
Catalin Marinas72c58392014-07-24 14:14:42 +0100148 asm volatile("msr_s " __stringify(ICC_SRE_EL1) ", %0" : : "r" (val));
Marc Zyngier021f6532014-06-30 16:01:31 +0100149 isb();
150
151 /*
152 * Need to check that the SRE bit has actually been set. If
153 * not, it means that SRE is disabled at EL2. We're going to
154 * die painfully, and there is nothing we can do about it.
155 *
156 * Kindly inform the luser.
157 */
Catalin Marinas72c58392014-07-24 14:14:42 +0100158 asm volatile("mrs_s %0, " __stringify(ICC_SRE_EL1) : "=r" (val));
Marc Zyngier021f6532014-06-30 16:01:31 +0100159 if (!(val & ICC_SRE_EL1_SRE))
160 pr_err("GIC: unable to set SRE (disabled at EL2), panic ahead\n");
161}
162
Sudeep Hollaa2c22512014-08-26 16:03:34 +0100163static void gic_enable_redist(bool enable)
Marc Zyngier021f6532014-06-30 16:01:31 +0100164{
165 void __iomem *rbase;
166 u32 count = 1000000; /* 1s! */
167 u32 val;
168
169 rbase = gic_data_rdist_rd_base();
170
Marc Zyngier021f6532014-06-30 16:01:31 +0100171 val = readl_relaxed(rbase + GICR_WAKER);
Sudeep Hollaa2c22512014-08-26 16:03:34 +0100172 if (enable)
173 /* Wake up this CPU redistributor */
174 val &= ~GICR_WAKER_ProcessorSleep;
175 else
176 val |= GICR_WAKER_ProcessorSleep;
Marc Zyngier021f6532014-06-30 16:01:31 +0100177 writel_relaxed(val, rbase + GICR_WAKER);
178
Sudeep Hollaa2c22512014-08-26 16:03:34 +0100179 if (!enable) { /* Check that GICR_WAKER is writeable */
180 val = readl_relaxed(rbase + GICR_WAKER);
181 if (!(val & GICR_WAKER_ProcessorSleep))
182 return; /* No PM support in this redistributor */
183 }
184
185 while (count--) {
186 val = readl_relaxed(rbase + GICR_WAKER);
187 if (enable ^ (val & GICR_WAKER_ChildrenAsleep))
188 break;
Marc Zyngier021f6532014-06-30 16:01:31 +0100189 cpu_relax();
190 udelay(1);
191 };
Sudeep Hollaa2c22512014-08-26 16:03:34 +0100192 if (!count)
193 pr_err_ratelimited("redistributor failed to %s...\n",
194 enable ? "wakeup" : "sleep");
Marc Zyngier021f6532014-06-30 16:01:31 +0100195}
196
197/*
198 * Routines to disable, enable, EOI and route interrupts
199 */
Marc Zyngierb594c6e2015-03-18 11:01:24 +0000200static int gic_peek_irq(struct irq_data *d, u32 offset)
201{
202 u32 mask = 1 << (gic_irq(d) % 32);
203 void __iomem *base;
204
205 if (gic_irq_in_rdist(d))
206 base = gic_data_rdist_sgi_base();
207 else
208 base = gic_data.dist_base;
209
210 return !!(readl_relaxed(base + offset + (gic_irq(d) / 32) * 4) & mask);
211}
212
Marc Zyngier021f6532014-06-30 16:01:31 +0100213static void gic_poke_irq(struct irq_data *d, u32 offset)
214{
215 u32 mask = 1 << (gic_irq(d) % 32);
216 void (*rwp_wait)(void);
217 void __iomem *base;
218
219 if (gic_irq_in_rdist(d)) {
220 base = gic_data_rdist_sgi_base();
221 rwp_wait = gic_redist_wait_for_rwp;
222 } else {
223 base = gic_data.dist_base;
224 rwp_wait = gic_dist_wait_for_rwp;
225 }
226
227 writel_relaxed(mask, base + offset + (gic_irq(d) / 32) * 4);
228 rwp_wait();
229}
230
Marc Zyngier021f6532014-06-30 16:01:31 +0100231static void gic_mask_irq(struct irq_data *d)
232{
233 gic_poke_irq(d, GICD_ICENABLER);
234}
235
Marc Zyngier0b6a3da2015-08-26 17:00:42 +0100236static void gic_eoimode1_mask_irq(struct irq_data *d)
237{
238 gic_mask_irq(d);
Marc Zyngier530bf352015-08-26 17:00:43 +0100239 /*
240 * When masking a forwarded interrupt, make sure it is
241 * deactivated as well.
242 *
243 * This ensures that an interrupt that is getting
244 * disabled/masked will not get "stuck", because there is
245 * noone to deactivate it (guest is being terminated).
246 */
Thomas Gleixner4df7f542015-09-15 13:19:16 +0200247 if (irqd_is_forwarded_to_vcpu(d))
Marc Zyngier530bf352015-08-26 17:00:43 +0100248 gic_poke_irq(d, GICD_ICACTIVER);
Marc Zyngier0b6a3da2015-08-26 17:00:42 +0100249}
250
Marc Zyngier021f6532014-06-30 16:01:31 +0100251static void gic_unmask_irq(struct irq_data *d)
252{
253 gic_poke_irq(d, GICD_ISENABLER);
254}
255
Marc Zyngierb594c6e2015-03-18 11:01:24 +0000256static int gic_irq_set_irqchip_state(struct irq_data *d,
257 enum irqchip_irq_state which, bool val)
258{
259 u32 reg;
260
261 if (d->hwirq >= gic_data.irq_nr) /* PPI/SPI only */
262 return -EINVAL;
263
264 switch (which) {
265 case IRQCHIP_STATE_PENDING:
266 reg = val ? GICD_ISPENDR : GICD_ICPENDR;
267 break;
268
269 case IRQCHIP_STATE_ACTIVE:
270 reg = val ? GICD_ISACTIVER : GICD_ICACTIVER;
271 break;
272
273 case IRQCHIP_STATE_MASKED:
274 reg = val ? GICD_ICENABLER : GICD_ISENABLER;
275 break;
276
277 default:
278 return -EINVAL;
279 }
280
281 gic_poke_irq(d, reg);
282 return 0;
283}
284
285static int gic_irq_get_irqchip_state(struct irq_data *d,
286 enum irqchip_irq_state which, bool *val)
287{
288 if (d->hwirq >= gic_data.irq_nr) /* PPI/SPI only */
289 return -EINVAL;
290
291 switch (which) {
292 case IRQCHIP_STATE_PENDING:
293 *val = gic_peek_irq(d, GICD_ISPENDR);
294 break;
295
296 case IRQCHIP_STATE_ACTIVE:
297 *val = gic_peek_irq(d, GICD_ISACTIVER);
298 break;
299
300 case IRQCHIP_STATE_MASKED:
301 *val = !gic_peek_irq(d, GICD_ISENABLER);
302 break;
303
304 default:
305 return -EINVAL;
306 }
307
308 return 0;
309}
310
Marc Zyngier021f6532014-06-30 16:01:31 +0100311static void gic_eoi_irq(struct irq_data *d)
312{
313 gic_write_eoir(gic_irq(d));
314}
315
Marc Zyngier0b6a3da2015-08-26 17:00:42 +0100316static void gic_eoimode1_eoi_irq(struct irq_data *d)
317{
318 /*
Marc Zyngier530bf352015-08-26 17:00:43 +0100319 * No need to deactivate an LPI, or an interrupt that
320 * is is getting forwarded to a vcpu.
Marc Zyngier0b6a3da2015-08-26 17:00:42 +0100321 */
Thomas Gleixner4df7f542015-09-15 13:19:16 +0200322 if (gic_irq(d) >= 8192 || irqd_is_forwarded_to_vcpu(d))
Marc Zyngier0b6a3da2015-08-26 17:00:42 +0100323 return;
324 gic_write_dir(gic_irq(d));
325}
326
Marc Zyngier021f6532014-06-30 16:01:31 +0100327static int gic_set_type(struct irq_data *d, unsigned int type)
328{
329 unsigned int irq = gic_irq(d);
330 void (*rwp_wait)(void);
331 void __iomem *base;
332
333 /* Interrupt configuration for SGIs can't be changed */
334 if (irq < 16)
335 return -EINVAL;
336
Liviu Dudaufb7e7de2015-01-20 16:52:59 +0000337 /* SPIs have restrictions on the supported types */
338 if (irq >= 32 && type != IRQ_TYPE_LEVEL_HIGH &&
339 type != IRQ_TYPE_EDGE_RISING)
Marc Zyngier021f6532014-06-30 16:01:31 +0100340 return -EINVAL;
341
342 if (gic_irq_in_rdist(d)) {
343 base = gic_data_rdist_sgi_base();
344 rwp_wait = gic_redist_wait_for_rwp;
345 } else {
346 base = gic_data.dist_base;
347 rwp_wait = gic_dist_wait_for_rwp;
348 }
349
Liviu Dudaufb7e7de2015-01-20 16:52:59 +0000350 return gic_configure_irq(irq, type, base, rwp_wait);
Marc Zyngier021f6532014-06-30 16:01:31 +0100351}
352
Marc Zyngier530bf352015-08-26 17:00:43 +0100353static int gic_irq_set_vcpu_affinity(struct irq_data *d, void *vcpu)
354{
Thomas Gleixner4df7f542015-09-15 13:19:16 +0200355 if (vcpu)
356 irqd_set_forwarded_to_vcpu(d);
357 else
358 irqd_clr_forwarded_to_vcpu(d);
Marc Zyngier530bf352015-08-26 17:00:43 +0100359 return 0;
360}
361
Marc Zyngier021f6532014-06-30 16:01:31 +0100362static u64 gic_mpidr_to_affinity(u64 mpidr)
363{
364 u64 aff;
365
366 aff = (MPIDR_AFFINITY_LEVEL(mpidr, 3) << 32 |
367 MPIDR_AFFINITY_LEVEL(mpidr, 2) << 16 |
368 MPIDR_AFFINITY_LEVEL(mpidr, 1) << 8 |
369 MPIDR_AFFINITY_LEVEL(mpidr, 0));
370
371 return aff;
372}
373
374static asmlinkage void __exception_irq_entry gic_handle_irq(struct pt_regs *regs)
375{
376 u64 irqnr;
377
378 do {
379 irqnr = gic_read_iar();
380
Marc Zyngierda33f312014-11-24 14:35:18 +0000381 if (likely(irqnr > 15 && irqnr < 1020) || irqnr >= 8192) {
Marc Zyngierebc6de02014-08-26 11:03:33 +0100382 int err;
Marc Zyngier0b6a3da2015-08-26 17:00:42 +0100383
384 if (static_key_true(&supports_deactivate))
385 gic_write_eoir(irqnr);
386
Marc Zyngierebc6de02014-08-26 11:03:33 +0100387 err = handle_domain_irq(gic_data.domain, irqnr, regs);
388 if (err) {
Marc Zyngierda33f312014-11-24 14:35:18 +0000389 WARN_ONCE(true, "Unexpected interrupt received!\n");
Marc Zyngier0b6a3da2015-08-26 17:00:42 +0100390 if (static_key_true(&supports_deactivate)) {
391 if (irqnr < 8192)
392 gic_write_dir(irqnr);
393 } else {
394 gic_write_eoir(irqnr);
395 }
Marc Zyngier021f6532014-06-30 16:01:31 +0100396 }
Marc Zyngierebc6de02014-08-26 11:03:33 +0100397 continue;
Marc Zyngier021f6532014-06-30 16:01:31 +0100398 }
399 if (irqnr < 16) {
400 gic_write_eoir(irqnr);
Marc Zyngier0b6a3da2015-08-26 17:00:42 +0100401 if (static_key_true(&supports_deactivate))
402 gic_write_dir(irqnr);
Marc Zyngier021f6532014-06-30 16:01:31 +0100403#ifdef CONFIG_SMP
404 handle_IPI(irqnr, regs);
405#else
406 WARN_ONCE(true, "Unexpected SGI received!\n");
407#endif
408 continue;
409 }
410 } while (irqnr != ICC_IAR1_EL1_SPURIOUS);
411}
412
413static void __init gic_dist_init(void)
414{
415 unsigned int i;
416 u64 affinity;
417 void __iomem *base = gic_data.dist_base;
418
419 /* Disable the distributor */
420 writel_relaxed(0, base + GICD_CTLR);
421 gic_dist_wait_for_rwp();
422
423 gic_dist_config(base, gic_data.irq_nr, gic_dist_wait_for_rwp);
424
425 /* Enable distributor with ARE, Group1 */
426 writel_relaxed(GICD_CTLR_ARE_NS | GICD_CTLR_ENABLE_G1A | GICD_CTLR_ENABLE_G1,
427 base + GICD_CTLR);
428
429 /*
430 * Set all global interrupts to the boot CPU only. ARE must be
431 * enabled.
432 */
433 affinity = gic_mpidr_to_affinity(cpu_logical_map(smp_processor_id()));
434 for (i = 32; i < gic_data.irq_nr; i++)
435 writeq_relaxed(affinity, base + GICD_IROUTER + i * 8);
436}
437
438static int gic_populate_rdist(void)
439{
440 u64 mpidr = cpu_logical_map(smp_processor_id());
441 u64 typer;
442 u32 aff;
443 int i;
444
445 /*
446 * Convert affinity to a 32bit value that can be matched to
447 * GICR_TYPER bits [63:32].
448 */
449 aff = (MPIDR_AFFINITY_LEVEL(mpidr, 3) << 24 |
450 MPIDR_AFFINITY_LEVEL(mpidr, 2) << 16 |
451 MPIDR_AFFINITY_LEVEL(mpidr, 1) << 8 |
452 MPIDR_AFFINITY_LEVEL(mpidr, 0));
453
Marc Zyngierf5c14342014-11-24 14:35:10 +0000454 for (i = 0; i < gic_data.nr_redist_regions; i++) {
455 void __iomem *ptr = gic_data.redist_regions[i].redist_base;
Marc Zyngier021f6532014-06-30 16:01:31 +0100456 u32 reg;
457
458 reg = readl_relaxed(ptr + GICR_PIDR2) & GIC_PIDR2_ARCH_MASK;
459 if (reg != GIC_PIDR2_ARCH_GICv3 &&
460 reg != GIC_PIDR2_ARCH_GICv4) { /* We're in trouble... */
461 pr_warn("No redistributor present @%p\n", ptr);
462 break;
463 }
464
465 do {
466 typer = readq_relaxed(ptr + GICR_TYPER);
467 if ((typer >> 32) == aff) {
Marc Zyngierf5c14342014-11-24 14:35:10 +0000468 u64 offset = ptr - gic_data.redist_regions[i].redist_base;
Marc Zyngier021f6532014-06-30 16:01:31 +0100469 gic_data_rdist_rd_base() = ptr;
Marc Zyngierf5c14342014-11-24 14:35:10 +0000470 gic_data_rdist()->phys_base = gic_data.redist_regions[i].phys_base + offset;
471 pr_info("CPU%d: found redistributor %llx region %d:%pa\n",
Marc Zyngier021f6532014-06-30 16:01:31 +0100472 smp_processor_id(),
Marc Zyngierf5c14342014-11-24 14:35:10 +0000473 (unsigned long long)mpidr,
474 i, &gic_data_rdist()->phys_base);
Marc Zyngier021f6532014-06-30 16:01:31 +0100475 return 0;
476 }
477
478 if (gic_data.redist_stride) {
479 ptr += gic_data.redist_stride;
480 } else {
481 ptr += SZ_64K * 2; /* Skip RD_base + SGI_base */
482 if (typer & GICR_TYPER_VLPIS)
483 ptr += SZ_64K * 2; /* Skip VLPI_base + reserved page */
484 }
485 } while (!(typer & GICR_TYPER_LAST));
486 }
487
488 /* We couldn't even deal with ourselves... */
489 WARN(true, "CPU%d: mpidr %llx has no re-distributor!\n",
490 smp_processor_id(), (unsigned long long)mpidr);
491 return -ENODEV;
492}
493
Sudeep Holla3708d522014-08-26 16:03:35 +0100494static void gic_cpu_sys_reg_init(void)
Marc Zyngier021f6532014-06-30 16:01:31 +0100495{
Marc Zyngier021f6532014-06-30 16:01:31 +0100496 /* Enable system registers */
497 gic_enable_sre();
498
499 /* Set priority mask register */
500 gic_write_pmr(DEFAULT_PMR_VALUE);
501
Marc Zyngier0b6a3da2015-08-26 17:00:42 +0100502 if (static_key_true(&supports_deactivate)) {
503 /* EOI drops priority only (mode 1) */
504 gic_write_ctlr(ICC_CTLR_EL1_EOImode_drop);
505 } else {
506 /* EOI deactivates interrupt too (mode 0) */
507 gic_write_ctlr(ICC_CTLR_EL1_EOImode_drop_dir);
508 }
Marc Zyngier021f6532014-06-30 16:01:31 +0100509
510 /* ... and let's hit the road... */
511 gic_write_grpen1(1);
512}
513
Marc Zyngierda33f312014-11-24 14:35:18 +0000514static int gic_dist_supports_lpis(void)
515{
516 return !!(readl_relaxed(gic_data.dist_base + GICD_TYPER) & GICD_TYPER_LPIS);
517}
518
Marc Zyngier021f6532014-06-30 16:01:31 +0100519static void gic_cpu_init(void)
520{
521 void __iomem *rbase;
522
523 /* Register ourselves with the rest of the world */
524 if (gic_populate_rdist())
525 return;
526
Sudeep Hollaa2c22512014-08-26 16:03:34 +0100527 gic_enable_redist(true);
Marc Zyngier021f6532014-06-30 16:01:31 +0100528
529 rbase = gic_data_rdist_sgi_base();
530
531 gic_cpu_config(rbase, gic_redist_wait_for_rwp);
532
Marc Zyngierda33f312014-11-24 14:35:18 +0000533 /* Give LPIs a spin */
534 if (IS_ENABLED(CONFIG_ARM_GIC_V3_ITS) && gic_dist_supports_lpis())
535 its_cpu_init();
536
Sudeep Holla3708d522014-08-26 16:03:35 +0100537 /* initialise system registers */
538 gic_cpu_sys_reg_init();
Marc Zyngier021f6532014-06-30 16:01:31 +0100539}
540
541#ifdef CONFIG_SMP
Marc Zyngier021f6532014-06-30 16:01:31 +0100542static int gic_secondary_init(struct notifier_block *nfb,
543 unsigned long action, void *hcpu)
544{
545 if (action == CPU_STARTING || action == CPU_STARTING_FROZEN)
546 gic_cpu_init();
547 return NOTIFY_OK;
548}
549
550/*
551 * Notifier for enabling the GIC CPU interface. Set an arbitrarily high
552 * priority because the GIC needs to be up before the ARM generic timers.
553 */
554static struct notifier_block gic_cpu_notifier = {
555 .notifier_call = gic_secondary_init,
556 .priority = 100,
557};
558
559static u16 gic_compute_target_list(int *base_cpu, const struct cpumask *mask,
560 u64 cluster_id)
561{
562 int cpu = *base_cpu;
563 u64 mpidr = cpu_logical_map(cpu);
564 u16 tlist = 0;
565
566 while (cpu < nr_cpu_ids) {
567 /*
568 * If we ever get a cluster of more than 16 CPUs, just
569 * scream and skip that CPU.
570 */
571 if (WARN_ON((mpidr & 0xff) >= 16))
572 goto out;
573
574 tlist |= 1 << (mpidr & 0xf);
575
576 cpu = cpumask_next(cpu, mask);
Vladimir Murzin614be382015-03-06 16:37:45 +0000577 if (cpu >= nr_cpu_ids)
Marc Zyngier021f6532014-06-30 16:01:31 +0100578 goto out;
579
580 mpidr = cpu_logical_map(cpu);
581
582 if (cluster_id != (mpidr & ~0xffUL)) {
583 cpu--;
584 goto out;
585 }
586 }
587out:
588 *base_cpu = cpu;
589 return tlist;
590}
591
Andre Przywara7e580272014-11-12 13:46:06 +0000592#define MPIDR_TO_SGI_AFFINITY(cluster_id, level) \
593 (MPIDR_AFFINITY_LEVEL(cluster_id, level) \
594 << ICC_SGI1R_AFFINITY_## level ##_SHIFT)
595
Marc Zyngier021f6532014-06-30 16:01:31 +0100596static void gic_send_sgi(u64 cluster_id, u16 tlist, unsigned int irq)
597{
598 u64 val;
599
Andre Przywara7e580272014-11-12 13:46:06 +0000600 val = (MPIDR_TO_SGI_AFFINITY(cluster_id, 3) |
601 MPIDR_TO_SGI_AFFINITY(cluster_id, 2) |
602 irq << ICC_SGI1R_SGI_ID_SHIFT |
603 MPIDR_TO_SGI_AFFINITY(cluster_id, 1) |
604 tlist << ICC_SGI1R_TARGET_LIST_SHIFT);
Marc Zyngier021f6532014-06-30 16:01:31 +0100605
606 pr_debug("CPU%d: ICC_SGI1R_EL1 %llx\n", smp_processor_id(), val);
607 gic_write_sgi1r(val);
608}
609
610static void gic_raise_softirq(const struct cpumask *mask, unsigned int irq)
611{
612 int cpu;
613
614 if (WARN_ON(irq >= 16))
615 return;
616
617 /*
618 * Ensure that stores to Normal memory are visible to the
619 * other CPUs before issuing the IPI.
620 */
621 smp_wmb();
622
Rusty Russellf9b531f2015-03-05 10:49:16 +1030623 for_each_cpu(cpu, mask) {
Marc Zyngier021f6532014-06-30 16:01:31 +0100624 u64 cluster_id = cpu_logical_map(cpu) & ~0xffUL;
625 u16 tlist;
626
627 tlist = gic_compute_target_list(&cpu, mask, cluster_id);
628 gic_send_sgi(cluster_id, tlist, irq);
629 }
630
631 /* Force the above writes to ICC_SGI1R_EL1 to be executed */
632 isb();
633}
634
635static void gic_smp_init(void)
636{
637 set_smp_cross_call(gic_raise_softirq);
638 register_cpu_notifier(&gic_cpu_notifier);
639}
640
641static int gic_set_affinity(struct irq_data *d, const struct cpumask *mask_val,
642 bool force)
643{
644 unsigned int cpu = cpumask_any_and(mask_val, cpu_online_mask);
645 void __iomem *reg;
646 int enabled;
647 u64 val;
648
649 if (gic_irq_in_rdist(d))
650 return -EINVAL;
651
652 /* If interrupt was enabled, disable it first */
653 enabled = gic_peek_irq(d, GICD_ISENABLER);
654 if (enabled)
655 gic_mask_irq(d);
656
657 reg = gic_dist_base(d) + GICD_IROUTER + (gic_irq(d) * 8);
658 val = gic_mpidr_to_affinity(cpu_logical_map(cpu));
659
660 writeq_relaxed(val, reg);
661
662 /*
663 * If the interrupt was enabled, enabled it again. Otherwise,
664 * just wait for the distributor to have digested our changes.
665 */
666 if (enabled)
667 gic_unmask_irq(d);
668 else
669 gic_dist_wait_for_rwp();
670
671 return IRQ_SET_MASK_OK;
672}
673#else
674#define gic_set_affinity NULL
675#define gic_smp_init() do { } while(0)
676#endif
677
Sudeep Holla3708d522014-08-26 16:03:35 +0100678#ifdef CONFIG_CPU_PM
679static int gic_cpu_pm_notifier(struct notifier_block *self,
680 unsigned long cmd, void *v)
681{
682 if (cmd == CPU_PM_EXIT) {
683 gic_enable_redist(true);
684 gic_cpu_sys_reg_init();
685 } else if (cmd == CPU_PM_ENTER) {
686 gic_write_grpen1(0);
687 gic_enable_redist(false);
688 }
689 return NOTIFY_OK;
690}
691
692static struct notifier_block gic_cpu_pm_notifier_block = {
693 .notifier_call = gic_cpu_pm_notifier,
694};
695
696static void gic_cpu_pm_init(void)
697{
698 cpu_pm_register_notifier(&gic_cpu_pm_notifier_block);
699}
700
701#else
702static inline void gic_cpu_pm_init(void) { }
703#endif /* CONFIG_CPU_PM */
704
Marc Zyngier021f6532014-06-30 16:01:31 +0100705static struct irq_chip gic_chip = {
706 .name = "GICv3",
707 .irq_mask = gic_mask_irq,
708 .irq_unmask = gic_unmask_irq,
709 .irq_eoi = gic_eoi_irq,
710 .irq_set_type = gic_set_type,
711 .irq_set_affinity = gic_set_affinity,
Marc Zyngierb594c6e2015-03-18 11:01:24 +0000712 .irq_get_irqchip_state = gic_irq_get_irqchip_state,
713 .irq_set_irqchip_state = gic_irq_set_irqchip_state,
Sudeep Holla55963c92015-06-05 11:59:57 +0100714 .flags = IRQCHIP_SET_TYPE_MASKED,
Marc Zyngier021f6532014-06-30 16:01:31 +0100715};
716
Marc Zyngier0b6a3da2015-08-26 17:00:42 +0100717static struct irq_chip gic_eoimode1_chip = {
718 .name = "GICv3",
719 .irq_mask = gic_eoimode1_mask_irq,
720 .irq_unmask = gic_unmask_irq,
721 .irq_eoi = gic_eoimode1_eoi_irq,
722 .irq_set_type = gic_set_type,
723 .irq_set_affinity = gic_set_affinity,
724 .irq_get_irqchip_state = gic_irq_get_irqchip_state,
725 .irq_set_irqchip_state = gic_irq_set_irqchip_state,
Marc Zyngier530bf352015-08-26 17:00:43 +0100726 .irq_set_vcpu_affinity = gic_irq_set_vcpu_affinity,
Marc Zyngier0b6a3da2015-08-26 17:00:42 +0100727 .flags = IRQCHIP_SET_TYPE_MASKED,
728};
729
Marc Zyngierda33f312014-11-24 14:35:18 +0000730#define GIC_ID_NR (1U << gic_data.rdists.id_bits)
731
Marc Zyngier021f6532014-06-30 16:01:31 +0100732static int gic_irq_domain_map(struct irq_domain *d, unsigned int irq,
733 irq_hw_number_t hw)
734{
Marc Zyngier0b6a3da2015-08-26 17:00:42 +0100735 struct irq_chip *chip = &gic_chip;
736
737 if (static_key_true(&supports_deactivate))
738 chip = &gic_eoimode1_chip;
739
Marc Zyngier021f6532014-06-30 16:01:31 +0100740 /* SGIs are private to the core kernel */
741 if (hw < 16)
742 return -EPERM;
Marc Zyngierda33f312014-11-24 14:35:18 +0000743 /* Nothing here */
744 if (hw >= gic_data.irq_nr && hw < 8192)
745 return -EPERM;
746 /* Off limits */
747 if (hw >= GIC_ID_NR)
748 return -EPERM;
749
Marc Zyngier021f6532014-06-30 16:01:31 +0100750 /* PPIs */
751 if (hw < 32) {
752 irq_set_percpu_devid(irq);
Marc Zyngier0b6a3da2015-08-26 17:00:42 +0100753 irq_domain_set_info(d, irq, hw, chip, d->host_data,
Marc Zyngier443acc42014-11-24 14:35:09 +0000754 handle_percpu_devid_irq, NULL, NULL);
Rob Herringd17cab42015-08-29 18:01:22 -0500755 irq_set_status_flags(irq, IRQ_NOAUTOEN);
Marc Zyngier021f6532014-06-30 16:01:31 +0100756 }
757 /* SPIs */
758 if (hw >= 32 && hw < gic_data.irq_nr) {
Marc Zyngier0b6a3da2015-08-26 17:00:42 +0100759 irq_domain_set_info(d, irq, hw, chip, d->host_data,
Marc Zyngier443acc42014-11-24 14:35:09 +0000760 handle_fasteoi_irq, NULL, NULL);
Rob Herringd17cab42015-08-29 18:01:22 -0500761 irq_set_probe(irq);
Marc Zyngier021f6532014-06-30 16:01:31 +0100762 }
Marc Zyngierda33f312014-11-24 14:35:18 +0000763 /* LPIs */
764 if (hw >= 8192 && hw < GIC_ID_NR) {
765 if (!gic_dist_supports_lpis())
766 return -EPERM;
Marc Zyngier0b6a3da2015-08-26 17:00:42 +0100767 irq_domain_set_info(d, irq, hw, chip, d->host_data,
Marc Zyngierda33f312014-11-24 14:35:18 +0000768 handle_fasteoi_irq, NULL, NULL);
Marc Zyngierda33f312014-11-24 14:35:18 +0000769 }
770
Marc Zyngier021f6532014-06-30 16:01:31 +0100771 return 0;
772}
773
774static int gic_irq_domain_xlate(struct irq_domain *d,
775 struct device_node *controller,
776 const u32 *intspec, unsigned int intsize,
777 unsigned long *out_hwirq, unsigned int *out_type)
778{
779 if (d->of_node != controller)
780 return -EINVAL;
781 if (intsize < 3)
782 return -EINVAL;
783
784 switch(intspec[0]) {
785 case 0: /* SPI */
786 *out_hwirq = intspec[1] + 32;
787 break;
788 case 1: /* PPI */
789 *out_hwirq = intspec[1] + 16;
790 break;
Marc Zyngierda33f312014-11-24 14:35:18 +0000791 case GIC_IRQ_TYPE_LPI: /* LPI */
792 *out_hwirq = intspec[1];
793 break;
Marc Zyngier021f6532014-06-30 16:01:31 +0100794 default:
795 return -EINVAL;
796 }
797
798 *out_type = intspec[2] & IRQ_TYPE_SENSE_MASK;
799 return 0;
800}
801
Marc Zyngier443acc42014-11-24 14:35:09 +0000802static int gic_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
803 unsigned int nr_irqs, void *arg)
804{
805 int i, ret;
806 irq_hw_number_t hwirq;
807 unsigned int type = IRQ_TYPE_NONE;
808 struct of_phandle_args *irq_data = arg;
809
810 ret = gic_irq_domain_xlate(domain, irq_data->np, irq_data->args,
811 irq_data->args_count, &hwirq, &type);
812 if (ret)
813 return ret;
814
815 for (i = 0; i < nr_irqs; i++)
816 gic_irq_domain_map(domain, virq + i, hwirq + i);
817
818 return 0;
819}
820
821static void gic_irq_domain_free(struct irq_domain *domain, unsigned int virq,
822 unsigned int nr_irqs)
823{
824 int i;
825
826 for (i = 0; i < nr_irqs; i++) {
827 struct irq_data *d = irq_domain_get_irq_data(domain, virq + i);
828 irq_set_handler(virq + i, NULL);
829 irq_domain_reset_irq_data(d);
830 }
831}
832
Marc Zyngier021f6532014-06-30 16:01:31 +0100833static const struct irq_domain_ops gic_irq_domain_ops = {
Marc Zyngier021f6532014-06-30 16:01:31 +0100834 .xlate = gic_irq_domain_xlate,
Marc Zyngier443acc42014-11-24 14:35:09 +0000835 .alloc = gic_irq_domain_alloc,
836 .free = gic_irq_domain_free,
Marc Zyngier021f6532014-06-30 16:01:31 +0100837};
838
839static int __init gic_of_init(struct device_node *node, struct device_node *parent)
840{
841 void __iomem *dist_base;
Marc Zyngierf5c14342014-11-24 14:35:10 +0000842 struct redist_region *rdist_regs;
Marc Zyngier021f6532014-06-30 16:01:31 +0100843 u64 redist_stride;
Marc Zyngierf5c14342014-11-24 14:35:10 +0000844 u32 nr_redist_regions;
845 u32 typer;
Marc Zyngier021f6532014-06-30 16:01:31 +0100846 u32 reg;
847 int gic_irqs;
848 int err;
849 int i;
850
851 dist_base = of_iomap(node, 0);
852 if (!dist_base) {
853 pr_err("%s: unable to map gic dist registers\n",
854 node->full_name);
855 return -ENXIO;
856 }
857
858 reg = readl_relaxed(dist_base + GICD_PIDR2) & GIC_PIDR2_ARCH_MASK;
859 if (reg != GIC_PIDR2_ARCH_GICv3 && reg != GIC_PIDR2_ARCH_GICv4) {
860 pr_err("%s: no distributor detected, giving up\n",
861 node->full_name);
862 err = -ENODEV;
863 goto out_unmap_dist;
864 }
865
Marc Zyngierf5c14342014-11-24 14:35:10 +0000866 if (of_property_read_u32(node, "#redistributor-regions", &nr_redist_regions))
867 nr_redist_regions = 1;
Marc Zyngier021f6532014-06-30 16:01:31 +0100868
Marc Zyngierf5c14342014-11-24 14:35:10 +0000869 rdist_regs = kzalloc(sizeof(*rdist_regs) * nr_redist_regions, GFP_KERNEL);
870 if (!rdist_regs) {
Marc Zyngier021f6532014-06-30 16:01:31 +0100871 err = -ENOMEM;
872 goto out_unmap_dist;
873 }
874
Marc Zyngierf5c14342014-11-24 14:35:10 +0000875 for (i = 0; i < nr_redist_regions; i++) {
876 struct resource res;
877 int ret;
878
879 ret = of_address_to_resource(node, 1 + i, &res);
880 rdist_regs[i].redist_base = of_iomap(node, 1 + i);
881 if (ret || !rdist_regs[i].redist_base) {
Marc Zyngier021f6532014-06-30 16:01:31 +0100882 pr_err("%s: couldn't map region %d\n",
883 node->full_name, i);
884 err = -ENODEV;
885 goto out_unmap_rdist;
886 }
Marc Zyngierf5c14342014-11-24 14:35:10 +0000887 rdist_regs[i].phys_base = res.start;
Marc Zyngier021f6532014-06-30 16:01:31 +0100888 }
889
890 if (of_property_read_u64(node, "redistributor-stride", &redist_stride))
891 redist_stride = 0;
892
Marc Zyngier0b6a3da2015-08-26 17:00:42 +0100893 if (!is_hyp_mode_available())
894 static_key_slow_dec(&supports_deactivate);
895
896 if (static_key_true(&supports_deactivate))
897 pr_info("GIC: Using split EOI/Deactivate mode\n");
898
Marc Zyngier021f6532014-06-30 16:01:31 +0100899 gic_data.dist_base = dist_base;
Marc Zyngierf5c14342014-11-24 14:35:10 +0000900 gic_data.redist_regions = rdist_regs;
901 gic_data.nr_redist_regions = nr_redist_regions;
Marc Zyngier021f6532014-06-30 16:01:31 +0100902 gic_data.redist_stride = redist_stride;
903
904 /*
905 * Find out how many interrupts are supported.
906 * The GIC only supports up to 1020 interrupt sources (SGI+PPI+SPI)
907 */
Marc Zyngierf5c14342014-11-24 14:35:10 +0000908 typer = readl_relaxed(gic_data.dist_base + GICD_TYPER);
909 gic_data.rdists.id_bits = GICD_TYPER_ID_BITS(typer);
910 gic_irqs = GICD_TYPER_IRQS(typer);
Marc Zyngier021f6532014-06-30 16:01:31 +0100911 if (gic_irqs > 1020)
912 gic_irqs = 1020;
913 gic_data.irq_nr = gic_irqs;
914
915 gic_data.domain = irq_domain_add_tree(node, &gic_irq_domain_ops,
916 &gic_data);
Marc Zyngierf5c14342014-11-24 14:35:10 +0000917 gic_data.rdists.rdist = alloc_percpu(typeof(*gic_data.rdists.rdist));
Marc Zyngier021f6532014-06-30 16:01:31 +0100918
Marc Zyngierf5c14342014-11-24 14:35:10 +0000919 if (WARN_ON(!gic_data.domain) || WARN_ON(!gic_data.rdists.rdist)) {
Marc Zyngier021f6532014-06-30 16:01:31 +0100920 err = -ENOMEM;
921 goto out_free;
922 }
923
924 set_handle_irq(gic_handle_irq);
925
Marc Zyngierda33f312014-11-24 14:35:18 +0000926 if (IS_ENABLED(CONFIG_ARM_GIC_V3_ITS) && gic_dist_supports_lpis())
927 its_init(node, &gic_data.rdists, gic_data.domain);
928
Marc Zyngier021f6532014-06-30 16:01:31 +0100929 gic_smp_init();
930 gic_dist_init();
931 gic_cpu_init();
Sudeep Holla3708d522014-08-26 16:03:35 +0100932 gic_cpu_pm_init();
Marc Zyngier021f6532014-06-30 16:01:31 +0100933
934 return 0;
935
936out_free:
937 if (gic_data.domain)
938 irq_domain_remove(gic_data.domain);
Marc Zyngierf5c14342014-11-24 14:35:10 +0000939 free_percpu(gic_data.rdists.rdist);
Marc Zyngier021f6532014-06-30 16:01:31 +0100940out_unmap_rdist:
Marc Zyngierf5c14342014-11-24 14:35:10 +0000941 for (i = 0; i < nr_redist_regions; i++)
942 if (rdist_regs[i].redist_base)
943 iounmap(rdist_regs[i].redist_base);
944 kfree(rdist_regs);
Marc Zyngier021f6532014-06-30 16:01:31 +0100945out_unmap_dist:
946 iounmap(dist_base);
947 return err;
948}
949
950IRQCHIP_DECLARE(gic_v3, "arm,gic-v3", gic_of_init);