blob: 04b6f0732c1afcb1aa9d20cbe0d6649c02bbe822 [file] [log] [blame]
Marc Zyngier021f6532014-06-30 16:01:31 +01001/*
2 * Copyright (C) 2013, 2014 ARM Limited, All Rights Reserved.
3 * Author: Marc Zyngier <marc.zyngier@arm.com>
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License version 2 as
7 * published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program. If not, see <http://www.gnu.org/licenses/>.
16 */
17
18#include <linux/cpu.h>
Sudeep Holla3708d522014-08-26 16:03:35 +010019#include <linux/cpu_pm.h>
Marc Zyngier021f6532014-06-30 16:01:31 +010020#include <linux/delay.h>
21#include <linux/interrupt.h>
22#include <linux/of.h>
23#include <linux/of_address.h>
24#include <linux/of_irq.h>
25#include <linux/percpu.h>
26#include <linux/slab.h>
27
28#include <linux/irqchip/arm-gic-v3.h>
29
30#include <asm/cputype.h>
31#include <asm/exception.h>
32#include <asm/smp_plat.h>
33
34#include "irq-gic-common.h"
35#include "irqchip.h"
36
Marc Zyngierf5c14342014-11-24 14:35:10 +000037struct redist_region {
38 void __iomem *redist_base;
39 phys_addr_t phys_base;
40};
41
Marc Zyngier021f6532014-06-30 16:01:31 +010042struct gic_chip_data {
43 void __iomem *dist_base;
Marc Zyngierf5c14342014-11-24 14:35:10 +000044 struct redist_region *redist_regions;
45 struct rdists rdists;
Marc Zyngier021f6532014-06-30 16:01:31 +010046 struct irq_domain *domain;
47 u64 redist_stride;
Marc Zyngierf5c14342014-11-24 14:35:10 +000048 u32 nr_redist_regions;
Marc Zyngier021f6532014-06-30 16:01:31 +010049 unsigned int irq_nr;
50};
51
52static struct gic_chip_data gic_data __read_mostly;
53
Marc Zyngierf5c14342014-11-24 14:35:10 +000054#define gic_data_rdist() (this_cpu_ptr(gic_data.rdists.rdist))
55#define gic_data_rdist_rd_base() (gic_data_rdist()->rd_base)
Marc Zyngier021f6532014-06-30 16:01:31 +010056#define gic_data_rdist_sgi_base() (gic_data_rdist_rd_base() + SZ_64K)
57
58/* Our default, arbitrary priority value. Linux only uses one anyway. */
59#define DEFAULT_PMR_VALUE 0xf0
60
61static inline unsigned int gic_irq(struct irq_data *d)
62{
63 return d->hwirq;
64}
65
66static inline int gic_irq_in_rdist(struct irq_data *d)
67{
68 return gic_irq(d) < 32;
69}
70
71static inline void __iomem *gic_dist_base(struct irq_data *d)
72{
73 if (gic_irq_in_rdist(d)) /* SGI+PPI -> SGI_base for this CPU */
74 return gic_data_rdist_sgi_base();
75
76 if (d->hwirq <= 1023) /* SPI -> dist_base */
77 return gic_data.dist_base;
78
Marc Zyngier021f6532014-06-30 16:01:31 +010079 return NULL;
80}
81
82static void gic_do_wait_for_rwp(void __iomem *base)
83{
84 u32 count = 1000000; /* 1s! */
85
86 while (readl_relaxed(base + GICD_CTLR) & GICD_CTLR_RWP) {
87 count--;
88 if (!count) {
89 pr_err_ratelimited("RWP timeout, gone fishing\n");
90 return;
91 }
92 cpu_relax();
93 udelay(1);
94 };
95}
96
97/* Wait for completion of a distributor change */
98static void gic_dist_wait_for_rwp(void)
99{
100 gic_do_wait_for_rwp(gic_data.dist_base);
101}
102
103/* Wait for completion of a redistributor change */
104static void gic_redist_wait_for_rwp(void)
105{
106 gic_do_wait_for_rwp(gic_data_rdist_rd_base());
107}
108
109/* Low level accessors */
Mark Brownc44e9d72014-07-30 20:23:14 +0100110static u64 __maybe_unused gic_read_iar(void)
Marc Zyngier021f6532014-06-30 16:01:31 +0100111{
112 u64 irqstat;
113
Catalin Marinas72c58392014-07-24 14:14:42 +0100114 asm volatile("mrs_s %0, " __stringify(ICC_IAR1_EL1) : "=r" (irqstat));
Marc Zyngier021f6532014-06-30 16:01:31 +0100115 return irqstat;
116}
117
Mark Brownc44e9d72014-07-30 20:23:14 +0100118static void __maybe_unused gic_write_pmr(u64 val)
Marc Zyngier021f6532014-06-30 16:01:31 +0100119{
Catalin Marinas72c58392014-07-24 14:14:42 +0100120 asm volatile("msr_s " __stringify(ICC_PMR_EL1) ", %0" : : "r" (val));
Marc Zyngier021f6532014-06-30 16:01:31 +0100121}
122
Mark Brownc44e9d72014-07-30 20:23:14 +0100123static void __maybe_unused gic_write_ctlr(u64 val)
Marc Zyngier021f6532014-06-30 16:01:31 +0100124{
Catalin Marinas72c58392014-07-24 14:14:42 +0100125 asm volatile("msr_s " __stringify(ICC_CTLR_EL1) ", %0" : : "r" (val));
Marc Zyngier021f6532014-06-30 16:01:31 +0100126 isb();
127}
128
Mark Brownc44e9d72014-07-30 20:23:14 +0100129static void __maybe_unused gic_write_grpen1(u64 val)
Marc Zyngier021f6532014-06-30 16:01:31 +0100130{
Catalin Marinas72c58392014-07-24 14:14:42 +0100131 asm volatile("msr_s " __stringify(ICC_GRPEN1_EL1) ", %0" : : "r" (val));
Marc Zyngier021f6532014-06-30 16:01:31 +0100132 isb();
133}
134
Mark Brownc44e9d72014-07-30 20:23:14 +0100135static void __maybe_unused gic_write_sgi1r(u64 val)
Marc Zyngier021f6532014-06-30 16:01:31 +0100136{
Catalin Marinas72c58392014-07-24 14:14:42 +0100137 asm volatile("msr_s " __stringify(ICC_SGI1R_EL1) ", %0" : : "r" (val));
Marc Zyngier021f6532014-06-30 16:01:31 +0100138}
139
140static void gic_enable_sre(void)
141{
142 u64 val;
143
Catalin Marinas72c58392014-07-24 14:14:42 +0100144 asm volatile("mrs_s %0, " __stringify(ICC_SRE_EL1) : "=r" (val));
Marc Zyngier021f6532014-06-30 16:01:31 +0100145 val |= ICC_SRE_EL1_SRE;
Catalin Marinas72c58392014-07-24 14:14:42 +0100146 asm volatile("msr_s " __stringify(ICC_SRE_EL1) ", %0" : : "r" (val));
Marc Zyngier021f6532014-06-30 16:01:31 +0100147 isb();
148
149 /*
150 * Need to check that the SRE bit has actually been set. If
151 * not, it means that SRE is disabled at EL2. We're going to
152 * die painfully, and there is nothing we can do about it.
153 *
154 * Kindly inform the luser.
155 */
Catalin Marinas72c58392014-07-24 14:14:42 +0100156 asm volatile("mrs_s %0, " __stringify(ICC_SRE_EL1) : "=r" (val));
Marc Zyngier021f6532014-06-30 16:01:31 +0100157 if (!(val & ICC_SRE_EL1_SRE))
158 pr_err("GIC: unable to set SRE (disabled at EL2), panic ahead\n");
159}
160
Sudeep Hollaa2c22512014-08-26 16:03:34 +0100161static void gic_enable_redist(bool enable)
Marc Zyngier021f6532014-06-30 16:01:31 +0100162{
163 void __iomem *rbase;
164 u32 count = 1000000; /* 1s! */
165 u32 val;
166
167 rbase = gic_data_rdist_rd_base();
168
Marc Zyngier021f6532014-06-30 16:01:31 +0100169 val = readl_relaxed(rbase + GICR_WAKER);
Sudeep Hollaa2c22512014-08-26 16:03:34 +0100170 if (enable)
171 /* Wake up this CPU redistributor */
172 val &= ~GICR_WAKER_ProcessorSleep;
173 else
174 val |= GICR_WAKER_ProcessorSleep;
Marc Zyngier021f6532014-06-30 16:01:31 +0100175 writel_relaxed(val, rbase + GICR_WAKER);
176
Sudeep Hollaa2c22512014-08-26 16:03:34 +0100177 if (!enable) { /* Check that GICR_WAKER is writeable */
178 val = readl_relaxed(rbase + GICR_WAKER);
179 if (!(val & GICR_WAKER_ProcessorSleep))
180 return; /* No PM support in this redistributor */
181 }
182
183 while (count--) {
184 val = readl_relaxed(rbase + GICR_WAKER);
185 if (enable ^ (val & GICR_WAKER_ChildrenAsleep))
186 break;
Marc Zyngier021f6532014-06-30 16:01:31 +0100187 cpu_relax();
188 udelay(1);
189 };
Sudeep Hollaa2c22512014-08-26 16:03:34 +0100190 if (!count)
191 pr_err_ratelimited("redistributor failed to %s...\n",
192 enable ? "wakeup" : "sleep");
Marc Zyngier021f6532014-06-30 16:01:31 +0100193}
194
195/*
196 * Routines to disable, enable, EOI and route interrupts
197 */
198static void gic_poke_irq(struct irq_data *d, u32 offset)
199{
200 u32 mask = 1 << (gic_irq(d) % 32);
201 void (*rwp_wait)(void);
202 void __iomem *base;
203
204 if (gic_irq_in_rdist(d)) {
205 base = gic_data_rdist_sgi_base();
206 rwp_wait = gic_redist_wait_for_rwp;
207 } else {
208 base = gic_data.dist_base;
209 rwp_wait = gic_dist_wait_for_rwp;
210 }
211
212 writel_relaxed(mask, base + offset + (gic_irq(d) / 32) * 4);
213 rwp_wait();
214}
215
Marc Zyngier021f6532014-06-30 16:01:31 +0100216static void gic_mask_irq(struct irq_data *d)
217{
218 gic_poke_irq(d, GICD_ICENABLER);
219}
220
221static void gic_unmask_irq(struct irq_data *d)
222{
223 gic_poke_irq(d, GICD_ISENABLER);
224}
225
226static void gic_eoi_irq(struct irq_data *d)
227{
228 gic_write_eoir(gic_irq(d));
229}
230
231static int gic_set_type(struct irq_data *d, unsigned int type)
232{
233 unsigned int irq = gic_irq(d);
234 void (*rwp_wait)(void);
235 void __iomem *base;
236
237 /* Interrupt configuration for SGIs can't be changed */
238 if (irq < 16)
239 return -EINVAL;
240
Liviu Dudaufb7e7de2015-01-20 16:52:59 +0000241 /* SPIs have restrictions on the supported types */
242 if (irq >= 32 && type != IRQ_TYPE_LEVEL_HIGH &&
243 type != IRQ_TYPE_EDGE_RISING)
Marc Zyngier021f6532014-06-30 16:01:31 +0100244 return -EINVAL;
245
246 if (gic_irq_in_rdist(d)) {
247 base = gic_data_rdist_sgi_base();
248 rwp_wait = gic_redist_wait_for_rwp;
249 } else {
250 base = gic_data.dist_base;
251 rwp_wait = gic_dist_wait_for_rwp;
252 }
253
Liviu Dudaufb7e7de2015-01-20 16:52:59 +0000254 return gic_configure_irq(irq, type, base, rwp_wait);
Marc Zyngier021f6532014-06-30 16:01:31 +0100255}
256
257static u64 gic_mpidr_to_affinity(u64 mpidr)
258{
259 u64 aff;
260
261 aff = (MPIDR_AFFINITY_LEVEL(mpidr, 3) << 32 |
262 MPIDR_AFFINITY_LEVEL(mpidr, 2) << 16 |
263 MPIDR_AFFINITY_LEVEL(mpidr, 1) << 8 |
264 MPIDR_AFFINITY_LEVEL(mpidr, 0));
265
266 return aff;
267}
268
269static asmlinkage void __exception_irq_entry gic_handle_irq(struct pt_regs *regs)
270{
271 u64 irqnr;
272
273 do {
274 irqnr = gic_read_iar();
275
Marc Zyngierda33f312014-11-24 14:35:18 +0000276 if (likely(irqnr > 15 && irqnr < 1020) || irqnr >= 8192) {
Marc Zyngierebc6de02014-08-26 11:03:33 +0100277 int err;
278 err = handle_domain_irq(gic_data.domain, irqnr, regs);
279 if (err) {
Marc Zyngierda33f312014-11-24 14:35:18 +0000280 WARN_ONCE(true, "Unexpected interrupt received!\n");
Marc Zyngierebc6de02014-08-26 11:03:33 +0100281 gic_write_eoir(irqnr);
Marc Zyngier021f6532014-06-30 16:01:31 +0100282 }
Marc Zyngierebc6de02014-08-26 11:03:33 +0100283 continue;
Marc Zyngier021f6532014-06-30 16:01:31 +0100284 }
285 if (irqnr < 16) {
286 gic_write_eoir(irqnr);
287#ifdef CONFIG_SMP
288 handle_IPI(irqnr, regs);
289#else
290 WARN_ONCE(true, "Unexpected SGI received!\n");
291#endif
292 continue;
293 }
294 } while (irqnr != ICC_IAR1_EL1_SPURIOUS);
295}
296
297static void __init gic_dist_init(void)
298{
299 unsigned int i;
300 u64 affinity;
301 void __iomem *base = gic_data.dist_base;
302
303 /* Disable the distributor */
304 writel_relaxed(0, base + GICD_CTLR);
305 gic_dist_wait_for_rwp();
306
307 gic_dist_config(base, gic_data.irq_nr, gic_dist_wait_for_rwp);
308
309 /* Enable distributor with ARE, Group1 */
310 writel_relaxed(GICD_CTLR_ARE_NS | GICD_CTLR_ENABLE_G1A | GICD_CTLR_ENABLE_G1,
311 base + GICD_CTLR);
312
313 /*
314 * Set all global interrupts to the boot CPU only. ARE must be
315 * enabled.
316 */
317 affinity = gic_mpidr_to_affinity(cpu_logical_map(smp_processor_id()));
318 for (i = 32; i < gic_data.irq_nr; i++)
319 writeq_relaxed(affinity, base + GICD_IROUTER + i * 8);
320}
321
322static int gic_populate_rdist(void)
323{
324 u64 mpidr = cpu_logical_map(smp_processor_id());
325 u64 typer;
326 u32 aff;
327 int i;
328
329 /*
330 * Convert affinity to a 32bit value that can be matched to
331 * GICR_TYPER bits [63:32].
332 */
333 aff = (MPIDR_AFFINITY_LEVEL(mpidr, 3) << 24 |
334 MPIDR_AFFINITY_LEVEL(mpidr, 2) << 16 |
335 MPIDR_AFFINITY_LEVEL(mpidr, 1) << 8 |
336 MPIDR_AFFINITY_LEVEL(mpidr, 0));
337
Marc Zyngierf5c14342014-11-24 14:35:10 +0000338 for (i = 0; i < gic_data.nr_redist_regions; i++) {
339 void __iomem *ptr = gic_data.redist_regions[i].redist_base;
Marc Zyngier021f6532014-06-30 16:01:31 +0100340 u32 reg;
341
342 reg = readl_relaxed(ptr + GICR_PIDR2) & GIC_PIDR2_ARCH_MASK;
343 if (reg != GIC_PIDR2_ARCH_GICv3 &&
344 reg != GIC_PIDR2_ARCH_GICv4) { /* We're in trouble... */
345 pr_warn("No redistributor present @%p\n", ptr);
346 break;
347 }
348
349 do {
350 typer = readq_relaxed(ptr + GICR_TYPER);
351 if ((typer >> 32) == aff) {
Marc Zyngierf5c14342014-11-24 14:35:10 +0000352 u64 offset = ptr - gic_data.redist_regions[i].redist_base;
Marc Zyngier021f6532014-06-30 16:01:31 +0100353 gic_data_rdist_rd_base() = ptr;
Marc Zyngierf5c14342014-11-24 14:35:10 +0000354 gic_data_rdist()->phys_base = gic_data.redist_regions[i].phys_base + offset;
355 pr_info("CPU%d: found redistributor %llx region %d:%pa\n",
Marc Zyngier021f6532014-06-30 16:01:31 +0100356 smp_processor_id(),
Marc Zyngierf5c14342014-11-24 14:35:10 +0000357 (unsigned long long)mpidr,
358 i, &gic_data_rdist()->phys_base);
Marc Zyngier021f6532014-06-30 16:01:31 +0100359 return 0;
360 }
361
362 if (gic_data.redist_stride) {
363 ptr += gic_data.redist_stride;
364 } else {
365 ptr += SZ_64K * 2; /* Skip RD_base + SGI_base */
366 if (typer & GICR_TYPER_VLPIS)
367 ptr += SZ_64K * 2; /* Skip VLPI_base + reserved page */
368 }
369 } while (!(typer & GICR_TYPER_LAST));
370 }
371
372 /* We couldn't even deal with ourselves... */
373 WARN(true, "CPU%d: mpidr %llx has no re-distributor!\n",
374 smp_processor_id(), (unsigned long long)mpidr);
375 return -ENODEV;
376}
377
Sudeep Holla3708d522014-08-26 16:03:35 +0100378static void gic_cpu_sys_reg_init(void)
Marc Zyngier021f6532014-06-30 16:01:31 +0100379{
Marc Zyngier021f6532014-06-30 16:01:31 +0100380 /* Enable system registers */
381 gic_enable_sre();
382
383 /* Set priority mask register */
384 gic_write_pmr(DEFAULT_PMR_VALUE);
385
386 /* EOI deactivates interrupt too (mode 0) */
387 gic_write_ctlr(ICC_CTLR_EL1_EOImode_drop_dir);
388
389 /* ... and let's hit the road... */
390 gic_write_grpen1(1);
391}
392
Marc Zyngierda33f312014-11-24 14:35:18 +0000393static int gic_dist_supports_lpis(void)
394{
395 return !!(readl_relaxed(gic_data.dist_base + GICD_TYPER) & GICD_TYPER_LPIS);
396}
397
Marc Zyngier021f6532014-06-30 16:01:31 +0100398static void gic_cpu_init(void)
399{
400 void __iomem *rbase;
401
402 /* Register ourselves with the rest of the world */
403 if (gic_populate_rdist())
404 return;
405
Sudeep Hollaa2c22512014-08-26 16:03:34 +0100406 gic_enable_redist(true);
Marc Zyngier021f6532014-06-30 16:01:31 +0100407
408 rbase = gic_data_rdist_sgi_base();
409
410 gic_cpu_config(rbase, gic_redist_wait_for_rwp);
411
Marc Zyngierda33f312014-11-24 14:35:18 +0000412 /* Give LPIs a spin */
413 if (IS_ENABLED(CONFIG_ARM_GIC_V3_ITS) && gic_dist_supports_lpis())
414 its_cpu_init();
415
Sudeep Holla3708d522014-08-26 16:03:35 +0100416 /* initialise system registers */
417 gic_cpu_sys_reg_init();
Marc Zyngier021f6532014-06-30 16:01:31 +0100418}
419
420#ifdef CONFIG_SMP
Mark Brownddc86822014-07-27 17:57:04 +0100421static int gic_peek_irq(struct irq_data *d, u32 offset)
422{
423 u32 mask = 1 << (gic_irq(d) % 32);
424 void __iomem *base;
425
426 if (gic_irq_in_rdist(d))
427 base = gic_data_rdist_sgi_base();
428 else
429 base = gic_data.dist_base;
430
431 return !!(readl_relaxed(base + offset + (gic_irq(d) / 32) * 4) & mask);
432}
433
Marc Zyngier021f6532014-06-30 16:01:31 +0100434static int gic_secondary_init(struct notifier_block *nfb,
435 unsigned long action, void *hcpu)
436{
437 if (action == CPU_STARTING || action == CPU_STARTING_FROZEN)
438 gic_cpu_init();
439 return NOTIFY_OK;
440}
441
442/*
443 * Notifier for enabling the GIC CPU interface. Set an arbitrarily high
444 * priority because the GIC needs to be up before the ARM generic timers.
445 */
446static struct notifier_block gic_cpu_notifier = {
447 .notifier_call = gic_secondary_init,
448 .priority = 100,
449};
450
451static u16 gic_compute_target_list(int *base_cpu, const struct cpumask *mask,
452 u64 cluster_id)
453{
454 int cpu = *base_cpu;
455 u64 mpidr = cpu_logical_map(cpu);
456 u16 tlist = 0;
457
458 while (cpu < nr_cpu_ids) {
459 /*
460 * If we ever get a cluster of more than 16 CPUs, just
461 * scream and skip that CPU.
462 */
463 if (WARN_ON((mpidr & 0xff) >= 16))
464 goto out;
465
466 tlist |= 1 << (mpidr & 0xf);
467
468 cpu = cpumask_next(cpu, mask);
469 if (cpu == nr_cpu_ids)
470 goto out;
471
472 mpidr = cpu_logical_map(cpu);
473
474 if (cluster_id != (mpidr & ~0xffUL)) {
475 cpu--;
476 goto out;
477 }
478 }
479out:
480 *base_cpu = cpu;
481 return tlist;
482}
483
Andre Przywara7e580272014-11-12 13:46:06 +0000484#define MPIDR_TO_SGI_AFFINITY(cluster_id, level) \
485 (MPIDR_AFFINITY_LEVEL(cluster_id, level) \
486 << ICC_SGI1R_AFFINITY_## level ##_SHIFT)
487
Marc Zyngier021f6532014-06-30 16:01:31 +0100488static void gic_send_sgi(u64 cluster_id, u16 tlist, unsigned int irq)
489{
490 u64 val;
491
Andre Przywara7e580272014-11-12 13:46:06 +0000492 val = (MPIDR_TO_SGI_AFFINITY(cluster_id, 3) |
493 MPIDR_TO_SGI_AFFINITY(cluster_id, 2) |
494 irq << ICC_SGI1R_SGI_ID_SHIFT |
495 MPIDR_TO_SGI_AFFINITY(cluster_id, 1) |
496 tlist << ICC_SGI1R_TARGET_LIST_SHIFT);
Marc Zyngier021f6532014-06-30 16:01:31 +0100497
498 pr_debug("CPU%d: ICC_SGI1R_EL1 %llx\n", smp_processor_id(), val);
499 gic_write_sgi1r(val);
500}
501
502static void gic_raise_softirq(const struct cpumask *mask, unsigned int irq)
503{
504 int cpu;
505
506 if (WARN_ON(irq >= 16))
507 return;
508
509 /*
510 * Ensure that stores to Normal memory are visible to the
511 * other CPUs before issuing the IPI.
512 */
513 smp_wmb();
514
Rusty Russellf9b531f2015-03-05 10:49:16 +1030515 for_each_cpu(cpu, mask) {
Marc Zyngier021f6532014-06-30 16:01:31 +0100516 u64 cluster_id = cpu_logical_map(cpu) & ~0xffUL;
517 u16 tlist;
518
519 tlist = gic_compute_target_list(&cpu, mask, cluster_id);
520 gic_send_sgi(cluster_id, tlist, irq);
521 }
522
523 /* Force the above writes to ICC_SGI1R_EL1 to be executed */
524 isb();
525}
526
527static void gic_smp_init(void)
528{
529 set_smp_cross_call(gic_raise_softirq);
530 register_cpu_notifier(&gic_cpu_notifier);
531}
532
533static int gic_set_affinity(struct irq_data *d, const struct cpumask *mask_val,
534 bool force)
535{
536 unsigned int cpu = cpumask_any_and(mask_val, cpu_online_mask);
537 void __iomem *reg;
538 int enabled;
539 u64 val;
540
541 if (gic_irq_in_rdist(d))
542 return -EINVAL;
543
544 /* If interrupt was enabled, disable it first */
545 enabled = gic_peek_irq(d, GICD_ISENABLER);
546 if (enabled)
547 gic_mask_irq(d);
548
549 reg = gic_dist_base(d) + GICD_IROUTER + (gic_irq(d) * 8);
550 val = gic_mpidr_to_affinity(cpu_logical_map(cpu));
551
552 writeq_relaxed(val, reg);
553
554 /*
555 * If the interrupt was enabled, enabled it again. Otherwise,
556 * just wait for the distributor to have digested our changes.
557 */
558 if (enabled)
559 gic_unmask_irq(d);
560 else
561 gic_dist_wait_for_rwp();
562
563 return IRQ_SET_MASK_OK;
564}
565#else
566#define gic_set_affinity NULL
567#define gic_smp_init() do { } while(0)
568#endif
569
Sudeep Holla3708d522014-08-26 16:03:35 +0100570#ifdef CONFIG_CPU_PM
571static int gic_cpu_pm_notifier(struct notifier_block *self,
572 unsigned long cmd, void *v)
573{
574 if (cmd == CPU_PM_EXIT) {
575 gic_enable_redist(true);
576 gic_cpu_sys_reg_init();
577 } else if (cmd == CPU_PM_ENTER) {
578 gic_write_grpen1(0);
579 gic_enable_redist(false);
580 }
581 return NOTIFY_OK;
582}
583
584static struct notifier_block gic_cpu_pm_notifier_block = {
585 .notifier_call = gic_cpu_pm_notifier,
586};
587
588static void gic_cpu_pm_init(void)
589{
590 cpu_pm_register_notifier(&gic_cpu_pm_notifier_block);
591}
592
593#else
594static inline void gic_cpu_pm_init(void) { }
595#endif /* CONFIG_CPU_PM */
596
Marc Zyngier021f6532014-06-30 16:01:31 +0100597static struct irq_chip gic_chip = {
598 .name = "GICv3",
599 .irq_mask = gic_mask_irq,
600 .irq_unmask = gic_unmask_irq,
601 .irq_eoi = gic_eoi_irq,
602 .irq_set_type = gic_set_type,
603 .irq_set_affinity = gic_set_affinity,
604};
605
Marc Zyngierda33f312014-11-24 14:35:18 +0000606#define GIC_ID_NR (1U << gic_data.rdists.id_bits)
607
Marc Zyngier021f6532014-06-30 16:01:31 +0100608static int gic_irq_domain_map(struct irq_domain *d, unsigned int irq,
609 irq_hw_number_t hw)
610{
611 /* SGIs are private to the core kernel */
612 if (hw < 16)
613 return -EPERM;
Marc Zyngierda33f312014-11-24 14:35:18 +0000614 /* Nothing here */
615 if (hw >= gic_data.irq_nr && hw < 8192)
616 return -EPERM;
617 /* Off limits */
618 if (hw >= GIC_ID_NR)
619 return -EPERM;
620
Marc Zyngier021f6532014-06-30 16:01:31 +0100621 /* PPIs */
622 if (hw < 32) {
623 irq_set_percpu_devid(irq);
Marc Zyngier443acc42014-11-24 14:35:09 +0000624 irq_domain_set_info(d, irq, hw, &gic_chip, d->host_data,
625 handle_percpu_devid_irq, NULL, NULL);
Marc Zyngier021f6532014-06-30 16:01:31 +0100626 set_irq_flags(irq, IRQF_VALID | IRQF_NOAUTOEN);
627 }
628 /* SPIs */
629 if (hw >= 32 && hw < gic_data.irq_nr) {
Marc Zyngier443acc42014-11-24 14:35:09 +0000630 irq_domain_set_info(d, irq, hw, &gic_chip, d->host_data,
631 handle_fasteoi_irq, NULL, NULL);
Marc Zyngier021f6532014-06-30 16:01:31 +0100632 set_irq_flags(irq, IRQF_VALID | IRQF_PROBE);
633 }
Marc Zyngierda33f312014-11-24 14:35:18 +0000634 /* LPIs */
635 if (hw >= 8192 && hw < GIC_ID_NR) {
636 if (!gic_dist_supports_lpis())
637 return -EPERM;
638 irq_domain_set_info(d, irq, hw, &gic_chip, d->host_data,
639 handle_fasteoi_irq, NULL, NULL);
640 set_irq_flags(irq, IRQF_VALID);
641 }
642
Marc Zyngier021f6532014-06-30 16:01:31 +0100643 return 0;
644}
645
646static int gic_irq_domain_xlate(struct irq_domain *d,
647 struct device_node *controller,
648 const u32 *intspec, unsigned int intsize,
649 unsigned long *out_hwirq, unsigned int *out_type)
650{
651 if (d->of_node != controller)
652 return -EINVAL;
653 if (intsize < 3)
654 return -EINVAL;
655
656 switch(intspec[0]) {
657 case 0: /* SPI */
658 *out_hwirq = intspec[1] + 32;
659 break;
660 case 1: /* PPI */
661 *out_hwirq = intspec[1] + 16;
662 break;
Marc Zyngierda33f312014-11-24 14:35:18 +0000663 case GIC_IRQ_TYPE_LPI: /* LPI */
664 *out_hwirq = intspec[1];
665 break;
Marc Zyngier021f6532014-06-30 16:01:31 +0100666 default:
667 return -EINVAL;
668 }
669
670 *out_type = intspec[2] & IRQ_TYPE_SENSE_MASK;
671 return 0;
672}
673
Marc Zyngier443acc42014-11-24 14:35:09 +0000674static int gic_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
675 unsigned int nr_irqs, void *arg)
676{
677 int i, ret;
678 irq_hw_number_t hwirq;
679 unsigned int type = IRQ_TYPE_NONE;
680 struct of_phandle_args *irq_data = arg;
681
682 ret = gic_irq_domain_xlate(domain, irq_data->np, irq_data->args,
683 irq_data->args_count, &hwirq, &type);
684 if (ret)
685 return ret;
686
687 for (i = 0; i < nr_irqs; i++)
688 gic_irq_domain_map(domain, virq + i, hwirq + i);
689
690 return 0;
691}
692
693static void gic_irq_domain_free(struct irq_domain *domain, unsigned int virq,
694 unsigned int nr_irqs)
695{
696 int i;
697
698 for (i = 0; i < nr_irqs; i++) {
699 struct irq_data *d = irq_domain_get_irq_data(domain, virq + i);
700 irq_set_handler(virq + i, NULL);
701 irq_domain_reset_irq_data(d);
702 }
703}
704
Marc Zyngier021f6532014-06-30 16:01:31 +0100705static const struct irq_domain_ops gic_irq_domain_ops = {
Marc Zyngier021f6532014-06-30 16:01:31 +0100706 .xlate = gic_irq_domain_xlate,
Marc Zyngier443acc42014-11-24 14:35:09 +0000707 .alloc = gic_irq_domain_alloc,
708 .free = gic_irq_domain_free,
Marc Zyngier021f6532014-06-30 16:01:31 +0100709};
710
711static int __init gic_of_init(struct device_node *node, struct device_node *parent)
712{
713 void __iomem *dist_base;
Marc Zyngierf5c14342014-11-24 14:35:10 +0000714 struct redist_region *rdist_regs;
Marc Zyngier021f6532014-06-30 16:01:31 +0100715 u64 redist_stride;
Marc Zyngierf5c14342014-11-24 14:35:10 +0000716 u32 nr_redist_regions;
717 u32 typer;
Marc Zyngier021f6532014-06-30 16:01:31 +0100718 u32 reg;
719 int gic_irqs;
720 int err;
721 int i;
722
723 dist_base = of_iomap(node, 0);
724 if (!dist_base) {
725 pr_err("%s: unable to map gic dist registers\n",
726 node->full_name);
727 return -ENXIO;
728 }
729
730 reg = readl_relaxed(dist_base + GICD_PIDR2) & GIC_PIDR2_ARCH_MASK;
731 if (reg != GIC_PIDR2_ARCH_GICv3 && reg != GIC_PIDR2_ARCH_GICv4) {
732 pr_err("%s: no distributor detected, giving up\n",
733 node->full_name);
734 err = -ENODEV;
735 goto out_unmap_dist;
736 }
737
Marc Zyngierf5c14342014-11-24 14:35:10 +0000738 if (of_property_read_u32(node, "#redistributor-regions", &nr_redist_regions))
739 nr_redist_regions = 1;
Marc Zyngier021f6532014-06-30 16:01:31 +0100740
Marc Zyngierf5c14342014-11-24 14:35:10 +0000741 rdist_regs = kzalloc(sizeof(*rdist_regs) * nr_redist_regions, GFP_KERNEL);
742 if (!rdist_regs) {
Marc Zyngier021f6532014-06-30 16:01:31 +0100743 err = -ENOMEM;
744 goto out_unmap_dist;
745 }
746
Marc Zyngierf5c14342014-11-24 14:35:10 +0000747 for (i = 0; i < nr_redist_regions; i++) {
748 struct resource res;
749 int ret;
750
751 ret = of_address_to_resource(node, 1 + i, &res);
752 rdist_regs[i].redist_base = of_iomap(node, 1 + i);
753 if (ret || !rdist_regs[i].redist_base) {
Marc Zyngier021f6532014-06-30 16:01:31 +0100754 pr_err("%s: couldn't map region %d\n",
755 node->full_name, i);
756 err = -ENODEV;
757 goto out_unmap_rdist;
758 }
Marc Zyngierf5c14342014-11-24 14:35:10 +0000759 rdist_regs[i].phys_base = res.start;
Marc Zyngier021f6532014-06-30 16:01:31 +0100760 }
761
762 if (of_property_read_u64(node, "redistributor-stride", &redist_stride))
763 redist_stride = 0;
764
765 gic_data.dist_base = dist_base;
Marc Zyngierf5c14342014-11-24 14:35:10 +0000766 gic_data.redist_regions = rdist_regs;
767 gic_data.nr_redist_regions = nr_redist_regions;
Marc Zyngier021f6532014-06-30 16:01:31 +0100768 gic_data.redist_stride = redist_stride;
769
770 /*
771 * Find out how many interrupts are supported.
772 * The GIC only supports up to 1020 interrupt sources (SGI+PPI+SPI)
773 */
Marc Zyngierf5c14342014-11-24 14:35:10 +0000774 typer = readl_relaxed(gic_data.dist_base + GICD_TYPER);
775 gic_data.rdists.id_bits = GICD_TYPER_ID_BITS(typer);
776 gic_irqs = GICD_TYPER_IRQS(typer);
Marc Zyngier021f6532014-06-30 16:01:31 +0100777 if (gic_irqs > 1020)
778 gic_irqs = 1020;
779 gic_data.irq_nr = gic_irqs;
780
781 gic_data.domain = irq_domain_add_tree(node, &gic_irq_domain_ops,
782 &gic_data);
Marc Zyngierf5c14342014-11-24 14:35:10 +0000783 gic_data.rdists.rdist = alloc_percpu(typeof(*gic_data.rdists.rdist));
Marc Zyngier021f6532014-06-30 16:01:31 +0100784
Marc Zyngierf5c14342014-11-24 14:35:10 +0000785 if (WARN_ON(!gic_data.domain) || WARN_ON(!gic_data.rdists.rdist)) {
Marc Zyngier021f6532014-06-30 16:01:31 +0100786 err = -ENOMEM;
787 goto out_free;
788 }
789
790 set_handle_irq(gic_handle_irq);
791
Marc Zyngierda33f312014-11-24 14:35:18 +0000792 if (IS_ENABLED(CONFIG_ARM_GIC_V3_ITS) && gic_dist_supports_lpis())
793 its_init(node, &gic_data.rdists, gic_data.domain);
794
Marc Zyngier021f6532014-06-30 16:01:31 +0100795 gic_smp_init();
796 gic_dist_init();
797 gic_cpu_init();
Sudeep Holla3708d522014-08-26 16:03:35 +0100798 gic_cpu_pm_init();
Marc Zyngier021f6532014-06-30 16:01:31 +0100799
800 return 0;
801
802out_free:
803 if (gic_data.domain)
804 irq_domain_remove(gic_data.domain);
Marc Zyngierf5c14342014-11-24 14:35:10 +0000805 free_percpu(gic_data.rdists.rdist);
Marc Zyngier021f6532014-06-30 16:01:31 +0100806out_unmap_rdist:
Marc Zyngierf5c14342014-11-24 14:35:10 +0000807 for (i = 0; i < nr_redist_regions; i++)
808 if (rdist_regs[i].redist_base)
809 iounmap(rdist_regs[i].redist_base);
810 kfree(rdist_regs);
Marc Zyngier021f6532014-06-30 16:01:31 +0100811out_unmap_dist:
812 iounmap(dist_base);
813 return err;
814}
815
816IRQCHIP_DECLARE(gic_v3, "arm,gic-v3", gic_of_init);