Benjamin Herrenschmidt | 0b05ac6 | 2011-04-04 13:46:58 +1000 | [diff] [blame] | 1 | /* |
| 2 | * Copyright 2011 IBM Corporation. |
| 3 | * |
| 4 | * This program is free software; you can redistribute it and/or |
| 5 | * modify it under the terms of the GNU General Public License |
| 6 | * as published by the Free Software Foundation; either version |
| 7 | * 2 of the License, or (at your option) any later version. |
| 8 | * |
| 9 | */ |
| 10 | #include <linux/types.h> |
| 11 | #include <linux/threads.h> |
| 12 | #include <linux/kernel.h> |
| 13 | #include <linux/irq.h> |
| 14 | #include <linux/debugfs.h> |
| 15 | #include <linux/smp.h> |
| 16 | #include <linux/interrupt.h> |
| 17 | #include <linux/seq_file.h> |
| 18 | #include <linux/init.h> |
| 19 | #include <linux/cpu.h> |
| 20 | #include <linux/of.h> |
| 21 | #include <linux/slab.h> |
| 22 | #include <linux/spinlock.h> |
| 23 | |
| 24 | #include <asm/prom.h> |
| 25 | #include <asm/io.h> |
| 26 | #include <asm/smp.h> |
| 27 | #include <asm/machdep.h> |
| 28 | #include <asm/irq.h> |
| 29 | #include <asm/errno.h> |
| 30 | #include <asm/rtas.h> |
| 31 | #include <asm/xics.h> |
| 32 | #include <asm/firmware.h> |
| 33 | |
| 34 | /* Globals common to all ICP/ICS implementations */ |
| 35 | const struct icp_ops *icp_ops; |
| 36 | |
| 37 | unsigned int xics_default_server = 0xff; |
| 38 | unsigned int xics_default_distrib_server = 0; |
| 39 | unsigned int xics_interrupt_server_size = 8; |
| 40 | |
| 41 | DEFINE_PER_CPU(struct xics_cppr, xics_cppr); |
| 42 | |
Grant Likely | bae1d8f | 2012-02-14 14:06:50 -0700 | [diff] [blame] | 43 | struct irq_domain *xics_host; |
Benjamin Herrenschmidt | 0b05ac6 | 2011-04-04 13:46:58 +1000 | [diff] [blame] | 44 | |
| 45 | static LIST_HEAD(ics_list); |
| 46 | |
| 47 | void xics_update_irq_servers(void) |
| 48 | { |
| 49 | int i, j; |
| 50 | struct device_node *np; |
| 51 | u32 ilen; |
| 52 | const u32 *ireg; |
| 53 | u32 hcpuid; |
| 54 | |
| 55 | /* Find the server numbers for the boot cpu. */ |
| 56 | np = of_get_cpu_node(boot_cpuid, NULL); |
| 57 | BUG_ON(!np); |
| 58 | |
| 59 | hcpuid = get_hard_smp_processor_id(boot_cpuid); |
Benjamin Herrenschmidt | f6e17f9 | 2011-03-04 18:25:55 +1100 | [diff] [blame] | 60 | xics_default_server = xics_default_distrib_server = hcpuid; |
| 61 | |
| 62 | pr_devel("xics: xics_default_server = 0x%x\n", xics_default_server); |
Benjamin Herrenschmidt | 0b05ac6 | 2011-04-04 13:46:58 +1000 | [diff] [blame] | 63 | |
| 64 | ireg = of_get_property(np, "ibm,ppc-interrupt-gserver#s", &ilen); |
| 65 | if (!ireg) { |
| 66 | of_node_put(np); |
| 67 | return; |
| 68 | } |
| 69 | |
| 70 | i = ilen / sizeof(int); |
| 71 | |
| 72 | /* Global interrupt distribution server is specified in the last |
| 73 | * entry of "ibm,ppc-interrupt-gserver#s" property. Get the last |
| 74 | * entry fom this property for current boot cpu id and use it as |
| 75 | * default distribution server |
| 76 | */ |
| 77 | for (j = 0; j < i; j += 2) { |
| 78 | if (ireg[j] == hcpuid) { |
| 79 | xics_default_distrib_server = ireg[j+1]; |
Benjamin Herrenschmidt | f6e17f9 | 2011-03-04 18:25:55 +1100 | [diff] [blame] | 80 | break; |
Benjamin Herrenschmidt | 0b05ac6 | 2011-04-04 13:46:58 +1000 | [diff] [blame] | 81 | } |
| 82 | } |
Benjamin Herrenschmidt | f6e17f9 | 2011-03-04 18:25:55 +1100 | [diff] [blame] | 83 | pr_devel("xics: xics_default_distrib_server = 0x%x\n", |
| 84 | xics_default_distrib_server); |
Benjamin Herrenschmidt | 0b05ac6 | 2011-04-04 13:46:58 +1000 | [diff] [blame] | 85 | of_node_put(np); |
| 86 | } |
| 87 | |
| 88 | /* GIQ stuff, currently only supported on RTAS setups, will have |
| 89 | * to be sorted properly for bare metal |
| 90 | */ |
| 91 | void xics_set_cpu_giq(unsigned int gserver, unsigned int join) |
| 92 | { |
| 93 | #ifdef CONFIG_PPC_RTAS |
| 94 | int index; |
| 95 | int status; |
| 96 | |
| 97 | if (!rtas_indicator_present(GLOBAL_INTERRUPT_QUEUE, NULL)) |
| 98 | return; |
| 99 | |
| 100 | index = (1UL << xics_interrupt_server_size) - 1 - gserver; |
| 101 | |
| 102 | status = rtas_set_indicator_fast(GLOBAL_INTERRUPT_QUEUE, index, join); |
| 103 | |
| 104 | WARN(status < 0, "set-indicator(%d, %d, %u) returned %d\n", |
| 105 | GLOBAL_INTERRUPT_QUEUE, index, join, status); |
| 106 | #endif |
| 107 | } |
| 108 | |
| 109 | void xics_setup_cpu(void) |
| 110 | { |
| 111 | icp_ops->set_priority(LOWEST_PRIORITY); |
| 112 | |
| 113 | xics_set_cpu_giq(xics_default_distrib_server, 1); |
| 114 | } |
| 115 | |
| 116 | void xics_mask_unknown_vec(unsigned int vec) |
| 117 | { |
| 118 | struct ics *ics; |
| 119 | |
Benjamin Herrenschmidt | f6e17f9 | 2011-03-04 18:25:55 +1100 | [diff] [blame] | 120 | pr_err("Interrupt 0x%x (real) is invalid, disabling it.\n", vec); |
Benjamin Herrenschmidt | 0b05ac6 | 2011-04-04 13:46:58 +1000 | [diff] [blame] | 121 | |
| 122 | list_for_each_entry(ics, &ics_list, link) |
| 123 | ics->mask_unknown(ics, vec); |
| 124 | } |
| 125 | |
| 126 | |
| 127 | #ifdef CONFIG_SMP |
| 128 | |
Benjamin Herrenschmidt | 0b05ac6 | 2011-04-04 13:46:58 +1000 | [diff] [blame] | 129 | static void xics_request_ipi(void) |
| 130 | { |
| 131 | unsigned int ipi; |
| 132 | |
| 133 | ipi = irq_create_mapping(xics_host, XICS_IPI); |
| 134 | BUG_ON(ipi == NO_IRQ); |
| 135 | |
| 136 | /* |
Yong Zhang | a3a9f3b | 2011-10-21 23:56:27 +0000 | [diff] [blame] | 137 | * IPIs are marked IRQF_PERCPU. The handler was set in map. |
Benjamin Herrenschmidt | 0b05ac6 | 2011-04-04 13:46:58 +1000 | [diff] [blame] | 138 | */ |
Benjamin Herrenschmidt | 0b05ac6 | 2011-04-04 13:46:58 +1000 | [diff] [blame] | 139 | BUG_ON(request_irq(ipi, icp_ops->ipi_action, |
Thomas Gleixner | 3b5e16d | 2011-10-05 02:30:50 +0000 | [diff] [blame] | 140 | IRQF_PERCPU | IRQF_NO_THREAD, "IPI", NULL)); |
Benjamin Herrenschmidt | 0b05ac6 | 2011-04-04 13:46:58 +1000 | [diff] [blame] | 141 | } |
| 142 | |
| 143 | int __init xics_smp_probe(void) |
| 144 | { |
Milton Miller | 23d72bf | 2011-05-10 19:29:39 +0000 | [diff] [blame] | 145 | /* Setup cause_ipi callback based on which ICP is used */ |
| 146 | smp_ops->cause_ipi = icp_ops->cause_ipi; |
Benjamin Herrenschmidt | 0b05ac6 | 2011-04-04 13:46:58 +1000 | [diff] [blame] | 147 | |
| 148 | /* Register all the IPIs */ |
| 149 | xics_request_ipi(); |
| 150 | |
| 151 | return cpumask_weight(cpu_possible_mask); |
| 152 | } |
| 153 | |
| 154 | #endif /* CONFIG_SMP */ |
| 155 | |
| 156 | void xics_teardown_cpu(void) |
| 157 | { |
| 158 | struct xics_cppr *os_cppr = &__get_cpu_var(xics_cppr); |
| 159 | |
| 160 | /* |
| 161 | * we have to reset the cppr index to 0 because we're |
| 162 | * not going to return from the IPI |
| 163 | */ |
| 164 | os_cppr->index = 0; |
| 165 | icp_ops->set_priority(0); |
| 166 | icp_ops->teardown_cpu(); |
| 167 | } |
| 168 | |
| 169 | void xics_kexec_teardown_cpu(int secondary) |
| 170 | { |
| 171 | xics_teardown_cpu(); |
| 172 | |
| 173 | icp_ops->flush_ipi(); |
| 174 | |
| 175 | /* |
| 176 | * Some machines need to have at least one cpu in the GIQ, |
| 177 | * so leave the master cpu in the group. |
| 178 | */ |
| 179 | if (secondary) |
| 180 | xics_set_cpu_giq(xics_default_distrib_server, 0); |
| 181 | } |
| 182 | |
| 183 | |
| 184 | #ifdef CONFIG_HOTPLUG_CPU |
| 185 | |
| 186 | /* Interrupts are disabled. */ |
| 187 | void xics_migrate_irqs_away(void) |
| 188 | { |
| 189 | int cpu = smp_processor_id(), hw_cpu = hard_smp_processor_id(); |
| 190 | unsigned int irq, virq; |
| 191 | |
| 192 | /* If we used to be the default server, move to the new "boot_cpuid" */ |
| 193 | if (hw_cpu == xics_default_server) |
| 194 | xics_update_irq_servers(); |
| 195 | |
| 196 | /* Reject any interrupt that was queued to us... */ |
| 197 | icp_ops->set_priority(0); |
| 198 | |
| 199 | /* Remove ourselves from the global interrupt queue */ |
| 200 | xics_set_cpu_giq(xics_default_distrib_server, 0); |
| 201 | |
| 202 | /* Allow IPIs again... */ |
| 203 | icp_ops->set_priority(DEFAULT_PRIORITY); |
| 204 | |
| 205 | for_each_irq(virq) { |
| 206 | struct irq_desc *desc; |
| 207 | struct irq_chip *chip; |
| 208 | long server; |
| 209 | unsigned long flags; |
| 210 | struct ics *ics; |
| 211 | |
| 212 | /* We can't set affinity on ISA interrupts */ |
| 213 | if (virq < NUM_ISA_INTERRUPTS) |
| 214 | continue; |
Benjamin Herrenschmidt | 0b05ac6 | 2011-04-04 13:46:58 +1000 | [diff] [blame] | 215 | desc = irq_to_desc(virq); |
| 216 | /* We only need to migrate enabled IRQS */ |
| 217 | if (!desc || !desc->action) |
| 218 | continue; |
Grant Likely | 6d9285b | 2012-02-14 14:06:51 -0700 | [diff] [blame] | 219 | if (desc->irq_data.domain != xics_host) |
| 220 | continue; |
| 221 | irq = desc->irq_data.hwirq; |
| 222 | /* We need to get IPIs still. */ |
| 223 | if (irq == XICS_IPI || irq == XICS_IRQ_SPURIOUS) |
| 224 | continue; |
Benjamin Herrenschmidt | 0b05ac6 | 2011-04-04 13:46:58 +1000 | [diff] [blame] | 225 | chip = irq_desc_get_chip(desc); |
| 226 | if (!chip || !chip->irq_set_affinity) |
| 227 | continue; |
| 228 | |
| 229 | raw_spin_lock_irqsave(&desc->lock, flags); |
| 230 | |
| 231 | /* Locate interrupt server */ |
| 232 | server = -1; |
| 233 | ics = irq_get_chip_data(virq); |
| 234 | if (ics) |
| 235 | server = ics->get_server(ics, irq); |
| 236 | if (server < 0) { |
| 237 | printk(KERN_ERR "%s: Can't find server for irq %d\n", |
| 238 | __func__, irq); |
| 239 | goto unlock; |
| 240 | } |
| 241 | |
| 242 | /* We only support delivery to all cpus or to one cpu. |
| 243 | * The irq has to be migrated only in the single cpu |
| 244 | * case. |
| 245 | */ |
| 246 | if (server != hw_cpu) |
| 247 | goto unlock; |
| 248 | |
| 249 | /* This is expected during cpu offline. */ |
| 250 | if (cpu_online(cpu)) |
| 251 | pr_warning("IRQ %u affinity broken off cpu %u\n", |
| 252 | virq, cpu); |
| 253 | |
| 254 | /* Reset affinity to all cpus */ |
| 255 | raw_spin_unlock_irqrestore(&desc->lock, flags); |
| 256 | irq_set_affinity(virq, cpu_all_mask); |
| 257 | continue; |
| 258 | unlock: |
| 259 | raw_spin_unlock_irqrestore(&desc->lock, flags); |
| 260 | } |
| 261 | } |
| 262 | #endif /* CONFIG_HOTPLUG_CPU */ |
| 263 | |
| 264 | #ifdef CONFIG_SMP |
| 265 | /* |
| 266 | * For the moment we only implement delivery to all cpus or one cpu. |
| 267 | * |
| 268 | * If the requested affinity is cpu_all_mask, we set global affinity. |
| 269 | * If not we set it to the first cpu in the mask, even if multiple cpus |
| 270 | * are set. This is so things like irqbalance (which set core and package |
| 271 | * wide affinities) do the right thing. |
Benjamin Herrenschmidt | f6e17f9 | 2011-03-04 18:25:55 +1100 | [diff] [blame] | 272 | * |
| 273 | * We need to fix this to implement support for the links |
Benjamin Herrenschmidt | 0b05ac6 | 2011-04-04 13:46:58 +1000 | [diff] [blame] | 274 | */ |
| 275 | int xics_get_irq_server(unsigned int virq, const struct cpumask *cpumask, |
| 276 | unsigned int strict_check) |
| 277 | { |
| 278 | |
| 279 | if (!distribute_irqs) |
| 280 | return xics_default_server; |
| 281 | |
| 282 | if (!cpumask_subset(cpu_possible_mask, cpumask)) { |
| 283 | int server = cpumask_first_and(cpu_online_mask, cpumask); |
| 284 | |
| 285 | if (server < nr_cpu_ids) |
| 286 | return get_hard_smp_processor_id(server); |
| 287 | |
| 288 | if (strict_check) |
| 289 | return -1; |
| 290 | } |
| 291 | |
| 292 | /* |
| 293 | * Workaround issue with some versions of JS20 firmware that |
| 294 | * deliver interrupts to cpus which haven't been started. This |
| 295 | * happens when using the maxcpus= boot option. |
| 296 | */ |
| 297 | if (cpumask_equal(cpu_online_mask, cpu_present_mask)) |
| 298 | return xics_default_distrib_server; |
| 299 | |
| 300 | return xics_default_server; |
| 301 | } |
| 302 | #endif /* CONFIG_SMP */ |
| 303 | |
Grant Likely | bae1d8f | 2012-02-14 14:06:50 -0700 | [diff] [blame] | 304 | static int xics_host_match(struct irq_domain *h, struct device_node *node) |
Benjamin Herrenschmidt | 0b05ac6 | 2011-04-04 13:46:58 +1000 | [diff] [blame] | 305 | { |
Michael Ellerman | 5ca1237 | 2011-04-14 22:31:59 +0000 | [diff] [blame] | 306 | struct ics *ics; |
| 307 | |
| 308 | list_for_each_entry(ics, &ics_list, link) |
| 309 | if (ics->host_match(ics, node)) |
| 310 | return 1; |
| 311 | |
| 312 | return 0; |
Benjamin Herrenschmidt | 0b05ac6 | 2011-04-04 13:46:58 +1000 | [diff] [blame] | 313 | } |
| 314 | |
| 315 | /* Dummies */ |
| 316 | static void xics_ipi_unmask(struct irq_data *d) { } |
| 317 | static void xics_ipi_mask(struct irq_data *d) { } |
| 318 | |
| 319 | static struct irq_chip xics_ipi_chip = { |
| 320 | .name = "XICS", |
| 321 | .irq_eoi = NULL, /* Patched at init time */ |
| 322 | .irq_mask = xics_ipi_mask, |
| 323 | .irq_unmask = xics_ipi_unmask, |
| 324 | }; |
| 325 | |
Grant Likely | bae1d8f | 2012-02-14 14:06:50 -0700 | [diff] [blame] | 326 | static int xics_host_map(struct irq_domain *h, unsigned int virq, |
Benjamin Herrenschmidt | 0b05ac6 | 2011-04-04 13:46:58 +1000 | [diff] [blame] | 327 | irq_hw_number_t hw) |
| 328 | { |
| 329 | struct ics *ics; |
| 330 | |
| 331 | pr_devel("xics: map virq %d, hwirq 0x%lx\n", virq, hw); |
| 332 | |
| 333 | /* Insert the interrupt mapping into the radix tree for fast lookup */ |
| 334 | irq_radix_revmap_insert(xics_host, virq, hw); |
| 335 | |
| 336 | /* They aren't all level sensitive but we just don't really know */ |
| 337 | irq_set_status_flags(virq, IRQ_LEVEL); |
| 338 | |
| 339 | /* Don't call into ICS for IPIs */ |
| 340 | if (hw == XICS_IPI) { |
| 341 | irq_set_chip_and_handler(virq, &xics_ipi_chip, |
Milton Miller | e085255 | 2011-05-10 19:29:49 +0000 | [diff] [blame] | 342 | handle_percpu_irq); |
Benjamin Herrenschmidt | 0b05ac6 | 2011-04-04 13:46:58 +1000 | [diff] [blame] | 343 | return 0; |
| 344 | } |
| 345 | |
| 346 | /* Let the ICS setup the chip data */ |
| 347 | list_for_each_entry(ics, &ics_list, link) |
| 348 | if (ics->map(ics, virq) == 0) |
Milton Miller | e085255 | 2011-05-10 19:29:49 +0000 | [diff] [blame] | 349 | return 0; |
| 350 | |
| 351 | return -EINVAL; |
Benjamin Herrenschmidt | 0b05ac6 | 2011-04-04 13:46:58 +1000 | [diff] [blame] | 352 | } |
| 353 | |
Grant Likely | bae1d8f | 2012-02-14 14:06:50 -0700 | [diff] [blame] | 354 | static int xics_host_xlate(struct irq_domain *h, struct device_node *ct, |
Benjamin Herrenschmidt | 0b05ac6 | 2011-04-04 13:46:58 +1000 | [diff] [blame] | 355 | const u32 *intspec, unsigned int intsize, |
| 356 | irq_hw_number_t *out_hwirq, unsigned int *out_flags) |
| 357 | |
| 358 | { |
| 359 | /* Current xics implementation translates everything |
| 360 | * to level. It is not technically right for MSIs but this |
| 361 | * is irrelevant at this point. We might get smarter in the future |
| 362 | */ |
| 363 | *out_hwirq = intspec[0]; |
| 364 | *out_flags = IRQ_TYPE_LEVEL_LOW; |
| 365 | |
| 366 | return 0; |
| 367 | } |
| 368 | |
Grant Likely | bae1d8f | 2012-02-14 14:06:50 -0700 | [diff] [blame] | 369 | static struct irq_domain_ops xics_host_ops = { |
Benjamin Herrenschmidt | 0b05ac6 | 2011-04-04 13:46:58 +1000 | [diff] [blame] | 370 | .match = xics_host_match, |
| 371 | .map = xics_host_map, |
| 372 | .xlate = xics_host_xlate, |
| 373 | }; |
| 374 | |
| 375 | static void __init xics_init_host(void) |
| 376 | { |
Grant Likely | a8db8cf | 2012-02-14 14:06:54 -0700 | [diff] [blame] | 377 | xics_host = irq_domain_add_tree(NULL, &xics_host_ops, NULL); |
Benjamin Herrenschmidt | 0b05ac6 | 2011-04-04 13:46:58 +1000 | [diff] [blame] | 378 | BUG_ON(xics_host == NULL); |
| 379 | irq_set_default_host(xics_host); |
| 380 | } |
| 381 | |
| 382 | void __init xics_register_ics(struct ics *ics) |
| 383 | { |
| 384 | list_add(&ics->link, &ics_list); |
| 385 | } |
| 386 | |
| 387 | static void __init xics_get_server_size(void) |
| 388 | { |
| 389 | struct device_node *np; |
| 390 | const u32 *isize; |
| 391 | |
| 392 | /* We fetch the interrupt server size from the first ICS node |
| 393 | * we find if any |
| 394 | */ |
| 395 | np = of_find_compatible_node(NULL, NULL, "ibm,ppc-xics"); |
| 396 | if (!np) |
| 397 | return; |
| 398 | isize = of_get_property(np, "ibm,interrupt-server#-size", NULL); |
| 399 | if (!isize) |
| 400 | return; |
| 401 | xics_interrupt_server_size = *isize; |
| 402 | of_node_put(np); |
| 403 | } |
| 404 | |
| 405 | void __init xics_init(void) |
| 406 | { |
| 407 | int rc = -1; |
| 408 | |
| 409 | /* Fist locate ICP */ |
Benjamin Herrenschmidt | 0b05ac6 | 2011-04-04 13:46:58 +1000 | [diff] [blame] | 410 | if (firmware_has_feature(FW_FEATURE_LPAR)) |
| 411 | rc = icp_hv_init(); |
Benjamin Herrenschmidt | 0b05ac6 | 2011-04-04 13:46:58 +1000 | [diff] [blame] | 412 | if (rc < 0) |
| 413 | rc = icp_native_init(); |
Benjamin Herrenschmidt | 0b05ac6 | 2011-04-04 13:46:58 +1000 | [diff] [blame] | 414 | if (rc < 0) { |
| 415 | pr_warning("XICS: Cannot find a Presentation Controller !\n"); |
| 416 | return; |
| 417 | } |
| 418 | |
| 419 | /* Copy get_irq callback over to ppc_md */ |
| 420 | ppc_md.get_irq = icp_ops->get_irq; |
| 421 | |
| 422 | /* Patch up IPI chip EOI */ |
| 423 | xics_ipi_chip.irq_eoi = icp_ops->eoi; |
| 424 | |
| 425 | /* Now locate ICS */ |
Benjamin Herrenschmidt | 0b05ac6 | 2011-04-04 13:46:58 +1000 | [diff] [blame] | 426 | rc = ics_rtas_init(); |
Benjamin Herrenschmidt | 5c7c1e9 | 2011-09-19 17:45:02 +0000 | [diff] [blame] | 427 | if (rc < 0) |
| 428 | rc = ics_opal_init(); |
Benjamin Herrenschmidt | 0b05ac6 | 2011-04-04 13:46:58 +1000 | [diff] [blame] | 429 | if (rc < 0) |
| 430 | pr_warning("XICS: Cannot find a Source Controller !\n"); |
| 431 | |
| 432 | /* Initialize common bits */ |
| 433 | xics_get_server_size(); |
| 434 | xics_update_irq_servers(); |
| 435 | xics_init_host(); |
| 436 | xics_setup_cpu(); |
| 437 | } |