David S. Miller | 759f89e | 2007-10-11 03:16:13 -0700 | [diff] [blame] | 1 | /* pci_msi.c: Sparc64 MSI support common layer. |
| 2 | * |
| 3 | * Copyright (C) 2007 David S. Miller (davem@davemloft.net) |
| 4 | */ |
| 5 | #include <linux/kernel.h> |
| 6 | #include <linux/interrupt.h> |
Tejun Heo | 5a0e3ad | 2010-03-24 17:04:11 +0900 | [diff] [blame^] | 7 | #include <linux/slab.h> |
David S. Miller | 759f89e | 2007-10-11 03:16:13 -0700 | [diff] [blame] | 8 | #include <linux/irq.h> |
| 9 | |
| 10 | #include "pci_impl.h" |
| 11 | |
| 12 | static irqreturn_t sparc64_msiq_interrupt(int irq, void *cookie) |
| 13 | { |
| 14 | struct sparc64_msiq_cookie *msiq_cookie = cookie; |
| 15 | struct pci_pbm_info *pbm = msiq_cookie->pbm; |
| 16 | unsigned long msiqid = msiq_cookie->msiqid; |
| 17 | const struct sparc64_msiq_ops *ops; |
| 18 | unsigned long orig_head, head; |
| 19 | int err; |
| 20 | |
| 21 | ops = pbm->msi_ops; |
| 22 | |
| 23 | err = ops->get_head(pbm, msiqid, &head); |
| 24 | if (unlikely(err < 0)) |
| 25 | goto err_get_head; |
| 26 | |
| 27 | orig_head = head; |
| 28 | for (;;) { |
| 29 | unsigned long msi; |
| 30 | |
| 31 | err = ops->dequeue_msi(pbm, msiqid, &head, &msi); |
David S. Miller | 8d57d3a | 2007-10-22 02:16:45 -0700 | [diff] [blame] | 32 | if (likely(err > 0)) { |
| 33 | struct irq_desc *desc; |
| 34 | unsigned int virt_irq; |
| 35 | |
| 36 | virt_irq = pbm->msi_irq_table[msi - pbm->msi_first]; |
| 37 | desc = irq_desc + virt_irq; |
| 38 | |
| 39 | desc->handle_irq(virt_irq, desc); |
| 40 | } |
David S. Miller | 759f89e | 2007-10-11 03:16:13 -0700 | [diff] [blame] | 41 | |
| 42 | if (unlikely(err < 0)) |
| 43 | goto err_dequeue; |
| 44 | |
| 45 | if (err == 0) |
| 46 | break; |
| 47 | } |
| 48 | if (likely(head != orig_head)) { |
| 49 | err = ops->set_head(pbm, msiqid, head); |
| 50 | if (unlikely(err < 0)) |
| 51 | goto err_set_head; |
| 52 | } |
| 53 | return IRQ_HANDLED; |
| 54 | |
| 55 | err_get_head: |
| 56 | printk(KERN_EMERG "MSI: Get head on msiqid[%lu] gives error %d\n", |
| 57 | msiqid, err); |
| 58 | goto err_out; |
| 59 | |
| 60 | err_dequeue: |
| 61 | printk(KERN_EMERG "MSI: Dequeue head[%lu] from msiqid[%lu] " |
| 62 | "gives error %d\n", |
| 63 | head, msiqid, err); |
| 64 | goto err_out; |
| 65 | |
| 66 | err_set_head: |
| 67 | printk(KERN_EMERG "MSI: Set head[%lu] on msiqid[%lu] " |
| 68 | "gives error %d\n", |
| 69 | head, msiqid, err); |
| 70 | goto err_out; |
| 71 | |
| 72 | err_out: |
| 73 | return IRQ_NONE; |
| 74 | } |
| 75 | |
| 76 | static u32 pick_msiq(struct pci_pbm_info *pbm) |
| 77 | { |
| 78 | static DEFINE_SPINLOCK(rotor_lock); |
| 79 | unsigned long flags; |
| 80 | u32 ret, rotor; |
| 81 | |
| 82 | spin_lock_irqsave(&rotor_lock, flags); |
| 83 | |
| 84 | rotor = pbm->msiq_rotor; |
| 85 | ret = pbm->msiq_first + rotor; |
| 86 | |
| 87 | if (++rotor >= pbm->msiq_num) |
| 88 | rotor = 0; |
| 89 | pbm->msiq_rotor = rotor; |
| 90 | |
| 91 | spin_unlock_irqrestore(&rotor_lock, flags); |
| 92 | |
| 93 | return ret; |
| 94 | } |
| 95 | |
| 96 | |
| 97 | static int alloc_msi(struct pci_pbm_info *pbm) |
| 98 | { |
| 99 | int i; |
| 100 | |
| 101 | for (i = 0; i < pbm->msi_num; i++) { |
| 102 | if (!test_and_set_bit(i, pbm->msi_bitmap)) |
| 103 | return i + pbm->msi_first; |
| 104 | } |
| 105 | |
| 106 | return -ENOENT; |
| 107 | } |
| 108 | |
| 109 | static void free_msi(struct pci_pbm_info *pbm, int msi_num) |
| 110 | { |
| 111 | msi_num -= pbm->msi_first; |
| 112 | clear_bit(msi_num, pbm->msi_bitmap); |
| 113 | } |
| 114 | |
| 115 | static struct irq_chip msi_irq = { |
Thomas Gleixner | 89a7183 | 2009-11-17 12:50:40 +0000 | [diff] [blame] | 116 | .name = "PCI-MSI", |
David S. Miller | 759f89e | 2007-10-11 03:16:13 -0700 | [diff] [blame] | 117 | .mask = mask_msi_irq, |
| 118 | .unmask = unmask_msi_irq, |
| 119 | .enable = unmask_msi_irq, |
| 120 | .disable = mask_msi_irq, |
| 121 | /* XXX affinity XXX */ |
| 122 | }; |
| 123 | |
Adrian Bunk | 908f516 | 2008-06-05 11:42:40 -0700 | [diff] [blame] | 124 | static int sparc64_setup_msi_irq(unsigned int *virt_irq_p, |
| 125 | struct pci_dev *pdev, |
| 126 | struct msi_desc *entry) |
David S. Miller | 759f89e | 2007-10-11 03:16:13 -0700 | [diff] [blame] | 127 | { |
| 128 | struct pci_pbm_info *pbm = pdev->dev.archdata.host_controller; |
| 129 | const struct sparc64_msiq_ops *ops = pbm->msi_ops; |
| 130 | struct msi_msg msg; |
| 131 | int msi, err; |
| 132 | u32 msiqid; |
| 133 | |
David S. Miller | 256c1df | 2007-10-13 23:50:38 -0700 | [diff] [blame] | 134 | *virt_irq_p = virt_irq_alloc(0, 0); |
David S. Miller | 759f89e | 2007-10-11 03:16:13 -0700 | [diff] [blame] | 135 | err = -ENOMEM; |
| 136 | if (!*virt_irq_p) |
| 137 | goto out_err; |
| 138 | |
David S. Miller | 8d57d3a | 2007-10-22 02:16:45 -0700 | [diff] [blame] | 139 | set_irq_chip_and_handler_name(*virt_irq_p, &msi_irq, |
| 140 | handle_simple_irq, "MSI"); |
David S. Miller | 759f89e | 2007-10-11 03:16:13 -0700 | [diff] [blame] | 141 | |
| 142 | err = alloc_msi(pbm); |
| 143 | if (unlikely(err < 0)) |
| 144 | goto out_virt_irq_free; |
| 145 | |
| 146 | msi = err; |
| 147 | |
| 148 | msiqid = pick_msiq(pbm); |
| 149 | |
| 150 | err = ops->msi_setup(pbm, msiqid, msi, |
| 151 | (entry->msi_attrib.is_64 ? 1 : 0)); |
| 152 | if (err) |
| 153 | goto out_msi_free; |
| 154 | |
| 155 | pbm->msi_irq_table[msi - pbm->msi_first] = *virt_irq_p; |
| 156 | |
| 157 | if (entry->msi_attrib.is_64) { |
| 158 | msg.address_hi = pbm->msi64_start >> 32; |
| 159 | msg.address_lo = pbm->msi64_start & 0xffffffff; |
| 160 | } else { |
| 161 | msg.address_hi = 0; |
| 162 | msg.address_lo = pbm->msi32_start; |
| 163 | } |
| 164 | msg.data = msi; |
| 165 | |
| 166 | set_irq_msi(*virt_irq_p, entry); |
| 167 | write_msi_msg(*virt_irq_p, &msg); |
| 168 | |
| 169 | return 0; |
| 170 | |
| 171 | out_msi_free: |
| 172 | free_msi(pbm, msi); |
| 173 | |
| 174 | out_virt_irq_free: |
| 175 | set_irq_chip(*virt_irq_p, NULL); |
| 176 | virt_irq_free(*virt_irq_p); |
| 177 | *virt_irq_p = 0; |
| 178 | |
| 179 | out_err: |
| 180 | return err; |
| 181 | } |
| 182 | |
Adrian Bunk | 908f516 | 2008-06-05 11:42:40 -0700 | [diff] [blame] | 183 | static void sparc64_teardown_msi_irq(unsigned int virt_irq, |
| 184 | struct pci_dev *pdev) |
David S. Miller | 759f89e | 2007-10-11 03:16:13 -0700 | [diff] [blame] | 185 | { |
| 186 | struct pci_pbm_info *pbm = pdev->dev.archdata.host_controller; |
| 187 | const struct sparc64_msiq_ops *ops = pbm->msi_ops; |
| 188 | unsigned int msi_num; |
| 189 | int i, err; |
| 190 | |
| 191 | for (i = 0; i < pbm->msi_num; i++) { |
| 192 | if (pbm->msi_irq_table[i] == virt_irq) |
| 193 | break; |
| 194 | } |
| 195 | if (i >= pbm->msi_num) { |
| 196 | printk(KERN_ERR "%s: teardown: No MSI for irq %u\n", |
| 197 | pbm->name, virt_irq); |
| 198 | return; |
| 199 | } |
| 200 | |
| 201 | msi_num = pbm->msi_first + i; |
| 202 | pbm->msi_irq_table[i] = ~0U; |
| 203 | |
| 204 | err = ops->msi_teardown(pbm, msi_num); |
| 205 | if (err) { |
| 206 | printk(KERN_ERR "%s: teardown: ops->teardown() on MSI %u, " |
| 207 | "irq %u, gives error %d\n", |
| 208 | pbm->name, msi_num, virt_irq, err); |
| 209 | return; |
| 210 | } |
| 211 | |
| 212 | free_msi(pbm, msi_num); |
| 213 | |
| 214 | set_irq_chip(virt_irq, NULL); |
| 215 | virt_irq_free(virt_irq); |
| 216 | } |
| 217 | |
| 218 | static int msi_bitmap_alloc(struct pci_pbm_info *pbm) |
| 219 | { |
| 220 | unsigned long size, bits_per_ulong; |
| 221 | |
| 222 | bits_per_ulong = sizeof(unsigned long) * 8; |
| 223 | size = (pbm->msi_num + (bits_per_ulong - 1)) & ~(bits_per_ulong - 1); |
| 224 | size /= 8; |
| 225 | BUG_ON(size % sizeof(unsigned long)); |
| 226 | |
| 227 | pbm->msi_bitmap = kzalloc(size, GFP_KERNEL); |
| 228 | if (!pbm->msi_bitmap) |
| 229 | return -ENOMEM; |
| 230 | |
| 231 | return 0; |
| 232 | } |
| 233 | |
| 234 | static void msi_bitmap_free(struct pci_pbm_info *pbm) |
| 235 | { |
| 236 | kfree(pbm->msi_bitmap); |
| 237 | pbm->msi_bitmap = NULL; |
| 238 | } |
| 239 | |
| 240 | static int msi_table_alloc(struct pci_pbm_info *pbm) |
| 241 | { |
| 242 | int size, i; |
| 243 | |
| 244 | size = pbm->msiq_num * sizeof(struct sparc64_msiq_cookie); |
| 245 | pbm->msiq_irq_cookies = kzalloc(size, GFP_KERNEL); |
| 246 | if (!pbm->msiq_irq_cookies) |
| 247 | return -ENOMEM; |
| 248 | |
| 249 | for (i = 0; i < pbm->msiq_num; i++) { |
| 250 | struct sparc64_msiq_cookie *p; |
| 251 | |
| 252 | p = &pbm->msiq_irq_cookies[i]; |
| 253 | p->pbm = pbm; |
| 254 | p->msiqid = pbm->msiq_first + i; |
| 255 | } |
| 256 | |
| 257 | size = pbm->msi_num * sizeof(unsigned int); |
| 258 | pbm->msi_irq_table = kzalloc(size, GFP_KERNEL); |
| 259 | if (!pbm->msi_irq_table) { |
| 260 | kfree(pbm->msiq_irq_cookies); |
| 261 | pbm->msiq_irq_cookies = NULL; |
| 262 | return -ENOMEM; |
| 263 | } |
| 264 | |
| 265 | return 0; |
| 266 | } |
| 267 | |
| 268 | static void msi_table_free(struct pci_pbm_info *pbm) |
| 269 | { |
| 270 | kfree(pbm->msiq_irq_cookies); |
| 271 | pbm->msiq_irq_cookies = NULL; |
| 272 | |
| 273 | kfree(pbm->msi_irq_table); |
| 274 | pbm->msi_irq_table = NULL; |
| 275 | } |
| 276 | |
| 277 | static int bringup_one_msi_queue(struct pci_pbm_info *pbm, |
| 278 | const struct sparc64_msiq_ops *ops, |
| 279 | unsigned long msiqid, |
| 280 | unsigned long devino) |
| 281 | { |
| 282 | int irq = ops->msiq_build_irq(pbm, msiqid, devino); |
David S. Miller | c1b1a5f | 2008-03-19 04:52:48 -0700 | [diff] [blame] | 283 | int err, nid; |
David S. Miller | 759f89e | 2007-10-11 03:16:13 -0700 | [diff] [blame] | 284 | |
| 285 | if (irq < 0) |
| 286 | return irq; |
| 287 | |
David S. Miller | c1b1a5f | 2008-03-19 04:52:48 -0700 | [diff] [blame] | 288 | nid = pbm->numa_node; |
| 289 | if (nid != -1) { |
Rusty Russell | 96d76a7 | 2008-12-26 22:23:38 +1030 | [diff] [blame] | 290 | cpumask_t numa_mask = *cpumask_of_node(nid); |
David S. Miller | c1b1a5f | 2008-03-19 04:52:48 -0700 | [diff] [blame] | 291 | |
Rusty Russell | 0de2652 | 2008-12-13 21:20:26 +1030 | [diff] [blame] | 292 | irq_set_affinity(irq, &numa_mask); |
David S. Miller | c1b1a5f | 2008-03-19 04:52:48 -0700 | [diff] [blame] | 293 | } |
David S. Miller | 759f89e | 2007-10-11 03:16:13 -0700 | [diff] [blame] | 294 | err = request_irq(irq, sparc64_msiq_interrupt, 0, |
| 295 | "MSIQ", |
| 296 | &pbm->msiq_irq_cookies[msiqid - pbm->msiq_first]); |
| 297 | if (err) |
| 298 | return err; |
| 299 | |
| 300 | return 0; |
| 301 | } |
| 302 | |
| 303 | static int sparc64_bringup_msi_queues(struct pci_pbm_info *pbm, |
| 304 | const struct sparc64_msiq_ops *ops) |
| 305 | { |
| 306 | int i; |
| 307 | |
| 308 | for (i = 0; i < pbm->msiq_num; i++) { |
| 309 | unsigned long msiqid = i + pbm->msiq_first; |
| 310 | unsigned long devino = i + pbm->msiq_first_devino; |
| 311 | int err; |
| 312 | |
| 313 | err = bringup_one_msi_queue(pbm, ops, msiqid, devino); |
| 314 | if (err) |
| 315 | return err; |
| 316 | } |
| 317 | |
| 318 | return 0; |
| 319 | } |
| 320 | |
| 321 | void sparc64_pbm_msi_init(struct pci_pbm_info *pbm, |
| 322 | const struct sparc64_msiq_ops *ops) |
| 323 | { |
| 324 | const u32 *val; |
| 325 | int len; |
| 326 | |
David S. Miller | 22fecba | 2008-09-10 00:19:28 -0700 | [diff] [blame] | 327 | val = of_get_property(pbm->op->node, "#msi-eqs", &len); |
David S. Miller | 759f89e | 2007-10-11 03:16:13 -0700 | [diff] [blame] | 328 | if (!val || len != 4) |
| 329 | goto no_msi; |
| 330 | pbm->msiq_num = *val; |
| 331 | if (pbm->msiq_num) { |
| 332 | const struct msiq_prop { |
| 333 | u32 first_msiq; |
| 334 | u32 num_msiq; |
| 335 | u32 first_devino; |
| 336 | } *mqp; |
| 337 | const struct msi_range_prop { |
| 338 | u32 first_msi; |
| 339 | u32 num_msi; |
| 340 | } *mrng; |
| 341 | const struct addr_range_prop { |
| 342 | u32 msi32_high; |
| 343 | u32 msi32_low; |
| 344 | u32 msi32_len; |
| 345 | u32 msi64_high; |
| 346 | u32 msi64_low; |
| 347 | u32 msi64_len; |
| 348 | } *arng; |
| 349 | |
David S. Miller | 22fecba | 2008-09-10 00:19:28 -0700 | [diff] [blame] | 350 | val = of_get_property(pbm->op->node, "msi-eq-size", &len); |
David S. Miller | 759f89e | 2007-10-11 03:16:13 -0700 | [diff] [blame] | 351 | if (!val || len != 4) |
| 352 | goto no_msi; |
| 353 | |
| 354 | pbm->msiq_ent_count = *val; |
| 355 | |
David S. Miller | 22fecba | 2008-09-10 00:19:28 -0700 | [diff] [blame] | 356 | mqp = of_get_property(pbm->op->node, |
David S. Miller | 759f89e | 2007-10-11 03:16:13 -0700 | [diff] [blame] | 357 | "msi-eq-to-devino", &len); |
| 358 | if (!mqp) |
David S. Miller | 22fecba | 2008-09-10 00:19:28 -0700 | [diff] [blame] | 359 | mqp = of_get_property(pbm->op->node, |
David S. Miller | 759f89e | 2007-10-11 03:16:13 -0700 | [diff] [blame] | 360 | "msi-eq-devino", &len); |
| 361 | if (!mqp || len != sizeof(struct msiq_prop)) |
| 362 | goto no_msi; |
| 363 | |
| 364 | pbm->msiq_first = mqp->first_msiq; |
| 365 | pbm->msiq_first_devino = mqp->first_devino; |
| 366 | |
David S. Miller | 22fecba | 2008-09-10 00:19:28 -0700 | [diff] [blame] | 367 | val = of_get_property(pbm->op->node, "#msi", &len); |
David S. Miller | 759f89e | 2007-10-11 03:16:13 -0700 | [diff] [blame] | 368 | if (!val || len != 4) |
| 369 | goto no_msi; |
| 370 | pbm->msi_num = *val; |
| 371 | |
David S. Miller | 22fecba | 2008-09-10 00:19:28 -0700 | [diff] [blame] | 372 | mrng = of_get_property(pbm->op->node, "msi-ranges", &len); |
David S. Miller | 759f89e | 2007-10-11 03:16:13 -0700 | [diff] [blame] | 373 | if (!mrng || len != sizeof(struct msi_range_prop)) |
| 374 | goto no_msi; |
| 375 | pbm->msi_first = mrng->first_msi; |
| 376 | |
David S. Miller | 22fecba | 2008-09-10 00:19:28 -0700 | [diff] [blame] | 377 | val = of_get_property(pbm->op->node, "msi-data-mask", &len); |
David S. Miller | 759f89e | 2007-10-11 03:16:13 -0700 | [diff] [blame] | 378 | if (!val || len != 4) |
| 379 | goto no_msi; |
| 380 | pbm->msi_data_mask = *val; |
| 381 | |
David S. Miller | 22fecba | 2008-09-10 00:19:28 -0700 | [diff] [blame] | 382 | val = of_get_property(pbm->op->node, "msix-data-width", &len); |
David S. Miller | 759f89e | 2007-10-11 03:16:13 -0700 | [diff] [blame] | 383 | if (!val || len != 4) |
| 384 | goto no_msi; |
| 385 | pbm->msix_data_width = *val; |
| 386 | |
David S. Miller | 22fecba | 2008-09-10 00:19:28 -0700 | [diff] [blame] | 387 | arng = of_get_property(pbm->op->node, "msi-address-ranges", |
David S. Miller | 759f89e | 2007-10-11 03:16:13 -0700 | [diff] [blame] | 388 | &len); |
| 389 | if (!arng || len != sizeof(struct addr_range_prop)) |
| 390 | goto no_msi; |
| 391 | pbm->msi32_start = ((u64)arng->msi32_high << 32) | |
| 392 | (u64) arng->msi32_low; |
| 393 | pbm->msi64_start = ((u64)arng->msi64_high << 32) | |
| 394 | (u64) arng->msi64_low; |
| 395 | pbm->msi32_len = arng->msi32_len; |
| 396 | pbm->msi64_len = arng->msi64_len; |
| 397 | |
| 398 | if (msi_bitmap_alloc(pbm)) |
| 399 | goto no_msi; |
| 400 | |
| 401 | if (msi_table_alloc(pbm)) { |
| 402 | msi_bitmap_free(pbm); |
| 403 | goto no_msi; |
| 404 | } |
| 405 | |
| 406 | if (ops->msiq_alloc(pbm)) { |
| 407 | msi_table_free(pbm); |
| 408 | msi_bitmap_free(pbm); |
| 409 | goto no_msi; |
| 410 | } |
| 411 | |
| 412 | if (sparc64_bringup_msi_queues(pbm, ops)) { |
| 413 | ops->msiq_free(pbm); |
| 414 | msi_table_free(pbm); |
| 415 | msi_bitmap_free(pbm); |
| 416 | goto no_msi; |
| 417 | } |
| 418 | |
| 419 | printk(KERN_INFO "%s: MSI Queue first[%u] num[%u] count[%u] " |
| 420 | "devino[0x%x]\n", |
| 421 | pbm->name, |
| 422 | pbm->msiq_first, pbm->msiq_num, |
| 423 | pbm->msiq_ent_count, |
| 424 | pbm->msiq_first_devino); |
| 425 | printk(KERN_INFO "%s: MSI first[%u] num[%u] mask[0x%x] " |
| 426 | "width[%u]\n", |
| 427 | pbm->name, |
| 428 | pbm->msi_first, pbm->msi_num, pbm->msi_data_mask, |
| 429 | pbm->msix_data_width); |
Sam Ravnborg | 9018113 | 2009-01-06 13:19:28 -0800 | [diff] [blame] | 430 | printk(KERN_INFO "%s: MSI addr32[0x%llx:0x%x] " |
| 431 | "addr64[0x%llx:0x%x]\n", |
David S. Miller | 759f89e | 2007-10-11 03:16:13 -0700 | [diff] [blame] | 432 | pbm->name, |
| 433 | pbm->msi32_start, pbm->msi32_len, |
| 434 | pbm->msi64_start, pbm->msi64_len); |
| 435 | printk(KERN_INFO "%s: MSI queues at RA [%016lx]\n", |
| 436 | pbm->name, |
| 437 | __pa(pbm->msi_queues)); |
| 438 | |
| 439 | pbm->msi_ops = ops; |
| 440 | pbm->setup_msi_irq = sparc64_setup_msi_irq; |
| 441 | pbm->teardown_msi_irq = sparc64_teardown_msi_irq; |
| 442 | } |
| 443 | return; |
| 444 | |
| 445 | no_msi: |
| 446 | pbm->msiq_num = 0; |
| 447 | printk(KERN_INFO "%s: No MSI support.\n", pbm->name); |
| 448 | } |