Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1 | /* |
| 2 | * File: msi.c |
| 3 | * Purpose: PCI Message Signaled Interrupt (MSI) |
| 4 | * |
| 5 | * Copyright (C) 2003-2004 Intel |
| 6 | * Copyright (C) Tom Long Nguyen (tom.l.nguyen@intel.com) |
Christoph Hellwig | aff1716 | 2016-07-12 18:20:17 +0900 | [diff] [blame] | 7 | * Copyright (C) 2016 Christoph Hellwig. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8 | */ |
| 9 | |
Eric W. Biederman | 1ce0337 | 2006-10-04 02:16:41 -0700 | [diff] [blame] | 10 | #include <linux/err.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 11 | #include <linux/mm.h> |
| 12 | #include <linux/irq.h> |
| 13 | #include <linux/interrupt.h> |
Paul Gortmaker | 363c75d | 2011-05-27 09:37:25 -0400 | [diff] [blame] | 14 | #include <linux/export.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 15 | #include <linux/ioport.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 16 | #include <linux/pci.h> |
| 17 | #include <linux/proc_fs.h> |
Eric W. Biederman | 3b7d192 | 2006-10-04 02:16:59 -0700 | [diff] [blame] | 18 | #include <linux/msi.h> |
Dan Williams | 4fdadeb | 2007-04-26 18:21:38 -0700 | [diff] [blame] | 19 | #include <linux/smp.h> |
Hidetoshi Seto | 500559a | 2009-08-10 10:14:15 +0900 | [diff] [blame] | 20 | #include <linux/errno.h> |
| 21 | #include <linux/io.h> |
Tejun Heo | 5a0e3ad | 2010-03-24 17:04:11 +0900 | [diff] [blame] | 22 | #include <linux/slab.h> |
Jiang Liu | 3878eae | 2014-11-11 21:02:18 +0800 | [diff] [blame] | 23 | #include <linux/irqdomain.h> |
David Daney | b6eec9b | 2015-10-08 15:10:49 -0700 | [diff] [blame] | 24 | #include <linux/of_irq.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 25 | |
| 26 | #include "pci.h" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 27 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 28 | static int pci_msi_enable = 1; |
Yijing Wang | 38737d8 | 2014-10-27 10:44:36 +0800 | [diff] [blame] | 29 | int pci_msi_ignore_mask; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 30 | |
Bjorn Helgaas | 527eee2 | 2013-04-17 17:44:48 -0600 | [diff] [blame] | 31 | #define msix_table_size(flags) ((flags & PCI_MSIX_FLAGS_QSIZE) + 1) |
| 32 | |
Jiang Liu | 8e047ad | 2014-11-15 22:24:07 +0800 | [diff] [blame] | 33 | #ifdef CONFIG_PCI_MSI_IRQ_DOMAIN |
| 34 | static struct irq_domain *pci_msi_default_domain; |
| 35 | static DEFINE_MUTEX(pci_msi_domain_lock); |
| 36 | |
| 37 | struct irq_domain * __weak arch_get_pci_msi_domain(struct pci_dev *dev) |
| 38 | { |
| 39 | return pci_msi_default_domain; |
| 40 | } |
| 41 | |
Marc Zyngier | 020c312 | 2014-11-15 10:49:12 +0000 | [diff] [blame] | 42 | static struct irq_domain *pci_msi_get_domain(struct pci_dev *dev) |
| 43 | { |
Marc Zyngier | d8a1cb7 | 2015-07-28 14:46:14 +0100 | [diff] [blame] | 44 | struct irq_domain *domain; |
Marc Zyngier | 020c312 | 2014-11-15 10:49:12 +0000 | [diff] [blame] | 45 | |
Marc Zyngier | d8a1cb7 | 2015-07-28 14:46:14 +0100 | [diff] [blame] | 46 | domain = dev_get_msi_domain(&dev->dev); |
| 47 | if (domain) |
| 48 | return domain; |
Marc Zyngier | 020c312 | 2014-11-15 10:49:12 +0000 | [diff] [blame] | 49 | |
Marc Zyngier | d8a1cb7 | 2015-07-28 14:46:14 +0100 | [diff] [blame] | 50 | return arch_get_pci_msi_domain(dev); |
Marc Zyngier | 020c312 | 2014-11-15 10:49:12 +0000 | [diff] [blame] | 51 | } |
| 52 | |
Jiang Liu | 8e047ad | 2014-11-15 22:24:07 +0800 | [diff] [blame] | 53 | static int pci_msi_setup_msi_irqs(struct pci_dev *dev, int nvec, int type) |
| 54 | { |
| 55 | struct irq_domain *domain; |
| 56 | |
Marc Zyngier | 020c312 | 2014-11-15 10:49:12 +0000 | [diff] [blame] | 57 | domain = pci_msi_get_domain(dev); |
Marc Zyngier | 3845d29 | 2015-12-04 10:28:14 -0600 | [diff] [blame] | 58 | if (domain && irq_domain_is_hierarchy(domain)) |
Jiang Liu | 8e047ad | 2014-11-15 22:24:07 +0800 | [diff] [blame] | 59 | return pci_msi_domain_alloc_irqs(domain, dev, nvec, type); |
| 60 | |
| 61 | return arch_setup_msi_irqs(dev, nvec, type); |
| 62 | } |
| 63 | |
| 64 | static void pci_msi_teardown_msi_irqs(struct pci_dev *dev) |
| 65 | { |
| 66 | struct irq_domain *domain; |
| 67 | |
Marc Zyngier | 020c312 | 2014-11-15 10:49:12 +0000 | [diff] [blame] | 68 | domain = pci_msi_get_domain(dev); |
Marc Zyngier | 3845d29 | 2015-12-04 10:28:14 -0600 | [diff] [blame] | 69 | if (domain && irq_domain_is_hierarchy(domain)) |
Jiang Liu | 8e047ad | 2014-11-15 22:24:07 +0800 | [diff] [blame] | 70 | pci_msi_domain_free_irqs(domain, dev); |
| 71 | else |
| 72 | arch_teardown_msi_irqs(dev); |
| 73 | } |
| 74 | #else |
| 75 | #define pci_msi_setup_msi_irqs arch_setup_msi_irqs |
| 76 | #define pci_msi_teardown_msi_irqs arch_teardown_msi_irqs |
| 77 | #endif |
Bjorn Helgaas | 527eee2 | 2013-04-17 17:44:48 -0600 | [diff] [blame] | 78 | |
Adrian Bunk | 6a9e7f2 | 2007-12-11 23:19:41 +0100 | [diff] [blame] | 79 | /* Arch hooks */ |
| 80 | |
Thomas Petazzoni | 4287d82 | 2013-08-09 22:27:06 +0200 | [diff] [blame] | 81 | int __weak arch_setup_msi_irq(struct pci_dev *dev, struct msi_desc *desc) |
| 82 | { |
Lorenzo Pieralisi | 2291ec0 | 2015-08-03 22:04:06 -0500 | [diff] [blame] | 83 | struct msi_controller *chip = dev->bus->msi; |
Thierry Reding | 0cbdcfc | 2013-08-09 22:27:08 +0200 | [diff] [blame] | 84 | int err; |
| 85 | |
| 86 | if (!chip || !chip->setup_irq) |
| 87 | return -EINVAL; |
| 88 | |
| 89 | err = chip->setup_irq(chip, dev, desc); |
| 90 | if (err < 0) |
| 91 | return err; |
| 92 | |
| 93 | irq_set_chip_data(desc->irq, chip); |
| 94 | |
| 95 | return 0; |
Thomas Petazzoni | 4287d82 | 2013-08-09 22:27:06 +0200 | [diff] [blame] | 96 | } |
| 97 | |
| 98 | void __weak arch_teardown_msi_irq(unsigned int irq) |
| 99 | { |
Yijing Wang | c2791b8 | 2014-11-11 17:45:45 -0700 | [diff] [blame] | 100 | struct msi_controller *chip = irq_get_chip_data(irq); |
Thierry Reding | 0cbdcfc | 2013-08-09 22:27:08 +0200 | [diff] [blame] | 101 | |
| 102 | if (!chip || !chip->teardown_irq) |
| 103 | return; |
| 104 | |
| 105 | chip->teardown_irq(chip, irq); |
Thomas Petazzoni | 4287d82 | 2013-08-09 22:27:06 +0200 | [diff] [blame] | 106 | } |
| 107 | |
Thomas Petazzoni | 4287d82 | 2013-08-09 22:27:06 +0200 | [diff] [blame] | 108 | int __weak arch_setup_msi_irqs(struct pci_dev *dev, int nvec, int type) |
Adrian Bunk | 6a9e7f2 | 2007-12-11 23:19:41 +0100 | [diff] [blame] | 109 | { |
Lucas Stach | 339e5b4 | 2015-09-18 13:58:34 -0500 | [diff] [blame] | 110 | struct msi_controller *chip = dev->bus->msi; |
Adrian Bunk | 6a9e7f2 | 2007-12-11 23:19:41 +0100 | [diff] [blame] | 111 | struct msi_desc *entry; |
| 112 | int ret; |
| 113 | |
Lucas Stach | 339e5b4 | 2015-09-18 13:58:34 -0500 | [diff] [blame] | 114 | if (chip && chip->setup_irqs) |
| 115 | return chip->setup_irqs(chip, dev, nvec, type); |
Matthew Wilcox | 1c8d7b0 | 2009-03-17 08:54:10 -0400 | [diff] [blame] | 116 | /* |
| 117 | * If an architecture wants to support multiple MSI, it needs to |
| 118 | * override arch_setup_msi_irqs() |
| 119 | */ |
| 120 | if (type == PCI_CAP_ID_MSI && nvec > 1) |
| 121 | return 1; |
| 122 | |
Jiang Liu | 5004e98 | 2015-07-09 16:00:41 +0800 | [diff] [blame] | 123 | for_each_pci_msi_entry(entry, dev) { |
Adrian Bunk | 6a9e7f2 | 2007-12-11 23:19:41 +0100 | [diff] [blame] | 124 | ret = arch_setup_msi_irq(dev, entry); |
Michael Ellerman | b5fbf53 | 2009-02-11 22:27:02 +1100 | [diff] [blame] | 125 | if (ret < 0) |
Adrian Bunk | 6a9e7f2 | 2007-12-11 23:19:41 +0100 | [diff] [blame] | 126 | return ret; |
Michael Ellerman | b5fbf53 | 2009-02-11 22:27:02 +1100 | [diff] [blame] | 127 | if (ret > 0) |
| 128 | return -ENOSPC; |
Adrian Bunk | 6a9e7f2 | 2007-12-11 23:19:41 +0100 | [diff] [blame] | 129 | } |
| 130 | |
| 131 | return 0; |
| 132 | } |
| 133 | |
Thomas Petazzoni | 4287d82 | 2013-08-09 22:27:06 +0200 | [diff] [blame] | 134 | /* |
| 135 | * We have a default implementation available as a separate non-weak |
| 136 | * function, as it is used by the Xen x86 PCI code |
| 137 | */ |
Thomas Gleixner | 1525bf0 | 2010-10-06 16:05:35 -0400 | [diff] [blame] | 138 | void default_teardown_msi_irqs(struct pci_dev *dev) |
Adrian Bunk | 6a9e7f2 | 2007-12-11 23:19:41 +0100 | [diff] [blame] | 139 | { |
Jiang Liu | 63a7b17 | 2014-11-06 22:20:32 +0800 | [diff] [blame] | 140 | int i; |
Adrian Bunk | 6a9e7f2 | 2007-12-11 23:19:41 +0100 | [diff] [blame] | 141 | struct msi_desc *entry; |
| 142 | |
Jiang Liu | 5004e98 | 2015-07-09 16:00:41 +0800 | [diff] [blame] | 143 | for_each_pci_msi_entry(entry, dev) |
Jiang Liu | 63a7b17 | 2014-11-06 22:20:32 +0800 | [diff] [blame] | 144 | if (entry->irq) |
| 145 | for (i = 0; i < entry->nvec_used; i++) |
| 146 | arch_teardown_msi_irq(entry->irq + i); |
Adrian Bunk | 6a9e7f2 | 2007-12-11 23:19:41 +0100 | [diff] [blame] | 147 | } |
| 148 | |
Thomas Petazzoni | 4287d82 | 2013-08-09 22:27:06 +0200 | [diff] [blame] | 149 | void __weak arch_teardown_msi_irqs(struct pci_dev *dev) |
| 150 | { |
| 151 | return default_teardown_msi_irqs(dev); |
| 152 | } |
Konrad Rzeszutek Wilk | 76ccc29 | 2011-12-16 17:38:18 -0500 | [diff] [blame] | 153 | |
DuanZhenzhong | ac8344c | 2013-12-04 13:09:16 +0800 | [diff] [blame] | 154 | static void default_restore_msi_irq(struct pci_dev *dev, int irq) |
Konrad Rzeszutek Wilk | 76ccc29 | 2011-12-16 17:38:18 -0500 | [diff] [blame] | 155 | { |
| 156 | struct msi_desc *entry; |
| 157 | |
| 158 | entry = NULL; |
| 159 | if (dev->msix_enabled) { |
Jiang Liu | 5004e98 | 2015-07-09 16:00:41 +0800 | [diff] [blame] | 160 | for_each_pci_msi_entry(entry, dev) { |
Konrad Rzeszutek Wilk | 76ccc29 | 2011-12-16 17:38:18 -0500 | [diff] [blame] | 161 | if (irq == entry->irq) |
| 162 | break; |
| 163 | } |
| 164 | } else if (dev->msi_enabled) { |
| 165 | entry = irq_get_msi_desc(irq); |
| 166 | } |
| 167 | |
| 168 | if (entry) |
Jiang Liu | 83a1891 | 2014-11-09 23:10:34 +0800 | [diff] [blame] | 169 | __pci_write_msi_msg(entry, &entry->msg); |
Konrad Rzeszutek Wilk | 76ccc29 | 2011-12-16 17:38:18 -0500 | [diff] [blame] | 170 | } |
Thomas Petazzoni | 4287d82 | 2013-08-09 22:27:06 +0200 | [diff] [blame] | 171 | |
DuanZhenzhong | ac8344c | 2013-12-04 13:09:16 +0800 | [diff] [blame] | 172 | void __weak arch_restore_msi_irqs(struct pci_dev *dev) |
Thomas Petazzoni | 4287d82 | 2013-08-09 22:27:06 +0200 | [diff] [blame] | 173 | { |
DuanZhenzhong | ac8344c | 2013-12-04 13:09:16 +0800 | [diff] [blame] | 174 | return default_restore_msi_irqs(dev); |
Thomas Petazzoni | 4287d82 | 2013-08-09 22:27:06 +0200 | [diff] [blame] | 175 | } |
Konrad Rzeszutek Wilk | 76ccc29 | 2011-12-16 17:38:18 -0500 | [diff] [blame] | 176 | |
Matthew Wilcox | bffac3c | 2009-01-21 19:19:19 -0500 | [diff] [blame] | 177 | static inline __attribute_const__ u32 msi_mask(unsigned x) |
| 178 | { |
Matthew Wilcox | 0b49ec3 | 2009-02-08 20:27:47 -0700 | [diff] [blame] | 179 | /* Don't shift by >= width of type */ |
| 180 | if (x >= 5) |
| 181 | return 0xffffffff; |
| 182 | return (1 << (1 << x)) - 1; |
Matthew Wilcox | bffac3c | 2009-01-21 19:19:19 -0500 | [diff] [blame] | 183 | } |
| 184 | |
Matthew Wilcox | ce6fce4 | 2008-07-25 15:42:58 -0600 | [diff] [blame] | 185 | /* |
| 186 | * PCI 2.3 does not specify mask bits for each MSI interrupt. Attempting to |
| 187 | * mask all MSI interrupts by clearing the MSI enable bit does not work |
| 188 | * reliably as devices without an INTx disable bit will then generate a |
| 189 | * level IRQ which will never be cleared. |
Matthew Wilcox | ce6fce4 | 2008-07-25 15:42:58 -0600 | [diff] [blame] | 190 | */ |
Thomas Gleixner | 23ed8d5 | 2014-11-23 11:55:58 +0100 | [diff] [blame] | 191 | u32 __pci_msi_desc_mask_irq(struct msi_desc *desc, u32 mask, u32 flag) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 192 | { |
Matthew Wilcox | f2440d9 | 2009-03-17 08:54:09 -0400 | [diff] [blame] | 193 | u32 mask_bits = desc->masked; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 194 | |
Yijing Wang | 38737d8 | 2014-10-27 10:44:36 +0800 | [diff] [blame] | 195 | if (pci_msi_ignore_mask || !desc->msi_attrib.maskbit) |
Hidetoshi Seto | 12abb8b | 2009-06-24 12:08:09 +0900 | [diff] [blame] | 196 | return 0; |
Matthew Wilcox | f2440d9 | 2009-03-17 08:54:09 -0400 | [diff] [blame] | 197 | |
| 198 | mask_bits &= ~mask; |
| 199 | mask_bits |= flag; |
Jiang Liu | e39758e | 2015-07-09 16:00:43 +0800 | [diff] [blame] | 200 | pci_write_config_dword(msi_desc_to_pci_dev(desc), desc->mask_pos, |
| 201 | mask_bits); |
Hidetoshi Seto | 12abb8b | 2009-06-24 12:08:09 +0900 | [diff] [blame] | 202 | |
| 203 | return mask_bits; |
| 204 | } |
| 205 | |
| 206 | static void msi_mask_irq(struct msi_desc *desc, u32 mask, u32 flag) |
| 207 | { |
Thomas Gleixner | 23ed8d5 | 2014-11-23 11:55:58 +0100 | [diff] [blame] | 208 | desc->masked = __pci_msi_desc_mask_irq(desc, mask, flag); |
Matthew Wilcox | f2440d9 | 2009-03-17 08:54:09 -0400 | [diff] [blame] | 209 | } |
| 210 | |
Christoph Hellwig | 5eb6d66 | 2016-07-12 18:20:14 +0900 | [diff] [blame] | 211 | static void __iomem *pci_msix_desc_addr(struct msi_desc *desc) |
| 212 | { |
| 213 | return desc->mask_base + |
| 214 | desc->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE; |
| 215 | } |
| 216 | |
Matthew Wilcox | f2440d9 | 2009-03-17 08:54:09 -0400 | [diff] [blame] | 217 | /* |
| 218 | * This internal function does not flush PCI writes to the device. |
| 219 | * All users must ensure that they read from the device before either |
| 220 | * assuming that the device state is up to date, or returning out of this |
| 221 | * file. This saves a few milliseconds when initialising devices with lots |
| 222 | * of MSI-X interrupts. |
| 223 | */ |
Thomas Gleixner | 23ed8d5 | 2014-11-23 11:55:58 +0100 | [diff] [blame] | 224 | u32 __pci_msix_desc_mask_irq(struct msi_desc *desc, u32 flag) |
Matthew Wilcox | f2440d9 | 2009-03-17 08:54:09 -0400 | [diff] [blame] | 225 | { |
| 226 | u32 mask_bits = desc->masked; |
Yijing Wang | 38737d8 | 2014-10-27 10:44:36 +0800 | [diff] [blame] | 227 | |
| 228 | if (pci_msi_ignore_mask) |
| 229 | return 0; |
| 230 | |
Sheng Yang | 8d80528 | 2010-11-11 15:46:55 +0800 | [diff] [blame] | 231 | mask_bits &= ~PCI_MSIX_ENTRY_CTRL_MASKBIT; |
| 232 | if (flag) |
| 233 | mask_bits |= PCI_MSIX_ENTRY_CTRL_MASKBIT; |
Christoph Hellwig | 5eb6d66 | 2016-07-12 18:20:14 +0900 | [diff] [blame] | 234 | writel(mask_bits, pci_msix_desc_addr(desc) + PCI_MSIX_ENTRY_VECTOR_CTRL); |
Hidetoshi Seto | 12abb8b | 2009-06-24 12:08:09 +0900 | [diff] [blame] | 235 | |
| 236 | return mask_bits; |
| 237 | } |
| 238 | |
| 239 | static void msix_mask_irq(struct msi_desc *desc, u32 flag) |
| 240 | { |
Thomas Gleixner | 23ed8d5 | 2014-11-23 11:55:58 +0100 | [diff] [blame] | 241 | desc->masked = __pci_msix_desc_mask_irq(desc, flag); |
Matthew Wilcox | f2440d9 | 2009-03-17 08:54:09 -0400 | [diff] [blame] | 242 | } |
| 243 | |
Thomas Gleixner | 1c9db52 | 2010-09-28 16:46:51 +0200 | [diff] [blame] | 244 | static void msi_set_mask_bit(struct irq_data *data, u32 flag) |
Matthew Wilcox | f2440d9 | 2009-03-17 08:54:09 -0400 | [diff] [blame] | 245 | { |
Jiang Liu | c391f26 | 2015-06-01 16:05:41 +0800 | [diff] [blame] | 246 | struct msi_desc *desc = irq_data_get_msi_desc(data); |
Matthew Wilcox | f2440d9 | 2009-03-17 08:54:09 -0400 | [diff] [blame] | 247 | |
| 248 | if (desc->msi_attrib.is_msix) { |
| 249 | msix_mask_irq(desc, flag); |
| 250 | readl(desc->mask_base); /* Flush write to device */ |
Matthew Wilcox | 24d2755 | 2009-03-17 08:54:06 -0400 | [diff] [blame] | 251 | } else { |
Yijing Wang | a281b78 | 2014-07-08 10:08:55 +0800 | [diff] [blame] | 252 | unsigned offset = data->irq - desc->irq; |
Matthew Wilcox | 1c8d7b0 | 2009-03-17 08:54:10 -0400 | [diff] [blame] | 253 | msi_mask_irq(desc, 1 << offset, flag << offset); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 254 | } |
Matthew Wilcox | f2440d9 | 2009-03-17 08:54:09 -0400 | [diff] [blame] | 255 | } |
| 256 | |
Thomas Gleixner | 23ed8d5 | 2014-11-23 11:55:58 +0100 | [diff] [blame] | 257 | /** |
| 258 | * pci_msi_mask_irq - Generic irq chip callback to mask PCI/MSI interrupts |
| 259 | * @data: pointer to irqdata associated to that interrupt |
| 260 | */ |
| 261 | void pci_msi_mask_irq(struct irq_data *data) |
Matthew Wilcox | f2440d9 | 2009-03-17 08:54:09 -0400 | [diff] [blame] | 262 | { |
Thomas Gleixner | 1c9db52 | 2010-09-28 16:46:51 +0200 | [diff] [blame] | 263 | msi_set_mask_bit(data, 1); |
Matthew Wilcox | f2440d9 | 2009-03-17 08:54:09 -0400 | [diff] [blame] | 264 | } |
Jake Oshins | a4289dc | 2015-12-10 17:52:59 +0000 | [diff] [blame] | 265 | EXPORT_SYMBOL_GPL(pci_msi_mask_irq); |
Matthew Wilcox | f2440d9 | 2009-03-17 08:54:09 -0400 | [diff] [blame] | 266 | |
Thomas Gleixner | 23ed8d5 | 2014-11-23 11:55:58 +0100 | [diff] [blame] | 267 | /** |
| 268 | * pci_msi_unmask_irq - Generic irq chip callback to unmask PCI/MSI interrupts |
| 269 | * @data: pointer to irqdata associated to that interrupt |
| 270 | */ |
| 271 | void pci_msi_unmask_irq(struct irq_data *data) |
Matthew Wilcox | f2440d9 | 2009-03-17 08:54:09 -0400 | [diff] [blame] | 272 | { |
Thomas Gleixner | 1c9db52 | 2010-09-28 16:46:51 +0200 | [diff] [blame] | 273 | msi_set_mask_bit(data, 0); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 274 | } |
Jake Oshins | a4289dc | 2015-12-10 17:52:59 +0000 | [diff] [blame] | 275 | EXPORT_SYMBOL_GPL(pci_msi_unmask_irq); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 276 | |
DuanZhenzhong | ac8344c | 2013-12-04 13:09:16 +0800 | [diff] [blame] | 277 | void default_restore_msi_irqs(struct pci_dev *dev) |
| 278 | { |
| 279 | struct msi_desc *entry; |
| 280 | |
Jiang Liu | 5004e98 | 2015-07-09 16:00:41 +0800 | [diff] [blame] | 281 | for_each_pci_msi_entry(entry, dev) |
DuanZhenzhong | ac8344c | 2013-12-04 13:09:16 +0800 | [diff] [blame] | 282 | default_restore_msi_irq(dev, entry->irq); |
DuanZhenzhong | ac8344c | 2013-12-04 13:09:16 +0800 | [diff] [blame] | 283 | } |
| 284 | |
Jiang Liu | 891d4a4 | 2014-11-09 23:10:33 +0800 | [diff] [blame] | 285 | void __pci_read_msi_msg(struct msi_desc *entry, struct msi_msg *msg) |
Eric W. Biederman | 0366f8f | 2006-10-04 02:16:33 -0700 | [diff] [blame] | 286 | { |
Jiang Liu | e39758e | 2015-07-09 16:00:43 +0800 | [diff] [blame] | 287 | struct pci_dev *dev = msi_desc_to_pci_dev(entry); |
| 288 | |
| 289 | BUG_ON(dev->current_state != PCI_D0); |
Eric W. Biederman | 0366f8f | 2006-10-04 02:16:33 -0700 | [diff] [blame] | 290 | |
Ben Hutchings | 30da552 | 2010-07-23 14:56:28 +0100 | [diff] [blame] | 291 | if (entry->msi_attrib.is_msix) { |
Christoph Hellwig | 5eb6d66 | 2016-07-12 18:20:14 +0900 | [diff] [blame] | 292 | void __iomem *base = pci_msix_desc_addr(entry); |
Ben Hutchings | 30da552 | 2010-07-23 14:56:28 +0100 | [diff] [blame] | 293 | |
| 294 | msg->address_lo = readl(base + PCI_MSIX_ENTRY_LOWER_ADDR); |
| 295 | msg->address_hi = readl(base + PCI_MSIX_ENTRY_UPPER_ADDR); |
| 296 | msg->data = readl(base + PCI_MSIX_ENTRY_DATA); |
| 297 | } else { |
Bjorn Helgaas | f532216 | 2013-04-17 17:34:36 -0600 | [diff] [blame] | 298 | int pos = dev->msi_cap; |
Ben Hutchings | 30da552 | 2010-07-23 14:56:28 +0100 | [diff] [blame] | 299 | u16 data; |
| 300 | |
Bjorn Helgaas | 9925ad0 | 2013-04-17 17:39:57 -0600 | [diff] [blame] | 301 | pci_read_config_dword(dev, pos + PCI_MSI_ADDRESS_LO, |
| 302 | &msg->address_lo); |
Ben Hutchings | 30da552 | 2010-07-23 14:56:28 +0100 | [diff] [blame] | 303 | if (entry->msi_attrib.is_64) { |
Bjorn Helgaas | 9925ad0 | 2013-04-17 17:39:57 -0600 | [diff] [blame] | 304 | pci_read_config_dword(dev, pos + PCI_MSI_ADDRESS_HI, |
| 305 | &msg->address_hi); |
Bjorn Helgaas | 2f22134 | 2013-04-17 17:41:13 -0600 | [diff] [blame] | 306 | pci_read_config_word(dev, pos + PCI_MSI_DATA_64, &data); |
Ben Hutchings | 30da552 | 2010-07-23 14:56:28 +0100 | [diff] [blame] | 307 | } else { |
| 308 | msg->address_hi = 0; |
Bjorn Helgaas | 2f22134 | 2013-04-17 17:41:13 -0600 | [diff] [blame] | 309 | pci_read_config_word(dev, pos + PCI_MSI_DATA_32, &data); |
Ben Hutchings | 30da552 | 2010-07-23 14:56:28 +0100 | [diff] [blame] | 310 | } |
| 311 | msg->data = data; |
| 312 | } |
Eric W. Biederman | 0366f8f | 2006-10-04 02:16:33 -0700 | [diff] [blame] | 313 | } |
| 314 | |
Jiang Liu | 83a1891 | 2014-11-09 23:10:34 +0800 | [diff] [blame] | 315 | void __pci_write_msi_msg(struct msi_desc *entry, struct msi_msg *msg) |
Yinghai Lu | 3145e94 | 2008-12-05 18:58:34 -0800 | [diff] [blame] | 316 | { |
Jiang Liu | e39758e | 2015-07-09 16:00:43 +0800 | [diff] [blame] | 317 | struct pci_dev *dev = msi_desc_to_pci_dev(entry); |
| 318 | |
| 319 | if (dev->current_state != PCI_D0) { |
Ben Hutchings | fcd097f | 2010-06-17 20:16:36 +0100 | [diff] [blame] | 320 | /* Don't touch the hardware now */ |
| 321 | } else if (entry->msi_attrib.is_msix) { |
Christoph Hellwig | 5eb6d66 | 2016-07-12 18:20:14 +0900 | [diff] [blame] | 322 | void __iomem *base = pci_msix_desc_addr(entry); |
Matthew Wilcox | 24d2755 | 2009-03-17 08:54:06 -0400 | [diff] [blame] | 323 | |
Hidetoshi Seto | 2c21fd4 | 2009-06-23 17:40:04 +0900 | [diff] [blame] | 324 | writel(msg->address_lo, base + PCI_MSIX_ENTRY_LOWER_ADDR); |
| 325 | writel(msg->address_hi, base + PCI_MSIX_ENTRY_UPPER_ADDR); |
| 326 | writel(msg->data, base + PCI_MSIX_ENTRY_DATA); |
Matthew Wilcox | 24d2755 | 2009-03-17 08:54:06 -0400 | [diff] [blame] | 327 | } else { |
Bjorn Helgaas | f532216 | 2013-04-17 17:34:36 -0600 | [diff] [blame] | 328 | int pos = dev->msi_cap; |
Matthew Wilcox | 1c8d7b0 | 2009-03-17 08:54:10 -0400 | [diff] [blame] | 329 | u16 msgctl; |
| 330 | |
Bjorn Helgaas | f84ecd2 | 2013-04-17 17:38:32 -0600 | [diff] [blame] | 331 | pci_read_config_word(dev, pos + PCI_MSI_FLAGS, &msgctl); |
Matthew Wilcox | 1c8d7b0 | 2009-03-17 08:54:10 -0400 | [diff] [blame] | 332 | msgctl &= ~PCI_MSI_FLAGS_QSIZE; |
| 333 | msgctl |= entry->msi_attrib.multiple << 4; |
Bjorn Helgaas | f84ecd2 | 2013-04-17 17:38:32 -0600 | [diff] [blame] | 334 | pci_write_config_word(dev, pos + PCI_MSI_FLAGS, msgctl); |
Eric W. Biederman | 0366f8f | 2006-10-04 02:16:33 -0700 | [diff] [blame] | 335 | |
Bjorn Helgaas | 9925ad0 | 2013-04-17 17:39:57 -0600 | [diff] [blame] | 336 | pci_write_config_dword(dev, pos + PCI_MSI_ADDRESS_LO, |
| 337 | msg->address_lo); |
Eric W. Biederman | 0366f8f | 2006-10-04 02:16:33 -0700 | [diff] [blame] | 338 | if (entry->msi_attrib.is_64) { |
Bjorn Helgaas | 9925ad0 | 2013-04-17 17:39:57 -0600 | [diff] [blame] | 339 | pci_write_config_dword(dev, pos + PCI_MSI_ADDRESS_HI, |
| 340 | msg->address_hi); |
Bjorn Helgaas | 2f22134 | 2013-04-17 17:41:13 -0600 | [diff] [blame] | 341 | pci_write_config_word(dev, pos + PCI_MSI_DATA_64, |
| 342 | msg->data); |
Eric W. Biederman | 0366f8f | 2006-10-04 02:16:33 -0700 | [diff] [blame] | 343 | } else { |
Bjorn Helgaas | 2f22134 | 2013-04-17 17:41:13 -0600 | [diff] [blame] | 344 | pci_write_config_word(dev, pos + PCI_MSI_DATA_32, |
| 345 | msg->data); |
Eric W. Biederman | 0366f8f | 2006-10-04 02:16:33 -0700 | [diff] [blame] | 346 | } |
Eric W. Biederman | 0366f8f | 2006-10-04 02:16:33 -0700 | [diff] [blame] | 347 | } |
Eric W. Biederman | 392ee1e | 2007-03-08 13:04:57 -0700 | [diff] [blame] | 348 | entry->msg = *msg; |
Eric W. Biederman | 0366f8f | 2006-10-04 02:16:33 -0700 | [diff] [blame] | 349 | } |
| 350 | |
Jiang Liu | 83a1891 | 2014-11-09 23:10:34 +0800 | [diff] [blame] | 351 | void pci_write_msi_msg(unsigned int irq, struct msi_msg *msg) |
Yinghai Lu | 3145e94 | 2008-12-05 18:58:34 -0800 | [diff] [blame] | 352 | { |
Thomas Gleixner | dced35a | 2011-03-28 17:49:12 +0200 | [diff] [blame] | 353 | struct msi_desc *entry = irq_get_msi_desc(irq); |
Yinghai Lu | 3145e94 | 2008-12-05 18:58:34 -0800 | [diff] [blame] | 354 | |
Jiang Liu | 83a1891 | 2014-11-09 23:10:34 +0800 | [diff] [blame] | 355 | __pci_write_msi_msg(entry, msg); |
Yinghai Lu | 3145e94 | 2008-12-05 18:58:34 -0800 | [diff] [blame] | 356 | } |
Jiang Liu | 83a1891 | 2014-11-09 23:10:34 +0800 | [diff] [blame] | 357 | EXPORT_SYMBOL_GPL(pci_write_msi_msg); |
Yinghai Lu | 3145e94 | 2008-12-05 18:58:34 -0800 | [diff] [blame] | 358 | |
Hidetoshi Seto | f56e448 | 2009-08-06 11:32:51 +0900 | [diff] [blame] | 359 | static void free_msi_irqs(struct pci_dev *dev) |
| 360 | { |
Jiang Liu | 5004e98 | 2015-07-09 16:00:41 +0800 | [diff] [blame] | 361 | struct list_head *msi_list = dev_to_msi_list(&dev->dev); |
Hidetoshi Seto | f56e448 | 2009-08-06 11:32:51 +0900 | [diff] [blame] | 362 | struct msi_desc *entry, *tmp; |
Greg Kroah-Hartman | 1c51b50 | 2013-12-19 12:30:17 -0800 | [diff] [blame] | 363 | struct attribute **msi_attrs; |
| 364 | struct device_attribute *dev_attr; |
Jiang Liu | 63a7b17 | 2014-11-06 22:20:32 +0800 | [diff] [blame] | 365 | int i, count = 0; |
Hidetoshi Seto | f56e448 | 2009-08-06 11:32:51 +0900 | [diff] [blame] | 366 | |
Jiang Liu | 5004e98 | 2015-07-09 16:00:41 +0800 | [diff] [blame] | 367 | for_each_pci_msi_entry(entry, dev) |
Jiang Liu | 63a7b17 | 2014-11-06 22:20:32 +0800 | [diff] [blame] | 368 | if (entry->irq) |
| 369 | for (i = 0; i < entry->nvec_used; i++) |
| 370 | BUG_ON(irq_has_action(entry->irq + i)); |
Hidetoshi Seto | f56e448 | 2009-08-06 11:32:51 +0900 | [diff] [blame] | 371 | |
Jiang Liu | 8e047ad | 2014-11-15 22:24:07 +0800 | [diff] [blame] | 372 | pci_msi_teardown_msi_irqs(dev); |
Hidetoshi Seto | f56e448 | 2009-08-06 11:32:51 +0900 | [diff] [blame] | 373 | |
Jiang Liu | 5004e98 | 2015-07-09 16:00:41 +0800 | [diff] [blame] | 374 | list_for_each_entry_safe(entry, tmp, msi_list, list) { |
Hidetoshi Seto | f56e448 | 2009-08-06 11:32:51 +0900 | [diff] [blame] | 375 | if (entry->msi_attrib.is_msix) { |
Jiang Liu | 5004e98 | 2015-07-09 16:00:41 +0800 | [diff] [blame] | 376 | if (list_is_last(&entry->list, msi_list)) |
Hidetoshi Seto | f56e448 | 2009-08-06 11:32:51 +0900 | [diff] [blame] | 377 | iounmap(entry->mask_base); |
| 378 | } |
Neil Horman | 424eb39 | 2012-01-03 10:29:54 -0500 | [diff] [blame] | 379 | |
Hidetoshi Seto | f56e448 | 2009-08-06 11:32:51 +0900 | [diff] [blame] | 380 | list_del(&entry->list); |
| 381 | kfree(entry); |
| 382 | } |
Greg Kroah-Hartman | 1c51b50 | 2013-12-19 12:30:17 -0800 | [diff] [blame] | 383 | |
| 384 | if (dev->msi_irq_groups) { |
| 385 | sysfs_remove_groups(&dev->dev.kobj, dev->msi_irq_groups); |
| 386 | msi_attrs = dev->msi_irq_groups[0]->attrs; |
Alexei Starovoitov | b701c0b | 2014-06-04 15:49:50 -0700 | [diff] [blame] | 387 | while (msi_attrs[count]) { |
Greg Kroah-Hartman | 1c51b50 | 2013-12-19 12:30:17 -0800 | [diff] [blame] | 388 | dev_attr = container_of(msi_attrs[count], |
| 389 | struct device_attribute, attr); |
| 390 | kfree(dev_attr->attr.name); |
| 391 | kfree(dev_attr); |
| 392 | ++count; |
| 393 | } |
| 394 | kfree(msi_attrs); |
| 395 | kfree(dev->msi_irq_groups[0]); |
| 396 | kfree(dev->msi_irq_groups); |
| 397 | dev->msi_irq_groups = NULL; |
| 398 | } |
Hidetoshi Seto | f56e448 | 2009-08-06 11:32:51 +0900 | [diff] [blame] | 399 | } |
Satoru Takeuchi | c54c187 | 2007-01-18 13:50:05 +0900 | [diff] [blame] | 400 | |
David Miller | ba698ad | 2007-10-25 01:16:30 -0700 | [diff] [blame] | 401 | static void pci_intx_for_msi(struct pci_dev *dev, int enable) |
| 402 | { |
| 403 | if (!(dev->dev_flags & PCI_DEV_FLAGS_MSI_INTX_DISABLE_BUG)) |
| 404 | pci_intx(dev, enable); |
| 405 | } |
| 406 | |
Michael Ellerman | 8fed4b6 | 2007-01-25 19:34:08 +1100 | [diff] [blame] | 407 | static void __pci_restore_msi_state(struct pci_dev *dev) |
Shaohua Li | 41017f0 | 2006-02-08 17:11:38 +0800 | [diff] [blame] | 408 | { |
Shaohua Li | 41017f0 | 2006-02-08 17:11:38 +0800 | [diff] [blame] | 409 | u16 control; |
Eric W. Biederman | 392ee1e | 2007-03-08 13:04:57 -0700 | [diff] [blame] | 410 | struct msi_desc *entry; |
Shaohua Li | 41017f0 | 2006-02-08 17:11:38 +0800 | [diff] [blame] | 411 | |
Eric W. Biederman | b1cbf4e | 2007-03-05 00:30:10 -0800 | [diff] [blame] | 412 | if (!dev->msi_enabled) |
| 413 | return; |
| 414 | |
Thomas Gleixner | dced35a | 2011-03-28 17:49:12 +0200 | [diff] [blame] | 415 | entry = irq_get_msi_desc(dev->irq); |
Shaohua Li | 41017f0 | 2006-02-08 17:11:38 +0800 | [diff] [blame] | 416 | |
David Miller | ba698ad | 2007-10-25 01:16:30 -0700 | [diff] [blame] | 417 | pci_intx_for_msi(dev, 0); |
Michael S. Tsirkin | 61b64ab | 2015-05-07 09:52:21 -0500 | [diff] [blame] | 418 | pci_msi_set_enable(dev, 0); |
DuanZhenzhong | ac8344c | 2013-12-04 13:09:16 +0800 | [diff] [blame] | 419 | arch_restore_msi_irqs(dev); |
Eric W. Biederman | 392ee1e | 2007-03-08 13:04:57 -0700 | [diff] [blame] | 420 | |
Bjorn Helgaas | f532216 | 2013-04-17 17:34:36 -0600 | [diff] [blame] | 421 | pci_read_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, &control); |
Yijing Wang | 31ea5d4 | 2014-06-19 16:30:30 +0800 | [diff] [blame] | 422 | msi_mask_irq(entry, msi_mask(entry->msi_attrib.multi_cap), |
| 423 | entry->masked); |
Jesse Barnes | abad2ec | 2008-08-07 08:52:37 -0700 | [diff] [blame] | 424 | control &= ~PCI_MSI_FLAGS_QSIZE; |
Matthew Wilcox | 1c8d7b0 | 2009-03-17 08:54:10 -0400 | [diff] [blame] | 425 | control |= (entry->msi_attrib.multiple << 4) | PCI_MSI_FLAGS_ENABLE; |
Bjorn Helgaas | f532216 | 2013-04-17 17:34:36 -0600 | [diff] [blame] | 426 | pci_write_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, control); |
Michael Ellerman | 8fed4b6 | 2007-01-25 19:34:08 +1100 | [diff] [blame] | 427 | } |
| 428 | |
| 429 | static void __pci_restore_msix_state(struct pci_dev *dev) |
Shaohua Li | 41017f0 | 2006-02-08 17:11:38 +0800 | [diff] [blame] | 430 | { |
Shaohua Li | 41017f0 | 2006-02-08 17:11:38 +0800 | [diff] [blame] | 431 | struct msi_desc *entry; |
Shaohua Li | 41017f0 | 2006-02-08 17:11:38 +0800 | [diff] [blame] | 432 | |
Eric W. Biederman | ded86d8 | 2007-01-28 12:42:52 -0700 | [diff] [blame] | 433 | if (!dev->msix_enabled) |
| 434 | return; |
Jiang Liu | 5004e98 | 2015-07-09 16:00:41 +0800 | [diff] [blame] | 435 | BUG_ON(list_empty(dev_to_msi_list(&dev->dev))); |
Eric W. Biederman | ded86d8 | 2007-01-28 12:42:52 -0700 | [diff] [blame] | 436 | |
Shaohua Li | 41017f0 | 2006-02-08 17:11:38 +0800 | [diff] [blame] | 437 | /* route the table */ |
David Miller | ba698ad | 2007-10-25 01:16:30 -0700 | [diff] [blame] | 438 | pci_intx_for_msi(dev, 0); |
Michael S. Tsirkin | 61b64ab | 2015-05-07 09:52:21 -0500 | [diff] [blame] | 439 | pci_msix_clear_and_set_ctrl(dev, 0, |
Yijing Wang | 66f0d0c | 2014-06-19 16:29:53 +0800 | [diff] [blame] | 440 | PCI_MSIX_FLAGS_ENABLE | PCI_MSIX_FLAGS_MASKALL); |
Shaohua Li | 41017f0 | 2006-02-08 17:11:38 +0800 | [diff] [blame] | 441 | |
DuanZhenzhong | ac8344c | 2013-12-04 13:09:16 +0800 | [diff] [blame] | 442 | arch_restore_msi_irqs(dev); |
Jiang Liu | 5004e98 | 2015-07-09 16:00:41 +0800 | [diff] [blame] | 443 | for_each_pci_msi_entry(entry, dev) |
Matthew Wilcox | f2440d9 | 2009-03-17 08:54:09 -0400 | [diff] [blame] | 444 | msix_mask_irq(entry, entry->masked); |
Shaohua Li | 41017f0 | 2006-02-08 17:11:38 +0800 | [diff] [blame] | 445 | |
Michael S. Tsirkin | 61b64ab | 2015-05-07 09:52:21 -0500 | [diff] [blame] | 446 | pci_msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_MASKALL, 0); |
Shaohua Li | 41017f0 | 2006-02-08 17:11:38 +0800 | [diff] [blame] | 447 | } |
Michael Ellerman | 8fed4b6 | 2007-01-25 19:34:08 +1100 | [diff] [blame] | 448 | |
| 449 | void pci_restore_msi_state(struct pci_dev *dev) |
| 450 | { |
| 451 | __pci_restore_msi_state(dev); |
| 452 | __pci_restore_msix_state(dev); |
| 453 | } |
Linas Vepstas | 94688cf | 2007-11-07 15:43:59 -0600 | [diff] [blame] | 454 | EXPORT_SYMBOL_GPL(pci_restore_msi_state); |
Shaohua Li | 41017f0 | 2006-02-08 17:11:38 +0800 | [diff] [blame] | 455 | |
Greg Kroah-Hartman | 1c51b50 | 2013-12-19 12:30:17 -0800 | [diff] [blame] | 456 | static ssize_t msi_mode_show(struct device *dev, struct device_attribute *attr, |
Neil Horman | da8d1c8 | 2011-10-06 14:08:18 -0400 | [diff] [blame] | 457 | char *buf) |
| 458 | { |
Greg Kroah-Hartman | 1c51b50 | 2013-12-19 12:30:17 -0800 | [diff] [blame] | 459 | struct msi_desc *entry; |
| 460 | unsigned long irq; |
| 461 | int retval; |
| 462 | |
| 463 | retval = kstrtoul(attr->attr.name, 10, &irq); |
| 464 | if (retval) |
| 465 | return retval; |
| 466 | |
Yijing Wang | e11ece5 | 2014-07-08 10:09:19 +0800 | [diff] [blame] | 467 | entry = irq_get_msi_desc(irq); |
| 468 | if (entry) |
| 469 | return sprintf(buf, "%s\n", |
| 470 | entry->msi_attrib.is_msix ? "msix" : "msi"); |
| 471 | |
Greg Kroah-Hartman | 1c51b50 | 2013-12-19 12:30:17 -0800 | [diff] [blame] | 472 | return -ENODEV; |
Neil Horman | da8d1c8 | 2011-10-06 14:08:18 -0400 | [diff] [blame] | 473 | } |
| 474 | |
Neil Horman | da8d1c8 | 2011-10-06 14:08:18 -0400 | [diff] [blame] | 475 | static int populate_msi_sysfs(struct pci_dev *pdev) |
| 476 | { |
Greg Kroah-Hartman | 1c51b50 | 2013-12-19 12:30:17 -0800 | [diff] [blame] | 477 | struct attribute **msi_attrs; |
| 478 | struct attribute *msi_attr; |
| 479 | struct device_attribute *msi_dev_attr; |
| 480 | struct attribute_group *msi_irq_group; |
| 481 | const struct attribute_group **msi_irq_groups; |
Neil Horman | da8d1c8 | 2011-10-06 14:08:18 -0400 | [diff] [blame] | 482 | struct msi_desc *entry; |
Greg Kroah-Hartman | 1c51b50 | 2013-12-19 12:30:17 -0800 | [diff] [blame] | 483 | int ret = -ENOMEM; |
| 484 | int num_msi = 0; |
Neil Horman | da8d1c8 | 2011-10-06 14:08:18 -0400 | [diff] [blame] | 485 | int count = 0; |
Romain Bezut | a867606 | 2015-09-24 01:31:16 +0200 | [diff] [blame] | 486 | int i; |
Neil Horman | da8d1c8 | 2011-10-06 14:08:18 -0400 | [diff] [blame] | 487 | |
Greg Kroah-Hartman | 1c51b50 | 2013-12-19 12:30:17 -0800 | [diff] [blame] | 488 | /* Determine how many msi entries we have */ |
Jiang Liu | 5004e98 | 2015-07-09 16:00:41 +0800 | [diff] [blame] | 489 | for_each_pci_msi_entry(entry, pdev) |
Romain Bezut | a867606 | 2015-09-24 01:31:16 +0200 | [diff] [blame] | 490 | num_msi += entry->nvec_used; |
Greg Kroah-Hartman | 1c51b50 | 2013-12-19 12:30:17 -0800 | [diff] [blame] | 491 | if (!num_msi) |
| 492 | return 0; |
| 493 | |
| 494 | /* Dynamically create the MSI attributes for the PCI device */ |
| 495 | msi_attrs = kzalloc(sizeof(void *) * (num_msi + 1), GFP_KERNEL); |
| 496 | if (!msi_attrs) |
| 497 | return -ENOMEM; |
Jiang Liu | 5004e98 | 2015-07-09 16:00:41 +0800 | [diff] [blame] | 498 | for_each_pci_msi_entry(entry, pdev) { |
Romain Bezut | a867606 | 2015-09-24 01:31:16 +0200 | [diff] [blame] | 499 | for (i = 0; i < entry->nvec_used; i++) { |
| 500 | msi_dev_attr = kzalloc(sizeof(*msi_dev_attr), GFP_KERNEL); |
| 501 | if (!msi_dev_attr) |
| 502 | goto error_attrs; |
| 503 | msi_attrs[count] = &msi_dev_attr->attr; |
Greg Kroah-Hartman | 86bb4f6 | 2014-02-13 10:47:20 -0700 | [diff] [blame] | 504 | |
Romain Bezut | a867606 | 2015-09-24 01:31:16 +0200 | [diff] [blame] | 505 | sysfs_attr_init(&msi_dev_attr->attr); |
| 506 | msi_dev_attr->attr.name = kasprintf(GFP_KERNEL, "%d", |
| 507 | entry->irq + i); |
| 508 | if (!msi_dev_attr->attr.name) |
| 509 | goto error_attrs; |
| 510 | msi_dev_attr->attr.mode = S_IRUGO; |
| 511 | msi_dev_attr->show = msi_mode_show; |
| 512 | ++count; |
| 513 | } |
Greg Kroah-Hartman | 1c51b50 | 2013-12-19 12:30:17 -0800 | [diff] [blame] | 514 | } |
| 515 | |
| 516 | msi_irq_group = kzalloc(sizeof(*msi_irq_group), GFP_KERNEL); |
| 517 | if (!msi_irq_group) |
| 518 | goto error_attrs; |
| 519 | msi_irq_group->name = "msi_irqs"; |
| 520 | msi_irq_group->attrs = msi_attrs; |
| 521 | |
| 522 | msi_irq_groups = kzalloc(sizeof(void *) * 2, GFP_KERNEL); |
| 523 | if (!msi_irq_groups) |
| 524 | goto error_irq_group; |
| 525 | msi_irq_groups[0] = msi_irq_group; |
| 526 | |
| 527 | ret = sysfs_create_groups(&pdev->dev.kobj, msi_irq_groups); |
| 528 | if (ret) |
| 529 | goto error_irq_groups; |
| 530 | pdev->msi_irq_groups = msi_irq_groups; |
Neil Horman | da8d1c8 | 2011-10-06 14:08:18 -0400 | [diff] [blame] | 531 | |
| 532 | return 0; |
| 533 | |
Greg Kroah-Hartman | 1c51b50 | 2013-12-19 12:30:17 -0800 | [diff] [blame] | 534 | error_irq_groups: |
| 535 | kfree(msi_irq_groups); |
| 536 | error_irq_group: |
| 537 | kfree(msi_irq_group); |
| 538 | error_attrs: |
| 539 | count = 0; |
| 540 | msi_attr = msi_attrs[count]; |
| 541 | while (msi_attr) { |
| 542 | msi_dev_attr = container_of(msi_attr, struct device_attribute, attr); |
| 543 | kfree(msi_attr->name); |
| 544 | kfree(msi_dev_attr); |
| 545 | ++count; |
| 546 | msi_attr = msi_attrs[count]; |
Neil Horman | da8d1c8 | 2011-10-06 14:08:18 -0400 | [diff] [blame] | 547 | } |
Greg Kroah-Hartman | 2923775 | 2014-02-13 10:47:35 -0700 | [diff] [blame] | 548 | kfree(msi_attrs); |
Neil Horman | da8d1c8 | 2011-10-06 14:08:18 -0400 | [diff] [blame] | 549 | return ret; |
| 550 | } |
| 551 | |
Jiang Liu | 63a7b17 | 2014-11-06 22:20:32 +0800 | [diff] [blame] | 552 | static struct msi_desc *msi_setup_entry(struct pci_dev *dev, int nvec) |
Yijing Wang | d873b4d | 2014-07-08 10:07:23 +0800 | [diff] [blame] | 553 | { |
| 554 | u16 control; |
| 555 | struct msi_desc *entry; |
| 556 | |
| 557 | /* MSI Entry Initialization */ |
Jiang Liu | aa48b6f | 2015-07-09 16:00:47 +0800 | [diff] [blame] | 558 | entry = alloc_msi_entry(&dev->dev); |
Yijing Wang | d873b4d | 2014-07-08 10:07:23 +0800 | [diff] [blame] | 559 | if (!entry) |
| 560 | return NULL; |
| 561 | |
| 562 | pci_read_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, &control); |
| 563 | |
| 564 | entry->msi_attrib.is_msix = 0; |
| 565 | entry->msi_attrib.is_64 = !!(control & PCI_MSI_FLAGS_64BIT); |
| 566 | entry->msi_attrib.entry_nr = 0; |
| 567 | entry->msi_attrib.maskbit = !!(control & PCI_MSI_FLAGS_MASKBIT); |
| 568 | entry->msi_attrib.default_irq = dev->irq; /* Save IOAPIC IRQ */ |
Yijing Wang | d873b4d | 2014-07-08 10:07:23 +0800 | [diff] [blame] | 569 | entry->msi_attrib.multi_cap = (control & PCI_MSI_FLAGS_QMASK) >> 1; |
Jiang Liu | 63a7b17 | 2014-11-06 22:20:32 +0800 | [diff] [blame] | 570 | entry->msi_attrib.multiple = ilog2(__roundup_pow_of_two(nvec)); |
| 571 | entry->nvec_used = nvec; |
Christoph Hellwig | 4ef3368 | 2016-07-12 18:20:18 +0900 | [diff] [blame] | 572 | entry->affinity = dev->irq_affinity; |
Yijing Wang | d873b4d | 2014-07-08 10:07:23 +0800 | [diff] [blame] | 573 | |
| 574 | if (control & PCI_MSI_FLAGS_64BIT) |
| 575 | entry->mask_pos = dev->msi_cap + PCI_MSI_MASK_64; |
| 576 | else |
| 577 | entry->mask_pos = dev->msi_cap + PCI_MSI_MASK_32; |
| 578 | |
| 579 | /* Save the initial mask status */ |
| 580 | if (entry->msi_attrib.maskbit) |
| 581 | pci_read_config_dword(dev, entry->mask_pos, &entry->masked); |
| 582 | |
| 583 | return entry; |
| 584 | } |
| 585 | |
Benjamin Herrenschmidt | f144d14 | 2014-10-03 15:13:24 +1000 | [diff] [blame] | 586 | static int msi_verify_entries(struct pci_dev *dev) |
| 587 | { |
| 588 | struct msi_desc *entry; |
| 589 | |
Jiang Liu | 5004e98 | 2015-07-09 16:00:41 +0800 | [diff] [blame] | 590 | for_each_pci_msi_entry(entry, dev) { |
Benjamin Herrenschmidt | f144d14 | 2014-10-03 15:13:24 +1000 | [diff] [blame] | 591 | if (!dev->no_64bit_msi || !entry->msg.address_hi) |
| 592 | continue; |
| 593 | dev_err(&dev->dev, "Device has broken 64-bit MSI but arch" |
| 594 | " tried to assign one above 4G\n"); |
| 595 | return -EIO; |
| 596 | } |
| 597 | return 0; |
| 598 | } |
| 599 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 600 | /** |
| 601 | * msi_capability_init - configure device's MSI capability structure |
| 602 | * @dev: pointer to the pci_dev data structure of MSI device function |
Matthew Wilcox | 1c8d7b0 | 2009-03-17 08:54:10 -0400 | [diff] [blame] | 603 | * @nvec: number of interrupts to allocate |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 604 | * |
Matthew Wilcox | 1c8d7b0 | 2009-03-17 08:54:10 -0400 | [diff] [blame] | 605 | * Setup the MSI capability structure of the device with the requested |
| 606 | * number of interrupts. A return value of zero indicates the successful |
| 607 | * setup of an entry with the new MSI irq. A negative return value indicates |
| 608 | * an error, and a positive return value indicates the number of interrupts |
| 609 | * which could have been allocated. |
| 610 | */ |
| 611 | static int msi_capability_init(struct pci_dev *dev, int nvec) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 612 | { |
| 613 | struct msi_desc *entry; |
Gavin Shan | f465136 | 2013-04-04 16:54:32 +0000 | [diff] [blame] | 614 | int ret; |
Matthew Wilcox | f2440d9 | 2009-03-17 08:54:09 -0400 | [diff] [blame] | 615 | unsigned mask; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 616 | |
Michael S. Tsirkin | 61b64ab | 2015-05-07 09:52:21 -0500 | [diff] [blame] | 617 | pci_msi_set_enable(dev, 0); /* Disable MSI during set up */ |
Matthew Wilcox | 110828c | 2009-06-16 06:31:45 -0600 | [diff] [blame] | 618 | |
Jiang Liu | 63a7b17 | 2014-11-06 22:20:32 +0800 | [diff] [blame] | 619 | entry = msi_setup_entry(dev, nvec); |
Eric W. Biederman | f7feaca | 2007-01-28 12:56:37 -0700 | [diff] [blame] | 620 | if (!entry) |
| 621 | return -ENOMEM; |
Eric W. Biederman | 1ce0337 | 2006-10-04 02:16:41 -0700 | [diff] [blame] | 622 | |
Matthew Wilcox | f2440d9 | 2009-03-17 08:54:09 -0400 | [diff] [blame] | 623 | /* All MSIs are unmasked by default, Mask them all */ |
Yijing Wang | 31ea5d4 | 2014-06-19 16:30:30 +0800 | [diff] [blame] | 624 | mask = msi_mask(entry->msi_attrib.multi_cap); |
Matthew Wilcox | f2440d9 | 2009-03-17 08:54:09 -0400 | [diff] [blame] | 625 | msi_mask_irq(entry, mask, mask); |
| 626 | |
Jiang Liu | 5004e98 | 2015-07-09 16:00:41 +0800 | [diff] [blame] | 627 | list_add_tail(&entry->list, dev_to_msi_list(&dev->dev)); |
Michael Ellerman | 9c83133 | 2007-04-18 19:39:21 +1000 | [diff] [blame] | 628 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 629 | /* Configure MSI capability structure */ |
Jiang Liu | 8e047ad | 2014-11-15 22:24:07 +0800 | [diff] [blame] | 630 | ret = pci_msi_setup_msi_irqs(dev, nvec, PCI_CAP_ID_MSI); |
Michael Ellerman | 7fe3730 | 2007-04-18 19:39:21 +1000 | [diff] [blame] | 631 | if (ret) { |
Hidetoshi Seto | 7ba1930 | 2009-06-23 17:39:27 +0900 | [diff] [blame] | 632 | msi_mask_irq(entry, mask, ~mask); |
Hidetoshi Seto | f56e448 | 2009-08-06 11:32:51 +0900 | [diff] [blame] | 633 | free_msi_irqs(dev); |
Michael Ellerman | 7fe3730 | 2007-04-18 19:39:21 +1000 | [diff] [blame] | 634 | return ret; |
Mark Maule | fd58e55 | 2006-04-10 21:17:48 -0500 | [diff] [blame] | 635 | } |
Eric W. Biederman | f7feaca | 2007-01-28 12:56:37 -0700 | [diff] [blame] | 636 | |
Benjamin Herrenschmidt | f144d14 | 2014-10-03 15:13:24 +1000 | [diff] [blame] | 637 | ret = msi_verify_entries(dev); |
| 638 | if (ret) { |
| 639 | msi_mask_irq(entry, mask, ~mask); |
| 640 | free_msi_irqs(dev); |
| 641 | return ret; |
| 642 | } |
| 643 | |
Neil Horman | da8d1c8 | 2011-10-06 14:08:18 -0400 | [diff] [blame] | 644 | ret = populate_msi_sysfs(dev); |
| 645 | if (ret) { |
| 646 | msi_mask_irq(entry, mask, ~mask); |
| 647 | free_msi_irqs(dev); |
| 648 | return ret; |
| 649 | } |
| 650 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 651 | /* Set MSI enabled bits */ |
David Miller | ba698ad | 2007-10-25 01:16:30 -0700 | [diff] [blame] | 652 | pci_intx_for_msi(dev, 0); |
Michael S. Tsirkin | 61b64ab | 2015-05-07 09:52:21 -0500 | [diff] [blame] | 653 | pci_msi_set_enable(dev, 1); |
Eric W. Biederman | b1cbf4e | 2007-03-05 00:30:10 -0800 | [diff] [blame] | 654 | dev->msi_enabled = 1; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 655 | |
Jiang Liu | 5f22699 | 2015-07-30 14:00:08 -0500 | [diff] [blame] | 656 | pcibios_free_irq(dev); |
Michael Ellerman | 7fe3730 | 2007-04-18 19:39:21 +1000 | [diff] [blame] | 657 | dev->irq = entry->irq; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 658 | return 0; |
| 659 | } |
| 660 | |
Gavin Shan | 520fe9d | 2013-04-04 16:54:33 +0000 | [diff] [blame] | 661 | static void __iomem *msix_map_region(struct pci_dev *dev, unsigned nr_entries) |
Hidetoshi Seto | 5a05a9d | 2009-08-06 11:34:34 +0900 | [diff] [blame] | 662 | { |
Kenji Kaneshige | 4302e0f | 2010-06-17 10:42:44 +0900 | [diff] [blame] | 663 | resource_size_t phys_addr; |
Hidetoshi Seto | 5a05a9d | 2009-08-06 11:34:34 +0900 | [diff] [blame] | 664 | u32 table_offset; |
Yijing Wang | 6a878e5 | 2015-01-28 09:52:17 +0800 | [diff] [blame] | 665 | unsigned long flags; |
Hidetoshi Seto | 5a05a9d | 2009-08-06 11:34:34 +0900 | [diff] [blame] | 666 | u8 bir; |
| 667 | |
Bjorn Helgaas | 909094c | 2013-04-17 17:43:40 -0600 | [diff] [blame] | 668 | pci_read_config_dword(dev, dev->msix_cap + PCI_MSIX_TABLE, |
| 669 | &table_offset); |
Bjorn Helgaas | 4d18760 | 2013-04-17 18:10:07 -0600 | [diff] [blame] | 670 | bir = (u8)(table_offset & PCI_MSIX_TABLE_BIR); |
Yijing Wang | 6a878e5 | 2015-01-28 09:52:17 +0800 | [diff] [blame] | 671 | flags = pci_resource_flags(dev, bir); |
| 672 | if (!flags || (flags & IORESOURCE_UNSET)) |
| 673 | return NULL; |
| 674 | |
Bjorn Helgaas | 4d18760 | 2013-04-17 18:10:07 -0600 | [diff] [blame] | 675 | table_offset &= PCI_MSIX_TABLE_OFFSET; |
Hidetoshi Seto | 5a05a9d | 2009-08-06 11:34:34 +0900 | [diff] [blame] | 676 | phys_addr = pci_resource_start(dev, bir) + table_offset; |
| 677 | |
| 678 | return ioremap_nocache(phys_addr, nr_entries * PCI_MSIX_ENTRY_SIZE); |
| 679 | } |
| 680 | |
Gavin Shan | 520fe9d | 2013-04-04 16:54:33 +0000 | [diff] [blame] | 681 | static int msix_setup_entries(struct pci_dev *dev, void __iomem *base, |
| 682 | struct msix_entry *entries, int nvec) |
Hidetoshi Seto | d9d7070 | 2009-08-06 11:35:48 +0900 | [diff] [blame] | 683 | { |
Christoph Hellwig | 4ef3368 | 2016-07-12 18:20:18 +0900 | [diff] [blame] | 684 | const struct cpumask *mask = NULL; |
Hidetoshi Seto | d9d7070 | 2009-08-06 11:35:48 +0900 | [diff] [blame] | 685 | struct msi_desc *entry; |
Christoph Hellwig | 4ef3368 | 2016-07-12 18:20:18 +0900 | [diff] [blame] | 686 | int cpu = -1, i; |
Hidetoshi Seto | d9d7070 | 2009-08-06 11:35:48 +0900 | [diff] [blame] | 687 | |
| 688 | for (i = 0; i < nvec; i++) { |
Christoph Hellwig | 4ef3368 | 2016-07-12 18:20:18 +0900 | [diff] [blame] | 689 | if (dev->irq_affinity) { |
| 690 | cpu = cpumask_next(cpu, dev->irq_affinity); |
| 691 | if (cpu >= nr_cpu_ids) |
| 692 | cpu = cpumask_first(dev->irq_affinity); |
| 693 | mask = cpumask_of(cpu); |
| 694 | } |
| 695 | |
Jiang Liu | aa48b6f | 2015-07-09 16:00:47 +0800 | [diff] [blame] | 696 | entry = alloc_msi_entry(&dev->dev); |
Hidetoshi Seto | d9d7070 | 2009-08-06 11:35:48 +0900 | [diff] [blame] | 697 | if (!entry) { |
| 698 | if (!i) |
| 699 | iounmap(base); |
| 700 | else |
| 701 | free_msi_irqs(dev); |
| 702 | /* No enough memory. Don't try again */ |
| 703 | return -ENOMEM; |
| 704 | } |
| 705 | |
| 706 | entry->msi_attrib.is_msix = 1; |
| 707 | entry->msi_attrib.is_64 = 1; |
Christoph Hellwig | 3ac020e | 2016-07-12 18:20:16 +0900 | [diff] [blame] | 708 | if (entries) |
| 709 | entry->msi_attrib.entry_nr = entries[i].entry; |
| 710 | else |
| 711 | entry->msi_attrib.entry_nr = i; |
Hidetoshi Seto | d9d7070 | 2009-08-06 11:35:48 +0900 | [diff] [blame] | 712 | entry->msi_attrib.default_irq = dev->irq; |
Hidetoshi Seto | d9d7070 | 2009-08-06 11:35:48 +0900 | [diff] [blame] | 713 | entry->mask_base = base; |
Jiang Liu | 63a7b17 | 2014-11-06 22:20:32 +0800 | [diff] [blame] | 714 | entry->nvec_used = 1; |
Christoph Hellwig | 4ef3368 | 2016-07-12 18:20:18 +0900 | [diff] [blame] | 715 | entry->affinity = mask; |
Hidetoshi Seto | d9d7070 | 2009-08-06 11:35:48 +0900 | [diff] [blame] | 716 | |
Jiang Liu | 5004e98 | 2015-07-09 16:00:41 +0800 | [diff] [blame] | 717 | list_add_tail(&entry->list, dev_to_msi_list(&dev->dev)); |
Hidetoshi Seto | d9d7070 | 2009-08-06 11:35:48 +0900 | [diff] [blame] | 718 | } |
| 719 | |
| 720 | return 0; |
| 721 | } |
| 722 | |
Hidetoshi Seto | 75cb342 | 2009-08-06 11:35:10 +0900 | [diff] [blame] | 723 | static void msix_program_entries(struct pci_dev *dev, |
Gavin Shan | 520fe9d | 2013-04-04 16:54:33 +0000 | [diff] [blame] | 724 | struct msix_entry *entries) |
Hidetoshi Seto | 75cb342 | 2009-08-06 11:35:10 +0900 | [diff] [blame] | 725 | { |
| 726 | struct msi_desc *entry; |
| 727 | int i = 0; |
| 728 | |
Jiang Liu | 5004e98 | 2015-07-09 16:00:41 +0800 | [diff] [blame] | 729 | for_each_pci_msi_entry(entry, dev) { |
Christoph Hellwig | 3ac020e | 2016-07-12 18:20:16 +0900 | [diff] [blame] | 730 | if (entries) |
| 731 | entries[i++].vector = entry->irq; |
Christoph Hellwig | 12eb21d | 2016-07-12 18:20:15 +0900 | [diff] [blame] | 732 | entry->masked = readl(pci_msix_desc_addr(entry) + |
| 733 | PCI_MSIX_ENTRY_VECTOR_CTRL); |
Hidetoshi Seto | 75cb342 | 2009-08-06 11:35:10 +0900 | [diff] [blame] | 734 | msix_mask_irq(entry, 1); |
Hidetoshi Seto | 75cb342 | 2009-08-06 11:35:10 +0900 | [diff] [blame] | 735 | } |
| 736 | } |
| 737 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 738 | /** |
| 739 | * msix_capability_init - configure device's MSI-X capability |
| 740 | * @dev: pointer to the pci_dev data structure of MSI-X device function |
Randy Dunlap | 8f7020d | 2005-10-23 11:57:38 -0700 | [diff] [blame] | 741 | * @entries: pointer to an array of struct msix_entry entries |
| 742 | * @nvec: number of @entries |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 743 | * |
Steven Cole | eaae4b3 | 2005-05-03 18:38:30 -0600 | [diff] [blame] | 744 | * Setup the MSI-X capability structure of device function with a |
Eric W. Biederman | 1ce0337 | 2006-10-04 02:16:41 -0700 | [diff] [blame] | 745 | * single MSI-X irq. A return of zero indicates the successful setup of |
| 746 | * requested MSI-X entries with allocated irqs or non-zero for otherwise. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 747 | **/ |
| 748 | static int msix_capability_init(struct pci_dev *dev, |
| 749 | struct msix_entry *entries, int nvec) |
| 750 | { |
Gavin Shan | 520fe9d | 2013-04-04 16:54:33 +0000 | [diff] [blame] | 751 | int ret; |
Hidetoshi Seto | 5a05a9d | 2009-08-06 11:34:34 +0900 | [diff] [blame] | 752 | u16 control; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 753 | void __iomem *base; |
| 754 | |
Matthew Wilcox | f598282 | 2009-06-18 19:15:59 -0700 | [diff] [blame] | 755 | /* Ensure MSI-X is disabled while it is set up */ |
Michael S. Tsirkin | 61b64ab | 2015-05-07 09:52:21 -0500 | [diff] [blame] | 756 | pci_msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_ENABLE, 0); |
Matthew Wilcox | f598282 | 2009-06-18 19:15:59 -0700 | [diff] [blame] | 757 | |
Yijing Wang | 66f0d0c | 2014-06-19 16:29:53 +0800 | [diff] [blame] | 758 | pci_read_config_word(dev, dev->msix_cap + PCI_MSIX_FLAGS, &control); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 759 | /* Request & Map MSI-X table region */ |
Bjorn Helgaas | 527eee2 | 2013-04-17 17:44:48 -0600 | [diff] [blame] | 760 | base = msix_map_region(dev, msix_table_size(control)); |
Hidetoshi Seto | 5a05a9d | 2009-08-06 11:34:34 +0900 | [diff] [blame] | 761 | if (!base) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 762 | return -ENOMEM; |
| 763 | |
Gavin Shan | 520fe9d | 2013-04-04 16:54:33 +0000 | [diff] [blame] | 764 | ret = msix_setup_entries(dev, base, entries, nvec); |
Hidetoshi Seto | d9d7070 | 2009-08-06 11:35:48 +0900 | [diff] [blame] | 765 | if (ret) |
| 766 | return ret; |
Michael Ellerman | 9c83133 | 2007-04-18 19:39:21 +1000 | [diff] [blame] | 767 | |
Jiang Liu | 8e047ad | 2014-11-15 22:24:07 +0800 | [diff] [blame] | 768 | ret = pci_msi_setup_msi_irqs(dev, nvec, PCI_CAP_ID_MSIX); |
Hidetoshi Seto | 583871d | 2009-08-06 11:33:39 +0900 | [diff] [blame] | 769 | if (ret) |
Alexander Gordeev | 2adc790 | 2013-12-16 09:34:56 +0100 | [diff] [blame] | 770 | goto out_avail; |
Michael Ellerman | 9c83133 | 2007-04-18 19:39:21 +1000 | [diff] [blame] | 771 | |
Benjamin Herrenschmidt | f144d14 | 2014-10-03 15:13:24 +1000 | [diff] [blame] | 772 | /* Check if all MSI entries honor device restrictions */ |
| 773 | ret = msi_verify_entries(dev); |
| 774 | if (ret) |
| 775 | goto out_free; |
| 776 | |
Matthew Wilcox | f598282 | 2009-06-18 19:15:59 -0700 | [diff] [blame] | 777 | /* |
| 778 | * Some devices require MSI-X to be enabled before we can touch the |
| 779 | * MSI-X registers. We need to mask all the vectors to prevent |
| 780 | * interrupts coming in before they're fully set up. |
| 781 | */ |
Michael S. Tsirkin | 61b64ab | 2015-05-07 09:52:21 -0500 | [diff] [blame] | 782 | pci_msix_clear_and_set_ctrl(dev, 0, |
Yijing Wang | 66f0d0c | 2014-06-19 16:29:53 +0800 | [diff] [blame] | 783 | PCI_MSIX_FLAGS_MASKALL | PCI_MSIX_FLAGS_ENABLE); |
Matthew Wilcox | f598282 | 2009-06-18 19:15:59 -0700 | [diff] [blame] | 784 | |
Hidetoshi Seto | 75cb342 | 2009-08-06 11:35:10 +0900 | [diff] [blame] | 785 | msix_program_entries(dev, entries); |
Matthew Wilcox | f598282 | 2009-06-18 19:15:59 -0700 | [diff] [blame] | 786 | |
Neil Horman | da8d1c8 | 2011-10-06 14:08:18 -0400 | [diff] [blame] | 787 | ret = populate_msi_sysfs(dev); |
Alexander Gordeev | 2adc790 | 2013-12-16 09:34:56 +0100 | [diff] [blame] | 788 | if (ret) |
| 789 | goto out_free; |
Neil Horman | da8d1c8 | 2011-10-06 14:08:18 -0400 | [diff] [blame] | 790 | |
Matthew Wilcox | f598282 | 2009-06-18 19:15:59 -0700 | [diff] [blame] | 791 | /* Set MSI-X enabled bits and unmask the function */ |
David Miller | ba698ad | 2007-10-25 01:16:30 -0700 | [diff] [blame] | 792 | pci_intx_for_msi(dev, 0); |
Eric W. Biederman | b1cbf4e | 2007-03-05 00:30:10 -0800 | [diff] [blame] | 793 | dev->msix_enabled = 1; |
Michael S. Tsirkin | 61b64ab | 2015-05-07 09:52:21 -0500 | [diff] [blame] | 794 | pci_msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_MASKALL, 0); |
Matthew Wilcox | 8d18101 | 2009-05-08 07:13:33 -0600 | [diff] [blame] | 795 | |
Jiang Liu | 5f22699 | 2015-07-30 14:00:08 -0500 | [diff] [blame] | 796 | pcibios_free_irq(dev); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 797 | return 0; |
Hidetoshi Seto | 583871d | 2009-08-06 11:33:39 +0900 | [diff] [blame] | 798 | |
Alexander Gordeev | 2adc790 | 2013-12-16 09:34:56 +0100 | [diff] [blame] | 799 | out_avail: |
Hidetoshi Seto | 583871d | 2009-08-06 11:33:39 +0900 | [diff] [blame] | 800 | if (ret < 0) { |
| 801 | /* |
| 802 | * If we had some success, report the number of irqs |
| 803 | * we succeeded in setting up. |
| 804 | */ |
Hidetoshi Seto | d9d7070 | 2009-08-06 11:35:48 +0900 | [diff] [blame] | 805 | struct msi_desc *entry; |
Hidetoshi Seto | 583871d | 2009-08-06 11:33:39 +0900 | [diff] [blame] | 806 | int avail = 0; |
| 807 | |
Jiang Liu | 5004e98 | 2015-07-09 16:00:41 +0800 | [diff] [blame] | 808 | for_each_pci_msi_entry(entry, dev) { |
Hidetoshi Seto | 583871d | 2009-08-06 11:33:39 +0900 | [diff] [blame] | 809 | if (entry->irq != 0) |
| 810 | avail++; |
| 811 | } |
| 812 | if (avail != 0) |
| 813 | ret = avail; |
| 814 | } |
| 815 | |
Alexander Gordeev | 2adc790 | 2013-12-16 09:34:56 +0100 | [diff] [blame] | 816 | out_free: |
Hidetoshi Seto | 583871d | 2009-08-06 11:33:39 +0900 | [diff] [blame] | 817 | free_msi_irqs(dev); |
| 818 | |
| 819 | return ret; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 820 | } |
| 821 | |
| 822 | /** |
Alexander Gordeev | a06cd74 | 2014-09-23 12:45:58 -0600 | [diff] [blame] | 823 | * pci_msi_supported - check whether MSI may be enabled on a device |
Brice Goglin | 24334a1 | 2006-08-31 01:55:07 -0400 | [diff] [blame] | 824 | * @dev: pointer to the pci_dev data structure of MSI device function |
Michael Ellerman | c9953a7 | 2007-04-05 17:19:08 +1000 | [diff] [blame] | 825 | * @nvec: how many MSIs have been requested ? |
Brice Goglin | 24334a1 | 2006-08-31 01:55:07 -0400 | [diff] [blame] | 826 | * |
Bjorn Helgaas | f762598 | 2013-11-14 11:28:18 -0700 | [diff] [blame] | 827 | * Look at global flags, the device itself, and its parent buses |
Michael Ellerman | 17bbc12 | 2007-04-05 17:19:07 +1000 | [diff] [blame] | 828 | * to determine if MSI/-X are supported for the device. If MSI/-X is |
Alexander Gordeev | a06cd74 | 2014-09-23 12:45:58 -0600 | [diff] [blame] | 829 | * supported return 1, else return 0. |
Brice Goglin | 24334a1 | 2006-08-31 01:55:07 -0400 | [diff] [blame] | 830 | **/ |
Alexander Gordeev | a06cd74 | 2014-09-23 12:45:58 -0600 | [diff] [blame] | 831 | static int pci_msi_supported(struct pci_dev *dev, int nvec) |
Brice Goglin | 24334a1 | 2006-08-31 01:55:07 -0400 | [diff] [blame] | 832 | { |
| 833 | struct pci_bus *bus; |
| 834 | |
Brice Goglin | 0306ebf | 2006-10-05 10:24:31 +0200 | [diff] [blame] | 835 | /* MSI must be globally enabled and supported by the device */ |
Alexander Gordeev | 27e2060 | 2014-09-23 14:25:11 -0600 | [diff] [blame] | 836 | if (!pci_msi_enable) |
Alexander Gordeev | a06cd74 | 2014-09-23 12:45:58 -0600 | [diff] [blame] | 837 | return 0; |
Alexander Gordeev | 27e2060 | 2014-09-23 14:25:11 -0600 | [diff] [blame] | 838 | |
| 839 | if (!dev || dev->no_msi || dev->current_state != PCI_D0) |
Alexander Gordeev | a06cd74 | 2014-09-23 12:45:58 -0600 | [diff] [blame] | 840 | return 0; |
Brice Goglin | 24334a1 | 2006-08-31 01:55:07 -0400 | [diff] [blame] | 841 | |
Michael Ellerman | 314e77b | 2007-04-05 17:19:12 +1000 | [diff] [blame] | 842 | /* |
| 843 | * You can't ask to have 0 or less MSIs configured. |
| 844 | * a) it's stupid .. |
| 845 | * b) the list manipulation code assumes nvec >= 1. |
| 846 | */ |
| 847 | if (nvec < 1) |
Alexander Gordeev | a06cd74 | 2014-09-23 12:45:58 -0600 | [diff] [blame] | 848 | return 0; |
Michael Ellerman | 314e77b | 2007-04-05 17:19:12 +1000 | [diff] [blame] | 849 | |
Hidetoshi Seto | 500559a | 2009-08-10 10:14:15 +0900 | [diff] [blame] | 850 | /* |
| 851 | * Any bridge which does NOT route MSI transactions from its |
| 852 | * secondary bus to its primary bus must set NO_MSI flag on |
Brice Goglin | 0306ebf | 2006-10-05 10:24:31 +0200 | [diff] [blame] | 853 | * the secondary pci_bus. |
| 854 | * We expect only arch-specific PCI host bus controller driver |
| 855 | * or quirks for specific PCI bridges to be setting NO_MSI. |
| 856 | */ |
Brice Goglin | 24334a1 | 2006-08-31 01:55:07 -0400 | [diff] [blame] | 857 | for (bus = dev->bus; bus; bus = bus->parent) |
| 858 | if (bus->bus_flags & PCI_BUS_FLAGS_NO_MSI) |
Alexander Gordeev | a06cd74 | 2014-09-23 12:45:58 -0600 | [diff] [blame] | 859 | return 0; |
Brice Goglin | 24334a1 | 2006-08-31 01:55:07 -0400 | [diff] [blame] | 860 | |
Alexander Gordeev | a06cd74 | 2014-09-23 12:45:58 -0600 | [diff] [blame] | 861 | return 1; |
Brice Goglin | 24334a1 | 2006-08-31 01:55:07 -0400 | [diff] [blame] | 862 | } |
| 863 | |
| 864 | /** |
Alexander Gordeev | d1ac1d2 | 2013-12-30 08:28:13 +0100 | [diff] [blame] | 865 | * pci_msi_vec_count - Return the number of MSI vectors a device can send |
| 866 | * @dev: device to report about |
| 867 | * |
| 868 | * This function returns the number of MSI vectors a device requested via |
| 869 | * Multiple Message Capable register. It returns a negative errno if the |
| 870 | * device is not capable sending MSI interrupts. Otherwise, the call succeeds |
| 871 | * and returns a power of two, up to a maximum of 2^5 (32), according to the |
| 872 | * MSI specification. |
| 873 | **/ |
| 874 | int pci_msi_vec_count(struct pci_dev *dev) |
| 875 | { |
| 876 | int ret; |
| 877 | u16 msgctl; |
| 878 | |
| 879 | if (!dev->msi_cap) |
| 880 | return -EINVAL; |
| 881 | |
| 882 | pci_read_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, &msgctl); |
| 883 | ret = 1 << ((msgctl & PCI_MSI_FLAGS_QMASK) >> 1); |
| 884 | |
| 885 | return ret; |
| 886 | } |
| 887 | EXPORT_SYMBOL(pci_msi_vec_count); |
| 888 | |
Matthew Wilcox | f2440d9 | 2009-03-17 08:54:09 -0400 | [diff] [blame] | 889 | void pci_msi_shutdown(struct pci_dev *dev) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 890 | { |
Matthew Wilcox | f2440d9 | 2009-03-17 08:54:09 -0400 | [diff] [blame] | 891 | struct msi_desc *desc; |
| 892 | u32 mask; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 893 | |
Michael Ellerman | 128bc5f | 2007-03-22 21:51:39 +1100 | [diff] [blame] | 894 | if (!pci_msi_enable || !dev || !dev->msi_enabled) |
Eric W. Biederman | ded86d8 | 2007-01-28 12:42:52 -0700 | [diff] [blame] | 895 | return; |
| 896 | |
Jiang Liu | 5004e98 | 2015-07-09 16:00:41 +0800 | [diff] [blame] | 897 | BUG_ON(list_empty(dev_to_msi_list(&dev->dev))); |
Jiang Liu | 4a7cc83 | 2015-07-09 16:00:44 +0800 | [diff] [blame] | 898 | desc = first_pci_msi_entry(dev); |
Matthew Wilcox | 110828c | 2009-06-16 06:31:45 -0600 | [diff] [blame] | 899 | |
Michael S. Tsirkin | 61b64ab | 2015-05-07 09:52:21 -0500 | [diff] [blame] | 900 | pci_msi_set_enable(dev, 0); |
David Miller | ba698ad | 2007-10-25 01:16:30 -0700 | [diff] [blame] | 901 | pci_intx_for_msi(dev, 1); |
Eric W. Biederman | b1cbf4e | 2007-03-05 00:30:10 -0800 | [diff] [blame] | 902 | dev->msi_enabled = 0; |
Eric W. Biederman | 7bd007e | 2006-10-04 02:16:31 -0700 | [diff] [blame] | 903 | |
Hidetoshi Seto | 12abb8b | 2009-06-24 12:08:09 +0900 | [diff] [blame] | 904 | /* Return the device with MSI unmasked as initial states */ |
Yijing Wang | 31ea5d4 | 2014-06-19 16:30:30 +0800 | [diff] [blame] | 905 | mask = msi_mask(desc->msi_attrib.multi_cap); |
Hidetoshi Seto | 12abb8b | 2009-06-24 12:08:09 +0900 | [diff] [blame] | 906 | /* Keep cached state to be restored */ |
Thomas Gleixner | 23ed8d5 | 2014-11-23 11:55:58 +0100 | [diff] [blame] | 907 | __pci_msi_desc_mask_irq(desc, mask, ~mask); |
Michael Ellerman | e387b9e | 2007-03-22 21:51:27 +1100 | [diff] [blame] | 908 | |
| 909 | /* Restore dev->irq to its default pin-assertion irq */ |
Matthew Wilcox | f2440d9 | 2009-03-17 08:54:09 -0400 | [diff] [blame] | 910 | dev->irq = desc->msi_attrib.default_irq; |
Jiang Liu | 5f22699 | 2015-07-30 14:00:08 -0500 | [diff] [blame] | 911 | pcibios_alloc_irq(dev); |
Yinghai Lu | d52877c | 2008-04-23 14:58:09 -0700 | [diff] [blame] | 912 | } |
Matthew Wilcox | 24d2755 | 2009-03-17 08:54:06 -0400 | [diff] [blame] | 913 | |
Hidetoshi Seto | 500559a | 2009-08-10 10:14:15 +0900 | [diff] [blame] | 914 | void pci_disable_msi(struct pci_dev *dev) |
Yinghai Lu | d52877c | 2008-04-23 14:58:09 -0700 | [diff] [blame] | 915 | { |
Yinghai Lu | d52877c | 2008-04-23 14:58:09 -0700 | [diff] [blame] | 916 | if (!pci_msi_enable || !dev || !dev->msi_enabled) |
| 917 | return; |
| 918 | |
| 919 | pci_msi_shutdown(dev); |
Hidetoshi Seto | f56e448 | 2009-08-06 11:32:51 +0900 | [diff] [blame] | 920 | free_msi_irqs(dev); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 921 | } |
Michael Ellerman | 4cc086f | 2007-03-22 21:51:34 +1100 | [diff] [blame] | 922 | EXPORT_SYMBOL(pci_disable_msi); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 923 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 924 | /** |
Alexander Gordeev | ff1aa43 | 2013-12-30 08:28:15 +0100 | [diff] [blame] | 925 | * pci_msix_vec_count - return the number of device's MSI-X table entries |
Rafael J. Wysocki | a52e2e3 | 2009-01-24 00:21:14 +0100 | [diff] [blame] | 926 | * @dev: pointer to the pci_dev data structure of MSI-X device function |
Alexander Gordeev | ff1aa43 | 2013-12-30 08:28:15 +0100 | [diff] [blame] | 927 | * This function returns the number of device's MSI-X table entries and |
| 928 | * therefore the number of MSI-X vectors device is capable of sending. |
| 929 | * It returns a negative errno if the device is not capable of sending MSI-X |
| 930 | * interrupts. |
| 931 | **/ |
| 932 | int pci_msix_vec_count(struct pci_dev *dev) |
Rafael J. Wysocki | a52e2e3 | 2009-01-24 00:21:14 +0100 | [diff] [blame] | 933 | { |
Rafael J. Wysocki | a52e2e3 | 2009-01-24 00:21:14 +0100 | [diff] [blame] | 934 | u16 control; |
| 935 | |
Gavin Shan | 520fe9d | 2013-04-04 16:54:33 +0000 | [diff] [blame] | 936 | if (!dev->msix_cap) |
Alexander Gordeev | ff1aa43 | 2013-12-30 08:28:15 +0100 | [diff] [blame] | 937 | return -EINVAL; |
Rafael J. Wysocki | a52e2e3 | 2009-01-24 00:21:14 +0100 | [diff] [blame] | 938 | |
Bjorn Helgaas | f84ecd2 | 2013-04-17 17:38:32 -0600 | [diff] [blame] | 939 | pci_read_config_word(dev, dev->msix_cap + PCI_MSIX_FLAGS, &control); |
Bjorn Helgaas | 527eee2 | 2013-04-17 17:44:48 -0600 | [diff] [blame] | 940 | return msix_table_size(control); |
Rafael J. Wysocki | a52e2e3 | 2009-01-24 00:21:14 +0100 | [diff] [blame] | 941 | } |
Alexander Gordeev | ff1aa43 | 2013-12-30 08:28:15 +0100 | [diff] [blame] | 942 | EXPORT_SYMBOL(pci_msix_vec_count); |
Rafael J. Wysocki | a52e2e3 | 2009-01-24 00:21:14 +0100 | [diff] [blame] | 943 | |
| 944 | /** |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 945 | * pci_enable_msix - configure device's MSI-X capability structure |
| 946 | * @dev: pointer to the pci_dev data structure of MSI-X device function |
Christoph Hellwig | 3ac020e | 2016-07-12 18:20:16 +0900 | [diff] [blame] | 947 | * @entries: pointer to an array of MSI-X entries (optional) |
Eric W. Biederman | 1ce0337 | 2006-10-04 02:16:41 -0700 | [diff] [blame] | 948 | * @nvec: number of MSI-X irqs requested for allocation by device driver |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 949 | * |
| 950 | * Setup the MSI-X capability structure of device function with the number |
Eric W. Biederman | 1ce0337 | 2006-10-04 02:16:41 -0700 | [diff] [blame] | 951 | * of requested irqs upon its software driver call to request for |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 952 | * MSI-X mode enabled on its hardware device function. A return of zero |
| 953 | * indicates the successful configuration of MSI-X capability structure |
Eric W. Biederman | 1ce0337 | 2006-10-04 02:16:41 -0700 | [diff] [blame] | 954 | * with new allocated MSI-X irqs. A return of < 0 indicates a failure. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 955 | * Or a return of > 0 indicates that driver request is exceeding the number |
Michael S. Tsirkin | 57fbf52 | 2009-05-07 11:28:41 +0300 | [diff] [blame] | 956 | * of irqs or MSI-X vectors available. Driver should use the returned value to |
| 957 | * re-send its request. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 958 | **/ |
Hidetoshi Seto | 500559a | 2009-08-10 10:14:15 +0900 | [diff] [blame] | 959 | int pci_enable_msix(struct pci_dev *dev, struct msix_entry *entries, int nvec) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 960 | { |
Bjorn Helgaas | 5ec0940 | 2014-09-23 14:38:28 -0600 | [diff] [blame] | 961 | int nr_entries; |
Eric W. Biederman | ded86d8 | 2007-01-28 12:42:52 -0700 | [diff] [blame] | 962 | int i, j; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 963 | |
Alexander Gordeev | a06cd74 | 2014-09-23 12:45:58 -0600 | [diff] [blame] | 964 | if (!pci_msi_supported(dev, nvec)) |
| 965 | return -EINVAL; |
Michael Ellerman | c9953a7 | 2007-04-05 17:19:08 +1000 | [diff] [blame] | 966 | |
Alexander Gordeev | ff1aa43 | 2013-12-30 08:28:15 +0100 | [diff] [blame] | 967 | nr_entries = pci_msix_vec_count(dev); |
| 968 | if (nr_entries < 0) |
| 969 | return nr_entries; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 970 | if (nvec > nr_entries) |
Michael S. Tsirkin | 57fbf52 | 2009-05-07 11:28:41 +0300 | [diff] [blame] | 971 | return nr_entries; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 972 | |
Christoph Hellwig | 3ac020e | 2016-07-12 18:20:16 +0900 | [diff] [blame] | 973 | if (entries) { |
| 974 | /* Check for any invalid entries */ |
| 975 | for (i = 0; i < nvec; i++) { |
| 976 | if (entries[i].entry >= nr_entries) |
| 977 | return -EINVAL; /* invalid entry */ |
| 978 | for (j = i + 1; j < nvec; j++) { |
| 979 | if (entries[i].entry == entries[j].entry) |
| 980 | return -EINVAL; /* duplicate entry */ |
| 981 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 982 | } |
| 983 | } |
Eric W. Biederman | ded86d8 | 2007-01-28 12:42:52 -0700 | [diff] [blame] | 984 | WARN_ON(!!dev->msix_enabled); |
Eric W. Biederman | 7bd007e | 2006-10-04 02:16:31 -0700 | [diff] [blame] | 985 | |
Eric W. Biederman | 1ce0337 | 2006-10-04 02:16:41 -0700 | [diff] [blame] | 986 | /* Check whether driver already requested for MSI irq */ |
Hidetoshi Seto | 500559a | 2009-08-10 10:14:15 +0900 | [diff] [blame] | 987 | if (dev->msi_enabled) { |
Ryan Desfosses | 227f064 | 2014-04-18 20:13:50 -0400 | [diff] [blame] | 988 | dev_info(&dev->dev, "can't enable MSI-X (MSI IRQ already assigned)\n"); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 989 | return -EINVAL; |
| 990 | } |
Bjorn Helgaas | 5ec0940 | 2014-09-23 14:38:28 -0600 | [diff] [blame] | 991 | return msix_capability_init(dev, entries, nvec); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 992 | } |
Michael Ellerman | 4cc086f | 2007-03-22 21:51:34 +1100 | [diff] [blame] | 993 | EXPORT_SYMBOL(pci_enable_msix); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 994 | |
Hidetoshi Seto | 500559a | 2009-08-10 10:14:15 +0900 | [diff] [blame] | 995 | void pci_msix_shutdown(struct pci_dev *dev) |
Michael Ellerman | fc4afc7 | 2007-03-22 21:51:33 +1100 | [diff] [blame] | 996 | { |
Hidetoshi Seto | 12abb8b | 2009-06-24 12:08:09 +0900 | [diff] [blame] | 997 | struct msi_desc *entry; |
| 998 | |
Michael Ellerman | 128bc5f | 2007-03-22 21:51:39 +1100 | [diff] [blame] | 999 | if (!pci_msi_enable || !dev || !dev->msix_enabled) |
Eric W. Biederman | ded86d8 | 2007-01-28 12:42:52 -0700 | [diff] [blame] | 1000 | return; |
| 1001 | |
Hidetoshi Seto | 12abb8b | 2009-06-24 12:08:09 +0900 | [diff] [blame] | 1002 | /* Return the device with MSI-X masked as initial states */ |
Jiang Liu | 5004e98 | 2015-07-09 16:00:41 +0800 | [diff] [blame] | 1003 | for_each_pci_msi_entry(entry, dev) { |
Hidetoshi Seto | 12abb8b | 2009-06-24 12:08:09 +0900 | [diff] [blame] | 1004 | /* Keep cached states to be restored */ |
Thomas Gleixner | 23ed8d5 | 2014-11-23 11:55:58 +0100 | [diff] [blame] | 1005 | __pci_msix_desc_mask_irq(entry, 1); |
Hidetoshi Seto | 12abb8b | 2009-06-24 12:08:09 +0900 | [diff] [blame] | 1006 | } |
| 1007 | |
Michael S. Tsirkin | 61b64ab | 2015-05-07 09:52:21 -0500 | [diff] [blame] | 1008 | pci_msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_ENABLE, 0); |
David Miller | ba698ad | 2007-10-25 01:16:30 -0700 | [diff] [blame] | 1009 | pci_intx_for_msi(dev, 1); |
Eric W. Biederman | b1cbf4e | 2007-03-05 00:30:10 -0800 | [diff] [blame] | 1010 | dev->msix_enabled = 0; |
Jiang Liu | 5f22699 | 2015-07-30 14:00:08 -0500 | [diff] [blame] | 1011 | pcibios_alloc_irq(dev); |
Yinghai Lu | d52877c | 2008-04-23 14:58:09 -0700 | [diff] [blame] | 1012 | } |
Hidetoshi Seto | c901851 | 2009-08-06 11:31:27 +0900 | [diff] [blame] | 1013 | |
Hidetoshi Seto | 500559a | 2009-08-10 10:14:15 +0900 | [diff] [blame] | 1014 | void pci_disable_msix(struct pci_dev *dev) |
Yinghai Lu | d52877c | 2008-04-23 14:58:09 -0700 | [diff] [blame] | 1015 | { |
| 1016 | if (!pci_msi_enable || !dev || !dev->msix_enabled) |
| 1017 | return; |
| 1018 | |
| 1019 | pci_msix_shutdown(dev); |
Hidetoshi Seto | f56e448 | 2009-08-06 11:32:51 +0900 | [diff] [blame] | 1020 | free_msi_irqs(dev); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1021 | } |
Michael Ellerman | 4cc086f | 2007-03-22 21:51:34 +1100 | [diff] [blame] | 1022 | EXPORT_SYMBOL(pci_disable_msix); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1023 | |
Matthew Wilcox | 309e57d | 2006-03-05 22:33:34 -0700 | [diff] [blame] | 1024 | void pci_no_msi(void) |
| 1025 | { |
| 1026 | pci_msi_enable = 0; |
| 1027 | } |
Michael Ellerman | c9953a7 | 2007-04-05 17:19:08 +1000 | [diff] [blame] | 1028 | |
Andrew Patterson | 07ae95f | 2008-11-10 15:31:05 -0700 | [diff] [blame] | 1029 | /** |
| 1030 | * pci_msi_enabled - is MSI enabled? |
| 1031 | * |
| 1032 | * Returns true if MSI has not been disabled by the command-line option |
| 1033 | * pci=nomsi. |
| 1034 | **/ |
| 1035 | int pci_msi_enabled(void) |
| 1036 | { |
| 1037 | return pci_msi_enable; |
| 1038 | } |
| 1039 | EXPORT_SYMBOL(pci_msi_enabled); |
| 1040 | |
Christoph Hellwig | 4ef3368 | 2016-07-12 18:20:18 +0900 | [diff] [blame] | 1041 | static int __pci_enable_msi_range(struct pci_dev *dev, int minvec, int maxvec, |
| 1042 | unsigned int flags) |
Alexander Gordeev | 302a252 | 2013-12-30 08:28:16 +0100 | [diff] [blame] | 1043 | { |
Alexander Gordeev | 034cd97 | 2014-04-14 15:28:35 +0200 | [diff] [blame] | 1044 | int nvec; |
Alexander Gordeev | 302a252 | 2013-12-30 08:28:16 +0100 | [diff] [blame] | 1045 | int rc; |
| 1046 | |
Alexander Gordeev | a06cd74 | 2014-09-23 12:45:58 -0600 | [diff] [blame] | 1047 | if (!pci_msi_supported(dev, minvec)) |
| 1048 | return -EINVAL; |
Alexander Gordeev | 034cd97 | 2014-04-14 15:28:35 +0200 | [diff] [blame] | 1049 | |
| 1050 | WARN_ON(!!dev->msi_enabled); |
| 1051 | |
| 1052 | /* Check whether driver already requested MSI-X irqs */ |
| 1053 | if (dev->msix_enabled) { |
| 1054 | dev_info(&dev->dev, |
| 1055 | "can't enable MSI (MSI-X already enabled)\n"); |
| 1056 | return -EINVAL; |
| 1057 | } |
| 1058 | |
Alexander Gordeev | 302a252 | 2013-12-30 08:28:16 +0100 | [diff] [blame] | 1059 | if (maxvec < minvec) |
| 1060 | return -ERANGE; |
| 1061 | |
Alexander Gordeev | 034cd97 | 2014-04-14 15:28:35 +0200 | [diff] [blame] | 1062 | nvec = pci_msi_vec_count(dev); |
| 1063 | if (nvec < 0) |
| 1064 | return nvec; |
Christoph Hellwig | 4ef3368 | 2016-07-12 18:20:18 +0900 | [diff] [blame] | 1065 | if (nvec < minvec) |
Alexander Gordeev | 034cd97 | 2014-04-14 15:28:35 +0200 | [diff] [blame] | 1066 | return -EINVAL; |
Christoph Hellwig | 4ef3368 | 2016-07-12 18:20:18 +0900 | [diff] [blame] | 1067 | |
| 1068 | if (nvec > maxvec) |
Alexander Gordeev | 034cd97 | 2014-04-14 15:28:35 +0200 | [diff] [blame] | 1069 | nvec = maxvec; |
| 1070 | |
Christoph Hellwig | 4ef3368 | 2016-07-12 18:20:18 +0900 | [diff] [blame] | 1071 | for (;;) { |
Christoph Hellwig | 4fe0d15 | 2016-08-11 07:11:04 -0700 | [diff] [blame] | 1072 | if (flags & PCI_IRQ_AFFINITY) { |
Christoph Hellwig | 4ef3368 | 2016-07-12 18:20:18 +0900 | [diff] [blame] | 1073 | dev->irq_affinity = irq_create_affinity_mask(&nvec); |
| 1074 | if (nvec < minvec) |
Alexander Gordeev | 302a252 | 2013-12-30 08:28:16 +0100 | [diff] [blame] | 1075 | return -ENOSPC; |
Alexander Gordeev | 302a252 | 2013-12-30 08:28:16 +0100 | [diff] [blame] | 1076 | } |
Alexander Gordeev | 302a252 | 2013-12-30 08:28:16 +0100 | [diff] [blame] | 1077 | |
Christoph Hellwig | 4ef3368 | 2016-07-12 18:20:18 +0900 | [diff] [blame] | 1078 | rc = msi_capability_init(dev, nvec); |
| 1079 | if (rc == 0) |
| 1080 | return nvec; |
| 1081 | |
| 1082 | kfree(dev->irq_affinity); |
| 1083 | dev->irq_affinity = NULL; |
| 1084 | |
| 1085 | if (rc < 0) |
| 1086 | return rc; |
| 1087 | if (rc < minvec) |
| 1088 | return -ENOSPC; |
| 1089 | |
| 1090 | nvec = rc; |
| 1091 | } |
| 1092 | } |
| 1093 | |
| 1094 | /** |
| 1095 | * pci_enable_msi_range - configure device's MSI capability structure |
| 1096 | * @dev: device to configure |
| 1097 | * @minvec: minimal number of interrupts to configure |
| 1098 | * @maxvec: maximum number of interrupts to configure |
| 1099 | * |
| 1100 | * This function tries to allocate a maximum possible number of interrupts in a |
| 1101 | * range between @minvec and @maxvec. It returns a negative errno if an error |
| 1102 | * occurs. If it succeeds, it returns the actual number of interrupts allocated |
| 1103 | * and updates the @dev's irq member to the lowest new interrupt number; |
| 1104 | * the other interrupt numbers allocated to this device are consecutive. |
| 1105 | **/ |
| 1106 | int pci_enable_msi_range(struct pci_dev *dev, int minvec, int maxvec) |
| 1107 | { |
Christoph Hellwig | 4fe0d15 | 2016-08-11 07:11:04 -0700 | [diff] [blame] | 1108 | return __pci_enable_msi_range(dev, minvec, maxvec, 0); |
Alexander Gordeev | 302a252 | 2013-12-30 08:28:16 +0100 | [diff] [blame] | 1109 | } |
| 1110 | EXPORT_SYMBOL(pci_enable_msi_range); |
| 1111 | |
Christoph Hellwig | 4ef3368 | 2016-07-12 18:20:18 +0900 | [diff] [blame] | 1112 | static int __pci_enable_msix_range(struct pci_dev *dev, |
| 1113 | struct msix_entry *entries, int minvec, int maxvec, |
| 1114 | unsigned int flags) |
| 1115 | { |
| 1116 | int nvec = maxvec; |
| 1117 | int rc; |
| 1118 | |
| 1119 | if (maxvec < minvec) |
| 1120 | return -ERANGE; |
| 1121 | |
| 1122 | for (;;) { |
Christoph Hellwig | 4fe0d15 | 2016-08-11 07:11:04 -0700 | [diff] [blame] | 1123 | if (flags & PCI_IRQ_AFFINITY) { |
Christoph Hellwig | 4ef3368 | 2016-07-12 18:20:18 +0900 | [diff] [blame] | 1124 | dev->irq_affinity = irq_create_affinity_mask(&nvec); |
| 1125 | if (nvec < minvec) |
| 1126 | return -ENOSPC; |
| 1127 | } |
| 1128 | |
| 1129 | rc = pci_enable_msix(dev, entries, nvec); |
| 1130 | if (rc == 0) |
| 1131 | return nvec; |
| 1132 | |
| 1133 | kfree(dev->irq_affinity); |
| 1134 | dev->irq_affinity = NULL; |
| 1135 | |
| 1136 | if (rc < 0) |
| 1137 | return rc; |
| 1138 | if (rc < minvec) |
| 1139 | return -ENOSPC; |
| 1140 | |
| 1141 | nvec = rc; |
| 1142 | } |
| 1143 | } |
| 1144 | |
Alexander Gordeev | 302a252 | 2013-12-30 08:28:16 +0100 | [diff] [blame] | 1145 | /** |
| 1146 | * pci_enable_msix_range - configure device's MSI-X capability structure |
| 1147 | * @dev: pointer to the pci_dev data structure of MSI-X device function |
| 1148 | * @entries: pointer to an array of MSI-X entries |
| 1149 | * @minvec: minimum number of MSI-X irqs requested |
| 1150 | * @maxvec: maximum number of MSI-X irqs requested |
| 1151 | * |
| 1152 | * Setup the MSI-X capability structure of device function with a maximum |
| 1153 | * possible number of interrupts in the range between @minvec and @maxvec |
| 1154 | * upon its software driver call to request for MSI-X mode enabled on its |
| 1155 | * hardware device function. It returns a negative errno if an error occurs. |
| 1156 | * If it succeeds, it returns the actual number of interrupts allocated and |
| 1157 | * indicates the successful configuration of MSI-X capability structure |
| 1158 | * with new allocated MSI-X interrupts. |
| 1159 | **/ |
| 1160 | int pci_enable_msix_range(struct pci_dev *dev, struct msix_entry *entries, |
Christoph Hellwig | 4ef3368 | 2016-07-12 18:20:18 +0900 | [diff] [blame] | 1161 | int minvec, int maxvec) |
Alexander Gordeev | 302a252 | 2013-12-30 08:28:16 +0100 | [diff] [blame] | 1162 | { |
Christoph Hellwig | 4fe0d15 | 2016-08-11 07:11:04 -0700 | [diff] [blame] | 1163 | return __pci_enable_msix_range(dev, entries, minvec, maxvec, 0); |
Alexander Gordeev | 302a252 | 2013-12-30 08:28:16 +0100 | [diff] [blame] | 1164 | } |
| 1165 | EXPORT_SYMBOL(pci_enable_msix_range); |
Jiang Liu | 3878eae | 2014-11-11 21:02:18 +0800 | [diff] [blame] | 1166 | |
Christoph Hellwig | aff1716 | 2016-07-12 18:20:17 +0900 | [diff] [blame] | 1167 | /** |
| 1168 | * pci_alloc_irq_vectors - allocate multiple IRQs for a device |
| 1169 | * @dev: PCI device to operate on |
| 1170 | * @min_vecs: minimum number of vectors required (must be >= 1) |
| 1171 | * @max_vecs: maximum (desired) number of vectors |
| 1172 | * @flags: flags or quirks for the allocation |
| 1173 | * |
| 1174 | * Allocate up to @max_vecs interrupt vectors for @dev, using MSI-X or MSI |
| 1175 | * vectors if available, and fall back to a single legacy vector |
| 1176 | * if neither is available. Return the number of vectors allocated, |
| 1177 | * (which might be smaller than @max_vecs) if successful, or a negative |
| 1178 | * error code on error. If less than @min_vecs interrupt vectors are |
| 1179 | * available for @dev the function will fail with -ENOSPC. |
| 1180 | * |
| 1181 | * To get the Linux IRQ number used for a vector that can be passed to |
| 1182 | * request_irq() use the pci_irq_vector() helper. |
| 1183 | */ |
| 1184 | int pci_alloc_irq_vectors(struct pci_dev *dev, unsigned int min_vecs, |
| 1185 | unsigned int max_vecs, unsigned int flags) |
| 1186 | { |
| 1187 | int vecs = -ENOSPC; |
| 1188 | |
Christoph Hellwig | 4fe0d15 | 2016-08-11 07:11:04 -0700 | [diff] [blame] | 1189 | if (flags & PCI_IRQ_MSIX) { |
Christoph Hellwig | 4ef3368 | 2016-07-12 18:20:18 +0900 | [diff] [blame] | 1190 | vecs = __pci_enable_msix_range(dev, NULL, min_vecs, max_vecs, |
| 1191 | flags); |
Christoph Hellwig | aff1716 | 2016-07-12 18:20:17 +0900 | [diff] [blame] | 1192 | if (vecs > 0) |
| 1193 | return vecs; |
| 1194 | } |
| 1195 | |
Christoph Hellwig | 4fe0d15 | 2016-08-11 07:11:04 -0700 | [diff] [blame] | 1196 | if (flags & PCI_IRQ_MSI) { |
Christoph Hellwig | 4ef3368 | 2016-07-12 18:20:18 +0900 | [diff] [blame] | 1197 | vecs = __pci_enable_msi_range(dev, min_vecs, max_vecs, flags); |
Christoph Hellwig | aff1716 | 2016-07-12 18:20:17 +0900 | [diff] [blame] | 1198 | if (vecs > 0) |
| 1199 | return vecs; |
| 1200 | } |
| 1201 | |
| 1202 | /* use legacy irq if allowed */ |
Christoph Hellwig | 5d0bdf2 | 2016-08-11 07:11:05 -0700 | [diff] [blame] | 1203 | if ((flags & PCI_IRQ_LEGACY) && min_vecs == 1) { |
| 1204 | pci_intx(dev, 1); |
Christoph Hellwig | aff1716 | 2016-07-12 18:20:17 +0900 | [diff] [blame] | 1205 | return 1; |
Christoph Hellwig | 5d0bdf2 | 2016-08-11 07:11:05 -0700 | [diff] [blame] | 1206 | } |
| 1207 | |
Christoph Hellwig | aff1716 | 2016-07-12 18:20:17 +0900 | [diff] [blame] | 1208 | return vecs; |
| 1209 | } |
| 1210 | EXPORT_SYMBOL(pci_alloc_irq_vectors); |
| 1211 | |
| 1212 | /** |
| 1213 | * pci_free_irq_vectors - free previously allocated IRQs for a device |
| 1214 | * @dev: PCI device to operate on |
| 1215 | * |
| 1216 | * Undoes the allocations and enabling in pci_alloc_irq_vectors(). |
| 1217 | */ |
| 1218 | void pci_free_irq_vectors(struct pci_dev *dev) |
| 1219 | { |
| 1220 | pci_disable_msix(dev); |
| 1221 | pci_disable_msi(dev); |
| 1222 | } |
| 1223 | EXPORT_SYMBOL(pci_free_irq_vectors); |
| 1224 | |
| 1225 | /** |
| 1226 | * pci_irq_vector - return Linux IRQ number of a device vector |
| 1227 | * @dev: PCI device to operate on |
| 1228 | * @nr: device-relative interrupt vector index (0-based). |
| 1229 | */ |
| 1230 | int pci_irq_vector(struct pci_dev *dev, unsigned int nr) |
| 1231 | { |
| 1232 | if (dev->msix_enabled) { |
| 1233 | struct msi_desc *entry; |
| 1234 | int i = 0; |
| 1235 | |
| 1236 | for_each_pci_msi_entry(entry, dev) { |
| 1237 | if (i == nr) |
| 1238 | return entry->irq; |
| 1239 | i++; |
| 1240 | } |
| 1241 | WARN_ON_ONCE(1); |
| 1242 | return -EINVAL; |
| 1243 | } |
| 1244 | |
| 1245 | if (dev->msi_enabled) { |
| 1246 | struct msi_desc *entry = first_pci_msi_entry(dev); |
| 1247 | |
| 1248 | if (WARN_ON_ONCE(nr >= entry->nvec_used)) |
| 1249 | return -EINVAL; |
| 1250 | } else { |
| 1251 | if (WARN_ON_ONCE(nr > 0)) |
| 1252 | return -EINVAL; |
| 1253 | } |
| 1254 | |
| 1255 | return dev->irq + nr; |
| 1256 | } |
| 1257 | EXPORT_SYMBOL(pci_irq_vector); |
| 1258 | |
Jiang Liu | 25a98bd | 2015-07-09 16:00:45 +0800 | [diff] [blame] | 1259 | struct pci_dev *msi_desc_to_pci_dev(struct msi_desc *desc) |
| 1260 | { |
| 1261 | return to_pci_dev(desc->dev); |
| 1262 | } |
Jake Oshins | a4289dc | 2015-12-10 17:52:59 +0000 | [diff] [blame] | 1263 | EXPORT_SYMBOL(msi_desc_to_pci_dev); |
Jiang Liu | 25a98bd | 2015-07-09 16:00:45 +0800 | [diff] [blame] | 1264 | |
Jiang Liu | c179c9b | 2015-07-09 16:00:36 +0800 | [diff] [blame] | 1265 | void *msi_desc_to_pci_sysdata(struct msi_desc *desc) |
| 1266 | { |
| 1267 | struct pci_dev *dev = msi_desc_to_pci_dev(desc); |
| 1268 | |
| 1269 | return dev->bus->sysdata; |
| 1270 | } |
| 1271 | EXPORT_SYMBOL_GPL(msi_desc_to_pci_sysdata); |
| 1272 | |
Jiang Liu | 3878eae | 2014-11-11 21:02:18 +0800 | [diff] [blame] | 1273 | #ifdef CONFIG_PCI_MSI_IRQ_DOMAIN |
| 1274 | /** |
| 1275 | * pci_msi_domain_write_msg - Helper to write MSI message to PCI config space |
| 1276 | * @irq_data: Pointer to interrupt data of the MSI interrupt |
| 1277 | * @msg: Pointer to the message |
| 1278 | */ |
| 1279 | void pci_msi_domain_write_msg(struct irq_data *irq_data, struct msi_msg *msg) |
| 1280 | { |
Jiang Liu | 507a883 | 2015-06-01 16:05:42 +0800 | [diff] [blame] | 1281 | struct msi_desc *desc = irq_data_get_msi_desc(irq_data); |
Jiang Liu | 3878eae | 2014-11-11 21:02:18 +0800 | [diff] [blame] | 1282 | |
| 1283 | /* |
| 1284 | * For MSI-X desc->irq is always equal to irq_data->irq. For |
| 1285 | * MSI only the first interrupt of MULTI MSI passes the test. |
| 1286 | */ |
| 1287 | if (desc->irq == irq_data->irq) |
| 1288 | __pci_write_msi_msg(desc, msg); |
| 1289 | } |
| 1290 | |
| 1291 | /** |
| 1292 | * pci_msi_domain_calc_hwirq - Generate a unique ID for an MSI source |
| 1293 | * @dev: Pointer to the PCI device |
| 1294 | * @desc: Pointer to the msi descriptor |
| 1295 | * |
| 1296 | * The ID number is only used within the irqdomain. |
| 1297 | */ |
| 1298 | irq_hw_number_t pci_msi_domain_calc_hwirq(struct pci_dev *dev, |
| 1299 | struct msi_desc *desc) |
| 1300 | { |
| 1301 | return (irq_hw_number_t)desc->msi_attrib.entry_nr | |
| 1302 | PCI_DEVID(dev->bus->number, dev->devfn) << 11 | |
| 1303 | (pci_domain_nr(dev->bus) & 0xFFFFFFFF) << 27; |
| 1304 | } |
| 1305 | |
| 1306 | static inline bool pci_msi_desc_is_multi_msi(struct msi_desc *desc) |
| 1307 | { |
| 1308 | return !desc->msi_attrib.is_msix && desc->nvec_used > 1; |
| 1309 | } |
| 1310 | |
| 1311 | /** |
| 1312 | * pci_msi_domain_check_cap - Verify that @domain supports the capabilities for @dev |
| 1313 | * @domain: The interrupt domain to check |
| 1314 | * @info: The domain info for verification |
| 1315 | * @dev: The device to check |
| 1316 | * |
| 1317 | * Returns: |
| 1318 | * 0 if the functionality is supported |
| 1319 | * 1 if Multi MSI is requested, but the domain does not support it |
| 1320 | * -ENOTSUPP otherwise |
| 1321 | */ |
| 1322 | int pci_msi_domain_check_cap(struct irq_domain *domain, |
| 1323 | struct msi_domain_info *info, struct device *dev) |
| 1324 | { |
| 1325 | struct msi_desc *desc = first_pci_msi_entry(to_pci_dev(dev)); |
| 1326 | |
| 1327 | /* Special handling to support pci_enable_msi_range() */ |
| 1328 | if (pci_msi_desc_is_multi_msi(desc) && |
| 1329 | !(info->flags & MSI_FLAG_MULTI_PCI_MSI)) |
| 1330 | return 1; |
| 1331 | else if (desc->msi_attrib.is_msix && !(info->flags & MSI_FLAG_PCI_MSIX)) |
| 1332 | return -ENOTSUPP; |
| 1333 | |
| 1334 | return 0; |
| 1335 | } |
| 1336 | |
| 1337 | static int pci_msi_domain_handle_error(struct irq_domain *domain, |
| 1338 | struct msi_desc *desc, int error) |
| 1339 | { |
| 1340 | /* Special handling to support pci_enable_msi_range() */ |
| 1341 | if (pci_msi_desc_is_multi_msi(desc) && error == -ENOSPC) |
| 1342 | return 1; |
| 1343 | |
| 1344 | return error; |
| 1345 | } |
| 1346 | |
| 1347 | #ifdef GENERIC_MSI_DOMAIN_OPS |
| 1348 | static void pci_msi_domain_set_desc(msi_alloc_info_t *arg, |
| 1349 | struct msi_desc *desc) |
| 1350 | { |
| 1351 | arg->desc = desc; |
| 1352 | arg->hwirq = pci_msi_domain_calc_hwirq(msi_desc_to_pci_dev(desc), |
| 1353 | desc); |
| 1354 | } |
| 1355 | #else |
| 1356 | #define pci_msi_domain_set_desc NULL |
| 1357 | #endif |
| 1358 | |
| 1359 | static struct msi_domain_ops pci_msi_domain_ops_default = { |
| 1360 | .set_desc = pci_msi_domain_set_desc, |
| 1361 | .msi_check = pci_msi_domain_check_cap, |
| 1362 | .handle_error = pci_msi_domain_handle_error, |
| 1363 | }; |
| 1364 | |
| 1365 | static void pci_msi_domain_update_dom_ops(struct msi_domain_info *info) |
| 1366 | { |
| 1367 | struct msi_domain_ops *ops = info->ops; |
| 1368 | |
| 1369 | if (ops == NULL) { |
| 1370 | info->ops = &pci_msi_domain_ops_default; |
| 1371 | } else { |
| 1372 | if (ops->set_desc == NULL) |
| 1373 | ops->set_desc = pci_msi_domain_set_desc; |
| 1374 | if (ops->msi_check == NULL) |
| 1375 | ops->msi_check = pci_msi_domain_check_cap; |
| 1376 | if (ops->handle_error == NULL) |
| 1377 | ops->handle_error = pci_msi_domain_handle_error; |
| 1378 | } |
| 1379 | } |
| 1380 | |
| 1381 | static void pci_msi_domain_update_chip_ops(struct msi_domain_info *info) |
| 1382 | { |
| 1383 | struct irq_chip *chip = info->chip; |
| 1384 | |
| 1385 | BUG_ON(!chip); |
| 1386 | if (!chip->irq_write_msi_msg) |
| 1387 | chip->irq_write_msi_msg = pci_msi_domain_write_msg; |
Marc Zyngier | 0701c53 | 2015-10-13 19:14:45 +0100 | [diff] [blame] | 1388 | if (!chip->irq_mask) |
| 1389 | chip->irq_mask = pci_msi_mask_irq; |
| 1390 | if (!chip->irq_unmask) |
| 1391 | chip->irq_unmask = pci_msi_unmask_irq; |
Jiang Liu | 3878eae | 2014-11-11 21:02:18 +0800 | [diff] [blame] | 1392 | } |
| 1393 | |
| 1394 | /** |
Marc Zyngier | be5436c | 2015-10-13 12:51:44 +0100 | [diff] [blame] | 1395 | * pci_msi_create_irq_domain - Create a MSI interrupt domain |
| 1396 | * @fwnode: Optional fwnode of the interrupt controller |
Jiang Liu | 3878eae | 2014-11-11 21:02:18 +0800 | [diff] [blame] | 1397 | * @info: MSI domain info |
| 1398 | * @parent: Parent irq domain |
| 1399 | * |
| 1400 | * Updates the domain and chip ops and creates a MSI interrupt domain. |
| 1401 | * |
| 1402 | * Returns: |
| 1403 | * A domain pointer or NULL in case of failure. |
| 1404 | */ |
Marc Zyngier | be5436c | 2015-10-13 12:51:44 +0100 | [diff] [blame] | 1405 | struct irq_domain *pci_msi_create_irq_domain(struct fwnode_handle *fwnode, |
Jiang Liu | 3878eae | 2014-11-11 21:02:18 +0800 | [diff] [blame] | 1406 | struct msi_domain_info *info, |
| 1407 | struct irq_domain *parent) |
| 1408 | { |
Marc Zyngier | 0380839 | 2015-07-28 14:46:09 +0100 | [diff] [blame] | 1409 | struct irq_domain *domain; |
| 1410 | |
Jiang Liu | 3878eae | 2014-11-11 21:02:18 +0800 | [diff] [blame] | 1411 | if (info->flags & MSI_FLAG_USE_DEF_DOM_OPS) |
| 1412 | pci_msi_domain_update_dom_ops(info); |
| 1413 | if (info->flags & MSI_FLAG_USE_DEF_CHIP_OPS) |
| 1414 | pci_msi_domain_update_chip_ops(info); |
| 1415 | |
Marc Zyngier | f3b0946 | 2016-07-13 17:18:33 +0100 | [diff] [blame] | 1416 | info->flags |= MSI_FLAG_ACTIVATE_EARLY; |
| 1417 | |
Marc Zyngier | be5436c | 2015-10-13 12:51:44 +0100 | [diff] [blame] | 1418 | domain = msi_create_irq_domain(fwnode, info, parent); |
Marc Zyngier | 0380839 | 2015-07-28 14:46:09 +0100 | [diff] [blame] | 1419 | if (!domain) |
| 1420 | return NULL; |
| 1421 | |
| 1422 | domain->bus_token = DOMAIN_BUS_PCI_MSI; |
| 1423 | return domain; |
Jiang Liu | 3878eae | 2014-11-11 21:02:18 +0800 | [diff] [blame] | 1424 | } |
Jake Oshins | a4289dc | 2015-12-10 17:52:59 +0000 | [diff] [blame] | 1425 | EXPORT_SYMBOL_GPL(pci_msi_create_irq_domain); |
Jiang Liu | 3878eae | 2014-11-11 21:02:18 +0800 | [diff] [blame] | 1426 | |
| 1427 | /** |
| 1428 | * pci_msi_domain_alloc_irqs - Allocate interrupts for @dev in @domain |
| 1429 | * @domain: The interrupt domain to allocate from |
| 1430 | * @dev: The device for which to allocate |
| 1431 | * @nvec: The number of interrupts to allocate |
| 1432 | * @type: Unused to allow simpler migration from the arch_XXX interfaces |
| 1433 | * |
| 1434 | * Returns: |
| 1435 | * A virtual interrupt number or an error code in case of failure |
| 1436 | */ |
| 1437 | int pci_msi_domain_alloc_irqs(struct irq_domain *domain, struct pci_dev *dev, |
| 1438 | int nvec, int type) |
| 1439 | { |
| 1440 | return msi_domain_alloc_irqs(domain, &dev->dev, nvec); |
| 1441 | } |
| 1442 | |
| 1443 | /** |
| 1444 | * pci_msi_domain_free_irqs - Free interrupts for @dev in @domain |
| 1445 | * @domain: The interrupt domain |
| 1446 | * @dev: The device for which to free interrupts |
| 1447 | */ |
| 1448 | void pci_msi_domain_free_irqs(struct irq_domain *domain, struct pci_dev *dev) |
| 1449 | { |
| 1450 | msi_domain_free_irqs(domain, &dev->dev); |
| 1451 | } |
Jiang Liu | 8e047ad | 2014-11-15 22:24:07 +0800 | [diff] [blame] | 1452 | |
| 1453 | /** |
| 1454 | * pci_msi_create_default_irq_domain - Create a default MSI interrupt domain |
Marc Zyngier | be5436c | 2015-10-13 12:51:44 +0100 | [diff] [blame] | 1455 | * @fwnode: Optional fwnode of the interrupt controller |
Jiang Liu | 8e047ad | 2014-11-15 22:24:07 +0800 | [diff] [blame] | 1456 | * @info: MSI domain info |
| 1457 | * @parent: Parent irq domain |
| 1458 | * |
| 1459 | * Returns: A domain pointer or NULL in case of failure. If successful |
| 1460 | * the default PCI/MSI irqdomain pointer is updated. |
| 1461 | */ |
Marc Zyngier | be5436c | 2015-10-13 12:51:44 +0100 | [diff] [blame] | 1462 | struct irq_domain *pci_msi_create_default_irq_domain(struct fwnode_handle *fwnode, |
Jiang Liu | 8e047ad | 2014-11-15 22:24:07 +0800 | [diff] [blame] | 1463 | struct msi_domain_info *info, struct irq_domain *parent) |
| 1464 | { |
| 1465 | struct irq_domain *domain; |
| 1466 | |
| 1467 | mutex_lock(&pci_msi_domain_lock); |
| 1468 | if (pci_msi_default_domain) { |
| 1469 | pr_err("PCI: default irq domain for PCI MSI has already been created.\n"); |
| 1470 | domain = NULL; |
| 1471 | } else { |
Marc Zyngier | be5436c | 2015-10-13 12:51:44 +0100 | [diff] [blame] | 1472 | domain = pci_msi_create_irq_domain(fwnode, info, parent); |
Jiang Liu | 8e047ad | 2014-11-15 22:24:07 +0800 | [diff] [blame] | 1473 | pci_msi_default_domain = domain; |
| 1474 | } |
| 1475 | mutex_unlock(&pci_msi_domain_lock); |
| 1476 | |
| 1477 | return domain; |
| 1478 | } |
David Daney | b6eec9b | 2015-10-08 15:10:49 -0700 | [diff] [blame] | 1479 | |
| 1480 | static int get_msi_id_cb(struct pci_dev *pdev, u16 alias, void *data) |
| 1481 | { |
| 1482 | u32 *pa = data; |
| 1483 | |
| 1484 | *pa = alias; |
| 1485 | return 0; |
| 1486 | } |
| 1487 | /** |
| 1488 | * pci_msi_domain_get_msi_rid - Get the MSI requester id (RID) |
| 1489 | * @domain: The interrupt domain |
| 1490 | * @pdev: The PCI device. |
| 1491 | * |
| 1492 | * The RID for a device is formed from the alias, with a firmware |
| 1493 | * supplied mapping applied |
| 1494 | * |
| 1495 | * Returns: The RID. |
| 1496 | */ |
| 1497 | u32 pci_msi_domain_get_msi_rid(struct irq_domain *domain, struct pci_dev *pdev) |
| 1498 | { |
| 1499 | struct device_node *of_node; |
| 1500 | u32 rid = 0; |
| 1501 | |
| 1502 | pci_for_each_dma_alias(pdev, get_msi_id_cb, &rid); |
| 1503 | |
| 1504 | of_node = irq_domain_get_of_node(domain); |
| 1505 | if (of_node) |
| 1506 | rid = of_msi_map_rid(&pdev->dev, of_node, rid); |
| 1507 | |
| 1508 | return rid; |
| 1509 | } |
Marc Zyngier | 54fa97e | 2015-10-02 14:43:06 +0100 | [diff] [blame] | 1510 | |
| 1511 | /** |
| 1512 | * pci_msi_get_device_domain - Get the MSI domain for a given PCI device |
| 1513 | * @pdev: The PCI device |
| 1514 | * |
| 1515 | * Use the firmware data to find a device-specific MSI domain |
| 1516 | * (i.e. not one that is ste as a default). |
| 1517 | * |
| 1518 | * Returns: The coresponding MSI domain or NULL if none has been found. |
| 1519 | */ |
| 1520 | struct irq_domain *pci_msi_get_device_domain(struct pci_dev *pdev) |
| 1521 | { |
| 1522 | u32 rid = 0; |
| 1523 | |
| 1524 | pci_for_each_dma_alias(pdev, get_msi_id_cb, &rid); |
| 1525 | return of_msi_map_get_device_domain(&pdev->dev, rid); |
| 1526 | } |
Jiang Liu | 3878eae | 2014-11-11 21:02:18 +0800 | [diff] [blame] | 1527 | #endif /* CONFIG_PCI_MSI_IRQ_DOMAIN */ |