blob: d9cbd586ae4bc282815a6b7aad16ddb3ecb523df [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * File: msi.c
3 * Purpose: PCI Message Signaled Interrupt (MSI)
4 *
5 * Copyright (C) 2003-2004 Intel
6 * Copyright (C) Tom Long Nguyen (tom.l.nguyen@intel.com)
7 */
8
Eric W. Biederman1ce03372006-10-04 02:16:41 -07009#include <linux/err.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070010#include <linux/mm.h>
11#include <linux/irq.h>
12#include <linux/interrupt.h>
13#include <linux/init.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070014#include <linux/ioport.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070015#include <linux/pci.h>
16#include <linux/proc_fs.h>
Eric W. Biederman3b7d1922006-10-04 02:16:59 -070017#include <linux/msi.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070018
19#include <asm/errno.h>
20#include <asm/io.h>
21#include <asm/smp.h>
22
23#include "pci.h"
24#include "msi.h"
25
Linus Torvalds1da177e2005-04-16 15:20:36 -070026static int pci_msi_enable = 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -070027
Eric W. Biedermanb1cbf4e2007-03-05 00:30:10 -080028static void msi_set_enable(struct pci_dev *dev, int enable)
29{
30 int pos;
31 u16 control;
32
33 pos = pci_find_capability(dev, PCI_CAP_ID_MSI);
34 if (pos) {
35 pci_read_config_word(dev, pos + PCI_MSI_FLAGS, &control);
36 control &= ~PCI_MSI_FLAGS_ENABLE;
37 if (enable)
38 control |= PCI_MSI_FLAGS_ENABLE;
39 pci_write_config_word(dev, pos + PCI_MSI_FLAGS, control);
40 }
41}
42
43static void msix_set_enable(struct pci_dev *dev, int enable)
44{
45 int pos;
46 u16 control;
47
48 pos = pci_find_capability(dev, PCI_CAP_ID_MSIX);
49 if (pos) {
50 pci_read_config_word(dev, pos + PCI_MSIX_FLAGS, &control);
51 control &= ~PCI_MSIX_FLAGS_ENABLE;
52 if (enable)
53 control |= PCI_MSIX_FLAGS_ENABLE;
54 pci_write_config_word(dev, pos + PCI_MSIX_FLAGS, control);
55 }
56}
57
Mitch Williams988cbb12007-03-30 11:54:08 -070058static void msix_flush_writes(unsigned int irq)
59{
60 struct msi_desc *entry;
61
62 entry = get_irq_msi(irq);
63 BUG_ON(!entry || !entry->dev);
64 switch (entry->msi_attrib.type) {
65 case PCI_CAP_ID_MSI:
66 /* nothing to do */
67 break;
68 case PCI_CAP_ID_MSIX:
69 {
70 int offset = entry->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE +
71 PCI_MSIX_ENTRY_VECTOR_CTRL_OFFSET;
72 readl(entry->mask_base + offset);
73 break;
74 }
75 default:
76 BUG();
77 break;
78 }
79}
80
Eric W. Biederman1ce03372006-10-04 02:16:41 -070081static void msi_set_mask_bit(unsigned int irq, int flag)
Linus Torvalds1da177e2005-04-16 15:20:36 -070082{
83 struct msi_desc *entry;
84
Eric W. Biederman5b912c12007-01-28 12:52:03 -070085 entry = get_irq_msi(irq);
Eric W. Biederman277bc332006-10-04 02:16:57 -070086 BUG_ON(!entry || !entry->dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -070087 switch (entry->msi_attrib.type) {
88 case PCI_CAP_ID_MSI:
Eric W. Biederman277bc332006-10-04 02:16:57 -070089 if (entry->msi_attrib.maskbit) {
Satoru Takeuchic54c1872007-01-18 13:50:05 +090090 int pos;
91 u32 mask_bits;
Linus Torvalds1da177e2005-04-16 15:20:36 -070092
Eric W. Biederman277bc332006-10-04 02:16:57 -070093 pos = (long)entry->mask_base;
94 pci_read_config_dword(entry->dev, pos, &mask_bits);
95 mask_bits &= ~(1);
96 mask_bits |= flag;
97 pci_write_config_dword(entry->dev, pos, mask_bits);
Eric W. Biederman58e05432007-03-05 00:30:11 -080098 } else {
99 msi_set_enable(entry->dev, !flag);
Eric W. Biederman277bc332006-10-04 02:16:57 -0700100 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700101 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700102 case PCI_CAP_ID_MSIX:
103 {
104 int offset = entry->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE +
105 PCI_MSIX_ENTRY_VECTOR_CTRL_OFFSET;
106 writel(flag, entry->mask_base + offset);
Eric W. Biederman348e3fd2007-04-03 01:41:49 -0600107 readl(entry->mask_base + offset);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700108 break;
109 }
110 default:
Eric W. Biederman277bc332006-10-04 02:16:57 -0700111 BUG();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700112 break;
113 }
Eric W. Biederman392ee1e2007-03-08 13:04:57 -0700114 entry->msi_attrib.masked = !!flag;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700115}
116
Eric W. Biederman3b7d1922006-10-04 02:16:59 -0700117void read_msi_msg(unsigned int irq, struct msi_msg *msg)
Eric W. Biederman0366f8f2006-10-04 02:16:33 -0700118{
Eric W. Biederman5b912c12007-01-28 12:52:03 -0700119 struct msi_desc *entry = get_irq_msi(irq);
Eric W. Biederman0366f8f2006-10-04 02:16:33 -0700120 switch(entry->msi_attrib.type) {
121 case PCI_CAP_ID_MSI:
122 {
123 struct pci_dev *dev = entry->dev;
124 int pos = entry->msi_attrib.pos;
125 u16 data;
126
127 pci_read_config_dword(dev, msi_lower_address_reg(pos),
128 &msg->address_lo);
129 if (entry->msi_attrib.is_64) {
130 pci_read_config_dword(dev, msi_upper_address_reg(pos),
131 &msg->address_hi);
132 pci_read_config_word(dev, msi_data_reg(pos, 1), &data);
133 } else {
134 msg->address_hi = 0;
135 pci_read_config_word(dev, msi_data_reg(pos, 1), &data);
136 }
137 msg->data = data;
138 break;
139 }
140 case PCI_CAP_ID_MSIX:
141 {
142 void __iomem *base;
143 base = entry->mask_base +
144 entry->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE;
145
146 msg->address_lo = readl(base + PCI_MSIX_ENTRY_LOWER_ADDR_OFFSET);
147 msg->address_hi = readl(base + PCI_MSIX_ENTRY_UPPER_ADDR_OFFSET);
148 msg->data = readl(base + PCI_MSIX_ENTRY_DATA_OFFSET);
149 break;
150 }
151 default:
152 BUG();
153 }
154}
155
Eric W. Biederman3b7d1922006-10-04 02:16:59 -0700156void write_msi_msg(unsigned int irq, struct msi_msg *msg)
Eric W. Biederman0366f8f2006-10-04 02:16:33 -0700157{
Eric W. Biederman5b912c12007-01-28 12:52:03 -0700158 struct msi_desc *entry = get_irq_msi(irq);
Eric W. Biederman0366f8f2006-10-04 02:16:33 -0700159 switch (entry->msi_attrib.type) {
160 case PCI_CAP_ID_MSI:
161 {
162 struct pci_dev *dev = entry->dev;
163 int pos = entry->msi_attrib.pos;
164
165 pci_write_config_dword(dev, msi_lower_address_reg(pos),
166 msg->address_lo);
167 if (entry->msi_attrib.is_64) {
168 pci_write_config_dword(dev, msi_upper_address_reg(pos),
169 msg->address_hi);
170 pci_write_config_word(dev, msi_data_reg(pos, 1),
171 msg->data);
172 } else {
173 pci_write_config_word(dev, msi_data_reg(pos, 0),
174 msg->data);
175 }
176 break;
177 }
178 case PCI_CAP_ID_MSIX:
179 {
180 void __iomem *base;
181 base = entry->mask_base +
182 entry->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE;
183
184 writel(msg->address_lo,
185 base + PCI_MSIX_ENTRY_LOWER_ADDR_OFFSET);
186 writel(msg->address_hi,
187 base + PCI_MSIX_ENTRY_UPPER_ADDR_OFFSET);
188 writel(msg->data, base + PCI_MSIX_ENTRY_DATA_OFFSET);
189 break;
190 }
191 default:
192 BUG();
193 }
Eric W. Biederman392ee1e2007-03-08 13:04:57 -0700194 entry->msg = *msg;
Eric W. Biederman0366f8f2006-10-04 02:16:33 -0700195}
196
Eric W. Biederman3b7d1922006-10-04 02:16:59 -0700197void mask_msi_irq(unsigned int irq)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700198{
Eric W. Biederman1ce03372006-10-04 02:16:41 -0700199 msi_set_mask_bit(irq, 1);
Mitch Williams988cbb12007-03-30 11:54:08 -0700200 msix_flush_writes(irq);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700201}
202
Eric W. Biederman3b7d1922006-10-04 02:16:59 -0700203void unmask_msi_irq(unsigned int irq)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700204{
Eric W. Biederman1ce03372006-10-04 02:16:41 -0700205 msi_set_mask_bit(irq, 0);
Mitch Williams988cbb12007-03-30 11:54:08 -0700206 msix_flush_writes(irq);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700207}
208
Michael Ellerman032de8e2007-04-18 19:39:22 +1000209static int msi_free_irqs(struct pci_dev* dev);
Satoru Takeuchic54c1872007-01-18 13:50:05 +0900210
Linus Torvalds1da177e2005-04-16 15:20:36 -0700211
Linus Torvalds1da177e2005-04-16 15:20:36 -0700212static struct msi_desc* alloc_msi_entry(void)
213{
214 struct msi_desc *entry;
215
Michael Ellerman3e916c02007-03-22 21:51:36 +1100216 entry = kzalloc(sizeof(struct msi_desc), GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700217 if (!entry)
218 return NULL;
219
Michael Ellerman4aa9bc92007-04-05 17:19:10 +1000220 INIT_LIST_HEAD(&entry->list);
221 entry->irq = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700222 entry->dev = NULL;
223
224 return entry;
225}
226
Shaohua Li41017f02006-02-08 17:11:38 +0800227#ifdef CONFIG_PM
Michael Ellerman8fed4b62007-01-25 19:34:08 +1100228static void __pci_restore_msi_state(struct pci_dev *dev)
Shaohua Li41017f02006-02-08 17:11:38 +0800229{
Eric W. Biederman392ee1e2007-03-08 13:04:57 -0700230 int pos;
Shaohua Li41017f02006-02-08 17:11:38 +0800231 u16 control;
Eric W. Biederman392ee1e2007-03-08 13:04:57 -0700232 struct msi_desc *entry;
Shaohua Li41017f02006-02-08 17:11:38 +0800233
Eric W. Biedermanb1cbf4e2007-03-05 00:30:10 -0800234 if (!dev->msi_enabled)
235 return;
236
Eric W. Biederman392ee1e2007-03-08 13:04:57 -0700237 entry = get_irq_msi(dev->irq);
238 pos = entry->msi_attrib.pos;
Shaohua Li41017f02006-02-08 17:11:38 +0800239
Eric W. Biedermanb1cbf4e2007-03-05 00:30:10 -0800240 pci_intx(dev, 0); /* disable intx */
Eric W. Biedermanb1cbf4e2007-03-05 00:30:10 -0800241 msi_set_enable(dev, 0);
Eric W. Biederman392ee1e2007-03-08 13:04:57 -0700242 write_msi_msg(dev->irq, &entry->msg);
243 if (entry->msi_attrib.maskbit)
244 msi_set_mask_bit(dev->irq, entry->msi_attrib.masked);
245
246 pci_read_config_word(dev, pos + PCI_MSI_FLAGS, &control);
247 control &= ~(PCI_MSI_FLAGS_QSIZE | PCI_MSI_FLAGS_ENABLE);
248 if (entry->msi_attrib.maskbit || !entry->msi_attrib.masked)
249 control |= PCI_MSI_FLAGS_ENABLE;
Shaohua Li41017f02006-02-08 17:11:38 +0800250 pci_write_config_word(dev, pos + PCI_MSI_FLAGS, control);
Michael Ellerman8fed4b62007-01-25 19:34:08 +1100251}
252
253static void __pci_restore_msix_state(struct pci_dev *dev)
Shaohua Li41017f02006-02-08 17:11:38 +0800254{
Shaohua Li41017f02006-02-08 17:11:38 +0800255 int pos;
Shaohua Li41017f02006-02-08 17:11:38 +0800256 struct msi_desc *entry;
Eric W. Biederman392ee1e2007-03-08 13:04:57 -0700257 u16 control;
Shaohua Li41017f02006-02-08 17:11:38 +0800258
Eric W. Biedermanded86d82007-01-28 12:42:52 -0700259 if (!dev->msix_enabled)
260 return;
261
Shaohua Li41017f02006-02-08 17:11:38 +0800262 /* route the table */
Eric W. Biedermanb1cbf4e2007-03-05 00:30:10 -0800263 pci_intx(dev, 0); /* disable intx */
264 msix_set_enable(dev, 0);
Shaohua Li41017f02006-02-08 17:11:38 +0800265
Michael Ellerman4aa9bc92007-04-05 17:19:10 +1000266 list_for_each_entry(entry, &dev->msi_list, list) {
267 write_msi_msg(entry->irq, &entry->msg);
268 msi_set_mask_bit(entry->irq, entry->msi_attrib.masked);
Shaohua Li41017f02006-02-08 17:11:38 +0800269 }
Shaohua Li41017f02006-02-08 17:11:38 +0800270
Michael Ellerman314e77b2007-04-05 17:19:12 +1000271 BUG_ON(list_empty(&dev->msi_list));
272 entry = list_entry(dev->msi_list.next, struct msi_desc, list);
Michael Ellerman4aa9bc92007-04-05 17:19:10 +1000273 pos = entry->msi_attrib.pos;
Eric W. Biederman392ee1e2007-03-08 13:04:57 -0700274 pci_read_config_word(dev, pos + PCI_MSIX_FLAGS, &control);
275 control &= ~PCI_MSIX_FLAGS_MASKALL;
276 control |= PCI_MSIX_FLAGS_ENABLE;
277 pci_write_config_word(dev, pos + PCI_MSIX_FLAGS, control);
Shaohua Li41017f02006-02-08 17:11:38 +0800278}
Michael Ellerman8fed4b62007-01-25 19:34:08 +1100279
280void pci_restore_msi_state(struct pci_dev *dev)
281{
282 __pci_restore_msi_state(dev);
283 __pci_restore_msix_state(dev);
284}
Satoru Takeuchic54c1872007-01-18 13:50:05 +0900285#endif /* CONFIG_PM */
Shaohua Li41017f02006-02-08 17:11:38 +0800286
Linus Torvalds1da177e2005-04-16 15:20:36 -0700287/**
288 * msi_capability_init - configure device's MSI capability structure
289 * @dev: pointer to the pci_dev data structure of MSI device function
290 *
Steven Coleeaae4b32005-05-03 18:38:30 -0600291 * Setup the MSI capability structure of device function with a single
Eric W. Biederman1ce03372006-10-04 02:16:41 -0700292 * MSI irq, regardless of device function is capable of handling
Linus Torvalds1da177e2005-04-16 15:20:36 -0700293 * multiple messages. A return of zero indicates the successful setup
Eric W. Biederman1ce03372006-10-04 02:16:41 -0700294 * of an entry zero with the new MSI irq or non-zero for otherwise.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700295 **/
296static int msi_capability_init(struct pci_dev *dev)
297{
298 struct msi_desc *entry;
Michael Ellerman7fe37302007-04-18 19:39:21 +1000299 int pos, ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700300 u16 control;
301
Eric W. Biedermanb1cbf4e2007-03-05 00:30:10 -0800302 msi_set_enable(dev, 0); /* Ensure msi is disabled as I set it up */
303
Linus Torvalds1da177e2005-04-16 15:20:36 -0700304 pos = pci_find_capability(dev, PCI_CAP_ID_MSI);
305 pci_read_config_word(dev, msi_control_reg(pos), &control);
306 /* MSI Entry Initialization */
Eric W. Biedermanf7feaca2007-01-28 12:56:37 -0700307 entry = alloc_msi_entry();
308 if (!entry)
309 return -ENOMEM;
Eric W. Biederman1ce03372006-10-04 02:16:41 -0700310
Linus Torvalds1da177e2005-04-16 15:20:36 -0700311 entry->msi_attrib.type = PCI_CAP_ID_MSI;
Eric W. Biederman0366f8f2006-10-04 02:16:33 -0700312 entry->msi_attrib.is_64 = is_64bit_address(control);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700313 entry->msi_attrib.entry_nr = 0;
314 entry->msi_attrib.maskbit = is_mask_bit_support(control);
Eric W. Biederman392ee1e2007-03-08 13:04:57 -0700315 entry->msi_attrib.masked = 1;
Eric W. Biederman1ce03372006-10-04 02:16:41 -0700316 entry->msi_attrib.default_irq = dev->irq; /* Save IOAPIC IRQ */
Eric W. Biederman0366f8f2006-10-04 02:16:33 -0700317 entry->msi_attrib.pos = pos;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700318 if (is_mask_bit_support(control)) {
319 entry->mask_base = (void __iomem *)(long)msi_mask_bits_reg(pos,
320 is_64bit_address(control));
321 }
Eric W. Biederman3b7d1922006-10-04 02:16:59 -0700322 entry->dev = dev;
323 if (entry->msi_attrib.maskbit) {
324 unsigned int maskbits, temp;
325 /* All MSIs are unmasked by default, Mask them all */
326 pci_read_config_dword(dev,
327 msi_mask_bits_reg(pos, is_64bit_address(control)),
328 &maskbits);
329 temp = (1 << multi_msi_capable(control));
330 temp = ((temp - 1) & ~temp);
331 maskbits |= temp;
332 pci_write_config_dword(dev,
333 msi_mask_bits_reg(pos, is_64bit_address(control)),
334 maskbits);
335 }
Michael Ellerman9c831332007-04-18 19:39:21 +1000336 list_add(&entry->list, &dev->msi_list);
337
Linus Torvalds1da177e2005-04-16 15:20:36 -0700338 /* Configure MSI capability structure */
Michael Ellerman9c831332007-04-18 19:39:21 +1000339 ret = arch_setup_msi_irqs(dev, 1, PCI_CAP_ID_MSI);
Michael Ellerman7fe37302007-04-18 19:39:21 +1000340 if (ret) {
Michael Ellerman032de8e2007-04-18 19:39:22 +1000341 msi_free_irqs(dev);
Michael Ellerman7fe37302007-04-18 19:39:21 +1000342 return ret;
Mark Maulefd58e552006-04-10 21:17:48 -0500343 }
Eric W. Biedermanf7feaca2007-01-28 12:56:37 -0700344
Linus Torvalds1da177e2005-04-16 15:20:36 -0700345 /* Set MSI enabled bits */
Eric W. Biedermanb1cbf4e2007-03-05 00:30:10 -0800346 pci_intx(dev, 0); /* disable intx */
347 msi_set_enable(dev, 1);
348 dev->msi_enabled = 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700349
Michael Ellerman7fe37302007-04-18 19:39:21 +1000350 dev->irq = entry->irq;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700351 return 0;
352}
353
354/**
355 * msix_capability_init - configure device's MSI-X capability
356 * @dev: pointer to the pci_dev data structure of MSI-X device function
Randy Dunlap8f7020d2005-10-23 11:57:38 -0700357 * @entries: pointer to an array of struct msix_entry entries
358 * @nvec: number of @entries
Linus Torvalds1da177e2005-04-16 15:20:36 -0700359 *
Steven Coleeaae4b32005-05-03 18:38:30 -0600360 * Setup the MSI-X capability structure of device function with a
Eric W. Biederman1ce03372006-10-04 02:16:41 -0700361 * single MSI-X irq. A return of zero indicates the successful setup of
362 * requested MSI-X entries with allocated irqs or non-zero for otherwise.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700363 **/
364static int msix_capability_init(struct pci_dev *dev,
365 struct msix_entry *entries, int nvec)
366{
Michael Ellerman4aa9bc92007-04-05 17:19:10 +1000367 struct msi_desc *entry;
Michael Ellerman9c831332007-04-18 19:39:21 +1000368 int pos, i, j, nr_entries, ret;
Grant Grundlera0454b42006-02-16 23:58:29 -0800369 unsigned long phys_addr;
370 u32 table_offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700371 u16 control;
372 u8 bir;
373 void __iomem *base;
374
Eric W. Biedermanb1cbf4e2007-03-05 00:30:10 -0800375 msix_set_enable(dev, 0);/* Ensure msix is disabled as I set it up */
376
Linus Torvalds1da177e2005-04-16 15:20:36 -0700377 pos = pci_find_capability(dev, PCI_CAP_ID_MSIX);
378 /* Request & Map MSI-X table region */
379 pci_read_config_word(dev, msi_control_reg(pos), &control);
380 nr_entries = multi_msix_capable(control);
Grant Grundlera0454b42006-02-16 23:58:29 -0800381
382 pci_read_config_dword(dev, msix_table_offset_reg(pos), &table_offset);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700383 bir = (u8)(table_offset & PCI_MSIX_FLAGS_BIRMASK);
Grant Grundlera0454b42006-02-16 23:58:29 -0800384 table_offset &= ~PCI_MSIX_FLAGS_BIRMASK;
385 phys_addr = pci_resource_start (dev, bir) + table_offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700386 base = ioremap_nocache(phys_addr, nr_entries * PCI_MSIX_ENTRY_SIZE);
387 if (base == NULL)
388 return -ENOMEM;
389
390 /* MSI-X Table Initialization */
391 for (i = 0; i < nvec; i++) {
Eric W. Biedermanf7feaca2007-01-28 12:56:37 -0700392 entry = alloc_msi_entry();
393 if (!entry)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700394 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700395
396 j = entries[i].entry;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700397 entry->msi_attrib.type = PCI_CAP_ID_MSIX;
Eric W. Biederman0366f8f2006-10-04 02:16:33 -0700398 entry->msi_attrib.is_64 = 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700399 entry->msi_attrib.entry_nr = j;
400 entry->msi_attrib.maskbit = 1;
Eric W. Biederman392ee1e2007-03-08 13:04:57 -0700401 entry->msi_attrib.masked = 1;
Eric W. Biederman1ce03372006-10-04 02:16:41 -0700402 entry->msi_attrib.default_irq = dev->irq;
Eric W. Biederman0366f8f2006-10-04 02:16:33 -0700403 entry->msi_attrib.pos = pos;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700404 entry->dev = dev;
405 entry->mask_base = base;
Eric W. Biedermanf7feaca2007-01-28 12:56:37 -0700406
Michael Ellerman4aa9bc92007-04-05 17:19:10 +1000407 list_add(&entry->list, &dev->msi_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700408 }
Michael Ellerman9c831332007-04-18 19:39:21 +1000409
410 ret = arch_setup_msi_irqs(dev, nvec, PCI_CAP_ID_MSIX);
411 if (ret) {
412 int avail = 0;
413 list_for_each_entry(entry, &dev->msi_list, list) {
414 if (entry->irq != 0) {
415 avail++;
Michael Ellerman9c831332007-04-18 19:39:21 +1000416 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700417 }
Michael Ellerman9c831332007-04-18 19:39:21 +1000418
Michael Ellerman032de8e2007-04-18 19:39:22 +1000419 msi_free_irqs(dev);
420
Eric W. Biederman92db6d12006-10-04 02:16:35 -0700421 /* If we had some success report the number of irqs
422 * we succeeded in setting up.
423 */
Michael Ellerman9c831332007-04-18 19:39:21 +1000424 if (avail == 0)
425 avail = ret;
Eric W. Biederman92db6d12006-10-04 02:16:35 -0700426 return avail;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700427 }
Michael Ellerman9c831332007-04-18 19:39:21 +1000428
429 i = 0;
430 list_for_each_entry(entry, &dev->msi_list, list) {
431 entries[i].vector = entry->irq;
432 set_irq_msi(entry->irq, entry);
433 i++;
434 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700435 /* Set MSI-X enabled bits */
Eric W. Biedermanb1cbf4e2007-03-05 00:30:10 -0800436 pci_intx(dev, 0); /* disable intx */
437 msix_set_enable(dev, 1);
438 dev->msix_enabled = 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700439
440 return 0;
441}
442
443/**
Michael Ellerman17bbc122007-04-05 17:19:07 +1000444 * pci_msi_check_device - check whether MSI may be enabled on a device
Brice Goglin24334a12006-08-31 01:55:07 -0400445 * @dev: pointer to the pci_dev data structure of MSI device function
Michael Ellermanc9953a72007-04-05 17:19:08 +1000446 * @nvec: how many MSIs have been requested ?
Michael Ellermanb1e23032007-03-22 21:51:39 +1100447 * @type: are we checking for MSI or MSI-X ?
Brice Goglin24334a12006-08-31 01:55:07 -0400448 *
Brice Goglin0306ebf2006-10-05 10:24:31 +0200449 * Look at global flags, the device itself, and its parent busses
Michael Ellerman17bbc122007-04-05 17:19:07 +1000450 * to determine if MSI/-X are supported for the device. If MSI/-X is
451 * supported return 0, else return an error code.
Brice Goglin24334a12006-08-31 01:55:07 -0400452 **/
Michael Ellermanc9953a72007-04-05 17:19:08 +1000453static int pci_msi_check_device(struct pci_dev* dev, int nvec, int type)
Brice Goglin24334a12006-08-31 01:55:07 -0400454{
455 struct pci_bus *bus;
Michael Ellermanc9953a72007-04-05 17:19:08 +1000456 int ret;
Brice Goglin24334a12006-08-31 01:55:07 -0400457
Brice Goglin0306ebf2006-10-05 10:24:31 +0200458 /* MSI must be globally enabled and supported by the device */
Brice Goglin24334a12006-08-31 01:55:07 -0400459 if (!pci_msi_enable || !dev || dev->no_msi)
460 return -EINVAL;
461
Michael Ellerman314e77b2007-04-05 17:19:12 +1000462 /*
463 * You can't ask to have 0 or less MSIs configured.
464 * a) it's stupid ..
465 * b) the list manipulation code assumes nvec >= 1.
466 */
467 if (nvec < 1)
468 return -ERANGE;
469
Brice Goglin0306ebf2006-10-05 10:24:31 +0200470 /* Any bridge which does NOT route MSI transactions from it's
471 * secondary bus to it's primary bus must set NO_MSI flag on
472 * the secondary pci_bus.
473 * We expect only arch-specific PCI host bus controller driver
474 * or quirks for specific PCI bridges to be setting NO_MSI.
475 */
Brice Goglin24334a12006-08-31 01:55:07 -0400476 for (bus = dev->bus; bus; bus = bus->parent)
477 if (bus->bus_flags & PCI_BUS_FLAGS_NO_MSI)
478 return -EINVAL;
479
Michael Ellermanc9953a72007-04-05 17:19:08 +1000480 ret = arch_msi_check_device(dev, nvec, type);
481 if (ret)
482 return ret;
483
Michael Ellermanb1e23032007-03-22 21:51:39 +1100484 if (!pci_find_capability(dev, type))
485 return -EINVAL;
486
Brice Goglin24334a12006-08-31 01:55:07 -0400487 return 0;
488}
489
490/**
Linus Torvalds1da177e2005-04-16 15:20:36 -0700491 * pci_enable_msi - configure device's MSI capability structure
492 * @dev: pointer to the pci_dev data structure of MSI device function
493 *
494 * Setup the MSI capability structure of device function with
Eric W. Biederman1ce03372006-10-04 02:16:41 -0700495 * a single MSI irq upon its software driver call to request for
Linus Torvalds1da177e2005-04-16 15:20:36 -0700496 * MSI mode enabled on its hardware device function. A return of zero
497 * indicates the successful setup of an entry zero with the new MSI
Eric W. Biederman1ce03372006-10-04 02:16:41 -0700498 * irq or non-zero for otherwise.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700499 **/
500int pci_enable_msi(struct pci_dev* dev)
501{
Michael Ellermanb1e23032007-03-22 21:51:39 +1100502 int status;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700503
Michael Ellermanc9953a72007-04-05 17:19:08 +1000504 status = pci_msi_check_device(dev, 1, PCI_CAP_ID_MSI);
505 if (status)
506 return status;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700507
Eric W. Biedermanded86d82007-01-28 12:42:52 -0700508 WARN_ON(!!dev->msi_enabled);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700509
Eric W. Biederman1ce03372006-10-04 02:16:41 -0700510 /* Check whether driver already requested for MSI-X irqs */
Eric W. Biedermanb1cbf4e2007-03-05 00:30:10 -0800511 if (dev->msix_enabled) {
512 printk(KERN_INFO "PCI: %s: Can't enable MSI. "
513 "Device already has MSI-X enabled\n",
514 pci_name(dev));
515 return -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700516 }
517 status = msi_capability_init(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700518 return status;
519}
Michael Ellerman4cc086f2007-03-22 21:51:34 +1100520EXPORT_SYMBOL(pci_enable_msi);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700521
522void pci_disable_msi(struct pci_dev* dev)
523{
524 struct msi_desc *entry;
Eric W. Biedermanb1cbf4e2007-03-05 00:30:10 -0800525 int default_irq;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700526
Michael Ellerman128bc5f2007-03-22 21:51:39 +1100527 if (!pci_msi_enable || !dev || !dev->msi_enabled)
Eric W. Biedermanded86d82007-01-28 12:42:52 -0700528 return;
529
Eric W. Biedermanb1cbf4e2007-03-05 00:30:10 -0800530 msi_set_enable(dev, 0);
531 pci_intx(dev, 1); /* enable intx */
532 dev->msi_enabled = 0;
Eric W. Biederman7bd007e2006-10-04 02:16:31 -0700533
Michael Ellerman314e77b2007-04-05 17:19:12 +1000534 BUG_ON(list_empty(&dev->msi_list));
535 entry = list_entry(dev->msi_list.next, struct msi_desc, list);
536 if (!entry->dev || entry->msi_attrib.type != PCI_CAP_ID_MSI) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700537 return;
538 }
Eric W. Biederman7bd007e2006-10-04 02:16:31 -0700539
Michael Ellermane387b9e2007-03-22 21:51:27 +1100540 default_irq = entry->msi_attrib.default_irq;
Michael Ellerman032de8e2007-04-18 19:39:22 +1000541 msi_free_irqs(dev);
Michael Ellermane387b9e2007-03-22 21:51:27 +1100542
543 /* Restore dev->irq to its default pin-assertion irq */
544 dev->irq = default_irq;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700545}
Michael Ellerman4cc086f2007-03-22 21:51:34 +1100546EXPORT_SYMBOL(pci_disable_msi);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700547
Michael Ellerman032de8e2007-04-18 19:39:22 +1000548static int msi_free_irqs(struct pci_dev* dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700549{
Michael Ellerman032de8e2007-04-18 19:39:22 +1000550 struct msi_desc *entry, *tmp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700551
David Millerb3b7cc72007-05-11 13:26:44 -0700552 list_for_each_entry(entry, &dev->msi_list, list) {
553 if (entry->irq)
554 BUG_ON(irq_has_action(entry->irq));
555 }
Michael Ellerman7ede9c12007-03-22 21:51:34 +1100556
Michael Ellerman032de8e2007-04-18 19:39:22 +1000557 arch_teardown_msi_irqs(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700558
Michael Ellerman032de8e2007-04-18 19:39:22 +1000559 list_for_each_entry_safe(entry, tmp, &dev->msi_list, list) {
560 if (entry->msi_attrib.type == PCI_CAP_ID_MSIX) {
561 if (list_is_last(&entry->list, &dev->msi_list))
562 iounmap(entry->mask_base);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700563
Michael Ellerman032de8e2007-04-18 19:39:22 +1000564 writel(1, entry->mask_base + entry->msi_attrib.entry_nr
565 * PCI_MSIX_ENTRY_SIZE
566 + PCI_MSIX_ENTRY_VECTOR_CTRL_OFFSET);
567 }
568 list_del(&entry->list);
569 kfree(entry);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700570 }
571
572 return 0;
573}
574
Linus Torvalds1da177e2005-04-16 15:20:36 -0700575/**
576 * pci_enable_msix - configure device's MSI-X capability structure
577 * @dev: pointer to the pci_dev data structure of MSI-X device function
Greg Kroah-Hartman70549ad2005-06-06 23:07:46 -0700578 * @entries: pointer to an array of MSI-X entries
Eric W. Biederman1ce03372006-10-04 02:16:41 -0700579 * @nvec: number of MSI-X irqs requested for allocation by device driver
Linus Torvalds1da177e2005-04-16 15:20:36 -0700580 *
581 * Setup the MSI-X capability structure of device function with the number
Eric W. Biederman1ce03372006-10-04 02:16:41 -0700582 * of requested irqs upon its software driver call to request for
Linus Torvalds1da177e2005-04-16 15:20:36 -0700583 * MSI-X mode enabled on its hardware device function. A return of zero
584 * indicates the successful configuration of MSI-X capability structure
Eric W. Biederman1ce03372006-10-04 02:16:41 -0700585 * with new allocated MSI-X irqs. A return of < 0 indicates a failure.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700586 * Or a return of > 0 indicates that driver request is exceeding the number
Eric W. Biederman1ce03372006-10-04 02:16:41 -0700587 * of irqs available. Driver should use the returned value to re-send
Linus Torvalds1da177e2005-04-16 15:20:36 -0700588 * its request.
589 **/
590int pci_enable_msix(struct pci_dev* dev, struct msix_entry *entries, int nvec)
591{
Eric W. Biederman92db6d12006-10-04 02:16:35 -0700592 int status, pos, nr_entries;
Eric W. Biedermanded86d82007-01-28 12:42:52 -0700593 int i, j;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700594 u16 control;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700595
Michael Ellermanc9953a72007-04-05 17:19:08 +1000596 if (!entries)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700597 return -EINVAL;
598
Michael Ellermanc9953a72007-04-05 17:19:08 +1000599 status = pci_msi_check_device(dev, nvec, PCI_CAP_ID_MSIX);
600 if (status)
601 return status;
602
Grant Grundlerb64c05e2006-01-14 00:34:53 -0700603 pos = pci_find_capability(dev, PCI_CAP_ID_MSIX);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700604 pci_read_config_word(dev, msi_control_reg(pos), &control);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700605 nr_entries = multi_msix_capable(control);
606 if (nvec > nr_entries)
607 return -EINVAL;
608
609 /* Check for any invalid entries */
610 for (i = 0; i < nvec; i++) {
611 if (entries[i].entry >= nr_entries)
612 return -EINVAL; /* invalid entry */
613 for (j = i + 1; j < nvec; j++) {
614 if (entries[i].entry == entries[j].entry)
615 return -EINVAL; /* duplicate entry */
616 }
617 }
Eric W. Biedermanded86d82007-01-28 12:42:52 -0700618 WARN_ON(!!dev->msix_enabled);
Eric W. Biederman7bd007e2006-10-04 02:16:31 -0700619
Eric W. Biederman1ce03372006-10-04 02:16:41 -0700620 /* Check whether driver already requested for MSI irq */
Eric W. Biedermanb1cbf4e2007-03-05 00:30:10 -0800621 if (dev->msi_enabled) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700622 printk(KERN_INFO "PCI: %s: Can't enable MSI-X. "
Eric W. Biederman1ce03372006-10-04 02:16:41 -0700623 "Device already has an MSI irq assigned\n",
Linus Torvalds1da177e2005-04-16 15:20:36 -0700624 pci_name(dev));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700625 return -EINVAL;
626 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700627 status = msix_capability_init(dev, entries, nvec);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700628 return status;
629}
Michael Ellerman4cc086f2007-03-22 21:51:34 +1100630EXPORT_SYMBOL(pci_enable_msix);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700631
Michael Ellermanfc4afc72007-03-22 21:51:33 +1100632static void msix_free_all_irqs(struct pci_dev *dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700633{
Michael Ellerman032de8e2007-04-18 19:39:22 +1000634 msi_free_irqs(dev);
Michael Ellermanfc4afc72007-03-22 21:51:33 +1100635}
636
637void pci_disable_msix(struct pci_dev* dev)
638{
Michael Ellerman128bc5f2007-03-22 21:51:39 +1100639 if (!pci_msi_enable || !dev || !dev->msix_enabled)
Eric W. Biedermanded86d82007-01-28 12:42:52 -0700640 return;
641
Eric W. Biedermanb1cbf4e2007-03-05 00:30:10 -0800642 msix_set_enable(dev, 0);
643 pci_intx(dev, 1); /* enable intx */
644 dev->msix_enabled = 0;
Eric W. Biederman7bd007e2006-10-04 02:16:31 -0700645
Michael Ellermanfc4afc72007-03-22 21:51:33 +1100646 msix_free_all_irqs(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700647}
Michael Ellerman4cc086f2007-03-22 21:51:34 +1100648EXPORT_SYMBOL(pci_disable_msix);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700649
650/**
Eric W. Biederman1ce03372006-10-04 02:16:41 -0700651 * msi_remove_pci_irq_vectors - reclaim MSI(X) irqs to unused state
Linus Torvalds1da177e2005-04-16 15:20:36 -0700652 * @dev: pointer to the pci_dev data structure of MSI(X) device function
653 *
Steven Coleeaae4b32005-05-03 18:38:30 -0600654 * Being called during hotplug remove, from which the device function
Eric W. Biederman1ce03372006-10-04 02:16:41 -0700655 * is hot-removed. All previous assigned MSI/MSI-X irqs, if
Linus Torvalds1da177e2005-04-16 15:20:36 -0700656 * allocated for this device function, are reclaimed to unused state,
657 * which may be used later on.
658 **/
659void msi_remove_pci_irq_vectors(struct pci_dev* dev)
660{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700661 if (!pci_msi_enable || !dev)
662 return;
663
Michael Ellerman032de8e2007-04-18 19:39:22 +1000664 if (dev->msi_enabled)
665 msi_free_irqs(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700666
Michael Ellermanfc4afc72007-03-22 21:51:33 +1100667 if (dev->msix_enabled)
668 msix_free_all_irqs(dev);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700669}
670
Matthew Wilcox309e57d2006-03-05 22:33:34 -0700671void pci_no_msi(void)
672{
673 pci_msi_enable = 0;
674}
Michael Ellermanc9953a72007-04-05 17:19:08 +1000675
Michael Ellerman4aa9bc92007-04-05 17:19:10 +1000676void pci_msi_init_pci_dev(struct pci_dev *dev)
677{
678 INIT_LIST_HEAD(&dev->msi_list);
679}
680
Michael Ellermanc9953a72007-04-05 17:19:08 +1000681
682/* Arch hooks */
683
684int __attribute__ ((weak))
685arch_msi_check_device(struct pci_dev* dev, int nvec, int type)
686{
687 return 0;
688}
689
Michael Ellerman9c831332007-04-18 19:39:21 +1000690int __attribute__ ((weak))
691arch_setup_msi_irq(struct pci_dev *dev, struct msi_desc *entry)
692{
693 return 0;
694}
695
696int __attribute__ ((weak))
697arch_setup_msi_irqs(struct pci_dev *dev, int nvec, int type)
698{
699 struct msi_desc *entry;
700 int ret;
701
702 list_for_each_entry(entry, &dev->msi_list, list) {
703 ret = arch_setup_msi_irq(dev, entry);
704 if (ret)
705 return ret;
706 }
707
708 return 0;
709}
Michael Ellerman032de8e2007-04-18 19:39:22 +1000710
711void __attribute__ ((weak)) arch_teardown_msi_irq(unsigned int irq)
712{
713 return;
714}
715
716void __attribute__ ((weak))
717arch_teardown_msi_irqs(struct pci_dev *dev)
718{
719 struct msi_desc *entry;
720
721 list_for_each_entry(entry, &dev->msi_list, list) {
722 if (entry->irq != 0)
723 arch_teardown_msi_irq(entry->irq);
724 }
725}