blob: 202b7507a357092906680bd9e937e7882e428324 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * File: msi.c
3 * Purpose: PCI Message Signaled Interrupt (MSI)
4 *
5 * Copyright (C) 2003-2004 Intel
6 * Copyright (C) Tom Long Nguyen (tom.l.nguyen@intel.com)
7 */
8
9#include <linux/mm.h>
10#include <linux/irq.h>
11#include <linux/interrupt.h>
12#include <linux/init.h>
13#include <linux/config.h>
14#include <linux/ioport.h>
15#include <linux/smp_lock.h>
16#include <linux/pci.h>
17#include <linux/proc_fs.h>
18
19#include <asm/errno.h>
20#include <asm/io.h>
21#include <asm/smp.h>
22
23#include "pci.h"
24#include "msi.h"
25
Ashok Rajb4033c12005-11-08 21:42:33 -080026#define MSI_TARGET_CPU first_cpu(cpu_online_map)
27
Linus Torvalds1da177e2005-04-16 15:20:36 -070028static DEFINE_SPINLOCK(msi_lock);
29static struct msi_desc* msi_desc[NR_IRQS] = { [0 ... NR_IRQS-1] = NULL };
30static kmem_cache_t* msi_cachep;
31
32static int pci_msi_enable = 1;
Greg Kroah-Hartman70549ad2005-06-06 23:07:46 -070033static int last_alloc_vector;
34static int nr_released_vectors;
Linus Torvalds1da177e2005-04-16 15:20:36 -070035static int nr_reserved_vectors = NR_HP_RESERVED_VECTORS;
Greg Kroah-Hartman70549ad2005-06-06 23:07:46 -070036static int nr_msix_devices;
Linus Torvalds1da177e2005-04-16 15:20:36 -070037
38#ifndef CONFIG_X86_IO_APIC
39int vector_irq[NR_VECTORS] = { [0 ... NR_VECTORS - 1] = -1};
40u8 irq_vector[NR_IRQ_VECTORS] = { FIRST_DEVICE_VECTOR , 0 };
41#endif
42
43static void msi_cache_ctor(void *p, kmem_cache_t *cache, unsigned long flags)
44{
45 memset(p, 0, NR_IRQS * sizeof(struct msi_desc));
46}
47
48static int msi_cache_init(void)
49{
50 msi_cachep = kmem_cache_create("msi_cache",
51 NR_IRQS * sizeof(struct msi_desc),
52 0, SLAB_HWCACHE_ALIGN, msi_cache_ctor, NULL);
53 if (!msi_cachep)
54 return -ENOMEM;
55
56 return 0;
57}
58
59static void msi_set_mask_bit(unsigned int vector, int flag)
60{
61 struct msi_desc *entry;
62
63 entry = (struct msi_desc *)msi_desc[vector];
64 if (!entry || !entry->dev || !entry->mask_base)
65 return;
66 switch (entry->msi_attrib.type) {
67 case PCI_CAP_ID_MSI:
68 {
69 int pos;
70 u32 mask_bits;
71
72 pos = (long)entry->mask_base;
73 pci_read_config_dword(entry->dev, pos, &mask_bits);
74 mask_bits &= ~(1);
75 mask_bits |= flag;
76 pci_write_config_dword(entry->dev, pos, mask_bits);
77 break;
78 }
79 case PCI_CAP_ID_MSIX:
80 {
81 int offset = entry->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE +
82 PCI_MSIX_ENTRY_VECTOR_CTRL_OFFSET;
83 writel(flag, entry->mask_base + offset);
84 break;
85 }
86 default:
87 break;
88 }
89}
90
91#ifdef CONFIG_SMP
92static void set_msi_affinity(unsigned int vector, cpumask_t cpu_mask)
93{
94 struct msi_desc *entry;
95 struct msg_address address;
Ashok Raj54d5d422005-09-06 15:16:15 -070096 unsigned int irq = vector;
Ashok Rajb4033c12005-11-08 21:42:33 -080097 unsigned int dest_cpu = first_cpu(cpu_mask);
Linus Torvalds1da177e2005-04-16 15:20:36 -070098
99 entry = (struct msi_desc *)msi_desc[vector];
100 if (!entry || !entry->dev)
101 return;
102
103 switch (entry->msi_attrib.type) {
104 case PCI_CAP_ID_MSI:
105 {
106 int pos;
107
108 if (!(pos = pci_find_capability(entry->dev, PCI_CAP_ID_MSI)))
109 return;
110
111 pci_read_config_dword(entry->dev, msi_lower_address_reg(pos),
112 &address.lo_address.value);
113 address.lo_address.value &= MSI_ADDRESS_DEST_ID_MASK;
Ashok Rajb4033c12005-11-08 21:42:33 -0800114 address.lo_address.value |= (cpu_physical_id(dest_cpu) <<
115 MSI_TARGET_CPU_SHIFT);
116 entry->msi_attrib.current_cpu = cpu_physical_id(dest_cpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700117 pci_write_config_dword(entry->dev, msi_lower_address_reg(pos),
118 address.lo_address.value);
Ashok Raj54d5d422005-09-06 15:16:15 -0700119 set_native_irq_info(irq, cpu_mask);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700120 break;
121 }
122 case PCI_CAP_ID_MSIX:
123 {
124 int offset = entry->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE +
125 PCI_MSIX_ENTRY_LOWER_ADDR_OFFSET;
126
127 address.lo_address.value = readl(entry->mask_base + offset);
128 address.lo_address.value &= MSI_ADDRESS_DEST_ID_MASK;
Ashok Rajb4033c12005-11-08 21:42:33 -0800129 address.lo_address.value |= (cpu_physical_id(dest_cpu) <<
130 MSI_TARGET_CPU_SHIFT);
131 entry->msi_attrib.current_cpu = cpu_physical_id(dest_cpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700132 writel(address.lo_address.value, entry->mask_base + offset);
Ashok Raj54d5d422005-09-06 15:16:15 -0700133 set_native_irq_info(irq, cpu_mask);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700134 break;
135 }
136 default:
137 break;
138 }
139}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700140#endif /* CONFIG_SMP */
141
142static void mask_MSI_irq(unsigned int vector)
143{
144 msi_set_mask_bit(vector, 1);
145}
146
147static void unmask_MSI_irq(unsigned int vector)
148{
149 msi_set_mask_bit(vector, 0);
150}
151
152static unsigned int startup_msi_irq_wo_maskbit(unsigned int vector)
153{
154 struct msi_desc *entry;
155 unsigned long flags;
156
157 spin_lock_irqsave(&msi_lock, flags);
158 entry = msi_desc[vector];
159 if (!entry || !entry->dev) {
160 spin_unlock_irqrestore(&msi_lock, flags);
161 return 0;
162 }
163 entry->msi_attrib.state = 1; /* Mark it active */
164 spin_unlock_irqrestore(&msi_lock, flags);
165
166 return 0; /* never anything pending */
167}
168
Linus Torvalds1da177e2005-04-16 15:20:36 -0700169static unsigned int startup_msi_irq_w_maskbit(unsigned int vector)
170{
Greg Kroah-Hartman70549ad2005-06-06 23:07:46 -0700171 startup_msi_irq_wo_maskbit(vector);
172 unmask_MSI_irq(vector);
173 return 0; /* never anything pending */
174}
175
176static void shutdown_msi_irq(unsigned int vector)
177{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700178 struct msi_desc *entry;
179 unsigned long flags;
180
181 spin_lock_irqsave(&msi_lock, flags);
182 entry = msi_desc[vector];
Greg Kroah-Hartman70549ad2005-06-06 23:07:46 -0700183 if (entry && entry->dev)
184 entry->msi_attrib.state = 0; /* Mark it not active */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700185 spin_unlock_irqrestore(&msi_lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700186}
187
Greg Kroah-Hartman70549ad2005-06-06 23:07:46 -0700188static void end_msi_irq_wo_maskbit(unsigned int vector)
189{
Ashok Raj54d5d422005-09-06 15:16:15 -0700190 move_native_irq(vector);
Greg Kroah-Hartman70549ad2005-06-06 23:07:46 -0700191 ack_APIC_irq();
192}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700193
194static void end_msi_irq_w_maskbit(unsigned int vector)
195{
Ashok Raj54d5d422005-09-06 15:16:15 -0700196 move_native_irq(vector);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700197 unmask_MSI_irq(vector);
198 ack_APIC_irq();
199}
200
Greg Kroah-Hartman70549ad2005-06-06 23:07:46 -0700201static void do_nothing(unsigned int vector)
202{
203}
204
Linus Torvalds1da177e2005-04-16 15:20:36 -0700205/*
206 * Interrupt Type for MSI-X PCI/PCI-X/PCI-Express Devices,
207 * which implement the MSI-X Capability Structure.
208 */
209static struct hw_interrupt_type msix_irq_type = {
210 .typename = "PCI-MSI-X",
211 .startup = startup_msi_irq_w_maskbit,
Greg Kroah-Hartman70549ad2005-06-06 23:07:46 -0700212 .shutdown = shutdown_msi_irq,
213 .enable = unmask_MSI_irq,
214 .disable = mask_MSI_irq,
215 .ack = mask_MSI_irq,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700216 .end = end_msi_irq_w_maskbit,
217 .set_affinity = set_msi_irq_affinity
218};
219
220/*
221 * Interrupt Type for MSI PCI/PCI-X/PCI-Express Devices,
222 * which implement the MSI Capability Structure with
223 * Mask-and-Pending Bits.
224 */
225static struct hw_interrupt_type msi_irq_w_maskbit_type = {
226 .typename = "PCI-MSI",
227 .startup = startup_msi_irq_w_maskbit,
Greg Kroah-Hartman70549ad2005-06-06 23:07:46 -0700228 .shutdown = shutdown_msi_irq,
229 .enable = unmask_MSI_irq,
230 .disable = mask_MSI_irq,
231 .ack = mask_MSI_irq,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700232 .end = end_msi_irq_w_maskbit,
233 .set_affinity = set_msi_irq_affinity
234};
235
236/*
237 * Interrupt Type for MSI PCI/PCI-X/PCI-Express Devices,
238 * which implement the MSI Capability Structure without
239 * Mask-and-Pending Bits.
240 */
241static struct hw_interrupt_type msi_irq_wo_maskbit_type = {
242 .typename = "PCI-MSI",
243 .startup = startup_msi_irq_wo_maskbit,
Greg Kroah-Hartman70549ad2005-06-06 23:07:46 -0700244 .shutdown = shutdown_msi_irq,
245 .enable = do_nothing,
246 .disable = do_nothing,
247 .ack = do_nothing,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700248 .end = end_msi_irq_wo_maskbit,
249 .set_affinity = set_msi_irq_affinity
250};
251
252static void msi_data_init(struct msg_data *msi_data,
253 unsigned int vector)
254{
255 memset(msi_data, 0, sizeof(struct msg_data));
256 msi_data->vector = (u8)vector;
257 msi_data->delivery_mode = MSI_DELIVERY_MODE;
258 msi_data->level = MSI_LEVEL_MODE;
259 msi_data->trigger = MSI_TRIGGER_MODE;
260}
261
262static void msi_address_init(struct msg_address *msi_address)
263{
264 unsigned int dest_id;
Ashok Rajb4033c12005-11-08 21:42:33 -0800265 unsigned long dest_phys_id = cpu_physical_id(MSI_TARGET_CPU);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700266
267 memset(msi_address, 0, sizeof(struct msg_address));
268 msi_address->hi_address = (u32)0;
269 dest_id = (MSI_ADDRESS_HEADER << MSI_ADDRESS_HEADER_SHIFT);
Ashok Rajb4033c12005-11-08 21:42:33 -0800270 msi_address->lo_address.u.dest_mode = MSI_PHYSICAL_MODE;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700271 msi_address->lo_address.u.redirection_hint = MSI_REDIRECTION_HINT_MODE;
272 msi_address->lo_address.u.dest_id = dest_id;
Ashok Rajb4033c12005-11-08 21:42:33 -0800273 msi_address->lo_address.value |= (dest_phys_id << MSI_TARGET_CPU_SHIFT);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700274}
275
276static int msi_free_vector(struct pci_dev* dev, int vector, int reassign);
277static int assign_msi_vector(void)
278{
279 static int new_vector_avail = 1;
280 int vector;
281 unsigned long flags;
282
283 /*
284 * msi_lock is provided to ensure that successful allocation of MSI
285 * vector is assigned unique among drivers.
286 */
287 spin_lock_irqsave(&msi_lock, flags);
288
289 if (!new_vector_avail) {
290 int free_vector = 0;
291
292 /*
293 * vector_irq[] = -1 indicates that this specific vector is:
294 * - assigned for MSI (since MSI have no associated IRQ) or
295 * - assigned for legacy if less than 16, or
296 * - having no corresponding 1:1 vector-to-IOxAPIC IRQ mapping
297 * vector_irq[] = 0 indicates that this vector, previously
298 * assigned for MSI, is freed by hotplug removed operations.
299 * This vector will be reused for any subsequent hotplug added
300 * operations.
301 * vector_irq[] > 0 indicates that this vector is assigned for
302 * IOxAPIC IRQs. This vector and its value provides a 1-to-1
303 * vector-to-IOxAPIC IRQ mapping.
304 */
305 for (vector = FIRST_DEVICE_VECTOR; vector < NR_IRQS; vector++) {
306 if (vector_irq[vector] != 0)
307 continue;
308 free_vector = vector;
309 if (!msi_desc[vector])
310 break;
311 else
312 continue;
313 }
314 if (!free_vector) {
315 spin_unlock_irqrestore(&msi_lock, flags);
316 return -EBUSY;
317 }
318 vector_irq[free_vector] = -1;
319 nr_released_vectors--;
320 spin_unlock_irqrestore(&msi_lock, flags);
321 if (msi_desc[free_vector] != NULL) {
322 struct pci_dev *dev;
323 int tail;
324
325 /* free all linked vectors before re-assign */
326 do {
327 spin_lock_irqsave(&msi_lock, flags);
328 dev = msi_desc[free_vector]->dev;
329 tail = msi_desc[free_vector]->link.tail;
330 spin_unlock_irqrestore(&msi_lock, flags);
331 msi_free_vector(dev, tail, 1);
332 } while (free_vector != tail);
333 }
334
335 return free_vector;
336 }
337 vector = assign_irq_vector(AUTO_ASSIGN);
338 last_alloc_vector = vector;
339 if (vector == LAST_DEVICE_VECTOR)
340 new_vector_avail = 0;
341
342 spin_unlock_irqrestore(&msi_lock, flags);
343 return vector;
344}
345
346static int get_new_vector(void)
347{
348 int vector;
349
350 if ((vector = assign_msi_vector()) > 0)
351 set_intr_gate(vector, interrupt[vector]);
352
353 return vector;
354}
355
356static int msi_init(void)
357{
358 static int status = -ENOMEM;
359
360 if (!status)
361 return status;
362
363 if (pci_msi_quirk) {
364 pci_msi_enable = 0;
365 printk(KERN_WARNING "PCI: MSI quirk detected. MSI disabled.\n");
366 status = -EINVAL;
367 return status;
368 }
369
370 if ((status = msi_cache_init()) < 0) {
371 pci_msi_enable = 0;
372 printk(KERN_WARNING "PCI: MSI cache init failed\n");
373 return status;
374 }
375 last_alloc_vector = assign_irq_vector(AUTO_ASSIGN);
376 if (last_alloc_vector < 0) {
377 pci_msi_enable = 0;
378 printk(KERN_WARNING "PCI: No interrupt vectors available for MSI\n");
379 status = -EBUSY;
380 return status;
381 }
382 vector_irq[last_alloc_vector] = 0;
383 nr_released_vectors++;
384
385 return status;
386}
387
388static int get_msi_vector(struct pci_dev *dev)
389{
390 return get_new_vector();
391}
392
393static struct msi_desc* alloc_msi_entry(void)
394{
395 struct msi_desc *entry;
396
Greg Kroah-Hartman70549ad2005-06-06 23:07:46 -0700397 entry = kmem_cache_alloc(msi_cachep, SLAB_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700398 if (!entry)
399 return NULL;
400
401 memset(entry, 0, sizeof(struct msi_desc));
402 entry->link.tail = entry->link.head = 0; /* single message */
403 entry->dev = NULL;
404
405 return entry;
406}
407
408static void attach_msi_entry(struct msi_desc *entry, int vector)
409{
410 unsigned long flags;
411
412 spin_lock_irqsave(&msi_lock, flags);
413 msi_desc[vector] = entry;
414 spin_unlock_irqrestore(&msi_lock, flags);
415}
416
417static void irq_handler_init(int cap_id, int pos, int mask)
418{
419 spin_lock(&irq_desc[pos].lock);
420 if (cap_id == PCI_CAP_ID_MSIX)
421 irq_desc[pos].handler = &msix_irq_type;
422 else {
423 if (!mask)
424 irq_desc[pos].handler = &msi_irq_wo_maskbit_type;
425 else
426 irq_desc[pos].handler = &msi_irq_w_maskbit_type;
427 }
428 spin_unlock(&irq_desc[pos].lock);
429}
430
431static void enable_msi_mode(struct pci_dev *dev, int pos, int type)
432{
433 u16 control;
434
435 pci_read_config_word(dev, msi_control_reg(pos), &control);
436 if (type == PCI_CAP_ID_MSI) {
437 /* Set enabled bits to single MSI & enable MSI_enable bit */
438 msi_enable(control, 1);
439 pci_write_config_word(dev, msi_control_reg(pos), control);
440 } else {
441 msix_enable(control);
442 pci_write_config_word(dev, msi_control_reg(pos), control);
443 }
444 if (pci_find_capability(dev, PCI_CAP_ID_EXP)) {
445 /* PCI Express Endpoint device detected */
Brett M Russa04ce0f2005-08-15 15:23:41 -0400446 pci_intx(dev, 0); /* disable intx */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700447 }
448}
449
Kristen Accardi4602b882005-08-16 15:15:58 -0700450void disable_msi_mode(struct pci_dev *dev, int pos, int type)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700451{
452 u16 control;
453
454 pci_read_config_word(dev, msi_control_reg(pos), &control);
455 if (type == PCI_CAP_ID_MSI) {
456 /* Set enabled bits to single MSI & enable MSI_enable bit */
457 msi_disable(control);
458 pci_write_config_word(dev, msi_control_reg(pos), control);
459 } else {
460 msix_disable(control);
461 pci_write_config_word(dev, msi_control_reg(pos), control);
462 }
463 if (pci_find_capability(dev, PCI_CAP_ID_EXP)) {
464 /* PCI Express Endpoint device detected */
Brett M Russa04ce0f2005-08-15 15:23:41 -0400465 pci_intx(dev, 1); /* enable intx */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700466 }
467}
468
469static int msi_lookup_vector(struct pci_dev *dev, int type)
470{
471 int vector;
472 unsigned long flags;
473
474 spin_lock_irqsave(&msi_lock, flags);
475 for (vector = FIRST_DEVICE_VECTOR; vector < NR_IRQS; vector++) {
476 if (!msi_desc[vector] || msi_desc[vector]->dev != dev ||
477 msi_desc[vector]->msi_attrib.type != type ||
478 msi_desc[vector]->msi_attrib.default_vector != dev->irq)
479 continue;
480 spin_unlock_irqrestore(&msi_lock, flags);
481 /* This pre-assigned MSI vector for this device
482 already exits. Override dev->irq with this vector */
483 dev->irq = vector;
484 return 0;
485 }
486 spin_unlock_irqrestore(&msi_lock, flags);
487
488 return -EACCES;
489}
490
491void pci_scan_msi_device(struct pci_dev *dev)
492{
493 if (!dev)
494 return;
495
496 if (pci_find_capability(dev, PCI_CAP_ID_MSIX) > 0)
497 nr_msix_devices++;
498 else if (pci_find_capability(dev, PCI_CAP_ID_MSI) > 0)
499 nr_reserved_vectors++;
500}
501
502/**
503 * msi_capability_init - configure device's MSI capability structure
504 * @dev: pointer to the pci_dev data structure of MSI device function
505 *
Steven Coleeaae4b32005-05-03 18:38:30 -0600506 * Setup the MSI capability structure of device function with a single
Linus Torvalds1da177e2005-04-16 15:20:36 -0700507 * MSI vector, regardless of device function is capable of handling
508 * multiple messages. A return of zero indicates the successful setup
509 * of an entry zero with the new MSI vector or non-zero for otherwise.
510 **/
511static int msi_capability_init(struct pci_dev *dev)
512{
513 struct msi_desc *entry;
514 struct msg_address address;
515 struct msg_data data;
516 int pos, vector;
517 u16 control;
518
519 pos = pci_find_capability(dev, PCI_CAP_ID_MSI);
520 pci_read_config_word(dev, msi_control_reg(pos), &control);
521 /* MSI Entry Initialization */
522 if (!(entry = alloc_msi_entry()))
523 return -ENOMEM;
524
525 if ((vector = get_msi_vector(dev)) < 0) {
526 kmem_cache_free(msi_cachep, entry);
527 return -EBUSY;
528 }
529 entry->link.head = vector;
530 entry->link.tail = vector;
531 entry->msi_attrib.type = PCI_CAP_ID_MSI;
532 entry->msi_attrib.state = 0; /* Mark it not active */
533 entry->msi_attrib.entry_nr = 0;
534 entry->msi_attrib.maskbit = is_mask_bit_support(control);
535 entry->msi_attrib.default_vector = dev->irq; /* Save IOAPIC IRQ */
536 dev->irq = vector;
537 entry->dev = dev;
538 if (is_mask_bit_support(control)) {
539 entry->mask_base = (void __iomem *)(long)msi_mask_bits_reg(pos,
540 is_64bit_address(control));
541 }
542 /* Replace with MSI handler */
543 irq_handler_init(PCI_CAP_ID_MSI, vector, entry->msi_attrib.maskbit);
544 /* Configure MSI capability structure */
545 msi_address_init(&address);
546 msi_data_init(&data, vector);
547 entry->msi_attrib.current_cpu = ((address.lo_address.u.dest_id >>
548 MSI_TARGET_CPU_SHIFT) & MSI_TARGET_CPU_MASK);
549 pci_write_config_dword(dev, msi_lower_address_reg(pos),
550 address.lo_address.value);
551 if (is_64bit_address(control)) {
552 pci_write_config_dword(dev,
553 msi_upper_address_reg(pos), address.hi_address);
554 pci_write_config_word(dev,
555 msi_data_reg(pos, 1), *((u32*)&data));
556 } else
557 pci_write_config_word(dev,
558 msi_data_reg(pos, 0), *((u32*)&data));
559 if (entry->msi_attrib.maskbit) {
560 unsigned int maskbits, temp;
561 /* All MSIs are unmasked by default, Mask them all */
562 pci_read_config_dword(dev,
563 msi_mask_bits_reg(pos, is_64bit_address(control)),
564 &maskbits);
565 temp = (1 << multi_msi_capable(control));
566 temp = ((temp - 1) & ~temp);
567 maskbits |= temp;
568 pci_write_config_dword(dev,
569 msi_mask_bits_reg(pos, is_64bit_address(control)),
570 maskbits);
571 }
572 attach_msi_entry(entry, vector);
573 /* Set MSI enabled bits */
574 enable_msi_mode(dev, pos, PCI_CAP_ID_MSI);
575
576 return 0;
577}
578
579/**
580 * msix_capability_init - configure device's MSI-X capability
581 * @dev: pointer to the pci_dev data structure of MSI-X device function
Randy Dunlap8f7020d2005-10-23 11:57:38 -0700582 * @entries: pointer to an array of struct msix_entry entries
583 * @nvec: number of @entries
Linus Torvalds1da177e2005-04-16 15:20:36 -0700584 *
Steven Coleeaae4b32005-05-03 18:38:30 -0600585 * Setup the MSI-X capability structure of device function with a
Linus Torvalds1da177e2005-04-16 15:20:36 -0700586 * single MSI-X vector. A return of zero indicates the successful setup of
587 * requested MSI-X entries with allocated vectors or non-zero for otherwise.
588 **/
589static int msix_capability_init(struct pci_dev *dev,
590 struct msix_entry *entries, int nvec)
591{
592 struct msi_desc *head = NULL, *tail = NULL, *entry = NULL;
593 struct msg_address address;
594 struct msg_data data;
595 int vector, pos, i, j, nr_entries, temp = 0;
596 u32 phys_addr, table_offset;
597 u16 control;
598 u8 bir;
599 void __iomem *base;
600
601 pos = pci_find_capability(dev, PCI_CAP_ID_MSIX);
602 /* Request & Map MSI-X table region */
603 pci_read_config_word(dev, msi_control_reg(pos), &control);
604 nr_entries = multi_msix_capable(control);
605 pci_read_config_dword(dev, msix_table_offset_reg(pos),
606 &table_offset);
607 bir = (u8)(table_offset & PCI_MSIX_FLAGS_BIRMASK);
608 phys_addr = pci_resource_start (dev, bir);
609 phys_addr += (u32)(table_offset & ~PCI_MSIX_FLAGS_BIRMASK);
610 base = ioremap_nocache(phys_addr, nr_entries * PCI_MSIX_ENTRY_SIZE);
611 if (base == NULL)
612 return -ENOMEM;
613
614 /* MSI-X Table Initialization */
615 for (i = 0; i < nvec; i++) {
616 entry = alloc_msi_entry();
617 if (!entry)
618 break;
619 if ((vector = get_msi_vector(dev)) < 0)
620 break;
621
622 j = entries[i].entry;
623 entries[i].vector = vector;
624 entry->msi_attrib.type = PCI_CAP_ID_MSIX;
625 entry->msi_attrib.state = 0; /* Mark it not active */
626 entry->msi_attrib.entry_nr = j;
627 entry->msi_attrib.maskbit = 1;
628 entry->msi_attrib.default_vector = dev->irq;
629 entry->dev = dev;
630 entry->mask_base = base;
631 if (!head) {
632 entry->link.head = vector;
633 entry->link.tail = vector;
634 head = entry;
635 } else {
636 entry->link.head = temp;
637 entry->link.tail = tail->link.tail;
638 tail->link.tail = vector;
639 head->link.head = vector;
640 }
641 temp = vector;
642 tail = entry;
643 /* Replace with MSI-X handler */
644 irq_handler_init(PCI_CAP_ID_MSIX, vector, 1);
645 /* Configure MSI-X capability structure */
646 msi_address_init(&address);
647 msi_data_init(&data, vector);
648 entry->msi_attrib.current_cpu =
649 ((address.lo_address.u.dest_id >>
650 MSI_TARGET_CPU_SHIFT) & MSI_TARGET_CPU_MASK);
651 writel(address.lo_address.value,
652 base + j * PCI_MSIX_ENTRY_SIZE +
653 PCI_MSIX_ENTRY_LOWER_ADDR_OFFSET);
654 writel(address.hi_address,
655 base + j * PCI_MSIX_ENTRY_SIZE +
656 PCI_MSIX_ENTRY_UPPER_ADDR_OFFSET);
657 writel(*(u32*)&data,
658 base + j * PCI_MSIX_ENTRY_SIZE +
659 PCI_MSIX_ENTRY_DATA_OFFSET);
660 attach_msi_entry(entry, vector);
661 }
662 if (i != nvec) {
663 i--;
664 for (; i >= 0; i--) {
665 vector = (entries + i)->vector;
666 msi_free_vector(dev, vector, 0);
667 (entries + i)->vector = 0;
668 }
669 return -EBUSY;
670 }
671 /* Set MSI-X enabled bits */
672 enable_msi_mode(dev, pos, PCI_CAP_ID_MSIX);
673
674 return 0;
675}
676
677/**
678 * pci_enable_msi - configure device's MSI capability structure
679 * @dev: pointer to the pci_dev data structure of MSI device function
680 *
681 * Setup the MSI capability structure of device function with
682 * a single MSI vector upon its software driver call to request for
683 * MSI mode enabled on its hardware device function. A return of zero
684 * indicates the successful setup of an entry zero with the new MSI
685 * vector or non-zero for otherwise.
686 **/
687int pci_enable_msi(struct pci_dev* dev)
688{
689 int pos, temp, status = -EINVAL;
690 u16 control;
691
692 if (!pci_msi_enable || !dev)
693 return status;
694
Kristen Accardi4602b882005-08-16 15:15:58 -0700695 if (dev->no_msi)
696 return status;
697
Linus Torvalds1da177e2005-04-16 15:20:36 -0700698 temp = dev->irq;
699
700 if ((status = msi_init()) < 0)
701 return status;
702
703 if (!(pos = pci_find_capability(dev, PCI_CAP_ID_MSI)))
704 return -EINVAL;
705
706 pci_read_config_word(dev, msi_control_reg(pos), &control);
707 if (control & PCI_MSI_FLAGS_ENABLE)
708 return 0; /* Already in MSI mode */
709
710 if (!msi_lookup_vector(dev, PCI_CAP_ID_MSI)) {
711 /* Lookup Sucess */
712 unsigned long flags;
713
714 spin_lock_irqsave(&msi_lock, flags);
715 if (!vector_irq[dev->irq]) {
716 msi_desc[dev->irq]->msi_attrib.state = 0;
717 vector_irq[dev->irq] = -1;
718 nr_released_vectors--;
719 spin_unlock_irqrestore(&msi_lock, flags);
720 enable_msi_mode(dev, pos, PCI_CAP_ID_MSI);
721 return 0;
722 }
723 spin_unlock_irqrestore(&msi_lock, flags);
724 dev->irq = temp;
725 }
726 /* Check whether driver already requested for MSI-X vectors */
727 if ((pos = pci_find_capability(dev, PCI_CAP_ID_MSIX)) > 0 &&
728 !msi_lookup_vector(dev, PCI_CAP_ID_MSIX)) {
729 printk(KERN_INFO "PCI: %s: Can't enable MSI. "
730 "Device already has MSI-X vectors assigned\n",
731 pci_name(dev));
732 dev->irq = temp;
733 return -EINVAL;
734 }
735 status = msi_capability_init(dev);
736 if (!status) {
737 if (!pos)
738 nr_reserved_vectors--; /* Only MSI capable */
739 else if (nr_msix_devices > 0)
740 nr_msix_devices--; /* Both MSI and MSI-X capable,
741 but choose enabling MSI */
742 }
743
744 return status;
745}
746
747void pci_disable_msi(struct pci_dev* dev)
748{
749 struct msi_desc *entry;
750 int pos, default_vector;
751 u16 control;
752 unsigned long flags;
753
754 if (!dev || !(pos = pci_find_capability(dev, PCI_CAP_ID_MSI)))
755 return;
756
757 pci_read_config_word(dev, msi_control_reg(pos), &control);
758 if (!(control & PCI_MSI_FLAGS_ENABLE))
759 return;
760
761 spin_lock_irqsave(&msi_lock, flags);
762 entry = msi_desc[dev->irq];
763 if (!entry || !entry->dev || entry->msi_attrib.type != PCI_CAP_ID_MSI) {
764 spin_unlock_irqrestore(&msi_lock, flags);
765 return;
766 }
767 if (entry->msi_attrib.state) {
768 spin_unlock_irqrestore(&msi_lock, flags);
769 printk(KERN_WARNING "PCI: %s: pci_disable_msi() called without "
770 "free_irq() on MSI vector %d\n",
771 pci_name(dev), dev->irq);
772 BUG_ON(entry->msi_attrib.state > 0);
773 } else {
774 vector_irq[dev->irq] = 0; /* free it */
775 nr_released_vectors++;
776 default_vector = entry->msi_attrib.default_vector;
777 spin_unlock_irqrestore(&msi_lock, flags);
778 /* Restore dev->irq to its default pin-assertion vector */
779 dev->irq = default_vector;
780 disable_msi_mode(dev, pci_find_capability(dev, PCI_CAP_ID_MSI),
781 PCI_CAP_ID_MSI);
782 }
783}
784
Linus Torvalds1da177e2005-04-16 15:20:36 -0700785static int msi_free_vector(struct pci_dev* dev, int vector, int reassign)
786{
787 struct msi_desc *entry;
788 int head, entry_nr, type;
789 void __iomem *base;
790 unsigned long flags;
791
792 spin_lock_irqsave(&msi_lock, flags);
793 entry = msi_desc[vector];
794 if (!entry || entry->dev != dev) {
795 spin_unlock_irqrestore(&msi_lock, flags);
796 return -EINVAL;
797 }
798 type = entry->msi_attrib.type;
799 entry_nr = entry->msi_attrib.entry_nr;
800 head = entry->link.head;
801 base = entry->mask_base;
802 msi_desc[entry->link.head]->link.tail = entry->link.tail;
803 msi_desc[entry->link.tail]->link.head = entry->link.head;
804 entry->dev = NULL;
805 if (!reassign) {
806 vector_irq[vector] = 0;
807 nr_released_vectors++;
808 }
809 msi_desc[vector] = NULL;
810 spin_unlock_irqrestore(&msi_lock, flags);
811
812 kmem_cache_free(msi_cachep, entry);
813
814 if (type == PCI_CAP_ID_MSIX) {
815 if (!reassign)
816 writel(1, base +
817 entry_nr * PCI_MSIX_ENTRY_SIZE +
818 PCI_MSIX_ENTRY_VECTOR_CTRL_OFFSET);
819
820 if (head == vector) {
821 /*
822 * Detect last MSI-X vector to be released.
823 * Release the MSI-X memory-mapped table.
824 */
825 int pos, nr_entries;
826 u32 phys_addr, table_offset;
827 u16 control;
828 u8 bir;
829
830 pos = pci_find_capability(dev, PCI_CAP_ID_MSIX);
831 pci_read_config_word(dev, msi_control_reg(pos),
832 &control);
833 nr_entries = multi_msix_capable(control);
834 pci_read_config_dword(dev, msix_table_offset_reg(pos),
835 &table_offset);
836 bir = (u8)(table_offset & PCI_MSIX_FLAGS_BIRMASK);
837 phys_addr = pci_resource_start (dev, bir);
838 phys_addr += (u32)(table_offset &
839 ~PCI_MSIX_FLAGS_BIRMASK);
840 iounmap(base);
841 }
842 }
843
844 return 0;
845}
846
847static int reroute_msix_table(int head, struct msix_entry *entries, int *nvec)
848{
849 int vector = head, tail = 0;
850 int i, j = 0, nr_entries = 0;
851 void __iomem *base;
852 unsigned long flags;
853
854 spin_lock_irqsave(&msi_lock, flags);
855 while (head != tail) {
856 nr_entries++;
857 tail = msi_desc[vector]->link.tail;
858 if (entries[0].entry == msi_desc[vector]->msi_attrib.entry_nr)
859 j = vector;
860 vector = tail;
861 }
862 if (*nvec > nr_entries) {
863 spin_unlock_irqrestore(&msi_lock, flags);
864 *nvec = nr_entries;
865 return -EINVAL;
866 }
867 vector = ((j > 0) ? j : head);
868 for (i = 0; i < *nvec; i++) {
869 j = msi_desc[vector]->msi_attrib.entry_nr;
870 msi_desc[vector]->msi_attrib.state = 0; /* Mark it not active */
871 vector_irq[vector] = -1; /* Mark it busy */
872 nr_released_vectors--;
873 entries[i].vector = vector;
874 if (j != (entries + i)->entry) {
875 base = msi_desc[vector]->mask_base;
876 msi_desc[vector]->msi_attrib.entry_nr =
877 (entries + i)->entry;
878 writel( readl(base + j * PCI_MSIX_ENTRY_SIZE +
879 PCI_MSIX_ENTRY_LOWER_ADDR_OFFSET), base +
880 (entries + i)->entry * PCI_MSIX_ENTRY_SIZE +
881 PCI_MSIX_ENTRY_LOWER_ADDR_OFFSET);
882 writel( readl(base + j * PCI_MSIX_ENTRY_SIZE +
883 PCI_MSIX_ENTRY_UPPER_ADDR_OFFSET), base +
884 (entries + i)->entry * PCI_MSIX_ENTRY_SIZE +
885 PCI_MSIX_ENTRY_UPPER_ADDR_OFFSET);
886 writel( (readl(base + j * PCI_MSIX_ENTRY_SIZE +
887 PCI_MSIX_ENTRY_DATA_OFFSET) & 0xff00) | vector,
888 base + (entries+i)->entry*PCI_MSIX_ENTRY_SIZE +
889 PCI_MSIX_ENTRY_DATA_OFFSET);
890 }
891 vector = msi_desc[vector]->link.tail;
892 }
893 spin_unlock_irqrestore(&msi_lock, flags);
894
895 return 0;
896}
897
898/**
899 * pci_enable_msix - configure device's MSI-X capability structure
900 * @dev: pointer to the pci_dev data structure of MSI-X device function
Greg Kroah-Hartman70549ad2005-06-06 23:07:46 -0700901 * @entries: pointer to an array of MSI-X entries
Linus Torvalds1da177e2005-04-16 15:20:36 -0700902 * @nvec: number of MSI-X vectors requested for allocation by device driver
903 *
904 * Setup the MSI-X capability structure of device function with the number
905 * of requested vectors upon its software driver call to request for
906 * MSI-X mode enabled on its hardware device function. A return of zero
907 * indicates the successful configuration of MSI-X capability structure
908 * with new allocated MSI-X vectors. A return of < 0 indicates a failure.
909 * Or a return of > 0 indicates that driver request is exceeding the number
910 * of vectors available. Driver should use the returned value to re-send
911 * its request.
912 **/
913int pci_enable_msix(struct pci_dev* dev, struct msix_entry *entries, int nvec)
914{
915 int status, pos, nr_entries, free_vectors;
916 int i, j, temp;
917 u16 control;
918 unsigned long flags;
919
920 if (!pci_msi_enable || !dev || !entries)
921 return -EINVAL;
922
923 if ((status = msi_init()) < 0)
924 return status;
925
926 if (!(pos = pci_find_capability(dev, PCI_CAP_ID_MSIX)))
927 return -EINVAL;
928
929 pci_read_config_word(dev, msi_control_reg(pos), &control);
930 if (control & PCI_MSIX_FLAGS_ENABLE)
931 return -EINVAL; /* Already in MSI-X mode */
932
933 nr_entries = multi_msix_capable(control);
934 if (nvec > nr_entries)
935 return -EINVAL;
936
937 /* Check for any invalid entries */
938 for (i = 0; i < nvec; i++) {
939 if (entries[i].entry >= nr_entries)
940 return -EINVAL; /* invalid entry */
941 for (j = i + 1; j < nvec; j++) {
942 if (entries[i].entry == entries[j].entry)
943 return -EINVAL; /* duplicate entry */
944 }
945 }
946 temp = dev->irq;
947 if (!msi_lookup_vector(dev, PCI_CAP_ID_MSIX)) {
948 /* Lookup Sucess */
949 nr_entries = nvec;
950 /* Reroute MSI-X table */
951 if (reroute_msix_table(dev->irq, entries, &nr_entries)) {
952 /* #requested > #previous-assigned */
953 dev->irq = temp;
954 return nr_entries;
955 }
956 dev->irq = temp;
957 enable_msi_mode(dev, pos, PCI_CAP_ID_MSIX);
958 return 0;
959 }
960 /* Check whether driver already requested for MSI vector */
961 if (pci_find_capability(dev, PCI_CAP_ID_MSI) > 0 &&
962 !msi_lookup_vector(dev, PCI_CAP_ID_MSI)) {
963 printk(KERN_INFO "PCI: %s: Can't enable MSI-X. "
964 "Device already has an MSI vector assigned\n",
965 pci_name(dev));
966 dev->irq = temp;
967 return -EINVAL;
968 }
969
970 spin_lock_irqsave(&msi_lock, flags);
971 /*
972 * msi_lock is provided to ensure that enough vectors resources are
973 * available before granting.
974 */
975 free_vectors = pci_vector_resources(last_alloc_vector,
976 nr_released_vectors);
977 /* Ensure that each MSI/MSI-X device has one vector reserved by
978 default to avoid any MSI-X driver to take all available
979 resources */
980 free_vectors -= nr_reserved_vectors;
981 /* Find the average of free vectors among MSI-X devices */
982 if (nr_msix_devices > 0)
983 free_vectors /= nr_msix_devices;
984 spin_unlock_irqrestore(&msi_lock, flags);
985
986 if (nvec > free_vectors) {
987 if (free_vectors > 0)
988 return free_vectors;
989 else
990 return -EBUSY;
991 }
992
993 status = msix_capability_init(dev, entries, nvec);
994 if (!status && nr_msix_devices > 0)
995 nr_msix_devices--;
996
997 return status;
998}
999
1000void pci_disable_msix(struct pci_dev* dev)
1001{
1002 int pos, temp;
1003 u16 control;
1004
1005 if (!dev || !(pos = pci_find_capability(dev, PCI_CAP_ID_MSIX)))
1006 return;
1007
1008 pci_read_config_word(dev, msi_control_reg(pos), &control);
1009 if (!(control & PCI_MSIX_FLAGS_ENABLE))
1010 return;
1011
1012 temp = dev->irq;
1013 if (!msi_lookup_vector(dev, PCI_CAP_ID_MSIX)) {
1014 int state, vector, head, tail = 0, warning = 0;
1015 unsigned long flags;
1016
1017 vector = head = dev->irq;
1018 spin_lock_irqsave(&msi_lock, flags);
1019 while (head != tail) {
1020 state = msi_desc[vector]->msi_attrib.state;
1021 if (state)
1022 warning = 1;
1023 else {
1024 vector_irq[vector] = 0; /* free it */
1025 nr_released_vectors++;
1026 }
1027 tail = msi_desc[vector]->link.tail;
1028 vector = tail;
1029 }
1030 spin_unlock_irqrestore(&msi_lock, flags);
1031 if (warning) {
1032 dev->irq = temp;
1033 printk(KERN_WARNING "PCI: %s: pci_disable_msix() called without "
1034 "free_irq() on all MSI-X vectors\n",
1035 pci_name(dev));
1036 BUG_ON(warning > 0);
1037 } else {
1038 dev->irq = temp;
1039 disable_msi_mode(dev,
1040 pci_find_capability(dev, PCI_CAP_ID_MSIX),
1041 PCI_CAP_ID_MSIX);
1042
1043 }
1044 }
1045}
1046
1047/**
1048 * msi_remove_pci_irq_vectors - reclaim MSI(X) vectors to unused state
1049 * @dev: pointer to the pci_dev data structure of MSI(X) device function
1050 *
Steven Coleeaae4b32005-05-03 18:38:30 -06001051 * Being called during hotplug remove, from which the device function
Linus Torvalds1da177e2005-04-16 15:20:36 -07001052 * is hot-removed. All previous assigned MSI/MSI-X vectors, if
1053 * allocated for this device function, are reclaimed to unused state,
1054 * which may be used later on.
1055 **/
1056void msi_remove_pci_irq_vectors(struct pci_dev* dev)
1057{
1058 int state, pos, temp;
1059 unsigned long flags;
1060
1061 if (!pci_msi_enable || !dev)
1062 return;
1063
1064 temp = dev->irq; /* Save IOAPIC IRQ */
1065 if ((pos = pci_find_capability(dev, PCI_CAP_ID_MSI)) > 0 &&
1066 !msi_lookup_vector(dev, PCI_CAP_ID_MSI)) {
1067 spin_lock_irqsave(&msi_lock, flags);
1068 state = msi_desc[dev->irq]->msi_attrib.state;
1069 spin_unlock_irqrestore(&msi_lock, flags);
1070 if (state) {
1071 printk(KERN_WARNING "PCI: %s: msi_remove_pci_irq_vectors() "
1072 "called without free_irq() on MSI vector %d\n",
1073 pci_name(dev), dev->irq);
1074 BUG_ON(state > 0);
1075 } else /* Release MSI vector assigned to this device */
1076 msi_free_vector(dev, dev->irq, 0);
1077 dev->irq = temp; /* Restore IOAPIC IRQ */
1078 }
1079 if ((pos = pci_find_capability(dev, PCI_CAP_ID_MSIX)) > 0 &&
1080 !msi_lookup_vector(dev, PCI_CAP_ID_MSIX)) {
1081 int vector, head, tail = 0, warning = 0;
1082 void __iomem *base = NULL;
1083
1084 vector = head = dev->irq;
1085 while (head != tail) {
1086 spin_lock_irqsave(&msi_lock, flags);
1087 state = msi_desc[vector]->msi_attrib.state;
1088 tail = msi_desc[vector]->link.tail;
1089 base = msi_desc[vector]->mask_base;
1090 spin_unlock_irqrestore(&msi_lock, flags);
1091 if (state)
1092 warning = 1;
1093 else if (vector != head) /* Release MSI-X vector */
1094 msi_free_vector(dev, vector, 0);
1095 vector = tail;
1096 }
1097 msi_free_vector(dev, vector, 0);
1098 if (warning) {
1099 /* Force to release the MSI-X memory-mapped table */
1100 u32 phys_addr, table_offset;
1101 u16 control;
1102 u8 bir;
1103
1104 pci_read_config_word(dev, msi_control_reg(pos),
1105 &control);
1106 pci_read_config_dword(dev, msix_table_offset_reg(pos),
1107 &table_offset);
1108 bir = (u8)(table_offset & PCI_MSIX_FLAGS_BIRMASK);
1109 phys_addr = pci_resource_start (dev, bir);
1110 phys_addr += (u32)(table_offset &
1111 ~PCI_MSIX_FLAGS_BIRMASK);
1112 iounmap(base);
1113 printk(KERN_WARNING "PCI: %s: msi_remove_pci_irq_vectors() "
1114 "called without free_irq() on all MSI-X vectors\n",
1115 pci_name(dev));
1116 BUG_ON(warning > 0);
1117 }
1118 dev->irq = temp; /* Restore IOAPIC IRQ */
1119 }
1120}
1121
1122EXPORT_SYMBOL(pci_enable_msi);
1123EXPORT_SYMBOL(pci_disable_msi);
1124EXPORT_SYMBOL(pci_enable_msix);
1125EXPORT_SYMBOL(pci_disable_msix);