blob: 2360cb6a8896a258d79518c29caba9b2e001ce1f [file] [log] [blame]
Yinghai Lu5aeecaf2008-08-19 20:49:59 -07001#include <linux/interrupt.h>
Suresh Siddhaad3ad3f2008-07-10 11:16:40 -07002#include <linux/dmar.h>
Suresh Siddha2ae21012008-07-10 11:16:43 -07003#include <linux/spinlock.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +09004#include <linux/slab.h>
Suresh Siddha2ae21012008-07-10 11:16:43 -07005#include <linux/jiffies.h>
Suresh Siddha20f30972009-08-04 12:07:08 -07006#include <linux/hpet.h>
Suresh Siddha2ae21012008-07-10 11:16:43 -07007#include <linux/pci.h>
Suresh Siddhab6fcb332008-07-10 11:16:44 -07008#include <linux/irq.h>
Lv Zheng8b484632013-12-03 08:49:16 +08009#include <linux/intel-iommu.h>
10#include <linux/acpi.h>
Suresh Siddhaad3ad3f2008-07-10 11:16:40 -070011#include <asm/io_apic.h>
Yinghai Lu17483a12008-12-12 13:14:18 -080012#include <asm/smp.h>
Jaswinder Singh Rajput6d652ea2009-01-07 21:38:59 +053013#include <asm/cpu.h>
Suresh Siddha8a8f4222012-03-30 11:47:08 -070014#include <asm/irq_remapping.h>
Weidong Hanf007e992009-05-23 00:41:15 +080015#include <asm/pci-direct.h>
Joerg Roedel5e2b9302012-03-30 11:47:05 -070016#include <asm/msidef.h>
Suresh Siddhaad3ad3f2008-07-10 11:16:40 -070017
Suresh Siddha8a8f4222012-03-30 11:47:08 -070018#include "irq_remapping.h"
Joerg Roedel736baef2012-03-30 11:47:00 -070019
Joerg Roedeleef93fd2012-03-30 11:46:59 -070020struct ioapic_scope {
21 struct intel_iommu *iommu;
22 unsigned int id;
23 unsigned int bus; /* PCI bus number */
24 unsigned int devfn; /* PCI devfn number */
25};
26
27struct hpet_scope {
28 struct intel_iommu *iommu;
29 u8 id;
30 unsigned int bus;
31 unsigned int devfn;
32};
33
34#define IR_X2APIC_MODE(mode) (mode ? (1 << 11) : 0)
Joerg Roedel0c3f1732012-03-30 11:47:02 -070035#define IRTE_DEST(dest) ((x2apic_mode) ? dest : dest << 8)
Joerg Roedeleef93fd2012-03-30 11:46:59 -070036
Suresh Siddhaad3ad3f2008-07-10 11:16:40 -070037static struct ioapic_scope ir_ioapic[MAX_IO_APICS];
Suresh Siddha20f30972009-08-04 12:07:08 -070038static struct hpet_scope ir_hpet[MAX_HPET_TBS];
Chris Wrightd1423d52010-07-20 11:06:49 -070039
Jiang Liu3a5670e2014-02-19 14:07:33 +080040/*
41 * Lock ordering:
42 * ->dmar_global_lock
43 * ->irq_2_ir_lock
44 * ->qi->q_lock
45 * ->iommu->register_lock
46 * Note:
47 * intel_irq_remap_ops.{supported,prepare,enable,disable,reenable} are called
48 * in single-threaded environment with interrupt disabled, so no need to tabke
49 * the dmar_global_lock.
50 */
Thomas Gleixner96f8e982011-07-19 16:28:19 +020051static DEFINE_RAW_SPINLOCK(irq_2_ir_lock);
Thomas Gleixnerd585d062010-10-10 12:34:27 +020052
Jiang Liu694835d2014-01-06 14:18:16 +080053static int __init parse_ioapics_under_ir(void);
54
Yinghai Lue420dfb2008-08-19 20:50:21 -070055static struct irq_2_iommu *irq_2_iommu(unsigned int irq)
56{
Jiang Liu91411da2014-10-27 16:12:09 +080057 struct irq_cfg *cfg = irq_cfg(irq);
Thomas Gleixner349d6762010-10-10 12:29:27 +020058 return cfg ? &cfg->irq_2_iommu : NULL;
Yinghai Lu0b8f1ef2008-12-05 18:58:31 -080059}
60
Rashika Kheria6a7885c2013-12-18 12:04:27 +053061static int get_irte(int irq, struct irte *entry)
Suresh Siddhab6fcb332008-07-10 11:16:44 -070062{
Thomas Gleixnerd585d062010-10-10 12:34:27 +020063 struct irq_2_iommu *irq_iommu = irq_2_iommu(irq);
Suresh Siddha4c5502b2009-03-16 17:04:53 -070064 unsigned long flags;
Thomas Gleixnerd585d062010-10-10 12:34:27 +020065 int index;
Suresh Siddhab6fcb332008-07-10 11:16:44 -070066
Thomas Gleixnerd585d062010-10-10 12:34:27 +020067 if (!entry || !irq_iommu)
Suresh Siddhab6fcb332008-07-10 11:16:44 -070068 return -1;
69
Thomas Gleixner96f8e982011-07-19 16:28:19 +020070 raw_spin_lock_irqsave(&irq_2_ir_lock, flags);
Suresh Siddhab6fcb332008-07-10 11:16:44 -070071
Greg Edwardsaf437462014-07-23 10:13:26 -060072 if (unlikely(!irq_iommu->iommu)) {
73 raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags);
74 return -1;
75 }
76
Yinghai Lue420dfb2008-08-19 20:50:21 -070077 index = irq_iommu->irte_index + irq_iommu->sub_handle;
78 *entry = *(irq_iommu->iommu->ir_table->base + index);
Suresh Siddhab6fcb332008-07-10 11:16:44 -070079
Thomas Gleixner96f8e982011-07-19 16:28:19 +020080 raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags);
Suresh Siddhab6fcb332008-07-10 11:16:44 -070081 return 0;
82}
83
Joerg Roedel263b5e82012-03-30 11:47:06 -070084static int alloc_irte(struct intel_iommu *iommu, int irq, u16 count)
Suresh Siddhab6fcb332008-07-10 11:16:44 -070085{
86 struct ir_table *table = iommu->ir_table;
Thomas Gleixnerd585d062010-10-10 12:34:27 +020087 struct irq_2_iommu *irq_iommu = irq_2_iommu(irq);
Jiang Liu91411da2014-10-27 16:12:09 +080088 struct irq_cfg *cfg = irq_cfg(irq);
Suresh Siddhab6fcb332008-07-10 11:16:44 -070089 unsigned int mask = 0;
Suresh Siddha4c5502b2009-03-16 17:04:53 -070090 unsigned long flags;
Dan Carpenter9f4c7442014-01-09 08:32:36 +030091 int index;
Suresh Siddhab6fcb332008-07-10 11:16:44 -070092
Thomas Gleixnerd585d062010-10-10 12:34:27 +020093 if (!count || !irq_iommu)
Suresh Siddhab6fcb332008-07-10 11:16:44 -070094 return -1;
95
Suresh Siddhab6fcb332008-07-10 11:16:44 -070096 if (count > 1) {
97 count = __roundup_pow_of_two(count);
98 mask = ilog2(count);
99 }
100
101 if (mask > ecap_max_handle_mask(iommu->ecap)) {
102 printk(KERN_ERR
103 "Requested mask %x exceeds the max invalidation handle"
104 " mask value %Lx\n", mask,
105 ecap_max_handle_mask(iommu->ecap));
106 return -1;
107 }
108
Thomas Gleixner96f8e982011-07-19 16:28:19 +0200109 raw_spin_lock_irqsave(&irq_2_ir_lock, flags);
Jiang Liu360eb3c2014-01-06 14:18:08 +0800110 index = bitmap_find_free_region(table->bitmap,
111 INTR_REMAP_TABLE_ENTRIES, mask);
112 if (index < 0) {
113 pr_warn("IR%d: can't allocate an IRTE\n", iommu->seq_id);
114 } else {
115 cfg->remapped = 1;
116 irq_iommu->iommu = iommu;
117 irq_iommu->irte_index = index;
118 irq_iommu->sub_handle = 0;
119 irq_iommu->irte_mask = mask;
120 }
Thomas Gleixner96f8e982011-07-19 16:28:19 +0200121 raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags);
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700122
123 return index;
124}
125
Yu Zhao704126a2009-01-04 16:28:52 +0800126static int qi_flush_iec(struct intel_iommu *iommu, int index, int mask)
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700127{
128 struct qi_desc desc;
129
130 desc.low = QI_IEC_IIDEX(index) | QI_IEC_TYPE | QI_IEC_IM(mask)
131 | QI_IEC_SELECTIVE;
132 desc.high = 0;
133
Yu Zhao704126a2009-01-04 16:28:52 +0800134 return qi_submit_sync(&desc, iommu);
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700135}
136
Joerg Roedel263b5e82012-03-30 11:47:06 -0700137static int map_irq_to_irte_handle(int irq, u16 *sub_handle)
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700138{
Thomas Gleixnerd585d062010-10-10 12:34:27 +0200139 struct irq_2_iommu *irq_iommu = irq_2_iommu(irq);
Suresh Siddha4c5502b2009-03-16 17:04:53 -0700140 unsigned long flags;
Thomas Gleixnerd585d062010-10-10 12:34:27 +0200141 int index;
142
143 if (!irq_iommu)
144 return -1;
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700145
Thomas Gleixner96f8e982011-07-19 16:28:19 +0200146 raw_spin_lock_irqsave(&irq_2_ir_lock, flags);
Yinghai Lue420dfb2008-08-19 20:50:21 -0700147 *sub_handle = irq_iommu->sub_handle;
148 index = irq_iommu->irte_index;
Thomas Gleixner96f8e982011-07-19 16:28:19 +0200149 raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags);
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700150 return index;
151}
152
Joerg Roedel263b5e82012-03-30 11:47:06 -0700153static int set_irte_irq(int irq, struct intel_iommu *iommu, u16 index, u16 subhandle)
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700154{
Thomas Gleixnerd585d062010-10-10 12:34:27 +0200155 struct irq_2_iommu *irq_iommu = irq_2_iommu(irq);
Jiang Liu91411da2014-10-27 16:12:09 +0800156 struct irq_cfg *cfg = irq_cfg(irq);
Suresh Siddha4c5502b2009-03-16 17:04:53 -0700157 unsigned long flags;
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700158
Thomas Gleixnerd585d062010-10-10 12:34:27 +0200159 if (!irq_iommu)
Yinghai Lu0b8f1ef2008-12-05 18:58:31 -0800160 return -1;
Thomas Gleixnerd585d062010-10-10 12:34:27 +0200161
Thomas Gleixner96f8e982011-07-19 16:28:19 +0200162 raw_spin_lock_irqsave(&irq_2_ir_lock, flags);
Yinghai Lu0b8f1ef2008-12-05 18:58:31 -0800163
Joerg Roedel9b1b0e42012-09-26 12:44:45 +0200164 cfg->remapped = 1;
Yinghai Lue420dfb2008-08-19 20:50:21 -0700165 irq_iommu->iommu = iommu;
166 irq_iommu->irte_index = index;
167 irq_iommu->sub_handle = subhandle;
168 irq_iommu->irte_mask = 0;
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700169
Thomas Gleixner96f8e982011-07-19 16:28:19 +0200170 raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags);
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700171
172 return 0;
173}
174
Joerg Roedel263b5e82012-03-30 11:47:06 -0700175static int modify_irte(int irq, struct irte *irte_modified)
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700176{
Thomas Gleixnerd585d062010-10-10 12:34:27 +0200177 struct irq_2_iommu *irq_iommu = irq_2_iommu(irq);
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700178 struct intel_iommu *iommu;
Suresh Siddha4c5502b2009-03-16 17:04:53 -0700179 unsigned long flags;
Thomas Gleixnerd585d062010-10-10 12:34:27 +0200180 struct irte *irte;
181 int rc, index;
182
183 if (!irq_iommu)
184 return -1;
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700185
Thomas Gleixner96f8e982011-07-19 16:28:19 +0200186 raw_spin_lock_irqsave(&irq_2_ir_lock, flags);
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700187
Yinghai Lue420dfb2008-08-19 20:50:21 -0700188 iommu = irq_iommu->iommu;
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700189
Yinghai Lue420dfb2008-08-19 20:50:21 -0700190 index = irq_iommu->irte_index + irq_iommu->sub_handle;
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700191 irte = &iommu->ir_table->base[index];
192
Linus Torvaldsc513b672010-08-06 11:02:31 -0700193 set_64bit(&irte->low, irte_modified->low);
194 set_64bit(&irte->high, irte_modified->high);
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700195 __iommu_flush_cache(iommu, irte, sizeof(*irte));
196
Yu Zhao704126a2009-01-04 16:28:52 +0800197 rc = qi_flush_iec(iommu, index, 0);
Thomas Gleixner96f8e982011-07-19 16:28:19 +0200198 raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags);
Yu Zhao704126a2009-01-04 16:28:52 +0800199
200 return rc;
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700201}
202
Joerg Roedel263b5e82012-03-30 11:47:06 -0700203static struct intel_iommu *map_hpet_to_ir(u8 hpet_id)
Suresh Siddha20f30972009-08-04 12:07:08 -0700204{
205 int i;
206
207 for (i = 0; i < MAX_HPET_TBS; i++)
Jiang Liua7a3dad2014-11-09 22:48:00 +0800208 if (ir_hpet[i].id == hpet_id && ir_hpet[i].iommu)
Suresh Siddha20f30972009-08-04 12:07:08 -0700209 return ir_hpet[i].iommu;
210 return NULL;
211}
212
Joerg Roedel263b5e82012-03-30 11:47:06 -0700213static struct intel_iommu *map_ioapic_to_ir(int apic)
Suresh Siddha89027d32008-07-10 11:16:56 -0700214{
215 int i;
216
217 for (i = 0; i < MAX_IO_APICS; i++)
Jiang Liua7a3dad2014-11-09 22:48:00 +0800218 if (ir_ioapic[i].id == apic && ir_ioapic[i].iommu)
Suresh Siddha89027d32008-07-10 11:16:56 -0700219 return ir_ioapic[i].iommu;
220 return NULL;
221}
222
Joerg Roedel263b5e82012-03-30 11:47:06 -0700223static struct intel_iommu *map_dev_to_ir(struct pci_dev *dev)
Suresh Siddha75c46fa2008-07-10 11:16:57 -0700224{
225 struct dmar_drhd_unit *drhd;
226
227 drhd = dmar_find_matched_drhd_unit(dev);
228 if (!drhd)
229 return NULL;
230
231 return drhd->iommu;
232}
233
Weidong Hanc4658b42009-05-23 00:41:14 +0800234static int clear_entries(struct irq_2_iommu *irq_iommu)
235{
236 struct irte *start, *entry, *end;
237 struct intel_iommu *iommu;
238 int index;
239
240 if (irq_iommu->sub_handle)
241 return 0;
242
243 iommu = irq_iommu->iommu;
244 index = irq_iommu->irte_index + irq_iommu->sub_handle;
245
246 start = iommu->ir_table->base + index;
247 end = start + (1 << irq_iommu->irte_mask);
248
249 for (entry = start; entry < end; entry++) {
Linus Torvaldsc513b672010-08-06 11:02:31 -0700250 set_64bit(&entry->low, 0);
251 set_64bit(&entry->high, 0);
Weidong Hanc4658b42009-05-23 00:41:14 +0800252 }
Jiang Liu360eb3c2014-01-06 14:18:08 +0800253 bitmap_release_region(iommu->ir_table->bitmap, index,
254 irq_iommu->irte_mask);
Weidong Hanc4658b42009-05-23 00:41:14 +0800255
256 return qi_flush_iec(iommu, index, irq_iommu->irte_mask);
257}
258
Joerg Roedel9d619f62012-03-30 11:47:04 -0700259static int free_irte(int irq)
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700260{
Thomas Gleixnerd585d062010-10-10 12:34:27 +0200261 struct irq_2_iommu *irq_iommu = irq_2_iommu(irq);
Suresh Siddha4c5502b2009-03-16 17:04:53 -0700262 unsigned long flags;
Thomas Gleixnerd585d062010-10-10 12:34:27 +0200263 int rc;
264
265 if (!irq_iommu)
266 return -1;
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700267
Thomas Gleixner96f8e982011-07-19 16:28:19 +0200268 raw_spin_lock_irqsave(&irq_2_ir_lock, flags);
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700269
Weidong Hanc4658b42009-05-23 00:41:14 +0800270 rc = clear_entries(irq_iommu);
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700271
Yinghai Lue420dfb2008-08-19 20:50:21 -0700272 irq_iommu->iommu = NULL;
273 irq_iommu->irte_index = 0;
274 irq_iommu->sub_handle = 0;
275 irq_iommu->irte_mask = 0;
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700276
Thomas Gleixner96f8e982011-07-19 16:28:19 +0200277 raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags);
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700278
Yu Zhao704126a2009-01-04 16:28:52 +0800279 return rc;
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700280}
281
Weidong Hanf007e992009-05-23 00:41:15 +0800282/*
283 * source validation type
284 */
285#define SVT_NO_VERIFY 0x0 /* no verification is required */
Lucas De Marchi25985ed2011-03-30 22:57:33 -0300286#define SVT_VERIFY_SID_SQ 0x1 /* verify using SID and SQ fields */
Weidong Hanf007e992009-05-23 00:41:15 +0800287#define SVT_VERIFY_BUS 0x2 /* verify bus of request-id */
288
289/*
290 * source-id qualifier
291 */
292#define SQ_ALL_16 0x0 /* verify all 16 bits of request-id */
293#define SQ_13_IGNORE_1 0x1 /* verify most significant 13 bits, ignore
294 * the third least significant bit
295 */
296#define SQ_13_IGNORE_2 0x2 /* verify most significant 13 bits, ignore
297 * the second and third least significant bits
298 */
299#define SQ_13_IGNORE_3 0x3 /* verify most significant 13 bits, ignore
300 * the least three significant bits
301 */
302
303/*
304 * set SVT, SQ and SID fields of irte to verify
305 * source ids of interrupt requests
306 */
307static void set_irte_sid(struct irte *irte, unsigned int svt,
308 unsigned int sq, unsigned int sid)
309{
Chris Wrightd1423d52010-07-20 11:06:49 -0700310 if (disable_sourceid_checking)
311 svt = SVT_NO_VERIFY;
Weidong Hanf007e992009-05-23 00:41:15 +0800312 irte->svt = svt;
313 irte->sq = sq;
314 irte->sid = sid;
315}
316
Joerg Roedel263b5e82012-03-30 11:47:06 -0700317static int set_ioapic_sid(struct irte *irte, int apic)
Weidong Hanf007e992009-05-23 00:41:15 +0800318{
319 int i;
320 u16 sid = 0;
321
322 if (!irte)
323 return -1;
324
Jiang Liu3a5670e2014-02-19 14:07:33 +0800325 down_read(&dmar_global_lock);
Weidong Hanf007e992009-05-23 00:41:15 +0800326 for (i = 0; i < MAX_IO_APICS; i++) {
Jiang Liua7a3dad2014-11-09 22:48:00 +0800327 if (ir_ioapic[i].iommu && ir_ioapic[i].id == apic) {
Weidong Hanf007e992009-05-23 00:41:15 +0800328 sid = (ir_ioapic[i].bus << 8) | ir_ioapic[i].devfn;
329 break;
330 }
331 }
Jiang Liu3a5670e2014-02-19 14:07:33 +0800332 up_read(&dmar_global_lock);
Weidong Hanf007e992009-05-23 00:41:15 +0800333
334 if (sid == 0) {
335 pr_warning("Failed to set source-id of IOAPIC (%d)\n", apic);
336 return -1;
337 }
338
Jiang Liu2fe2c602014-01-06 14:18:17 +0800339 set_irte_sid(irte, SVT_VERIFY_SID_SQ, SQ_ALL_16, sid);
Weidong Hanf007e992009-05-23 00:41:15 +0800340
341 return 0;
342}
343
Joerg Roedel263b5e82012-03-30 11:47:06 -0700344static int set_hpet_sid(struct irte *irte, u8 id)
Suresh Siddha20f30972009-08-04 12:07:08 -0700345{
346 int i;
347 u16 sid = 0;
348
349 if (!irte)
350 return -1;
351
Jiang Liu3a5670e2014-02-19 14:07:33 +0800352 down_read(&dmar_global_lock);
Suresh Siddha20f30972009-08-04 12:07:08 -0700353 for (i = 0; i < MAX_HPET_TBS; i++) {
Jiang Liua7a3dad2014-11-09 22:48:00 +0800354 if (ir_hpet[i].iommu && ir_hpet[i].id == id) {
Suresh Siddha20f30972009-08-04 12:07:08 -0700355 sid = (ir_hpet[i].bus << 8) | ir_hpet[i].devfn;
356 break;
357 }
358 }
Jiang Liu3a5670e2014-02-19 14:07:33 +0800359 up_read(&dmar_global_lock);
Suresh Siddha20f30972009-08-04 12:07:08 -0700360
361 if (sid == 0) {
362 pr_warning("Failed to set source-id of HPET block (%d)\n", id);
363 return -1;
364 }
365
366 /*
367 * Should really use SQ_ALL_16. Some platforms are broken.
368 * While we figure out the right quirks for these broken platforms, use
369 * SQ_13_IGNORE_3 for now.
370 */
371 set_irte_sid(irte, SVT_VERIFY_SID_SQ, SQ_13_IGNORE_3, sid);
372
373 return 0;
374}
375
Alex Williamson579305f2014-07-03 09:51:43 -0600376struct set_msi_sid_data {
377 struct pci_dev *pdev;
378 u16 alias;
379};
380
381static int set_msi_sid_cb(struct pci_dev *pdev, u16 alias, void *opaque)
382{
383 struct set_msi_sid_data *data = opaque;
384
385 data->pdev = pdev;
386 data->alias = alias;
387
388 return 0;
389}
390
Joerg Roedel263b5e82012-03-30 11:47:06 -0700391static int set_msi_sid(struct irte *irte, struct pci_dev *dev)
Weidong Hanf007e992009-05-23 00:41:15 +0800392{
Alex Williamson579305f2014-07-03 09:51:43 -0600393 struct set_msi_sid_data data;
Weidong Hanf007e992009-05-23 00:41:15 +0800394
395 if (!irte || !dev)
396 return -1;
397
Alex Williamson579305f2014-07-03 09:51:43 -0600398 pci_for_each_dma_alias(dev, set_msi_sid_cb, &data);
Weidong Hanf007e992009-05-23 00:41:15 +0800399
Alex Williamson579305f2014-07-03 09:51:43 -0600400 /*
401 * DMA alias provides us with a PCI device and alias. The only case
402 * where the it will return an alias on a different bus than the
403 * device is the case of a PCIe-to-PCI bridge, where the alias is for
404 * the subordinate bus. In this case we can only verify the bus.
405 *
406 * If the alias device is on a different bus than our source device
407 * then we have a topology based alias, use it.
408 *
409 * Otherwise, the alias is for a device DMA quirk and we cannot
410 * assume that MSI uses the same requester ID. Therefore use the
411 * original device.
412 */
413 if (PCI_BUS_NUM(data.alias) != data.pdev->bus->number)
414 set_irte_sid(irte, SVT_VERIFY_BUS, SQ_ALL_16,
415 PCI_DEVID(PCI_BUS_NUM(data.alias),
416 dev->bus->number));
417 else if (data.pdev->bus->number != dev->bus->number)
418 set_irte_sid(irte, SVT_VERIFY_SID_SQ, SQ_ALL_16, data.alias);
419 else
420 set_irte_sid(irte, SVT_VERIFY_SID_SQ, SQ_ALL_16,
421 PCI_DEVID(dev->bus->number, dev->devfn));
Weidong Hanf007e992009-05-23 00:41:15 +0800422
423 return 0;
424}
425
Suresh Siddha95a02e92012-03-30 11:47:07 -0700426static void iommu_set_irq_remapping(struct intel_iommu *iommu, int mode)
Suresh Siddha2ae21012008-07-10 11:16:43 -0700427{
428 u64 addr;
David Woodhousec416daa2009-05-10 20:30:58 +0100429 u32 sts;
Suresh Siddha2ae21012008-07-10 11:16:43 -0700430 unsigned long flags;
431
432 addr = virt_to_phys((void *)iommu->ir_table->base);
433
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +0200434 raw_spin_lock_irqsave(&iommu->register_lock, flags);
Suresh Siddha2ae21012008-07-10 11:16:43 -0700435
436 dmar_writeq(iommu->reg + DMAR_IRTA_REG,
437 (addr) | IR_X2APIC_MODE(mode) | INTR_REMAP_TABLE_REG_SIZE);
438
439 /* Set interrupt-remapping table pointer */
Jan Kiszkaf63ef692014-08-11 13:13:25 +0200440 writel(iommu->gcmd | DMA_GCMD_SIRTP, iommu->reg + DMAR_GCMD_REG);
Suresh Siddha2ae21012008-07-10 11:16:43 -0700441
442 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
443 readl, (sts & DMA_GSTS_IRTPS), sts);
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +0200444 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
Suresh Siddha2ae21012008-07-10 11:16:43 -0700445
446 /*
447 * global invalidation of interrupt entry cache before enabling
448 * interrupt-remapping.
449 */
450 qi_global_iec(iommu);
451
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +0200452 raw_spin_lock_irqsave(&iommu->register_lock, flags);
Suresh Siddha2ae21012008-07-10 11:16:43 -0700453
454 /* Enable interrupt-remapping */
Suresh Siddha2ae21012008-07-10 11:16:43 -0700455 iommu->gcmd |= DMA_GCMD_IRE;
Andy Lutomirskiaf8d1022013-02-01 14:57:43 -0800456 iommu->gcmd &= ~DMA_GCMD_CFI; /* Block compatibility-format MSIs */
David Woodhousec416daa2009-05-10 20:30:58 +0100457 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
Suresh Siddha2ae21012008-07-10 11:16:43 -0700458
459 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
460 readl, (sts & DMA_GSTS_IRES), sts);
461
Andy Lutomirskiaf8d1022013-02-01 14:57:43 -0800462 /*
463 * With CFI clear in the Global Command register, we should be
464 * protected from dangerous (i.e. compatibility) interrupts
465 * regardless of x2apic status. Check just to be sure.
466 */
467 if (sts & DMA_GSTS_CFIS)
468 WARN(1, KERN_WARNING
469 "Compatibility-format IRQs enabled despite intr remapping;\n"
470 "you are vulnerable to IRQ injection.\n");
471
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +0200472 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
Suresh Siddha2ae21012008-07-10 11:16:43 -0700473}
474
Jiang Liua7a3dad2014-11-09 22:48:00 +0800475static int intel_setup_irq_remapping(struct intel_iommu *iommu)
Suresh Siddha2ae21012008-07-10 11:16:43 -0700476{
477 struct ir_table *ir_table;
478 struct page *pages;
Jiang Liu360eb3c2014-01-06 14:18:08 +0800479 unsigned long *bitmap;
Suresh Siddha2ae21012008-07-10 11:16:43 -0700480
Jiang Liua7a3dad2014-11-09 22:48:00 +0800481 if (iommu->ir_table)
482 return 0;
Suresh Siddha2ae21012008-07-10 11:16:43 -0700483
Jiang Liua7a3dad2014-11-09 22:48:00 +0800484 ir_table = kzalloc(sizeof(struct ir_table), GFP_ATOMIC);
485 if (!ir_table)
Suresh Siddha2ae21012008-07-10 11:16:43 -0700486 return -ENOMEM;
487
Suresh Siddha824cd752009-10-02 11:01:23 -0700488 pages = alloc_pages_node(iommu->node, GFP_ATOMIC | __GFP_ZERO,
489 INTR_REMAP_PAGE_ORDER);
Suresh Siddha2ae21012008-07-10 11:16:43 -0700490
491 if (!pages) {
Jiang Liu360eb3c2014-01-06 14:18:08 +0800492 pr_err("IR%d: failed to allocate pages of order %d\n",
493 iommu->seq_id, INTR_REMAP_PAGE_ORDER);
Jiang Liua7a3dad2014-11-09 22:48:00 +0800494 goto out_free_table;
Suresh Siddha2ae21012008-07-10 11:16:43 -0700495 }
496
Jiang Liu360eb3c2014-01-06 14:18:08 +0800497 bitmap = kcalloc(BITS_TO_LONGS(INTR_REMAP_TABLE_ENTRIES),
498 sizeof(long), GFP_ATOMIC);
499 if (bitmap == NULL) {
500 pr_err("IR%d: failed to allocate bitmap\n", iommu->seq_id);
Jiang Liua7a3dad2014-11-09 22:48:00 +0800501 goto out_free_pages;
Jiang Liu360eb3c2014-01-06 14:18:08 +0800502 }
503
Suresh Siddha2ae21012008-07-10 11:16:43 -0700504 ir_table->base = page_address(pages);
Jiang Liu360eb3c2014-01-06 14:18:08 +0800505 ir_table->bitmap = bitmap;
Jiang Liua7a3dad2014-11-09 22:48:00 +0800506 iommu->ir_table = ir_table;
Suresh Siddha2ae21012008-07-10 11:16:43 -0700507 return 0;
Jiang Liua7a3dad2014-11-09 22:48:00 +0800508
509out_free_pages:
510 __free_pages(pages, INTR_REMAP_PAGE_ORDER);
511out_free_table:
512 kfree(ir_table);
513 return -ENOMEM;
514}
515
516static void intel_teardown_irq_remapping(struct intel_iommu *iommu)
517{
518 if (iommu && iommu->ir_table) {
519 free_pages((unsigned long)iommu->ir_table->base,
520 INTR_REMAP_PAGE_ORDER);
521 kfree(iommu->ir_table->bitmap);
522 kfree(iommu->ir_table);
523 iommu->ir_table = NULL;
524 }
Suresh Siddha2ae21012008-07-10 11:16:43 -0700525}
526
Suresh Siddhaeba67e52009-03-16 17:04:56 -0700527/*
528 * Disable Interrupt Remapping.
529 */
Suresh Siddha95a02e92012-03-30 11:47:07 -0700530static void iommu_disable_irq_remapping(struct intel_iommu *iommu)
Suresh Siddhaeba67e52009-03-16 17:04:56 -0700531{
532 unsigned long flags;
533 u32 sts;
534
535 if (!ecap_ir_support(iommu->ecap))
536 return;
537
Fenghua Yub24696b2009-03-27 14:22:44 -0700538 /*
539 * global invalidation of interrupt entry cache before disabling
540 * interrupt-remapping.
541 */
542 qi_global_iec(iommu);
543
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +0200544 raw_spin_lock_irqsave(&iommu->register_lock, flags);
Suresh Siddhaeba67e52009-03-16 17:04:56 -0700545
546 sts = dmar_readq(iommu->reg + DMAR_GSTS_REG);
547 if (!(sts & DMA_GSTS_IRES))
548 goto end;
549
550 iommu->gcmd &= ~DMA_GCMD_IRE;
551 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
552
553 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
554 readl, !(sts & DMA_GSTS_IRES), sts);
555
556end:
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +0200557 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
Suresh Siddhaeba67e52009-03-16 17:04:56 -0700558}
559
Suresh Siddha41750d32011-08-23 17:05:18 -0700560static int __init dmar_x2apic_optout(void)
561{
562 struct acpi_table_dmar *dmar;
563 dmar = (struct acpi_table_dmar *)dmar_tbl;
564 if (!dmar || no_x2apic_optout)
565 return 0;
566 return dmar->flags & DMAR_X2APIC_OPT_OUT;
567}
568
Suresh Siddha95a02e92012-03-30 11:47:07 -0700569static int __init intel_irq_remapping_supported(void)
Weidong Han93758232009-04-17 16:42:14 +0800570{
571 struct dmar_drhd_unit *drhd;
Jiang Liu7c919772014-01-06 14:18:18 +0800572 struct intel_iommu *iommu;
Weidong Han93758232009-04-17 16:42:14 +0800573
Suresh Siddha95a02e92012-03-30 11:47:07 -0700574 if (disable_irq_remap)
Weidong Han03ea8152009-04-17 16:42:15 +0800575 return 0;
Neil Horman03bbcb22013-04-16 16:38:32 -0400576 if (irq_remap_broken) {
Neil Horman05104a42013-09-27 12:53:35 -0400577 printk(KERN_WARNING
578 "This system BIOS has enabled interrupt remapping\n"
579 "on a chipset that contains an erratum making that\n"
580 "feature unstable. To maintain system stability\n"
581 "interrupt remapping is being disabled. Please\n"
582 "contact your BIOS vendor for an update\n");
583 add_taint(TAINT_FIRMWARE_WORKAROUND, LOCKDEP_STILL_OK);
Neil Horman03bbcb22013-04-16 16:38:32 -0400584 disable_irq_remap = 1;
585 return 0;
586 }
Weidong Han03ea8152009-04-17 16:42:15 +0800587
Youquan Song074835f2009-09-09 12:05:39 -0400588 if (!dmar_ir_support())
589 return 0;
590
Jiang Liu7c919772014-01-06 14:18:18 +0800591 for_each_iommu(iommu, drhd)
Weidong Han93758232009-04-17 16:42:14 +0800592 if (!ecap_ir_support(iommu->ecap))
593 return 0;
Weidong Han93758232009-04-17 16:42:14 +0800594
595 return 1;
596}
597
Thomas Gleixner11190302015-01-07 15:31:29 +0800598static void __init intel_cleanup_irq_remapping(void)
599{
600 struct dmar_drhd_unit *drhd;
601 struct intel_iommu *iommu;
602
603 for_each_iommu(iommu, drhd) {
604 if (ecap_ir_support(iommu->ecap)) {
605 iommu_disable_irq_remapping(iommu);
606 intel_teardown_irq_remapping(iommu);
607 }
608 }
609
610 if (x2apic_supported())
611 pr_warn("Failed to enable irq remapping. You are vulnerable to irq-injection attacks.\n");
612}
613
614static int __init intel_prepare_irq_remapping(void)
615{
616 struct dmar_drhd_unit *drhd;
617 struct intel_iommu *iommu;
618
619 if (dmar_table_init() < 0)
620 return -1;
621
622 if (parse_ioapics_under_ir() != 1) {
623 printk(KERN_INFO "Not enabling interrupt remapping\n");
624 goto error;
625 }
626
627 for_each_iommu(iommu, drhd) {
628 if (!ecap_ir_support(iommu->ecap))
629 continue;
630
631 /* Do the allocations early */
632 if (intel_setup_irq_remapping(iommu))
633 goto error;
634 }
635 return 0;
636error:
637 intel_cleanup_irq_remapping();
638 return -1;
639}
640
Suresh Siddha95a02e92012-03-30 11:47:07 -0700641static int __init intel_enable_irq_remapping(void)
Suresh Siddha2ae21012008-07-10 11:16:43 -0700642{
643 struct dmar_drhd_unit *drhd;
Jiang Liu7c919772014-01-06 14:18:18 +0800644 struct intel_iommu *iommu;
Suresh Siddha2ae21012008-07-10 11:16:43 -0700645 int setup = 0;
Suresh Siddha41750d32011-08-23 17:05:18 -0700646 int eim = 0;
Suresh Siddha2ae21012008-07-10 11:16:43 -0700647
Thomas Gleixner11190302015-01-07 15:31:29 +0800648 if (x2apic_supported()) {
Jiang Liub977e732014-01-06 14:18:14 +0800649 pr_info("Queued invalidation will be enabled to support x2apic and Intr-remapping.\n");
650
Suresh Siddha41750d32011-08-23 17:05:18 -0700651 eim = !dmar_x2apic_optout();
Andy Lutomirskiaf8d1022013-02-01 14:57:43 -0800652 if (!eim)
653 printk(KERN_WARNING
654 "Your BIOS is broken and requested that x2apic be disabled.\n"
655 "This will slightly decrease performance.\n"
656 "Use 'intremap=no_x2apic_optout' to override BIOS request.\n");
Suresh Siddha41750d32011-08-23 17:05:18 -0700657 }
658
Jiang Liu7c919772014-01-06 14:18:18 +0800659 for_each_iommu(iommu, drhd) {
Suresh Siddha1531a6a2009-03-16 17:04:57 -0700660 /*
Han, Weidong34aaaa92009-04-04 17:21:26 +0800661 * If the queued invalidation is already initialized,
662 * shouldn't disable it.
663 */
664 if (iommu->qi)
665 continue;
666
667 /*
Suresh Siddha1531a6a2009-03-16 17:04:57 -0700668 * Clear previous faults.
669 */
670 dmar_fault(-1, iommu);
671
672 /*
673 * Disable intr remapping and queued invalidation, if already
674 * enabled prior to OS handover.
675 */
Suresh Siddha95a02e92012-03-30 11:47:07 -0700676 iommu_disable_irq_remapping(iommu);
Suresh Siddha1531a6a2009-03-16 17:04:57 -0700677
678 dmar_disable_qi(iommu);
679 }
680
Suresh Siddha2ae21012008-07-10 11:16:43 -0700681 /*
682 * check for the Interrupt-remapping support
683 */
Jiang Liu7c919772014-01-06 14:18:18 +0800684 for_each_iommu(iommu, drhd) {
Suresh Siddha2ae21012008-07-10 11:16:43 -0700685 if (!ecap_ir_support(iommu->ecap))
686 continue;
687
688 if (eim && !ecap_eim_support(iommu->ecap)) {
689 printk(KERN_INFO "DRHD %Lx: EIM not supported by DRHD, "
690 " ecap %Lx\n", drhd->reg_base_addr, iommu->ecap);
Andy Lutomirskiaf8d1022013-02-01 14:57:43 -0800691 goto error;
Suresh Siddha2ae21012008-07-10 11:16:43 -0700692 }
693 }
694
695 /*
696 * Enable queued invalidation for all the DRHD's.
697 */
Jiang Liu7c919772014-01-06 14:18:18 +0800698 for_each_iommu(iommu, drhd) {
699 int ret = dmar_enable_qi(iommu);
Suresh Siddha2ae21012008-07-10 11:16:43 -0700700
701 if (ret) {
702 printk(KERN_ERR "DRHD %Lx: failed to enable queued, "
703 " invalidation, ecap %Lx, ret %d\n",
704 drhd->reg_base_addr, iommu->ecap, ret);
Andy Lutomirskiaf8d1022013-02-01 14:57:43 -0800705 goto error;
Suresh Siddha2ae21012008-07-10 11:16:43 -0700706 }
707 }
708
709 /*
710 * Setup Interrupt-remapping for all the DRHD's now.
711 */
Jiang Liu7c919772014-01-06 14:18:18 +0800712 for_each_iommu(iommu, drhd) {
Suresh Siddha2ae21012008-07-10 11:16:43 -0700713 if (!ecap_ir_support(iommu->ecap))
714 continue;
715
Jiang Liua7a3dad2014-11-09 22:48:00 +0800716 iommu_set_irq_remapping(iommu, eim);
Suresh Siddha2ae21012008-07-10 11:16:43 -0700717 setup = 1;
718 }
719
720 if (!setup)
721 goto error;
722
Suresh Siddha95a02e92012-03-30 11:47:07 -0700723 irq_remapping_enabled = 1;
Joerg Roedelafcc8a42012-09-26 12:44:36 +0200724
725 /*
726 * VT-d has a different layout for IO-APIC entries when
727 * interrupt remapping is enabled. So it needs a special routine
728 * to print IO-APIC entries for debugging purposes too.
729 */
730 x86_io_apic_ops.print_entries = intel_ir_io_apic_print_entries;
731
Suresh Siddha41750d32011-08-23 17:05:18 -0700732 pr_info("Enabled IRQ remapping in %s mode\n", eim ? "x2apic" : "xapic");
Suresh Siddha2ae21012008-07-10 11:16:43 -0700733
Suresh Siddha41750d32011-08-23 17:05:18 -0700734 return eim ? IRQ_REMAP_X2APIC_MODE : IRQ_REMAP_XAPIC_MODE;
Suresh Siddha2ae21012008-07-10 11:16:43 -0700735
736error:
Thomas Gleixner11190302015-01-07 15:31:29 +0800737 intel_cleanup_irq_remapping();
Suresh Siddha2ae21012008-07-10 11:16:43 -0700738 return -1;
739}
Suresh Siddhaad3ad3f2008-07-10 11:16:40 -0700740
Jiang Liua7a3dad2014-11-09 22:48:00 +0800741static int ir_parse_one_hpet_scope(struct acpi_dmar_device_scope *scope,
742 struct intel_iommu *iommu,
743 struct acpi_dmar_hardware_unit *drhd)
Suresh Siddha20f30972009-08-04 12:07:08 -0700744{
745 struct acpi_dmar_pci_path *path;
746 u8 bus;
Jiang Liua7a3dad2014-11-09 22:48:00 +0800747 int count, free = -1;
Suresh Siddha20f30972009-08-04 12:07:08 -0700748
749 bus = scope->bus;
750 path = (struct acpi_dmar_pci_path *)(scope + 1);
751 count = (scope->length - sizeof(struct acpi_dmar_device_scope))
752 / sizeof(struct acpi_dmar_pci_path);
753
754 while (--count > 0) {
755 /*
756 * Access PCI directly due to the PCI
757 * subsystem isn't initialized yet.
758 */
Lv Zhengfa5f5082013-10-31 09:30:22 +0800759 bus = read_pci_config_byte(bus, path->device, path->function,
Suresh Siddha20f30972009-08-04 12:07:08 -0700760 PCI_SECONDARY_BUS);
761 path++;
762 }
Jiang Liua7a3dad2014-11-09 22:48:00 +0800763
764 for (count = 0; count < MAX_HPET_TBS; count++) {
765 if (ir_hpet[count].iommu == iommu &&
766 ir_hpet[count].id == scope->enumeration_id)
767 return 0;
768 else if (ir_hpet[count].iommu == NULL && free == -1)
769 free = count;
770 }
771 if (free == -1) {
772 pr_warn("Exceeded Max HPET blocks\n");
773 return -ENOSPC;
774 }
775
776 ir_hpet[free].iommu = iommu;
777 ir_hpet[free].id = scope->enumeration_id;
778 ir_hpet[free].bus = bus;
779 ir_hpet[free].devfn = PCI_DEVFN(path->device, path->function);
780 pr_info("HPET id %d under DRHD base 0x%Lx\n",
781 scope->enumeration_id, drhd->address);
782
783 return 0;
Suresh Siddha20f30972009-08-04 12:07:08 -0700784}
785
Jiang Liua7a3dad2014-11-09 22:48:00 +0800786static int ir_parse_one_ioapic_scope(struct acpi_dmar_device_scope *scope,
787 struct intel_iommu *iommu,
788 struct acpi_dmar_hardware_unit *drhd)
Weidong Hanf007e992009-05-23 00:41:15 +0800789{
790 struct acpi_dmar_pci_path *path;
791 u8 bus;
Jiang Liua7a3dad2014-11-09 22:48:00 +0800792 int count, free = -1;
Weidong Hanf007e992009-05-23 00:41:15 +0800793
794 bus = scope->bus;
795 path = (struct acpi_dmar_pci_path *)(scope + 1);
796 count = (scope->length - sizeof(struct acpi_dmar_device_scope))
797 / sizeof(struct acpi_dmar_pci_path);
798
799 while (--count > 0) {
800 /*
801 * Access PCI directly due to the PCI
802 * subsystem isn't initialized yet.
803 */
Lv Zhengfa5f5082013-10-31 09:30:22 +0800804 bus = read_pci_config_byte(bus, path->device, path->function,
Weidong Hanf007e992009-05-23 00:41:15 +0800805 PCI_SECONDARY_BUS);
806 path++;
807 }
808
Jiang Liua7a3dad2014-11-09 22:48:00 +0800809 for (count = 0; count < MAX_IO_APICS; count++) {
810 if (ir_ioapic[count].iommu == iommu &&
811 ir_ioapic[count].id == scope->enumeration_id)
812 return 0;
813 else if (ir_ioapic[count].iommu == NULL && free == -1)
814 free = count;
815 }
816 if (free == -1) {
817 pr_warn("Exceeded Max IO APICS\n");
818 return -ENOSPC;
819 }
820
821 ir_ioapic[free].bus = bus;
822 ir_ioapic[free].devfn = PCI_DEVFN(path->device, path->function);
823 ir_ioapic[free].iommu = iommu;
824 ir_ioapic[free].id = scope->enumeration_id;
825 pr_info("IOAPIC id %d under DRHD base 0x%Lx IOMMU %d\n",
826 scope->enumeration_id, drhd->address, iommu->seq_id);
827
828 return 0;
Weidong Hanf007e992009-05-23 00:41:15 +0800829}
830
Suresh Siddha20f30972009-08-04 12:07:08 -0700831static int ir_parse_ioapic_hpet_scope(struct acpi_dmar_header *header,
832 struct intel_iommu *iommu)
Suresh Siddhaad3ad3f2008-07-10 11:16:40 -0700833{
Jiang Liua7a3dad2014-11-09 22:48:00 +0800834 int ret = 0;
Suresh Siddhaad3ad3f2008-07-10 11:16:40 -0700835 struct acpi_dmar_hardware_unit *drhd;
836 struct acpi_dmar_device_scope *scope;
837 void *start, *end;
838
839 drhd = (struct acpi_dmar_hardware_unit *)header;
Suresh Siddhaad3ad3f2008-07-10 11:16:40 -0700840 start = (void *)(drhd + 1);
841 end = ((void *)drhd) + header->length;
842
Jiang Liua7a3dad2014-11-09 22:48:00 +0800843 while (start < end && ret == 0) {
Suresh Siddhaad3ad3f2008-07-10 11:16:40 -0700844 scope = start;
Jiang Liua7a3dad2014-11-09 22:48:00 +0800845 if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_IOAPIC)
846 ret = ir_parse_one_ioapic_scope(scope, iommu, drhd);
847 else if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_HPET)
848 ret = ir_parse_one_hpet_scope(scope, iommu, drhd);
Suresh Siddhaad3ad3f2008-07-10 11:16:40 -0700849 start += scope->length;
850 }
851
Jiang Liua7a3dad2014-11-09 22:48:00 +0800852 return ret;
853}
854
855static void ir_remove_ioapic_hpet_scope(struct intel_iommu *iommu)
856{
857 int i;
858
859 for (i = 0; i < MAX_HPET_TBS; i++)
860 if (ir_hpet[i].iommu == iommu)
861 ir_hpet[i].iommu = NULL;
862
863 for (i = 0; i < MAX_IO_APICS; i++)
864 if (ir_ioapic[i].iommu == iommu)
865 ir_ioapic[i].iommu = NULL;
Suresh Siddhaad3ad3f2008-07-10 11:16:40 -0700866}
867
868/*
869 * Finds the assocaition between IOAPIC's and its Interrupt-remapping
870 * hardware unit.
871 */
Jiang Liu694835d2014-01-06 14:18:16 +0800872static int __init parse_ioapics_under_ir(void)
Suresh Siddhaad3ad3f2008-07-10 11:16:40 -0700873{
874 struct dmar_drhd_unit *drhd;
Jiang Liu7c919772014-01-06 14:18:18 +0800875 struct intel_iommu *iommu;
Suresh Siddhaad3ad3f2008-07-10 11:16:40 -0700876 int ir_supported = 0;
Seth Forshee32ab31e2012-08-08 08:27:03 -0500877 int ioapic_idx;
Suresh Siddhaad3ad3f2008-07-10 11:16:40 -0700878
Jiang Liu7c919772014-01-06 14:18:18 +0800879 for_each_iommu(iommu, drhd)
Suresh Siddhaad3ad3f2008-07-10 11:16:40 -0700880 if (ecap_ir_support(iommu->ecap)) {
Suresh Siddha20f30972009-08-04 12:07:08 -0700881 if (ir_parse_ioapic_hpet_scope(drhd->hdr, iommu))
Suresh Siddhaad3ad3f2008-07-10 11:16:40 -0700882 return -1;
883
884 ir_supported = 1;
885 }
Suresh Siddhaad3ad3f2008-07-10 11:16:40 -0700886
Seth Forshee32ab31e2012-08-08 08:27:03 -0500887 if (!ir_supported)
888 return 0;
889
890 for (ioapic_idx = 0; ioapic_idx < nr_ioapics; ioapic_idx++) {
891 int ioapic_id = mpc_ioapic_id(ioapic_idx);
892 if (!map_ioapic_to_ir(ioapic_id)) {
893 pr_err(FW_BUG "ioapic %d has no mapping iommu, "
894 "interrupt remapping will be disabled\n",
895 ioapic_id);
896 return -1;
897 }
Suresh Siddhaad3ad3f2008-07-10 11:16:40 -0700898 }
899
Seth Forshee32ab31e2012-08-08 08:27:03 -0500900 return 1;
Suresh Siddhaad3ad3f2008-07-10 11:16:40 -0700901}
Fenghua Yub24696b2009-03-27 14:22:44 -0700902
Rashika Kheria6a7885c2013-12-18 12:04:27 +0530903static int __init ir_dev_scope_init(void)
Suresh Siddhac2c72862011-08-23 17:05:19 -0700904{
Jiang Liu3a5670e2014-02-19 14:07:33 +0800905 int ret;
906
Suresh Siddha95a02e92012-03-30 11:47:07 -0700907 if (!irq_remapping_enabled)
Suresh Siddhac2c72862011-08-23 17:05:19 -0700908 return 0;
909
Jiang Liu3a5670e2014-02-19 14:07:33 +0800910 down_write(&dmar_global_lock);
911 ret = dmar_dev_scope_init();
912 up_write(&dmar_global_lock);
913
914 return ret;
Suresh Siddhac2c72862011-08-23 17:05:19 -0700915}
916rootfs_initcall(ir_dev_scope_init);
917
Suresh Siddha95a02e92012-03-30 11:47:07 -0700918static void disable_irq_remapping(void)
Fenghua Yub24696b2009-03-27 14:22:44 -0700919{
920 struct dmar_drhd_unit *drhd;
921 struct intel_iommu *iommu = NULL;
922
923 /*
924 * Disable Interrupt-remapping for all the DRHD's now.
925 */
926 for_each_iommu(iommu, drhd) {
927 if (!ecap_ir_support(iommu->ecap))
928 continue;
929
Suresh Siddha95a02e92012-03-30 11:47:07 -0700930 iommu_disable_irq_remapping(iommu);
Fenghua Yub24696b2009-03-27 14:22:44 -0700931 }
932}
933
Suresh Siddha95a02e92012-03-30 11:47:07 -0700934static int reenable_irq_remapping(int eim)
Fenghua Yub24696b2009-03-27 14:22:44 -0700935{
936 struct dmar_drhd_unit *drhd;
937 int setup = 0;
938 struct intel_iommu *iommu = NULL;
939
940 for_each_iommu(iommu, drhd)
941 if (iommu->qi)
942 dmar_reenable_qi(iommu);
943
944 /*
945 * Setup Interrupt-remapping for all the DRHD's now.
946 */
947 for_each_iommu(iommu, drhd) {
948 if (!ecap_ir_support(iommu->ecap))
949 continue;
950
951 /* Set up interrupt remapping for iommu.*/
Suresh Siddha95a02e92012-03-30 11:47:07 -0700952 iommu_set_irq_remapping(iommu, eim);
Fenghua Yub24696b2009-03-27 14:22:44 -0700953 setup = 1;
954 }
955
956 if (!setup)
957 goto error;
958
959 return 0;
960
961error:
962 /*
963 * handle error condition gracefully here!
964 */
965 return -1;
966}
967
Joerg Roedel0c3f1732012-03-30 11:47:02 -0700968static void prepare_irte(struct irte *irte, int vector,
969 unsigned int dest)
970{
971 memset(irte, 0, sizeof(*irte));
972
973 irte->present = 1;
974 irte->dst_mode = apic->irq_dest_mode;
975 /*
976 * Trigger mode in the IRTE will always be edge, and for IO-APIC, the
977 * actual level or edge trigger will be setup in the IO-APIC
978 * RTE. This will help simplify level triggered irq migration.
979 * For more details, see the comments (in io_apic.c) explainig IO-APIC
980 * irq migration in the presence of interrupt-remapping.
981 */
982 irte->trigger_mode = 0;
983 irte->dlvry_mode = apic->irq_delivery_mode;
984 irte->vector = vector;
985 irte->dest_id = IRTE_DEST(dest);
986 irte->redir_hint = 1;
987}
988
989static int intel_setup_ioapic_entry(int irq,
990 struct IO_APIC_route_entry *route_entry,
991 unsigned int destination, int vector,
992 struct io_apic_irq_attr *attr)
993{
994 int ioapic_id = mpc_ioapic_id(attr->ioapic);
Jiang Liu3a5670e2014-02-19 14:07:33 +0800995 struct intel_iommu *iommu;
Joerg Roedel0c3f1732012-03-30 11:47:02 -0700996 struct IR_IO_APIC_route_entry *entry;
997 struct irte irte;
998 int index;
999
Jiang Liu3a5670e2014-02-19 14:07:33 +08001000 down_read(&dmar_global_lock);
1001 iommu = map_ioapic_to_ir(ioapic_id);
Joerg Roedel0c3f1732012-03-30 11:47:02 -07001002 if (!iommu) {
1003 pr_warn("No mapping iommu for ioapic %d\n", ioapic_id);
Jiang Liu3a5670e2014-02-19 14:07:33 +08001004 index = -ENODEV;
1005 } else {
1006 index = alloc_irte(iommu, irq, 1);
1007 if (index < 0) {
1008 pr_warn("Failed to allocate IRTE for ioapic %d\n",
1009 ioapic_id);
1010 index = -ENOMEM;
1011 }
Joerg Roedel0c3f1732012-03-30 11:47:02 -07001012 }
Jiang Liu3a5670e2014-02-19 14:07:33 +08001013 up_read(&dmar_global_lock);
1014 if (index < 0)
1015 return index;
Joerg Roedel0c3f1732012-03-30 11:47:02 -07001016
1017 prepare_irte(&irte, vector, destination);
1018
1019 /* Set source-id of interrupt request */
1020 set_ioapic_sid(&irte, ioapic_id);
1021
1022 modify_irte(irq, &irte);
1023
1024 apic_printk(APIC_VERBOSE, KERN_DEBUG "IOAPIC[%d]: "
1025 "Set IRTE entry (P:%d FPD:%d Dst_Mode:%d "
1026 "Redir_hint:%d Trig_Mode:%d Dlvry_Mode:%X "
1027 "Avail:%X Vector:%02X Dest:%08X "
1028 "SID:%04X SQ:%X SVT:%X)\n",
1029 attr->ioapic, irte.present, irte.fpd, irte.dst_mode,
1030 irte.redir_hint, irte.trigger_mode, irte.dlvry_mode,
1031 irte.avail, irte.vector, irte.dest_id,
1032 irte.sid, irte.sq, irte.svt);
1033
Jiang Liu3a5670e2014-02-19 14:07:33 +08001034 entry = (struct IR_IO_APIC_route_entry *)route_entry;
Joerg Roedel0c3f1732012-03-30 11:47:02 -07001035 memset(entry, 0, sizeof(*entry));
1036
1037 entry->index2 = (index >> 15) & 0x1;
1038 entry->zero = 0;
1039 entry->format = 1;
1040 entry->index = (index & 0x7fff);
1041 /*
1042 * IO-APIC RTE will be configured with virtual vector.
1043 * irq handler will do the explicit EOI to the io-apic.
1044 */
1045 entry->vector = attr->ioapic_pin;
1046 entry->mask = 0; /* enable IRQ */
1047 entry->trigger = attr->trigger;
1048 entry->polarity = attr->polarity;
1049
1050 /* Mask level triggered irqs.
1051 * Use IRQ_DELAYED_DISABLE for edge triggered irqs.
1052 */
1053 if (attr->trigger)
1054 entry->mask = 1;
1055
1056 return 0;
1057}
1058
Joerg Roedel4c1bad62012-03-30 11:47:03 -07001059/*
1060 * Migrate the IO-APIC irq in the presence of intr-remapping.
1061 *
1062 * For both level and edge triggered, irq migration is a simple atomic
1063 * update(of vector and cpu destination) of IRTE and flush the hardware cache.
1064 *
1065 * For level triggered, we eliminate the io-apic RTE modification (with the
1066 * updated vector information), by using a virtual vector (io-apic pin number).
1067 * Real vector that is used for interrupting cpu will be coming from
1068 * the interrupt-remapping table entry.
1069 *
1070 * As the migration is a simple atomic update of IRTE, the same mechanism
1071 * is used to migrate MSI irq's in the presence of interrupt-remapping.
1072 */
1073static int
1074intel_ioapic_set_affinity(struct irq_data *data, const struct cpumask *mask,
1075 bool force)
1076{
Jiang Liu91411da2014-10-27 16:12:09 +08001077 struct irq_cfg *cfg = irqd_cfg(data);
Joerg Roedel4c1bad62012-03-30 11:47:03 -07001078 unsigned int dest, irq = data->irq;
1079 struct irte irte;
Alexander Gordeevff164322012-06-07 15:15:59 +02001080 int err;
Joerg Roedel4c1bad62012-03-30 11:47:03 -07001081
Suresh Siddha7eb9ae02012-06-14 18:28:49 -07001082 if (!config_enabled(CONFIG_SMP))
1083 return -EINVAL;
1084
Joerg Roedel4c1bad62012-03-30 11:47:03 -07001085 if (!cpumask_intersects(mask, cpu_online_mask))
1086 return -EINVAL;
1087
1088 if (get_irte(irq, &irte))
1089 return -EBUSY;
1090
Alexander Gordeevff164322012-06-07 15:15:59 +02001091 err = assign_irq_vector(irq, cfg, mask);
1092 if (err)
1093 return err;
Joerg Roedel4c1bad62012-03-30 11:47:03 -07001094
Alexander Gordeevff164322012-06-07 15:15:59 +02001095 err = apic->cpu_mask_to_apicid_and(cfg->domain, mask, &dest);
1096 if (err) {
Dan Carpentered88bed2012-06-12 19:26:33 +03001097 if (assign_irq_vector(irq, cfg, data->affinity))
Alexander Gordeevff164322012-06-07 15:15:59 +02001098 pr_err("Failed to recover vector for irq %d\n", irq);
1099 return err;
1100 }
Joerg Roedel4c1bad62012-03-30 11:47:03 -07001101
1102 irte.vector = cfg->vector;
1103 irte.dest_id = IRTE_DEST(dest);
1104
1105 /*
1106 * Atomically updates the IRTE with the new destination, vector
1107 * and flushes the interrupt entry cache.
1108 */
1109 modify_irte(irq, &irte);
1110
1111 /*
1112 * After this point, all the interrupts will start arriving
1113 * at the new destination. So, time to cleanup the previous
1114 * vector allocation.
1115 */
1116 if (cfg->move_in_progress)
1117 send_cleanup_vector(cfg);
1118
1119 cpumask_copy(data->affinity, mask);
1120 return 0;
1121}
Joerg Roedel0c3f1732012-03-30 11:47:02 -07001122
Joerg Roedel5e2b9302012-03-30 11:47:05 -07001123static void intel_compose_msi_msg(struct pci_dev *pdev,
1124 unsigned int irq, unsigned int dest,
1125 struct msi_msg *msg, u8 hpet_id)
1126{
1127 struct irq_cfg *cfg;
1128 struct irte irte;
Suresh Siddhac558df42012-05-08 00:08:54 -07001129 u16 sub_handle = 0;
Joerg Roedel5e2b9302012-03-30 11:47:05 -07001130 int ir_index;
1131
Jiang Liu91411da2014-10-27 16:12:09 +08001132 cfg = irq_cfg(irq);
Joerg Roedel5e2b9302012-03-30 11:47:05 -07001133
1134 ir_index = map_irq_to_irte_handle(irq, &sub_handle);
1135 BUG_ON(ir_index == -1);
1136
1137 prepare_irte(&irte, cfg->vector, dest);
1138
1139 /* Set source-id of interrupt request */
1140 if (pdev)
1141 set_msi_sid(&irte, pdev);
1142 else
1143 set_hpet_sid(&irte, hpet_id);
1144
1145 modify_irte(irq, &irte);
1146
1147 msg->address_hi = MSI_ADDR_BASE_HI;
1148 msg->data = sub_handle;
1149 msg->address_lo = MSI_ADDR_BASE_LO | MSI_ADDR_IR_EXT_INT |
1150 MSI_ADDR_IR_SHV |
1151 MSI_ADDR_IR_INDEX1(ir_index) |
1152 MSI_ADDR_IR_INDEX2(ir_index);
1153}
1154
1155/*
1156 * Map the PCI dev to the corresponding remapping hardware unit
1157 * and allocate 'nvec' consecutive interrupt-remapping table entries
1158 * in it.
1159 */
1160static int intel_msi_alloc_irq(struct pci_dev *dev, int irq, int nvec)
1161{
1162 struct intel_iommu *iommu;
1163 int index;
1164
Jiang Liu3a5670e2014-02-19 14:07:33 +08001165 down_read(&dmar_global_lock);
Joerg Roedel5e2b9302012-03-30 11:47:05 -07001166 iommu = map_dev_to_ir(dev);
1167 if (!iommu) {
1168 printk(KERN_ERR
1169 "Unable to map PCI %s to iommu\n", pci_name(dev));
Jiang Liu3a5670e2014-02-19 14:07:33 +08001170 index = -ENOENT;
1171 } else {
1172 index = alloc_irte(iommu, irq, nvec);
1173 if (index < 0) {
1174 printk(KERN_ERR
1175 "Unable to allocate %d IRTE for PCI %s\n",
1176 nvec, pci_name(dev));
1177 index = -ENOSPC;
1178 }
Joerg Roedel5e2b9302012-03-30 11:47:05 -07001179 }
Jiang Liu3a5670e2014-02-19 14:07:33 +08001180 up_read(&dmar_global_lock);
Joerg Roedel5e2b9302012-03-30 11:47:05 -07001181
Joerg Roedel5e2b9302012-03-30 11:47:05 -07001182 return index;
1183}
1184
1185static int intel_msi_setup_irq(struct pci_dev *pdev, unsigned int irq,
1186 int index, int sub_handle)
1187{
1188 struct intel_iommu *iommu;
Jiang Liu3a5670e2014-02-19 14:07:33 +08001189 int ret = -ENOENT;
Joerg Roedel5e2b9302012-03-30 11:47:05 -07001190
Jiang Liu3a5670e2014-02-19 14:07:33 +08001191 down_read(&dmar_global_lock);
Joerg Roedel5e2b9302012-03-30 11:47:05 -07001192 iommu = map_dev_to_ir(pdev);
Jiang Liu3a5670e2014-02-19 14:07:33 +08001193 if (iommu) {
1194 /*
1195 * setup the mapping between the irq and the IRTE
1196 * base index, the sub_handle pointing to the
1197 * appropriate interrupt remap table entry.
1198 */
1199 set_irte_irq(irq, iommu, index, sub_handle);
1200 ret = 0;
1201 }
1202 up_read(&dmar_global_lock);
Joerg Roedel5e2b9302012-03-30 11:47:05 -07001203
Jiang Liu3a5670e2014-02-19 14:07:33 +08001204 return ret;
Joerg Roedel5e2b9302012-03-30 11:47:05 -07001205}
1206
Yijing Wang5fc24d82014-09-17 17:32:19 +08001207static int intel_alloc_hpet_msi(unsigned int irq, unsigned int id)
Joerg Roedel5e2b9302012-03-30 11:47:05 -07001208{
Jiang Liu3a5670e2014-02-19 14:07:33 +08001209 int ret = -1;
1210 struct intel_iommu *iommu;
Joerg Roedel5e2b9302012-03-30 11:47:05 -07001211 int index;
1212
Jiang Liu3a5670e2014-02-19 14:07:33 +08001213 down_read(&dmar_global_lock);
1214 iommu = map_hpet_to_ir(id);
1215 if (iommu) {
1216 index = alloc_irte(iommu, irq, 1);
1217 if (index >= 0)
1218 ret = 0;
1219 }
1220 up_read(&dmar_global_lock);
Joerg Roedel5e2b9302012-03-30 11:47:05 -07001221
Jiang Liu3a5670e2014-02-19 14:07:33 +08001222 return ret;
Joerg Roedel5e2b9302012-03-30 11:47:05 -07001223}
1224
Joerg Roedel736baef2012-03-30 11:47:00 -07001225struct irq_remap_ops intel_irq_remap_ops = {
Suresh Siddha95a02e92012-03-30 11:47:07 -07001226 .supported = intel_irq_remapping_supported,
Thomas Gleixner11190302015-01-07 15:31:29 +08001227 .prepare = intel_prepare_irq_remapping,
Suresh Siddha95a02e92012-03-30 11:47:07 -07001228 .enable = intel_enable_irq_remapping,
1229 .disable = disable_irq_remapping,
1230 .reenable = reenable_irq_remapping,
Joerg Roedel4f3d8b62012-03-30 11:47:01 -07001231 .enable_faulting = enable_drhd_fault_handling,
Joerg Roedel0c3f1732012-03-30 11:47:02 -07001232 .setup_ioapic_entry = intel_setup_ioapic_entry,
Joerg Roedel4c1bad62012-03-30 11:47:03 -07001233 .set_affinity = intel_ioapic_set_affinity,
Joerg Roedel9d619f62012-03-30 11:47:04 -07001234 .free_irq = free_irte,
Joerg Roedel5e2b9302012-03-30 11:47:05 -07001235 .compose_msi_msg = intel_compose_msi_msg,
1236 .msi_alloc_irq = intel_msi_alloc_irq,
1237 .msi_setup_irq = intel_msi_setup_irq,
Yijing Wang5fc24d82014-09-17 17:32:19 +08001238 .alloc_hpet_msi = intel_alloc_hpet_msi,
Joerg Roedel736baef2012-03-30 11:47:00 -07001239};
Jiang Liu6b197242014-11-09 22:47:58 +08001240
Jiang Liua7a3dad2014-11-09 22:48:00 +08001241/*
1242 * Support of Interrupt Remapping Unit Hotplug
1243 */
1244static int dmar_ir_add(struct dmar_drhd_unit *dmaru, struct intel_iommu *iommu)
1245{
1246 int ret;
1247 int eim = x2apic_enabled();
1248
1249 if (eim && !ecap_eim_support(iommu->ecap)) {
1250 pr_info("DRHD %Lx: EIM not supported by DRHD, ecap %Lx\n",
1251 iommu->reg_phys, iommu->ecap);
1252 return -ENODEV;
1253 }
1254
1255 if (ir_parse_ioapic_hpet_scope(dmaru->hdr, iommu)) {
1256 pr_warn("DRHD %Lx: failed to parse managed IOAPIC/HPET\n",
1257 iommu->reg_phys);
1258 return -ENODEV;
1259 }
1260
1261 /* TODO: check all IOAPICs are covered by IOMMU */
1262
1263 /* Setup Interrupt-remapping now. */
1264 ret = intel_setup_irq_remapping(iommu);
1265 if (ret) {
1266 pr_err("DRHD %Lx: failed to allocate resource\n",
1267 iommu->reg_phys);
1268 ir_remove_ioapic_hpet_scope(iommu);
1269 return ret;
1270 }
1271
1272 if (!iommu->qi) {
1273 /* Clear previous faults. */
1274 dmar_fault(-1, iommu);
1275 iommu_disable_irq_remapping(iommu);
1276 dmar_disable_qi(iommu);
1277 }
1278
1279 /* Enable queued invalidation */
1280 ret = dmar_enable_qi(iommu);
1281 if (!ret) {
1282 iommu_set_irq_remapping(iommu, eim);
1283 } else {
1284 pr_err("DRHD %Lx: failed to enable queued invalidation, ecap %Lx, ret %d\n",
1285 iommu->reg_phys, iommu->ecap, ret);
1286 intel_teardown_irq_remapping(iommu);
1287 ir_remove_ioapic_hpet_scope(iommu);
1288 }
1289
1290 return ret;
1291}
1292
Jiang Liu6b197242014-11-09 22:47:58 +08001293int dmar_ir_hotplug(struct dmar_drhd_unit *dmaru, bool insert)
1294{
Jiang Liua7a3dad2014-11-09 22:48:00 +08001295 int ret = 0;
1296 struct intel_iommu *iommu = dmaru->iommu;
1297
1298 if (!irq_remapping_enabled)
1299 return 0;
1300 if (iommu == NULL)
1301 return -EINVAL;
1302 if (!ecap_ir_support(iommu->ecap))
1303 return 0;
1304
1305 if (insert) {
1306 if (!iommu->ir_table)
1307 ret = dmar_ir_add(dmaru, iommu);
1308 } else {
1309 if (iommu->ir_table) {
1310 if (!bitmap_empty(iommu->ir_table->bitmap,
1311 INTR_REMAP_TABLE_ENTRIES)) {
1312 ret = -EBUSY;
1313 } else {
1314 iommu_disable_irq_remapping(iommu);
1315 intel_teardown_irq_remapping(iommu);
1316 ir_remove_ioapic_hpet_scope(iommu);
1317 }
1318 }
1319 }
1320
1321 return ret;
Jiang Liu6b197242014-11-09 22:47:58 +08001322}