blob: 7c80661b35c19c1ebd3f0eb5db5b3930de22a012 [file] [log] [blame]
Yinghai Lu5aeecaf2008-08-19 20:49:59 -07001#include <linux/interrupt.h>
Suresh Siddhaad3ad3f2008-07-10 11:16:40 -07002#include <linux/dmar.h>
Suresh Siddha2ae21012008-07-10 11:16:43 -07003#include <linux/spinlock.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +09004#include <linux/slab.h>
Suresh Siddha2ae21012008-07-10 11:16:43 -07005#include <linux/jiffies.h>
Suresh Siddha20f30972009-08-04 12:07:08 -07006#include <linux/hpet.h>
Suresh Siddha2ae21012008-07-10 11:16:43 -07007#include <linux/pci.h>
Suresh Siddhab6fcb332008-07-10 11:16:44 -07008#include <linux/irq.h>
Lv Zheng8b484632013-12-03 08:49:16 +08009#include <linux/intel-iommu.h>
10#include <linux/acpi.h>
Suresh Siddhaad3ad3f2008-07-10 11:16:40 -070011#include <asm/io_apic.h>
Yinghai Lu17483a12008-12-12 13:14:18 -080012#include <asm/smp.h>
Jaswinder Singh Rajput6d652ea2009-01-07 21:38:59 +053013#include <asm/cpu.h>
Suresh Siddha8a8f4222012-03-30 11:47:08 -070014#include <asm/irq_remapping.h>
Weidong Hanf007e992009-05-23 00:41:15 +080015#include <asm/pci-direct.h>
Joerg Roedel5e2b9302012-03-30 11:47:05 -070016#include <asm/msidef.h>
Suresh Siddhaad3ad3f2008-07-10 11:16:40 -070017
Suresh Siddha8a8f4222012-03-30 11:47:08 -070018#include "irq_remapping.h"
Joerg Roedel736baef2012-03-30 11:47:00 -070019
Joerg Roedeleef93fd2012-03-30 11:46:59 -070020struct ioapic_scope {
21 struct intel_iommu *iommu;
22 unsigned int id;
23 unsigned int bus; /* PCI bus number */
24 unsigned int devfn; /* PCI devfn number */
25};
26
27struct hpet_scope {
28 struct intel_iommu *iommu;
29 u8 id;
30 unsigned int bus;
31 unsigned int devfn;
32};
33
34#define IR_X2APIC_MODE(mode) (mode ? (1 << 11) : 0)
Joerg Roedel0c3f1732012-03-30 11:47:02 -070035#define IRTE_DEST(dest) ((x2apic_mode) ? dest : dest << 8)
Joerg Roedeleef93fd2012-03-30 11:46:59 -070036
Suresh Siddhaad3ad3f2008-07-10 11:16:40 -070037static struct ioapic_scope ir_ioapic[MAX_IO_APICS];
Suresh Siddha20f30972009-08-04 12:07:08 -070038static struct hpet_scope ir_hpet[MAX_HPET_TBS];
39static int ir_ioapic_num, ir_hpet_num;
Chris Wrightd1423d52010-07-20 11:06:49 -070040
Jiang Liu3a5670e2014-02-19 14:07:33 +080041/*
42 * Lock ordering:
43 * ->dmar_global_lock
44 * ->irq_2_ir_lock
45 * ->qi->q_lock
46 * ->iommu->register_lock
47 * Note:
48 * intel_irq_remap_ops.{supported,prepare,enable,disable,reenable} are called
49 * in single-threaded environment with interrupt disabled, so no need to tabke
50 * the dmar_global_lock.
51 */
Thomas Gleixner96f8e982011-07-19 16:28:19 +020052static DEFINE_RAW_SPINLOCK(irq_2_ir_lock);
Thomas Gleixnerd585d062010-10-10 12:34:27 +020053
Jiang Liu694835d2014-01-06 14:18:16 +080054static int __init parse_ioapics_under_ir(void);
55
Yinghai Lue420dfb2008-08-19 20:50:21 -070056static struct irq_2_iommu *irq_2_iommu(unsigned int irq)
57{
Thomas Gleixnerdced35a2011-03-28 17:49:12 +020058 struct irq_cfg *cfg = irq_get_chip_data(irq);
Thomas Gleixner349d6762010-10-10 12:29:27 +020059 return cfg ? &cfg->irq_2_iommu : NULL;
Yinghai Lu0b8f1ef2008-12-05 18:58:31 -080060}
61
Rashika Kheria6a7885c2013-12-18 12:04:27 +053062static int get_irte(int irq, struct irte *entry)
Suresh Siddhab6fcb332008-07-10 11:16:44 -070063{
Thomas Gleixnerd585d062010-10-10 12:34:27 +020064 struct irq_2_iommu *irq_iommu = irq_2_iommu(irq);
Suresh Siddha4c5502b2009-03-16 17:04:53 -070065 unsigned long flags;
Thomas Gleixnerd585d062010-10-10 12:34:27 +020066 int index;
Suresh Siddhab6fcb332008-07-10 11:16:44 -070067
Thomas Gleixnerd585d062010-10-10 12:34:27 +020068 if (!entry || !irq_iommu)
Suresh Siddhab6fcb332008-07-10 11:16:44 -070069 return -1;
70
Thomas Gleixner96f8e982011-07-19 16:28:19 +020071 raw_spin_lock_irqsave(&irq_2_ir_lock, flags);
Suresh Siddhab6fcb332008-07-10 11:16:44 -070072
Greg Edwardsaf437462014-07-23 10:13:26 -060073 if (unlikely(!irq_iommu->iommu)) {
74 raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags);
75 return -1;
76 }
77
Yinghai Lue420dfb2008-08-19 20:50:21 -070078 index = irq_iommu->irte_index + irq_iommu->sub_handle;
79 *entry = *(irq_iommu->iommu->ir_table->base + index);
Suresh Siddhab6fcb332008-07-10 11:16:44 -070080
Thomas Gleixner96f8e982011-07-19 16:28:19 +020081 raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags);
Suresh Siddhab6fcb332008-07-10 11:16:44 -070082 return 0;
83}
84
Joerg Roedel263b5e82012-03-30 11:47:06 -070085static int alloc_irte(struct intel_iommu *iommu, int irq, u16 count)
Suresh Siddhab6fcb332008-07-10 11:16:44 -070086{
87 struct ir_table *table = iommu->ir_table;
Thomas Gleixnerd585d062010-10-10 12:34:27 +020088 struct irq_2_iommu *irq_iommu = irq_2_iommu(irq);
Joerg Roedel9b1b0e42012-09-26 12:44:45 +020089 struct irq_cfg *cfg = irq_get_chip_data(irq);
Suresh Siddhab6fcb332008-07-10 11:16:44 -070090 unsigned int mask = 0;
Suresh Siddha4c5502b2009-03-16 17:04:53 -070091 unsigned long flags;
Dan Carpenter9f4c7442014-01-09 08:32:36 +030092 int index;
Suresh Siddhab6fcb332008-07-10 11:16:44 -070093
Thomas Gleixnerd585d062010-10-10 12:34:27 +020094 if (!count || !irq_iommu)
Suresh Siddhab6fcb332008-07-10 11:16:44 -070095 return -1;
96
Suresh Siddhab6fcb332008-07-10 11:16:44 -070097 if (count > 1) {
98 count = __roundup_pow_of_two(count);
99 mask = ilog2(count);
100 }
101
102 if (mask > ecap_max_handle_mask(iommu->ecap)) {
103 printk(KERN_ERR
104 "Requested mask %x exceeds the max invalidation handle"
105 " mask value %Lx\n", mask,
106 ecap_max_handle_mask(iommu->ecap));
107 return -1;
108 }
109
Thomas Gleixner96f8e982011-07-19 16:28:19 +0200110 raw_spin_lock_irqsave(&irq_2_ir_lock, flags);
Jiang Liu360eb3c2014-01-06 14:18:08 +0800111 index = bitmap_find_free_region(table->bitmap,
112 INTR_REMAP_TABLE_ENTRIES, mask);
113 if (index < 0) {
114 pr_warn("IR%d: can't allocate an IRTE\n", iommu->seq_id);
115 } else {
116 cfg->remapped = 1;
117 irq_iommu->iommu = iommu;
118 irq_iommu->irte_index = index;
119 irq_iommu->sub_handle = 0;
120 irq_iommu->irte_mask = mask;
121 }
Thomas Gleixner96f8e982011-07-19 16:28:19 +0200122 raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags);
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700123
124 return index;
125}
126
Yu Zhao704126a2009-01-04 16:28:52 +0800127static int qi_flush_iec(struct intel_iommu *iommu, int index, int mask)
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700128{
129 struct qi_desc desc;
130
131 desc.low = QI_IEC_IIDEX(index) | QI_IEC_TYPE | QI_IEC_IM(mask)
132 | QI_IEC_SELECTIVE;
133 desc.high = 0;
134
Yu Zhao704126a2009-01-04 16:28:52 +0800135 return qi_submit_sync(&desc, iommu);
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700136}
137
Joerg Roedel263b5e82012-03-30 11:47:06 -0700138static int map_irq_to_irte_handle(int irq, u16 *sub_handle)
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700139{
Thomas Gleixnerd585d062010-10-10 12:34:27 +0200140 struct irq_2_iommu *irq_iommu = irq_2_iommu(irq);
Suresh Siddha4c5502b2009-03-16 17:04:53 -0700141 unsigned long flags;
Thomas Gleixnerd585d062010-10-10 12:34:27 +0200142 int index;
143
144 if (!irq_iommu)
145 return -1;
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700146
Thomas Gleixner96f8e982011-07-19 16:28:19 +0200147 raw_spin_lock_irqsave(&irq_2_ir_lock, flags);
Yinghai Lue420dfb2008-08-19 20:50:21 -0700148 *sub_handle = irq_iommu->sub_handle;
149 index = irq_iommu->irte_index;
Thomas Gleixner96f8e982011-07-19 16:28:19 +0200150 raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags);
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700151 return index;
152}
153
Joerg Roedel263b5e82012-03-30 11:47:06 -0700154static int set_irte_irq(int irq, struct intel_iommu *iommu, u16 index, u16 subhandle)
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700155{
Thomas Gleixnerd585d062010-10-10 12:34:27 +0200156 struct irq_2_iommu *irq_iommu = irq_2_iommu(irq);
Joerg Roedel9b1b0e42012-09-26 12:44:45 +0200157 struct irq_cfg *cfg = irq_get_chip_data(irq);
Suresh Siddha4c5502b2009-03-16 17:04:53 -0700158 unsigned long flags;
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700159
Thomas Gleixnerd585d062010-10-10 12:34:27 +0200160 if (!irq_iommu)
Yinghai Lu0b8f1ef2008-12-05 18:58:31 -0800161 return -1;
Thomas Gleixnerd585d062010-10-10 12:34:27 +0200162
Thomas Gleixner96f8e982011-07-19 16:28:19 +0200163 raw_spin_lock_irqsave(&irq_2_ir_lock, flags);
Yinghai Lu0b8f1ef2008-12-05 18:58:31 -0800164
Joerg Roedel9b1b0e42012-09-26 12:44:45 +0200165 cfg->remapped = 1;
Yinghai Lue420dfb2008-08-19 20:50:21 -0700166 irq_iommu->iommu = iommu;
167 irq_iommu->irte_index = index;
168 irq_iommu->sub_handle = subhandle;
169 irq_iommu->irte_mask = 0;
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700170
Thomas Gleixner96f8e982011-07-19 16:28:19 +0200171 raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags);
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700172
173 return 0;
174}
175
Joerg Roedel263b5e82012-03-30 11:47:06 -0700176static int modify_irte(int irq, struct irte *irte_modified)
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700177{
Thomas Gleixnerd585d062010-10-10 12:34:27 +0200178 struct irq_2_iommu *irq_iommu = irq_2_iommu(irq);
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700179 struct intel_iommu *iommu;
Suresh Siddha4c5502b2009-03-16 17:04:53 -0700180 unsigned long flags;
Thomas Gleixnerd585d062010-10-10 12:34:27 +0200181 struct irte *irte;
182 int rc, index;
183
184 if (!irq_iommu)
185 return -1;
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700186
Thomas Gleixner96f8e982011-07-19 16:28:19 +0200187 raw_spin_lock_irqsave(&irq_2_ir_lock, flags);
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700188
Yinghai Lue420dfb2008-08-19 20:50:21 -0700189 iommu = irq_iommu->iommu;
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700190
Yinghai Lue420dfb2008-08-19 20:50:21 -0700191 index = irq_iommu->irte_index + irq_iommu->sub_handle;
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700192 irte = &iommu->ir_table->base[index];
193
Linus Torvaldsc513b672010-08-06 11:02:31 -0700194 set_64bit(&irte->low, irte_modified->low);
195 set_64bit(&irte->high, irte_modified->high);
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700196 __iommu_flush_cache(iommu, irte, sizeof(*irte));
197
Yu Zhao704126a2009-01-04 16:28:52 +0800198 rc = qi_flush_iec(iommu, index, 0);
Thomas Gleixner96f8e982011-07-19 16:28:19 +0200199 raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags);
Yu Zhao704126a2009-01-04 16:28:52 +0800200
201 return rc;
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700202}
203
Joerg Roedel263b5e82012-03-30 11:47:06 -0700204static struct intel_iommu *map_hpet_to_ir(u8 hpet_id)
Suresh Siddha20f30972009-08-04 12:07:08 -0700205{
206 int i;
207
208 for (i = 0; i < MAX_HPET_TBS; i++)
209 if (ir_hpet[i].id == hpet_id)
210 return ir_hpet[i].iommu;
211 return NULL;
212}
213
Joerg Roedel263b5e82012-03-30 11:47:06 -0700214static struct intel_iommu *map_ioapic_to_ir(int apic)
Suresh Siddha89027d32008-07-10 11:16:56 -0700215{
216 int i;
217
218 for (i = 0; i < MAX_IO_APICS; i++)
219 if (ir_ioapic[i].id == apic)
220 return ir_ioapic[i].iommu;
221 return NULL;
222}
223
Joerg Roedel263b5e82012-03-30 11:47:06 -0700224static struct intel_iommu *map_dev_to_ir(struct pci_dev *dev)
Suresh Siddha75c46fa2008-07-10 11:16:57 -0700225{
226 struct dmar_drhd_unit *drhd;
227
228 drhd = dmar_find_matched_drhd_unit(dev);
229 if (!drhd)
230 return NULL;
231
232 return drhd->iommu;
233}
234
Weidong Hanc4658b42009-05-23 00:41:14 +0800235static int clear_entries(struct irq_2_iommu *irq_iommu)
236{
237 struct irte *start, *entry, *end;
238 struct intel_iommu *iommu;
239 int index;
240
241 if (irq_iommu->sub_handle)
242 return 0;
243
244 iommu = irq_iommu->iommu;
245 index = irq_iommu->irte_index + irq_iommu->sub_handle;
246
247 start = iommu->ir_table->base + index;
248 end = start + (1 << irq_iommu->irte_mask);
249
250 for (entry = start; entry < end; entry++) {
Linus Torvaldsc513b672010-08-06 11:02:31 -0700251 set_64bit(&entry->low, 0);
252 set_64bit(&entry->high, 0);
Weidong Hanc4658b42009-05-23 00:41:14 +0800253 }
Jiang Liu360eb3c2014-01-06 14:18:08 +0800254 bitmap_release_region(iommu->ir_table->bitmap, index,
255 irq_iommu->irte_mask);
Weidong Hanc4658b42009-05-23 00:41:14 +0800256
257 return qi_flush_iec(iommu, index, irq_iommu->irte_mask);
258}
259
Joerg Roedel9d619f62012-03-30 11:47:04 -0700260static int free_irte(int irq)
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700261{
Thomas Gleixnerd585d062010-10-10 12:34:27 +0200262 struct irq_2_iommu *irq_iommu = irq_2_iommu(irq);
Suresh Siddha4c5502b2009-03-16 17:04:53 -0700263 unsigned long flags;
Thomas Gleixnerd585d062010-10-10 12:34:27 +0200264 int rc;
265
266 if (!irq_iommu)
267 return -1;
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700268
Thomas Gleixner96f8e982011-07-19 16:28:19 +0200269 raw_spin_lock_irqsave(&irq_2_ir_lock, flags);
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700270
Weidong Hanc4658b42009-05-23 00:41:14 +0800271 rc = clear_entries(irq_iommu);
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700272
Yinghai Lue420dfb2008-08-19 20:50:21 -0700273 irq_iommu->iommu = NULL;
274 irq_iommu->irte_index = 0;
275 irq_iommu->sub_handle = 0;
276 irq_iommu->irte_mask = 0;
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700277
Thomas Gleixner96f8e982011-07-19 16:28:19 +0200278 raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags);
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700279
Yu Zhao704126a2009-01-04 16:28:52 +0800280 return rc;
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700281}
282
Weidong Hanf007e992009-05-23 00:41:15 +0800283/*
284 * source validation type
285 */
286#define SVT_NO_VERIFY 0x0 /* no verification is required */
Lucas De Marchi25985ed2011-03-30 22:57:33 -0300287#define SVT_VERIFY_SID_SQ 0x1 /* verify using SID and SQ fields */
Weidong Hanf007e992009-05-23 00:41:15 +0800288#define SVT_VERIFY_BUS 0x2 /* verify bus of request-id */
289
290/*
291 * source-id qualifier
292 */
293#define SQ_ALL_16 0x0 /* verify all 16 bits of request-id */
294#define SQ_13_IGNORE_1 0x1 /* verify most significant 13 bits, ignore
295 * the third least significant bit
296 */
297#define SQ_13_IGNORE_2 0x2 /* verify most significant 13 bits, ignore
298 * the second and third least significant bits
299 */
300#define SQ_13_IGNORE_3 0x3 /* verify most significant 13 bits, ignore
301 * the least three significant bits
302 */
303
304/*
305 * set SVT, SQ and SID fields of irte to verify
306 * source ids of interrupt requests
307 */
308static void set_irte_sid(struct irte *irte, unsigned int svt,
309 unsigned int sq, unsigned int sid)
310{
Chris Wrightd1423d52010-07-20 11:06:49 -0700311 if (disable_sourceid_checking)
312 svt = SVT_NO_VERIFY;
Weidong Hanf007e992009-05-23 00:41:15 +0800313 irte->svt = svt;
314 irte->sq = sq;
315 irte->sid = sid;
316}
317
Joerg Roedel263b5e82012-03-30 11:47:06 -0700318static int set_ioapic_sid(struct irte *irte, int apic)
Weidong Hanf007e992009-05-23 00:41:15 +0800319{
320 int i;
321 u16 sid = 0;
322
323 if (!irte)
324 return -1;
325
Jiang Liu3a5670e2014-02-19 14:07:33 +0800326 down_read(&dmar_global_lock);
Weidong Hanf007e992009-05-23 00:41:15 +0800327 for (i = 0; i < MAX_IO_APICS; i++) {
328 if (ir_ioapic[i].id == apic) {
329 sid = (ir_ioapic[i].bus << 8) | ir_ioapic[i].devfn;
330 break;
331 }
332 }
Jiang Liu3a5670e2014-02-19 14:07:33 +0800333 up_read(&dmar_global_lock);
Weidong Hanf007e992009-05-23 00:41:15 +0800334
335 if (sid == 0) {
336 pr_warning("Failed to set source-id of IOAPIC (%d)\n", apic);
337 return -1;
338 }
339
Jiang Liu2fe2c602014-01-06 14:18:17 +0800340 set_irte_sid(irte, SVT_VERIFY_SID_SQ, SQ_ALL_16, sid);
Weidong Hanf007e992009-05-23 00:41:15 +0800341
342 return 0;
343}
344
Joerg Roedel263b5e82012-03-30 11:47:06 -0700345static int set_hpet_sid(struct irte *irte, u8 id)
Suresh Siddha20f30972009-08-04 12:07:08 -0700346{
347 int i;
348 u16 sid = 0;
349
350 if (!irte)
351 return -1;
352
Jiang Liu3a5670e2014-02-19 14:07:33 +0800353 down_read(&dmar_global_lock);
Suresh Siddha20f30972009-08-04 12:07:08 -0700354 for (i = 0; i < MAX_HPET_TBS; i++) {
355 if (ir_hpet[i].id == id) {
356 sid = (ir_hpet[i].bus << 8) | ir_hpet[i].devfn;
357 break;
358 }
359 }
Jiang Liu3a5670e2014-02-19 14:07:33 +0800360 up_read(&dmar_global_lock);
Suresh Siddha20f30972009-08-04 12:07:08 -0700361
362 if (sid == 0) {
363 pr_warning("Failed to set source-id of HPET block (%d)\n", id);
364 return -1;
365 }
366
367 /*
368 * Should really use SQ_ALL_16. Some platforms are broken.
369 * While we figure out the right quirks for these broken platforms, use
370 * SQ_13_IGNORE_3 for now.
371 */
372 set_irte_sid(irte, SVT_VERIFY_SID_SQ, SQ_13_IGNORE_3, sid);
373
374 return 0;
375}
376
Alex Williamson579305f2014-07-03 09:51:43 -0600377struct set_msi_sid_data {
378 struct pci_dev *pdev;
379 u16 alias;
380};
381
382static int set_msi_sid_cb(struct pci_dev *pdev, u16 alias, void *opaque)
383{
384 struct set_msi_sid_data *data = opaque;
385
386 data->pdev = pdev;
387 data->alias = alias;
388
389 return 0;
390}
391
Joerg Roedel263b5e82012-03-30 11:47:06 -0700392static int set_msi_sid(struct irte *irte, struct pci_dev *dev)
Weidong Hanf007e992009-05-23 00:41:15 +0800393{
Alex Williamson579305f2014-07-03 09:51:43 -0600394 struct set_msi_sid_data data;
Weidong Hanf007e992009-05-23 00:41:15 +0800395
396 if (!irte || !dev)
397 return -1;
398
Alex Williamson579305f2014-07-03 09:51:43 -0600399 pci_for_each_dma_alias(dev, set_msi_sid_cb, &data);
Weidong Hanf007e992009-05-23 00:41:15 +0800400
Alex Williamson579305f2014-07-03 09:51:43 -0600401 /*
402 * DMA alias provides us with a PCI device and alias. The only case
403 * where the it will return an alias on a different bus than the
404 * device is the case of a PCIe-to-PCI bridge, where the alias is for
405 * the subordinate bus. In this case we can only verify the bus.
406 *
407 * If the alias device is on a different bus than our source device
408 * then we have a topology based alias, use it.
409 *
410 * Otherwise, the alias is for a device DMA quirk and we cannot
411 * assume that MSI uses the same requester ID. Therefore use the
412 * original device.
413 */
414 if (PCI_BUS_NUM(data.alias) != data.pdev->bus->number)
415 set_irte_sid(irte, SVT_VERIFY_BUS, SQ_ALL_16,
416 PCI_DEVID(PCI_BUS_NUM(data.alias),
417 dev->bus->number));
418 else if (data.pdev->bus->number != dev->bus->number)
419 set_irte_sid(irte, SVT_VERIFY_SID_SQ, SQ_ALL_16, data.alias);
420 else
421 set_irte_sid(irte, SVT_VERIFY_SID_SQ, SQ_ALL_16,
422 PCI_DEVID(dev->bus->number, dev->devfn));
Weidong Hanf007e992009-05-23 00:41:15 +0800423
424 return 0;
425}
426
Suresh Siddha95a02e92012-03-30 11:47:07 -0700427static void iommu_set_irq_remapping(struct intel_iommu *iommu, int mode)
Suresh Siddha2ae21012008-07-10 11:16:43 -0700428{
429 u64 addr;
David Woodhousec416daa2009-05-10 20:30:58 +0100430 u32 sts;
Suresh Siddha2ae21012008-07-10 11:16:43 -0700431 unsigned long flags;
432
433 addr = virt_to_phys((void *)iommu->ir_table->base);
434
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +0200435 raw_spin_lock_irqsave(&iommu->register_lock, flags);
Suresh Siddha2ae21012008-07-10 11:16:43 -0700436
437 dmar_writeq(iommu->reg + DMAR_IRTA_REG,
438 (addr) | IR_X2APIC_MODE(mode) | INTR_REMAP_TABLE_REG_SIZE);
439
440 /* Set interrupt-remapping table pointer */
Jan Kiszkaf63ef692014-08-11 13:13:25 +0200441 writel(iommu->gcmd | DMA_GCMD_SIRTP, iommu->reg + DMAR_GCMD_REG);
Suresh Siddha2ae21012008-07-10 11:16:43 -0700442
443 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
444 readl, (sts & DMA_GSTS_IRTPS), sts);
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +0200445 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
Suresh Siddha2ae21012008-07-10 11:16:43 -0700446
447 /*
448 * global invalidation of interrupt entry cache before enabling
449 * interrupt-remapping.
450 */
451 qi_global_iec(iommu);
452
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +0200453 raw_spin_lock_irqsave(&iommu->register_lock, flags);
Suresh Siddha2ae21012008-07-10 11:16:43 -0700454
455 /* Enable interrupt-remapping */
Suresh Siddha2ae21012008-07-10 11:16:43 -0700456 iommu->gcmd |= DMA_GCMD_IRE;
Andy Lutomirskiaf8d1022013-02-01 14:57:43 -0800457 iommu->gcmd &= ~DMA_GCMD_CFI; /* Block compatibility-format MSIs */
David Woodhousec416daa2009-05-10 20:30:58 +0100458 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
Suresh Siddha2ae21012008-07-10 11:16:43 -0700459
460 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
461 readl, (sts & DMA_GSTS_IRES), sts);
462
Andy Lutomirskiaf8d1022013-02-01 14:57:43 -0800463 /*
464 * With CFI clear in the Global Command register, we should be
465 * protected from dangerous (i.e. compatibility) interrupts
466 * regardless of x2apic status. Check just to be sure.
467 */
468 if (sts & DMA_GSTS_CFIS)
469 WARN(1, KERN_WARNING
470 "Compatibility-format IRQs enabled despite intr remapping;\n"
471 "you are vulnerable to IRQ injection.\n");
472
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +0200473 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
Suresh Siddha2ae21012008-07-10 11:16:43 -0700474}
475
476
Suresh Siddha95a02e92012-03-30 11:47:07 -0700477static int intel_setup_irq_remapping(struct intel_iommu *iommu, int mode)
Suresh Siddha2ae21012008-07-10 11:16:43 -0700478{
479 struct ir_table *ir_table;
480 struct page *pages;
Jiang Liu360eb3c2014-01-06 14:18:08 +0800481 unsigned long *bitmap;
Suresh Siddha2ae21012008-07-10 11:16:43 -0700482
483 ir_table = iommu->ir_table = kzalloc(sizeof(struct ir_table),
Suresh Siddhafa4b57c2009-03-16 17:05:05 -0700484 GFP_ATOMIC);
Suresh Siddha2ae21012008-07-10 11:16:43 -0700485
486 if (!iommu->ir_table)
487 return -ENOMEM;
488
Suresh Siddha824cd752009-10-02 11:01:23 -0700489 pages = alloc_pages_node(iommu->node, GFP_ATOMIC | __GFP_ZERO,
490 INTR_REMAP_PAGE_ORDER);
Suresh Siddha2ae21012008-07-10 11:16:43 -0700491
492 if (!pages) {
Jiang Liu360eb3c2014-01-06 14:18:08 +0800493 pr_err("IR%d: failed to allocate pages of order %d\n",
494 iommu->seq_id, INTR_REMAP_PAGE_ORDER);
Suresh Siddha2ae21012008-07-10 11:16:43 -0700495 kfree(iommu->ir_table);
496 return -ENOMEM;
497 }
498
Jiang Liu360eb3c2014-01-06 14:18:08 +0800499 bitmap = kcalloc(BITS_TO_LONGS(INTR_REMAP_TABLE_ENTRIES),
500 sizeof(long), GFP_ATOMIC);
501 if (bitmap == NULL) {
502 pr_err("IR%d: failed to allocate bitmap\n", iommu->seq_id);
503 __free_pages(pages, INTR_REMAP_PAGE_ORDER);
504 kfree(ir_table);
505 return -ENOMEM;
506 }
507
Suresh Siddha2ae21012008-07-10 11:16:43 -0700508 ir_table->base = page_address(pages);
Jiang Liu360eb3c2014-01-06 14:18:08 +0800509 ir_table->bitmap = bitmap;
Suresh Siddha2ae21012008-07-10 11:16:43 -0700510
Suresh Siddha95a02e92012-03-30 11:47:07 -0700511 iommu_set_irq_remapping(iommu, mode);
Suresh Siddha2ae21012008-07-10 11:16:43 -0700512 return 0;
513}
514
Suresh Siddhaeba67e52009-03-16 17:04:56 -0700515/*
516 * Disable Interrupt Remapping.
517 */
Suresh Siddha95a02e92012-03-30 11:47:07 -0700518static void iommu_disable_irq_remapping(struct intel_iommu *iommu)
Suresh Siddhaeba67e52009-03-16 17:04:56 -0700519{
520 unsigned long flags;
521 u32 sts;
522
523 if (!ecap_ir_support(iommu->ecap))
524 return;
525
Fenghua Yub24696b2009-03-27 14:22:44 -0700526 /*
527 * global invalidation of interrupt entry cache before disabling
528 * interrupt-remapping.
529 */
530 qi_global_iec(iommu);
531
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +0200532 raw_spin_lock_irqsave(&iommu->register_lock, flags);
Suresh Siddhaeba67e52009-03-16 17:04:56 -0700533
534 sts = dmar_readq(iommu->reg + DMAR_GSTS_REG);
535 if (!(sts & DMA_GSTS_IRES))
536 goto end;
537
538 iommu->gcmd &= ~DMA_GCMD_IRE;
539 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
540
541 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
542 readl, !(sts & DMA_GSTS_IRES), sts);
543
544end:
Thomas Gleixner1f5b3c32011-07-19 16:19:51 +0200545 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
Suresh Siddhaeba67e52009-03-16 17:04:56 -0700546}
547
Suresh Siddha41750d32011-08-23 17:05:18 -0700548static int __init dmar_x2apic_optout(void)
549{
550 struct acpi_table_dmar *dmar;
551 dmar = (struct acpi_table_dmar *)dmar_tbl;
552 if (!dmar || no_x2apic_optout)
553 return 0;
554 return dmar->flags & DMAR_X2APIC_OPT_OUT;
555}
556
Suresh Siddha95a02e92012-03-30 11:47:07 -0700557static int __init intel_irq_remapping_supported(void)
Weidong Han93758232009-04-17 16:42:14 +0800558{
559 struct dmar_drhd_unit *drhd;
Jiang Liu7c919772014-01-06 14:18:18 +0800560 struct intel_iommu *iommu;
Weidong Han93758232009-04-17 16:42:14 +0800561
Suresh Siddha95a02e92012-03-30 11:47:07 -0700562 if (disable_irq_remap)
Weidong Han03ea8152009-04-17 16:42:15 +0800563 return 0;
Neil Horman03bbcb22013-04-16 16:38:32 -0400564 if (irq_remap_broken) {
Neil Horman05104a42013-09-27 12:53:35 -0400565 printk(KERN_WARNING
566 "This system BIOS has enabled interrupt remapping\n"
567 "on a chipset that contains an erratum making that\n"
568 "feature unstable. To maintain system stability\n"
569 "interrupt remapping is being disabled. Please\n"
570 "contact your BIOS vendor for an update\n");
571 add_taint(TAINT_FIRMWARE_WORKAROUND, LOCKDEP_STILL_OK);
Neil Horman03bbcb22013-04-16 16:38:32 -0400572 disable_irq_remap = 1;
573 return 0;
574 }
Weidong Han03ea8152009-04-17 16:42:15 +0800575
Youquan Song074835f2009-09-09 12:05:39 -0400576 if (!dmar_ir_support())
577 return 0;
578
Jiang Liu7c919772014-01-06 14:18:18 +0800579 for_each_iommu(iommu, drhd)
Weidong Han93758232009-04-17 16:42:14 +0800580 if (!ecap_ir_support(iommu->ecap))
581 return 0;
Weidong Han93758232009-04-17 16:42:14 +0800582
583 return 1;
584}
585
Suresh Siddha95a02e92012-03-30 11:47:07 -0700586static int __init intel_enable_irq_remapping(void)
Suresh Siddha2ae21012008-07-10 11:16:43 -0700587{
588 struct dmar_drhd_unit *drhd;
Jiang Liu7c919772014-01-06 14:18:18 +0800589 struct intel_iommu *iommu;
Andy Lutomirskiaf8d1022013-02-01 14:57:43 -0800590 bool x2apic_present;
Suresh Siddha2ae21012008-07-10 11:16:43 -0700591 int setup = 0;
Suresh Siddha41750d32011-08-23 17:05:18 -0700592 int eim = 0;
Suresh Siddha2ae21012008-07-10 11:16:43 -0700593
Andy Lutomirskiaf8d1022013-02-01 14:57:43 -0800594 x2apic_present = x2apic_supported();
595
Youquan Songe936d072009-09-07 10:58:07 -0400596 if (parse_ioapics_under_ir() != 1) {
597 printk(KERN_INFO "Not enable interrupt remapping\n");
Andy Lutomirskiaf8d1022013-02-01 14:57:43 -0800598 goto error;
Youquan Songe936d072009-09-07 10:58:07 -0400599 }
600
Andy Lutomirskiaf8d1022013-02-01 14:57:43 -0800601 if (x2apic_present) {
Jiang Liub977e732014-01-06 14:18:14 +0800602 pr_info("Queued invalidation will be enabled to support x2apic and Intr-remapping.\n");
603
Suresh Siddha41750d32011-08-23 17:05:18 -0700604 eim = !dmar_x2apic_optout();
Andy Lutomirskiaf8d1022013-02-01 14:57:43 -0800605 if (!eim)
606 printk(KERN_WARNING
607 "Your BIOS is broken and requested that x2apic be disabled.\n"
608 "This will slightly decrease performance.\n"
609 "Use 'intremap=no_x2apic_optout' to override BIOS request.\n");
Suresh Siddha41750d32011-08-23 17:05:18 -0700610 }
611
Jiang Liu7c919772014-01-06 14:18:18 +0800612 for_each_iommu(iommu, drhd) {
Suresh Siddha1531a6a2009-03-16 17:04:57 -0700613 /*
Han, Weidong34aaaa92009-04-04 17:21:26 +0800614 * If the queued invalidation is already initialized,
615 * shouldn't disable it.
616 */
617 if (iommu->qi)
618 continue;
619
620 /*
Suresh Siddha1531a6a2009-03-16 17:04:57 -0700621 * Clear previous faults.
622 */
623 dmar_fault(-1, iommu);
624
625 /*
626 * Disable intr remapping and queued invalidation, if already
627 * enabled prior to OS handover.
628 */
Suresh Siddha95a02e92012-03-30 11:47:07 -0700629 iommu_disable_irq_remapping(iommu);
Suresh Siddha1531a6a2009-03-16 17:04:57 -0700630
631 dmar_disable_qi(iommu);
632 }
633
Suresh Siddha2ae21012008-07-10 11:16:43 -0700634 /*
635 * check for the Interrupt-remapping support
636 */
Jiang Liu7c919772014-01-06 14:18:18 +0800637 for_each_iommu(iommu, drhd) {
Suresh Siddha2ae21012008-07-10 11:16:43 -0700638 if (!ecap_ir_support(iommu->ecap))
639 continue;
640
641 if (eim && !ecap_eim_support(iommu->ecap)) {
642 printk(KERN_INFO "DRHD %Lx: EIM not supported by DRHD, "
643 " ecap %Lx\n", drhd->reg_base_addr, iommu->ecap);
Andy Lutomirskiaf8d1022013-02-01 14:57:43 -0800644 goto error;
Suresh Siddha2ae21012008-07-10 11:16:43 -0700645 }
646 }
647
648 /*
649 * Enable queued invalidation for all the DRHD's.
650 */
Jiang Liu7c919772014-01-06 14:18:18 +0800651 for_each_iommu(iommu, drhd) {
652 int ret = dmar_enable_qi(iommu);
Suresh Siddha2ae21012008-07-10 11:16:43 -0700653
654 if (ret) {
655 printk(KERN_ERR "DRHD %Lx: failed to enable queued, "
656 " invalidation, ecap %Lx, ret %d\n",
657 drhd->reg_base_addr, iommu->ecap, ret);
Andy Lutomirskiaf8d1022013-02-01 14:57:43 -0800658 goto error;
Suresh Siddha2ae21012008-07-10 11:16:43 -0700659 }
660 }
661
662 /*
663 * Setup Interrupt-remapping for all the DRHD's now.
664 */
Jiang Liu7c919772014-01-06 14:18:18 +0800665 for_each_iommu(iommu, drhd) {
Suresh Siddha2ae21012008-07-10 11:16:43 -0700666 if (!ecap_ir_support(iommu->ecap))
667 continue;
668
Suresh Siddha95a02e92012-03-30 11:47:07 -0700669 if (intel_setup_irq_remapping(iommu, eim))
Suresh Siddha2ae21012008-07-10 11:16:43 -0700670 goto error;
671
672 setup = 1;
673 }
674
675 if (!setup)
676 goto error;
677
Suresh Siddha95a02e92012-03-30 11:47:07 -0700678 irq_remapping_enabled = 1;
Joerg Roedelafcc8a42012-09-26 12:44:36 +0200679
680 /*
681 * VT-d has a different layout for IO-APIC entries when
682 * interrupt remapping is enabled. So it needs a special routine
683 * to print IO-APIC entries for debugging purposes too.
684 */
685 x86_io_apic_ops.print_entries = intel_ir_io_apic_print_entries;
686
Suresh Siddha41750d32011-08-23 17:05:18 -0700687 pr_info("Enabled IRQ remapping in %s mode\n", eim ? "x2apic" : "xapic");
Suresh Siddha2ae21012008-07-10 11:16:43 -0700688
Suresh Siddha41750d32011-08-23 17:05:18 -0700689 return eim ? IRQ_REMAP_X2APIC_MODE : IRQ_REMAP_XAPIC_MODE;
Suresh Siddha2ae21012008-07-10 11:16:43 -0700690
691error:
692 /*
693 * handle error condition gracefully here!
694 */
Andy Lutomirskiaf8d1022013-02-01 14:57:43 -0800695
696 if (x2apic_present)
Andy Lutomirskid01140d2013-05-13 15:22:42 -0700697 pr_warn("Failed to enable irq remapping. You are vulnerable to irq-injection attacks.\n");
Andy Lutomirskiaf8d1022013-02-01 14:57:43 -0800698
Suresh Siddha2ae21012008-07-10 11:16:43 -0700699 return -1;
700}
Suresh Siddhaad3ad3f2008-07-10 11:16:40 -0700701
Suresh Siddha20f30972009-08-04 12:07:08 -0700702static void ir_parse_one_hpet_scope(struct acpi_dmar_device_scope *scope,
703 struct intel_iommu *iommu)
704{
705 struct acpi_dmar_pci_path *path;
706 u8 bus;
707 int count;
708
709 bus = scope->bus;
710 path = (struct acpi_dmar_pci_path *)(scope + 1);
711 count = (scope->length - sizeof(struct acpi_dmar_device_scope))
712 / sizeof(struct acpi_dmar_pci_path);
713
714 while (--count > 0) {
715 /*
716 * Access PCI directly due to the PCI
717 * subsystem isn't initialized yet.
718 */
Lv Zhengfa5f5082013-10-31 09:30:22 +0800719 bus = read_pci_config_byte(bus, path->device, path->function,
Suresh Siddha20f30972009-08-04 12:07:08 -0700720 PCI_SECONDARY_BUS);
721 path++;
722 }
723 ir_hpet[ir_hpet_num].bus = bus;
Lv Zhengfa5f5082013-10-31 09:30:22 +0800724 ir_hpet[ir_hpet_num].devfn = PCI_DEVFN(path->device, path->function);
Suresh Siddha20f30972009-08-04 12:07:08 -0700725 ir_hpet[ir_hpet_num].iommu = iommu;
726 ir_hpet[ir_hpet_num].id = scope->enumeration_id;
727 ir_hpet_num++;
728}
729
Weidong Hanf007e992009-05-23 00:41:15 +0800730static void ir_parse_one_ioapic_scope(struct acpi_dmar_device_scope *scope,
731 struct intel_iommu *iommu)
732{
733 struct acpi_dmar_pci_path *path;
734 u8 bus;
735 int count;
736
737 bus = scope->bus;
738 path = (struct acpi_dmar_pci_path *)(scope + 1);
739 count = (scope->length - sizeof(struct acpi_dmar_device_scope))
740 / sizeof(struct acpi_dmar_pci_path);
741
742 while (--count > 0) {
743 /*
744 * Access PCI directly due to the PCI
745 * subsystem isn't initialized yet.
746 */
Lv Zhengfa5f5082013-10-31 09:30:22 +0800747 bus = read_pci_config_byte(bus, path->device, path->function,
Weidong Hanf007e992009-05-23 00:41:15 +0800748 PCI_SECONDARY_BUS);
749 path++;
750 }
751
752 ir_ioapic[ir_ioapic_num].bus = bus;
Lv Zhengfa5f5082013-10-31 09:30:22 +0800753 ir_ioapic[ir_ioapic_num].devfn = PCI_DEVFN(path->device, path->function);
Weidong Hanf007e992009-05-23 00:41:15 +0800754 ir_ioapic[ir_ioapic_num].iommu = iommu;
755 ir_ioapic[ir_ioapic_num].id = scope->enumeration_id;
756 ir_ioapic_num++;
757}
758
Suresh Siddha20f30972009-08-04 12:07:08 -0700759static int ir_parse_ioapic_hpet_scope(struct acpi_dmar_header *header,
760 struct intel_iommu *iommu)
Suresh Siddhaad3ad3f2008-07-10 11:16:40 -0700761{
762 struct acpi_dmar_hardware_unit *drhd;
763 struct acpi_dmar_device_scope *scope;
764 void *start, *end;
765
766 drhd = (struct acpi_dmar_hardware_unit *)header;
767
768 start = (void *)(drhd + 1);
769 end = ((void *)drhd) + header->length;
770
771 while (start < end) {
772 scope = start;
773 if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_IOAPIC) {
774 if (ir_ioapic_num == MAX_IO_APICS) {
775 printk(KERN_WARNING "Exceeded Max IO APICS\n");
776 return -1;
777 }
778
Yinghai Lu680a7522010-04-08 19:58:23 +0100779 printk(KERN_INFO "IOAPIC id %d under DRHD base "
780 " 0x%Lx IOMMU %d\n", scope->enumeration_id,
781 drhd->address, iommu->seq_id);
Suresh Siddhaad3ad3f2008-07-10 11:16:40 -0700782
Weidong Hanf007e992009-05-23 00:41:15 +0800783 ir_parse_one_ioapic_scope(scope, iommu);
Suresh Siddha20f30972009-08-04 12:07:08 -0700784 } else if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_HPET) {
785 if (ir_hpet_num == MAX_HPET_TBS) {
786 printk(KERN_WARNING "Exceeded Max HPET blocks\n");
787 return -1;
788 }
789
790 printk(KERN_INFO "HPET id %d under DRHD base"
791 " 0x%Lx\n", scope->enumeration_id,
792 drhd->address);
793
794 ir_parse_one_hpet_scope(scope, iommu);
Suresh Siddhaad3ad3f2008-07-10 11:16:40 -0700795 }
796 start += scope->length;
797 }
798
799 return 0;
800}
801
802/*
803 * Finds the assocaition between IOAPIC's and its Interrupt-remapping
804 * hardware unit.
805 */
Jiang Liu694835d2014-01-06 14:18:16 +0800806static int __init parse_ioapics_under_ir(void)
Suresh Siddhaad3ad3f2008-07-10 11:16:40 -0700807{
808 struct dmar_drhd_unit *drhd;
Jiang Liu7c919772014-01-06 14:18:18 +0800809 struct intel_iommu *iommu;
Suresh Siddhaad3ad3f2008-07-10 11:16:40 -0700810 int ir_supported = 0;
Seth Forshee32ab31e2012-08-08 08:27:03 -0500811 int ioapic_idx;
Suresh Siddhaad3ad3f2008-07-10 11:16:40 -0700812
Jiang Liu7c919772014-01-06 14:18:18 +0800813 for_each_iommu(iommu, drhd)
Suresh Siddhaad3ad3f2008-07-10 11:16:40 -0700814 if (ecap_ir_support(iommu->ecap)) {
Suresh Siddha20f30972009-08-04 12:07:08 -0700815 if (ir_parse_ioapic_hpet_scope(drhd->hdr, iommu))
Suresh Siddhaad3ad3f2008-07-10 11:16:40 -0700816 return -1;
817
818 ir_supported = 1;
819 }
Suresh Siddhaad3ad3f2008-07-10 11:16:40 -0700820
Seth Forshee32ab31e2012-08-08 08:27:03 -0500821 if (!ir_supported)
822 return 0;
823
824 for (ioapic_idx = 0; ioapic_idx < nr_ioapics; ioapic_idx++) {
825 int ioapic_id = mpc_ioapic_id(ioapic_idx);
826 if (!map_ioapic_to_ir(ioapic_id)) {
827 pr_err(FW_BUG "ioapic %d has no mapping iommu, "
828 "interrupt remapping will be disabled\n",
829 ioapic_id);
830 return -1;
831 }
Suresh Siddhaad3ad3f2008-07-10 11:16:40 -0700832 }
833
Seth Forshee32ab31e2012-08-08 08:27:03 -0500834 return 1;
Suresh Siddhaad3ad3f2008-07-10 11:16:40 -0700835}
Fenghua Yub24696b2009-03-27 14:22:44 -0700836
Rashika Kheria6a7885c2013-12-18 12:04:27 +0530837static int __init ir_dev_scope_init(void)
Suresh Siddhac2c72862011-08-23 17:05:19 -0700838{
Jiang Liu3a5670e2014-02-19 14:07:33 +0800839 int ret;
840
Suresh Siddha95a02e92012-03-30 11:47:07 -0700841 if (!irq_remapping_enabled)
Suresh Siddhac2c72862011-08-23 17:05:19 -0700842 return 0;
843
Jiang Liu3a5670e2014-02-19 14:07:33 +0800844 down_write(&dmar_global_lock);
845 ret = dmar_dev_scope_init();
846 up_write(&dmar_global_lock);
847
848 return ret;
Suresh Siddhac2c72862011-08-23 17:05:19 -0700849}
850rootfs_initcall(ir_dev_scope_init);
851
Suresh Siddha95a02e92012-03-30 11:47:07 -0700852static void disable_irq_remapping(void)
Fenghua Yub24696b2009-03-27 14:22:44 -0700853{
854 struct dmar_drhd_unit *drhd;
855 struct intel_iommu *iommu = NULL;
856
857 /*
858 * Disable Interrupt-remapping for all the DRHD's now.
859 */
860 for_each_iommu(iommu, drhd) {
861 if (!ecap_ir_support(iommu->ecap))
862 continue;
863
Suresh Siddha95a02e92012-03-30 11:47:07 -0700864 iommu_disable_irq_remapping(iommu);
Fenghua Yub24696b2009-03-27 14:22:44 -0700865 }
866}
867
Suresh Siddha95a02e92012-03-30 11:47:07 -0700868static int reenable_irq_remapping(int eim)
Fenghua Yub24696b2009-03-27 14:22:44 -0700869{
870 struct dmar_drhd_unit *drhd;
871 int setup = 0;
872 struct intel_iommu *iommu = NULL;
873
874 for_each_iommu(iommu, drhd)
875 if (iommu->qi)
876 dmar_reenable_qi(iommu);
877
878 /*
879 * Setup Interrupt-remapping for all the DRHD's now.
880 */
881 for_each_iommu(iommu, drhd) {
882 if (!ecap_ir_support(iommu->ecap))
883 continue;
884
885 /* Set up interrupt remapping for iommu.*/
Suresh Siddha95a02e92012-03-30 11:47:07 -0700886 iommu_set_irq_remapping(iommu, eim);
Fenghua Yub24696b2009-03-27 14:22:44 -0700887 setup = 1;
888 }
889
890 if (!setup)
891 goto error;
892
893 return 0;
894
895error:
896 /*
897 * handle error condition gracefully here!
898 */
899 return -1;
900}
901
Joerg Roedel0c3f1732012-03-30 11:47:02 -0700902static void prepare_irte(struct irte *irte, int vector,
903 unsigned int dest)
904{
905 memset(irte, 0, sizeof(*irte));
906
907 irte->present = 1;
908 irte->dst_mode = apic->irq_dest_mode;
909 /*
910 * Trigger mode in the IRTE will always be edge, and for IO-APIC, the
911 * actual level or edge trigger will be setup in the IO-APIC
912 * RTE. This will help simplify level triggered irq migration.
913 * For more details, see the comments (in io_apic.c) explainig IO-APIC
914 * irq migration in the presence of interrupt-remapping.
915 */
916 irte->trigger_mode = 0;
917 irte->dlvry_mode = apic->irq_delivery_mode;
918 irte->vector = vector;
919 irte->dest_id = IRTE_DEST(dest);
920 irte->redir_hint = 1;
921}
922
923static int intel_setup_ioapic_entry(int irq,
924 struct IO_APIC_route_entry *route_entry,
925 unsigned int destination, int vector,
926 struct io_apic_irq_attr *attr)
927{
928 int ioapic_id = mpc_ioapic_id(attr->ioapic);
Jiang Liu3a5670e2014-02-19 14:07:33 +0800929 struct intel_iommu *iommu;
Joerg Roedel0c3f1732012-03-30 11:47:02 -0700930 struct IR_IO_APIC_route_entry *entry;
931 struct irte irte;
932 int index;
933
Jiang Liu3a5670e2014-02-19 14:07:33 +0800934 down_read(&dmar_global_lock);
935 iommu = map_ioapic_to_ir(ioapic_id);
Joerg Roedel0c3f1732012-03-30 11:47:02 -0700936 if (!iommu) {
937 pr_warn("No mapping iommu for ioapic %d\n", ioapic_id);
Jiang Liu3a5670e2014-02-19 14:07:33 +0800938 index = -ENODEV;
939 } else {
940 index = alloc_irte(iommu, irq, 1);
941 if (index < 0) {
942 pr_warn("Failed to allocate IRTE for ioapic %d\n",
943 ioapic_id);
944 index = -ENOMEM;
945 }
Joerg Roedel0c3f1732012-03-30 11:47:02 -0700946 }
Jiang Liu3a5670e2014-02-19 14:07:33 +0800947 up_read(&dmar_global_lock);
948 if (index < 0)
949 return index;
Joerg Roedel0c3f1732012-03-30 11:47:02 -0700950
951 prepare_irte(&irte, vector, destination);
952
953 /* Set source-id of interrupt request */
954 set_ioapic_sid(&irte, ioapic_id);
955
956 modify_irte(irq, &irte);
957
958 apic_printk(APIC_VERBOSE, KERN_DEBUG "IOAPIC[%d]: "
959 "Set IRTE entry (P:%d FPD:%d Dst_Mode:%d "
960 "Redir_hint:%d Trig_Mode:%d Dlvry_Mode:%X "
961 "Avail:%X Vector:%02X Dest:%08X "
962 "SID:%04X SQ:%X SVT:%X)\n",
963 attr->ioapic, irte.present, irte.fpd, irte.dst_mode,
964 irte.redir_hint, irte.trigger_mode, irte.dlvry_mode,
965 irte.avail, irte.vector, irte.dest_id,
966 irte.sid, irte.sq, irte.svt);
967
Jiang Liu3a5670e2014-02-19 14:07:33 +0800968 entry = (struct IR_IO_APIC_route_entry *)route_entry;
Joerg Roedel0c3f1732012-03-30 11:47:02 -0700969 memset(entry, 0, sizeof(*entry));
970
971 entry->index2 = (index >> 15) & 0x1;
972 entry->zero = 0;
973 entry->format = 1;
974 entry->index = (index & 0x7fff);
975 /*
976 * IO-APIC RTE will be configured with virtual vector.
977 * irq handler will do the explicit EOI to the io-apic.
978 */
979 entry->vector = attr->ioapic_pin;
980 entry->mask = 0; /* enable IRQ */
981 entry->trigger = attr->trigger;
982 entry->polarity = attr->polarity;
983
984 /* Mask level triggered irqs.
985 * Use IRQ_DELAYED_DISABLE for edge triggered irqs.
986 */
987 if (attr->trigger)
988 entry->mask = 1;
989
990 return 0;
991}
992
Joerg Roedel4c1bad62012-03-30 11:47:03 -0700993/*
994 * Migrate the IO-APIC irq in the presence of intr-remapping.
995 *
996 * For both level and edge triggered, irq migration is a simple atomic
997 * update(of vector and cpu destination) of IRTE and flush the hardware cache.
998 *
999 * For level triggered, we eliminate the io-apic RTE modification (with the
1000 * updated vector information), by using a virtual vector (io-apic pin number).
1001 * Real vector that is used for interrupting cpu will be coming from
1002 * the interrupt-remapping table entry.
1003 *
1004 * As the migration is a simple atomic update of IRTE, the same mechanism
1005 * is used to migrate MSI irq's in the presence of interrupt-remapping.
1006 */
1007static int
1008intel_ioapic_set_affinity(struct irq_data *data, const struct cpumask *mask,
1009 bool force)
1010{
1011 struct irq_cfg *cfg = data->chip_data;
1012 unsigned int dest, irq = data->irq;
1013 struct irte irte;
Alexander Gordeevff164322012-06-07 15:15:59 +02001014 int err;
Joerg Roedel4c1bad62012-03-30 11:47:03 -07001015
Suresh Siddha7eb9ae02012-06-14 18:28:49 -07001016 if (!config_enabled(CONFIG_SMP))
1017 return -EINVAL;
1018
Joerg Roedel4c1bad62012-03-30 11:47:03 -07001019 if (!cpumask_intersects(mask, cpu_online_mask))
1020 return -EINVAL;
1021
1022 if (get_irte(irq, &irte))
1023 return -EBUSY;
1024
Alexander Gordeevff164322012-06-07 15:15:59 +02001025 err = assign_irq_vector(irq, cfg, mask);
1026 if (err)
1027 return err;
Joerg Roedel4c1bad62012-03-30 11:47:03 -07001028
Alexander Gordeevff164322012-06-07 15:15:59 +02001029 err = apic->cpu_mask_to_apicid_and(cfg->domain, mask, &dest);
1030 if (err) {
Dan Carpentered88bed2012-06-12 19:26:33 +03001031 if (assign_irq_vector(irq, cfg, data->affinity))
Alexander Gordeevff164322012-06-07 15:15:59 +02001032 pr_err("Failed to recover vector for irq %d\n", irq);
1033 return err;
1034 }
Joerg Roedel4c1bad62012-03-30 11:47:03 -07001035
1036 irte.vector = cfg->vector;
1037 irte.dest_id = IRTE_DEST(dest);
1038
1039 /*
1040 * Atomically updates the IRTE with the new destination, vector
1041 * and flushes the interrupt entry cache.
1042 */
1043 modify_irte(irq, &irte);
1044
1045 /*
1046 * After this point, all the interrupts will start arriving
1047 * at the new destination. So, time to cleanup the previous
1048 * vector allocation.
1049 */
1050 if (cfg->move_in_progress)
1051 send_cleanup_vector(cfg);
1052
1053 cpumask_copy(data->affinity, mask);
1054 return 0;
1055}
Joerg Roedel0c3f1732012-03-30 11:47:02 -07001056
Joerg Roedel5e2b9302012-03-30 11:47:05 -07001057static void intel_compose_msi_msg(struct pci_dev *pdev,
1058 unsigned int irq, unsigned int dest,
1059 struct msi_msg *msg, u8 hpet_id)
1060{
1061 struct irq_cfg *cfg;
1062 struct irte irte;
Suresh Siddhac558df42012-05-08 00:08:54 -07001063 u16 sub_handle = 0;
Joerg Roedel5e2b9302012-03-30 11:47:05 -07001064 int ir_index;
1065
1066 cfg = irq_get_chip_data(irq);
1067
1068 ir_index = map_irq_to_irte_handle(irq, &sub_handle);
1069 BUG_ON(ir_index == -1);
1070
1071 prepare_irte(&irte, cfg->vector, dest);
1072
1073 /* Set source-id of interrupt request */
1074 if (pdev)
1075 set_msi_sid(&irte, pdev);
1076 else
1077 set_hpet_sid(&irte, hpet_id);
1078
1079 modify_irte(irq, &irte);
1080
1081 msg->address_hi = MSI_ADDR_BASE_HI;
1082 msg->data = sub_handle;
1083 msg->address_lo = MSI_ADDR_BASE_LO | MSI_ADDR_IR_EXT_INT |
1084 MSI_ADDR_IR_SHV |
1085 MSI_ADDR_IR_INDEX1(ir_index) |
1086 MSI_ADDR_IR_INDEX2(ir_index);
1087}
1088
1089/*
1090 * Map the PCI dev to the corresponding remapping hardware unit
1091 * and allocate 'nvec' consecutive interrupt-remapping table entries
1092 * in it.
1093 */
1094static int intel_msi_alloc_irq(struct pci_dev *dev, int irq, int nvec)
1095{
1096 struct intel_iommu *iommu;
1097 int index;
1098
Jiang Liu3a5670e2014-02-19 14:07:33 +08001099 down_read(&dmar_global_lock);
Joerg Roedel5e2b9302012-03-30 11:47:05 -07001100 iommu = map_dev_to_ir(dev);
1101 if (!iommu) {
1102 printk(KERN_ERR
1103 "Unable to map PCI %s to iommu\n", pci_name(dev));
Jiang Liu3a5670e2014-02-19 14:07:33 +08001104 index = -ENOENT;
1105 } else {
1106 index = alloc_irte(iommu, irq, nvec);
1107 if (index < 0) {
1108 printk(KERN_ERR
1109 "Unable to allocate %d IRTE for PCI %s\n",
1110 nvec, pci_name(dev));
1111 index = -ENOSPC;
1112 }
Joerg Roedel5e2b9302012-03-30 11:47:05 -07001113 }
Jiang Liu3a5670e2014-02-19 14:07:33 +08001114 up_read(&dmar_global_lock);
Joerg Roedel5e2b9302012-03-30 11:47:05 -07001115
Joerg Roedel5e2b9302012-03-30 11:47:05 -07001116 return index;
1117}
1118
1119static int intel_msi_setup_irq(struct pci_dev *pdev, unsigned int irq,
1120 int index, int sub_handle)
1121{
1122 struct intel_iommu *iommu;
Jiang Liu3a5670e2014-02-19 14:07:33 +08001123 int ret = -ENOENT;
Joerg Roedel5e2b9302012-03-30 11:47:05 -07001124
Jiang Liu3a5670e2014-02-19 14:07:33 +08001125 down_read(&dmar_global_lock);
Joerg Roedel5e2b9302012-03-30 11:47:05 -07001126 iommu = map_dev_to_ir(pdev);
Jiang Liu3a5670e2014-02-19 14:07:33 +08001127 if (iommu) {
1128 /*
1129 * setup the mapping between the irq and the IRTE
1130 * base index, the sub_handle pointing to the
1131 * appropriate interrupt remap table entry.
1132 */
1133 set_irte_irq(irq, iommu, index, sub_handle);
1134 ret = 0;
1135 }
1136 up_read(&dmar_global_lock);
Joerg Roedel5e2b9302012-03-30 11:47:05 -07001137
Jiang Liu3a5670e2014-02-19 14:07:33 +08001138 return ret;
Joerg Roedel5e2b9302012-03-30 11:47:05 -07001139}
1140
Yijing Wang5fc24d82014-09-17 17:32:19 +08001141static int intel_alloc_hpet_msi(unsigned int irq, unsigned int id)
Joerg Roedel5e2b9302012-03-30 11:47:05 -07001142{
Jiang Liu3a5670e2014-02-19 14:07:33 +08001143 int ret = -1;
1144 struct intel_iommu *iommu;
Joerg Roedel5e2b9302012-03-30 11:47:05 -07001145 int index;
1146
Jiang Liu3a5670e2014-02-19 14:07:33 +08001147 down_read(&dmar_global_lock);
1148 iommu = map_hpet_to_ir(id);
1149 if (iommu) {
1150 index = alloc_irte(iommu, irq, 1);
1151 if (index >= 0)
1152 ret = 0;
1153 }
1154 up_read(&dmar_global_lock);
Joerg Roedel5e2b9302012-03-30 11:47:05 -07001155
Jiang Liu3a5670e2014-02-19 14:07:33 +08001156 return ret;
Joerg Roedel5e2b9302012-03-30 11:47:05 -07001157}
1158
Joerg Roedel736baef2012-03-30 11:47:00 -07001159struct irq_remap_ops intel_irq_remap_ops = {
Suresh Siddha95a02e92012-03-30 11:47:07 -07001160 .supported = intel_irq_remapping_supported,
1161 .prepare = dmar_table_init,
1162 .enable = intel_enable_irq_remapping,
1163 .disable = disable_irq_remapping,
1164 .reenable = reenable_irq_remapping,
Joerg Roedel4f3d8b62012-03-30 11:47:01 -07001165 .enable_faulting = enable_drhd_fault_handling,
Joerg Roedel0c3f1732012-03-30 11:47:02 -07001166 .setup_ioapic_entry = intel_setup_ioapic_entry,
Joerg Roedel4c1bad62012-03-30 11:47:03 -07001167 .set_affinity = intel_ioapic_set_affinity,
Joerg Roedel9d619f62012-03-30 11:47:04 -07001168 .free_irq = free_irte,
Joerg Roedel5e2b9302012-03-30 11:47:05 -07001169 .compose_msi_msg = intel_compose_msi_msg,
1170 .msi_alloc_irq = intel_msi_alloc_irq,
1171 .msi_setup_irq = intel_msi_setup_irq,
Yijing Wang5fc24d82014-09-17 17:32:19 +08001172 .alloc_hpet_msi = intel_alloc_hpet_msi,
Joerg Roedel736baef2012-03-30 11:47:00 -07001173};