blob: f5e0ea724a6f53a12d26443fba84b01ab898f35d [file] [log] [blame]
Yinghai Lu5aeecaf2008-08-19 20:49:59 -07001#include <linux/interrupt.h>
Suresh Siddhaad3ad3f2008-07-10 11:16:40 -07002#include <linux/dmar.h>
Suresh Siddha2ae21012008-07-10 11:16:43 -07003#include <linux/spinlock.h>
4#include <linux/jiffies.h>
5#include <linux/pci.h>
Suresh Siddhab6fcb332008-07-10 11:16:44 -07006#include <linux/irq.h>
Suresh Siddhaad3ad3f2008-07-10 11:16:40 -07007#include <asm/io_apic.h>
Yinghai Lu17483a12008-12-12 13:14:18 -08008#include <asm/smp.h>
Jaswinder Singh Rajput6d652ea2009-01-07 21:38:59 +05309#include <asm/cpu.h>
Kay, Allen M38717942008-09-09 18:37:29 +030010#include <linux/intel-iommu.h>
Suresh Siddhaad3ad3f2008-07-10 11:16:40 -070011#include "intr_remapping.h"
Alexander Beregalov46f06b722009-04-06 16:45:28 +010012#include <acpi/acpi.h>
Suresh Siddhaad3ad3f2008-07-10 11:16:40 -070013
14static struct ioapic_scope ir_ioapic[MAX_IO_APICS];
15static int ir_ioapic_num;
Suresh Siddha2ae21012008-07-10 11:16:43 -070016int intr_remapping_enabled;
17
Yinghai Lu5aeecaf2008-08-19 20:49:59 -070018struct irq_2_iommu {
Suresh Siddhab6fcb332008-07-10 11:16:44 -070019 struct intel_iommu *iommu;
20 u16 irte_index;
21 u16 sub_handle;
22 u8 irte_mask;
Yinghai Lu5aeecaf2008-08-19 20:49:59 -070023};
24
Yinghai Lud7e51e62009-01-07 15:03:13 -080025#ifdef CONFIG_GENERIC_HARDIRQS
Yinghai Lu0b8f1ef2008-12-05 18:58:31 -080026static struct irq_2_iommu *get_one_free_irq_2_iommu(int cpu)
27{
28 struct irq_2_iommu *iommu;
29 int node;
30
31 node = cpu_to_node(cpu);
32
33 iommu = kzalloc_node(sizeof(*iommu), GFP_ATOMIC, node);
34 printk(KERN_DEBUG "alloc irq_2_iommu on cpu %d node %d\n", cpu, node);
35
36 return iommu;
37}
Yinghai Lue420dfb2008-08-19 20:50:21 -070038
39static struct irq_2_iommu *irq_2_iommu(unsigned int irq)
40{
Yinghai Lu0b8f1ef2008-12-05 18:58:31 -080041 struct irq_desc *desc;
42
43 desc = irq_to_desc(irq);
44
45 if (WARN_ON_ONCE(!desc))
46 return NULL;
47
48 return desc->irq_2_iommu;
49}
50
51static struct irq_2_iommu *irq_2_iommu_alloc_cpu(unsigned int irq, int cpu)
52{
53 struct irq_desc *desc;
54 struct irq_2_iommu *irq_iommu;
55
56 /*
57 * alloc irq desc if not allocated already.
58 */
59 desc = irq_to_desc_alloc_cpu(irq, cpu);
60 if (!desc) {
61 printk(KERN_INFO "can not get irq_desc for %d\n", irq);
62 return NULL;
63 }
64
65 irq_iommu = desc->irq_2_iommu;
66
67 if (!irq_iommu)
68 desc->irq_2_iommu = get_one_free_irq_2_iommu(cpu);
69
70 return desc->irq_2_iommu;
Yinghai Lue420dfb2008-08-19 20:50:21 -070071}
Thomas Gleixnerd6c88a52008-10-15 15:27:23 +020072
Yinghai Lue420dfb2008-08-19 20:50:21 -070073static struct irq_2_iommu *irq_2_iommu_alloc(unsigned int irq)
74{
Yinghai Lu0b8f1ef2008-12-05 18:58:31 -080075 return irq_2_iommu_alloc_cpu(irq, boot_cpu_id);
76}
77
78#else /* !CONFIG_SPARSE_IRQ */
79
80static struct irq_2_iommu irq_2_iommuX[NR_IRQS];
81
82static struct irq_2_iommu *irq_2_iommu(unsigned int irq)
83{
84 if (irq < nr_irqs)
85 return &irq_2_iommuX[irq];
86
87 return NULL;
88}
89static struct irq_2_iommu *irq_2_iommu_alloc(unsigned int irq)
90{
Yinghai Lue420dfb2008-08-19 20:50:21 -070091 return irq_2_iommu(irq);
92}
Yinghai Lu0b8f1ef2008-12-05 18:58:31 -080093#endif
Suresh Siddhab6fcb332008-07-10 11:16:44 -070094
95static DEFINE_SPINLOCK(irq_2_ir_lock);
96
Yinghai Lue420dfb2008-08-19 20:50:21 -070097static struct irq_2_iommu *valid_irq_2_iommu(unsigned int irq)
98{
99 struct irq_2_iommu *irq_iommu;
100
101 irq_iommu = irq_2_iommu(irq);
102
103 if (!irq_iommu)
104 return NULL;
105
106 if (!irq_iommu->iommu)
107 return NULL;
108
109 return irq_iommu;
110}
111
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700112int irq_remapped(int irq)
113{
Yinghai Lue420dfb2008-08-19 20:50:21 -0700114 return valid_irq_2_iommu(irq) != NULL;
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700115}
116
117int get_irte(int irq, struct irte *entry)
118{
119 int index;
Yinghai Lue420dfb2008-08-19 20:50:21 -0700120 struct irq_2_iommu *irq_iommu;
Suresh Siddha4c5502b2009-03-16 17:04:53 -0700121 unsigned long flags;
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700122
Yinghai Lue420dfb2008-08-19 20:50:21 -0700123 if (!entry)
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700124 return -1;
125
Suresh Siddha4c5502b2009-03-16 17:04:53 -0700126 spin_lock_irqsave(&irq_2_ir_lock, flags);
Yinghai Lue420dfb2008-08-19 20:50:21 -0700127 irq_iommu = valid_irq_2_iommu(irq);
128 if (!irq_iommu) {
Suresh Siddha4c5502b2009-03-16 17:04:53 -0700129 spin_unlock_irqrestore(&irq_2_ir_lock, flags);
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700130 return -1;
131 }
132
Yinghai Lue420dfb2008-08-19 20:50:21 -0700133 index = irq_iommu->irte_index + irq_iommu->sub_handle;
134 *entry = *(irq_iommu->iommu->ir_table->base + index);
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700135
Suresh Siddha4c5502b2009-03-16 17:04:53 -0700136 spin_unlock_irqrestore(&irq_2_ir_lock, flags);
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700137 return 0;
138}
139
140int alloc_irte(struct intel_iommu *iommu, int irq, u16 count)
141{
142 struct ir_table *table = iommu->ir_table;
Yinghai Lue420dfb2008-08-19 20:50:21 -0700143 struct irq_2_iommu *irq_iommu;
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700144 u16 index, start_index;
145 unsigned int mask = 0;
Suresh Siddha4c5502b2009-03-16 17:04:53 -0700146 unsigned long flags;
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700147 int i;
148
149 if (!count)
150 return -1;
151
Yinghai Lu0b8f1ef2008-12-05 18:58:31 -0800152#ifndef CONFIG_SPARSE_IRQ
Yinghai Lue420dfb2008-08-19 20:50:21 -0700153 /* protect irq_2_iommu_alloc later */
154 if (irq >= nr_irqs)
155 return -1;
Yinghai Lu0b8f1ef2008-12-05 18:58:31 -0800156#endif
Yinghai Lue420dfb2008-08-19 20:50:21 -0700157
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700158 /*
159 * start the IRTE search from index 0.
160 */
161 index = start_index = 0;
162
163 if (count > 1) {
164 count = __roundup_pow_of_two(count);
165 mask = ilog2(count);
166 }
167
168 if (mask > ecap_max_handle_mask(iommu->ecap)) {
169 printk(KERN_ERR
170 "Requested mask %x exceeds the max invalidation handle"
171 " mask value %Lx\n", mask,
172 ecap_max_handle_mask(iommu->ecap));
173 return -1;
174 }
175
Suresh Siddha4c5502b2009-03-16 17:04:53 -0700176 spin_lock_irqsave(&irq_2_ir_lock, flags);
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700177 do {
178 for (i = index; i < index + count; i++)
179 if (table->base[i].present)
180 break;
181 /* empty index found */
182 if (i == index + count)
183 break;
184
185 index = (index + count) % INTR_REMAP_TABLE_ENTRIES;
186
187 if (index == start_index) {
Suresh Siddha4c5502b2009-03-16 17:04:53 -0700188 spin_unlock_irqrestore(&irq_2_ir_lock, flags);
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700189 printk(KERN_ERR "can't allocate an IRTE\n");
190 return -1;
191 }
192 } while (1);
193
194 for (i = index; i < index + count; i++)
195 table->base[i].present = 1;
196
Yinghai Lue420dfb2008-08-19 20:50:21 -0700197 irq_iommu = irq_2_iommu_alloc(irq);
Yinghai Lu0b8f1ef2008-12-05 18:58:31 -0800198 if (!irq_iommu) {
Suresh Siddha4c5502b2009-03-16 17:04:53 -0700199 spin_unlock_irqrestore(&irq_2_ir_lock, flags);
Yinghai Lu0b8f1ef2008-12-05 18:58:31 -0800200 printk(KERN_ERR "can't allocate irq_2_iommu\n");
201 return -1;
202 }
203
Yinghai Lue420dfb2008-08-19 20:50:21 -0700204 irq_iommu->iommu = iommu;
205 irq_iommu->irte_index = index;
206 irq_iommu->sub_handle = 0;
207 irq_iommu->irte_mask = mask;
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700208
Suresh Siddha4c5502b2009-03-16 17:04:53 -0700209 spin_unlock_irqrestore(&irq_2_ir_lock, flags);
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700210
211 return index;
212}
213
Yu Zhao704126a2009-01-04 16:28:52 +0800214static int qi_flush_iec(struct intel_iommu *iommu, int index, int mask)
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700215{
216 struct qi_desc desc;
217
218 desc.low = QI_IEC_IIDEX(index) | QI_IEC_TYPE | QI_IEC_IM(mask)
219 | QI_IEC_SELECTIVE;
220 desc.high = 0;
221
Yu Zhao704126a2009-01-04 16:28:52 +0800222 return qi_submit_sync(&desc, iommu);
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700223}
224
225int map_irq_to_irte_handle(int irq, u16 *sub_handle)
226{
227 int index;
Yinghai Lue420dfb2008-08-19 20:50:21 -0700228 struct irq_2_iommu *irq_iommu;
Suresh Siddha4c5502b2009-03-16 17:04:53 -0700229 unsigned long flags;
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700230
Suresh Siddha4c5502b2009-03-16 17:04:53 -0700231 spin_lock_irqsave(&irq_2_ir_lock, flags);
Yinghai Lue420dfb2008-08-19 20:50:21 -0700232 irq_iommu = valid_irq_2_iommu(irq);
233 if (!irq_iommu) {
Suresh Siddha4c5502b2009-03-16 17:04:53 -0700234 spin_unlock_irqrestore(&irq_2_ir_lock, flags);
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700235 return -1;
236 }
237
Yinghai Lue420dfb2008-08-19 20:50:21 -0700238 *sub_handle = irq_iommu->sub_handle;
239 index = irq_iommu->irte_index;
Suresh Siddha4c5502b2009-03-16 17:04:53 -0700240 spin_unlock_irqrestore(&irq_2_ir_lock, flags);
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700241 return index;
242}
243
244int set_irte_irq(int irq, struct intel_iommu *iommu, u16 index, u16 subhandle)
245{
Yinghai Lue420dfb2008-08-19 20:50:21 -0700246 struct irq_2_iommu *irq_iommu;
Suresh Siddha4c5502b2009-03-16 17:04:53 -0700247 unsigned long flags;
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700248
Suresh Siddha4c5502b2009-03-16 17:04:53 -0700249 spin_lock_irqsave(&irq_2_ir_lock, flags);
Suresh Siddha7ddfb652008-08-20 17:22:51 -0700250
251 irq_iommu = irq_2_iommu_alloc(irq);
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700252
Yinghai Lu0b8f1ef2008-12-05 18:58:31 -0800253 if (!irq_iommu) {
Suresh Siddha4c5502b2009-03-16 17:04:53 -0700254 spin_unlock_irqrestore(&irq_2_ir_lock, flags);
Yinghai Lu0b8f1ef2008-12-05 18:58:31 -0800255 printk(KERN_ERR "can't allocate irq_2_iommu\n");
256 return -1;
257 }
258
Yinghai Lue420dfb2008-08-19 20:50:21 -0700259 irq_iommu->iommu = iommu;
260 irq_iommu->irte_index = index;
261 irq_iommu->sub_handle = subhandle;
262 irq_iommu->irte_mask = 0;
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700263
Suresh Siddha4c5502b2009-03-16 17:04:53 -0700264 spin_unlock_irqrestore(&irq_2_ir_lock, flags);
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700265
266 return 0;
267}
268
269int clear_irte_irq(int irq, struct intel_iommu *iommu, u16 index)
270{
Yinghai Lue420dfb2008-08-19 20:50:21 -0700271 struct irq_2_iommu *irq_iommu;
Suresh Siddha4c5502b2009-03-16 17:04:53 -0700272 unsigned long flags;
Yinghai Lue420dfb2008-08-19 20:50:21 -0700273
Suresh Siddha4c5502b2009-03-16 17:04:53 -0700274 spin_lock_irqsave(&irq_2_ir_lock, flags);
Yinghai Lue420dfb2008-08-19 20:50:21 -0700275 irq_iommu = valid_irq_2_iommu(irq);
276 if (!irq_iommu) {
Suresh Siddha4c5502b2009-03-16 17:04:53 -0700277 spin_unlock_irqrestore(&irq_2_ir_lock, flags);
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700278 return -1;
279 }
280
Yinghai Lue420dfb2008-08-19 20:50:21 -0700281 irq_iommu->iommu = NULL;
282 irq_iommu->irte_index = 0;
283 irq_iommu->sub_handle = 0;
284 irq_2_iommu(irq)->irte_mask = 0;
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700285
Suresh Siddha4c5502b2009-03-16 17:04:53 -0700286 spin_unlock_irqrestore(&irq_2_ir_lock, flags);
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700287
288 return 0;
289}
290
291int modify_irte(int irq, struct irte *irte_modified)
292{
Yu Zhao704126a2009-01-04 16:28:52 +0800293 int rc;
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700294 int index;
295 struct irte *irte;
296 struct intel_iommu *iommu;
Yinghai Lue420dfb2008-08-19 20:50:21 -0700297 struct irq_2_iommu *irq_iommu;
Suresh Siddha4c5502b2009-03-16 17:04:53 -0700298 unsigned long flags;
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700299
Suresh Siddha4c5502b2009-03-16 17:04:53 -0700300 spin_lock_irqsave(&irq_2_ir_lock, flags);
Yinghai Lue420dfb2008-08-19 20:50:21 -0700301 irq_iommu = valid_irq_2_iommu(irq);
302 if (!irq_iommu) {
Suresh Siddha4c5502b2009-03-16 17:04:53 -0700303 spin_unlock_irqrestore(&irq_2_ir_lock, flags);
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700304 return -1;
305 }
306
Yinghai Lue420dfb2008-08-19 20:50:21 -0700307 iommu = irq_iommu->iommu;
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700308
Yinghai Lue420dfb2008-08-19 20:50:21 -0700309 index = irq_iommu->irte_index + irq_iommu->sub_handle;
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700310 irte = &iommu->ir_table->base[index];
311
Suresh Siddha9d783ba2009-03-16 17:04:55 -0700312 set_64bit((unsigned long *)irte, irte_modified->low);
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700313 __iommu_flush_cache(iommu, irte, sizeof(*irte));
314
Yu Zhao704126a2009-01-04 16:28:52 +0800315 rc = qi_flush_iec(iommu, index, 0);
Suresh Siddha4c5502b2009-03-16 17:04:53 -0700316 spin_unlock_irqrestore(&irq_2_ir_lock, flags);
Yu Zhao704126a2009-01-04 16:28:52 +0800317
318 return rc;
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700319}
320
321int flush_irte(int irq)
322{
Yu Zhao704126a2009-01-04 16:28:52 +0800323 int rc;
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700324 int index;
325 struct intel_iommu *iommu;
Yinghai Lue420dfb2008-08-19 20:50:21 -0700326 struct irq_2_iommu *irq_iommu;
Suresh Siddha4c5502b2009-03-16 17:04:53 -0700327 unsigned long flags;
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700328
Suresh Siddha4c5502b2009-03-16 17:04:53 -0700329 spin_lock_irqsave(&irq_2_ir_lock, flags);
Yinghai Lue420dfb2008-08-19 20:50:21 -0700330 irq_iommu = valid_irq_2_iommu(irq);
331 if (!irq_iommu) {
Suresh Siddha4c5502b2009-03-16 17:04:53 -0700332 spin_unlock_irqrestore(&irq_2_ir_lock, flags);
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700333 return -1;
334 }
335
Yinghai Lue420dfb2008-08-19 20:50:21 -0700336 iommu = irq_iommu->iommu;
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700337
Yinghai Lue420dfb2008-08-19 20:50:21 -0700338 index = irq_iommu->irte_index + irq_iommu->sub_handle;
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700339
Yu Zhao704126a2009-01-04 16:28:52 +0800340 rc = qi_flush_iec(iommu, index, irq_iommu->irte_mask);
Suresh Siddha4c5502b2009-03-16 17:04:53 -0700341 spin_unlock_irqrestore(&irq_2_ir_lock, flags);
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700342
Yu Zhao704126a2009-01-04 16:28:52 +0800343 return rc;
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700344}
345
Suresh Siddha89027d32008-07-10 11:16:56 -0700346struct intel_iommu *map_ioapic_to_ir(int apic)
347{
348 int i;
349
350 for (i = 0; i < MAX_IO_APICS; i++)
351 if (ir_ioapic[i].id == apic)
352 return ir_ioapic[i].iommu;
353 return NULL;
354}
355
Suresh Siddha75c46fa2008-07-10 11:16:57 -0700356struct intel_iommu *map_dev_to_ir(struct pci_dev *dev)
357{
358 struct dmar_drhd_unit *drhd;
359
360 drhd = dmar_find_matched_drhd_unit(dev);
361 if (!drhd)
362 return NULL;
363
364 return drhd->iommu;
365}
366
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700367int free_irte(int irq)
368{
Yu Zhao704126a2009-01-04 16:28:52 +0800369 int rc = 0;
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700370 int index, i;
371 struct irte *irte;
372 struct intel_iommu *iommu;
Yinghai Lue420dfb2008-08-19 20:50:21 -0700373 struct irq_2_iommu *irq_iommu;
Suresh Siddha4c5502b2009-03-16 17:04:53 -0700374 unsigned long flags;
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700375
Suresh Siddha4c5502b2009-03-16 17:04:53 -0700376 spin_lock_irqsave(&irq_2_ir_lock, flags);
Yinghai Lue420dfb2008-08-19 20:50:21 -0700377 irq_iommu = valid_irq_2_iommu(irq);
378 if (!irq_iommu) {
Suresh Siddha4c5502b2009-03-16 17:04:53 -0700379 spin_unlock_irqrestore(&irq_2_ir_lock, flags);
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700380 return -1;
381 }
382
Yinghai Lue420dfb2008-08-19 20:50:21 -0700383 iommu = irq_iommu->iommu;
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700384
Yinghai Lue420dfb2008-08-19 20:50:21 -0700385 index = irq_iommu->irte_index + irq_iommu->sub_handle;
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700386 irte = &iommu->ir_table->base[index];
387
Yinghai Lue420dfb2008-08-19 20:50:21 -0700388 if (!irq_iommu->sub_handle) {
389 for (i = 0; i < (1 << irq_iommu->irte_mask); i++)
Suresh Siddha2e934562009-03-16 17:04:58 -0700390 set_64bit((unsigned long *)(irte + i), 0);
Yu Zhao704126a2009-01-04 16:28:52 +0800391 rc = qi_flush_iec(iommu, index, irq_iommu->irte_mask);
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700392 }
393
Yinghai Lue420dfb2008-08-19 20:50:21 -0700394 irq_iommu->iommu = NULL;
395 irq_iommu->irte_index = 0;
396 irq_iommu->sub_handle = 0;
397 irq_iommu->irte_mask = 0;
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700398
Suresh Siddha4c5502b2009-03-16 17:04:53 -0700399 spin_unlock_irqrestore(&irq_2_ir_lock, flags);
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700400
Yu Zhao704126a2009-01-04 16:28:52 +0800401 return rc;
Suresh Siddhab6fcb332008-07-10 11:16:44 -0700402}
403
Suresh Siddha2ae21012008-07-10 11:16:43 -0700404static void iommu_set_intr_remapping(struct intel_iommu *iommu, int mode)
405{
406 u64 addr;
407 u32 cmd, sts;
408 unsigned long flags;
409
410 addr = virt_to_phys((void *)iommu->ir_table->base);
411
412 spin_lock_irqsave(&iommu->register_lock, flags);
413
414 dmar_writeq(iommu->reg + DMAR_IRTA_REG,
415 (addr) | IR_X2APIC_MODE(mode) | INTR_REMAP_TABLE_REG_SIZE);
416
417 /* Set interrupt-remapping table pointer */
418 cmd = iommu->gcmd | DMA_GCMD_SIRTP;
Han, Weidong161fde02009-04-03 17:15:47 +0800419 iommu->gcmd |= DMA_GCMD_SIRTP;
Suresh Siddha2ae21012008-07-10 11:16:43 -0700420 writel(cmd, iommu->reg + DMAR_GCMD_REG);
421
422 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
423 readl, (sts & DMA_GSTS_IRTPS), sts);
424 spin_unlock_irqrestore(&iommu->register_lock, flags);
425
Han, Weidong161fde02009-04-03 17:15:47 +0800426 if (mode == 0) {
427 spin_lock_irqsave(&iommu->register_lock, flags);
428
429 /* enable comaptiblity format interrupt pass through */
430 cmd = iommu->gcmd | DMA_GCMD_CFI;
431 iommu->gcmd |= DMA_GCMD_CFI;
432 writel(cmd, iommu->reg + DMAR_GCMD_REG);
433
434 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
435 readl, (sts & DMA_GSTS_CFIS), sts);
436
437 spin_unlock_irqrestore(&iommu->register_lock, flags);
438 }
439
Suresh Siddha2ae21012008-07-10 11:16:43 -0700440 /*
441 * global invalidation of interrupt entry cache before enabling
442 * interrupt-remapping.
443 */
444 qi_global_iec(iommu);
445
446 spin_lock_irqsave(&iommu->register_lock, flags);
447
448 /* Enable interrupt-remapping */
449 cmd = iommu->gcmd | DMA_GCMD_IRE;
450 iommu->gcmd |= DMA_GCMD_IRE;
451 writel(cmd, iommu->reg + DMAR_GCMD_REG);
452
453 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
454 readl, (sts & DMA_GSTS_IRES), sts);
455
456 spin_unlock_irqrestore(&iommu->register_lock, flags);
457}
458
459
460static int setup_intr_remapping(struct intel_iommu *iommu, int mode)
461{
462 struct ir_table *ir_table;
463 struct page *pages;
464
465 ir_table = iommu->ir_table = kzalloc(sizeof(struct ir_table),
Suresh Siddhafa4b57c2009-03-16 17:05:05 -0700466 GFP_ATOMIC);
Suresh Siddha2ae21012008-07-10 11:16:43 -0700467
468 if (!iommu->ir_table)
469 return -ENOMEM;
470
Suresh Siddhafa4b57c2009-03-16 17:05:05 -0700471 pages = alloc_pages(GFP_ATOMIC | __GFP_ZERO, INTR_REMAP_PAGE_ORDER);
Suresh Siddha2ae21012008-07-10 11:16:43 -0700472
473 if (!pages) {
474 printk(KERN_ERR "failed to allocate pages of order %d\n",
475 INTR_REMAP_PAGE_ORDER);
476 kfree(iommu->ir_table);
477 return -ENOMEM;
478 }
479
480 ir_table->base = page_address(pages);
481
482 iommu_set_intr_remapping(iommu, mode);
483 return 0;
484}
485
Suresh Siddhaeba67e52009-03-16 17:04:56 -0700486/*
487 * Disable Interrupt Remapping.
488 */
Fenghua Yub24696b2009-03-27 14:22:44 -0700489static void iommu_disable_intr_remapping(struct intel_iommu *iommu)
Suresh Siddhaeba67e52009-03-16 17:04:56 -0700490{
491 unsigned long flags;
492 u32 sts;
493
494 if (!ecap_ir_support(iommu->ecap))
495 return;
496
Fenghua Yub24696b2009-03-27 14:22:44 -0700497 /*
498 * global invalidation of interrupt entry cache before disabling
499 * interrupt-remapping.
500 */
501 qi_global_iec(iommu);
502
Suresh Siddhaeba67e52009-03-16 17:04:56 -0700503 spin_lock_irqsave(&iommu->register_lock, flags);
504
505 sts = dmar_readq(iommu->reg + DMAR_GSTS_REG);
506 if (!(sts & DMA_GSTS_IRES))
507 goto end;
508
509 iommu->gcmd &= ~DMA_GCMD_IRE;
510 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
511
512 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
513 readl, !(sts & DMA_GSTS_IRES), sts);
514
515end:
516 spin_unlock_irqrestore(&iommu->register_lock, flags);
517}
518
Suresh Siddha2ae21012008-07-10 11:16:43 -0700519int __init enable_intr_remapping(int eim)
520{
521 struct dmar_drhd_unit *drhd;
522 int setup = 0;
523
Suresh Siddha1531a6a2009-03-16 17:04:57 -0700524 for_each_drhd_unit(drhd) {
525 struct intel_iommu *iommu = drhd->iommu;
526
527 /*
Han, Weidong34aaaa92009-04-04 17:21:26 +0800528 * If the queued invalidation is already initialized,
529 * shouldn't disable it.
530 */
531 if (iommu->qi)
532 continue;
533
534 /*
Suresh Siddha1531a6a2009-03-16 17:04:57 -0700535 * Clear previous faults.
536 */
537 dmar_fault(-1, iommu);
538
539 /*
540 * Disable intr remapping and queued invalidation, if already
541 * enabled prior to OS handover.
542 */
Fenghua Yub24696b2009-03-27 14:22:44 -0700543 iommu_disable_intr_remapping(iommu);
Suresh Siddha1531a6a2009-03-16 17:04:57 -0700544
545 dmar_disable_qi(iommu);
546 }
547
Suresh Siddha2ae21012008-07-10 11:16:43 -0700548 /*
549 * check for the Interrupt-remapping support
550 */
551 for_each_drhd_unit(drhd) {
552 struct intel_iommu *iommu = drhd->iommu;
553
554 if (!ecap_ir_support(iommu->ecap))
555 continue;
556
557 if (eim && !ecap_eim_support(iommu->ecap)) {
558 printk(KERN_INFO "DRHD %Lx: EIM not supported by DRHD, "
559 " ecap %Lx\n", drhd->reg_base_addr, iommu->ecap);
560 return -1;
561 }
562 }
563
564 /*
565 * Enable queued invalidation for all the DRHD's.
566 */
567 for_each_drhd_unit(drhd) {
568 int ret;
569 struct intel_iommu *iommu = drhd->iommu;
570 ret = dmar_enable_qi(iommu);
571
572 if (ret) {
573 printk(KERN_ERR "DRHD %Lx: failed to enable queued, "
574 " invalidation, ecap %Lx, ret %d\n",
575 drhd->reg_base_addr, iommu->ecap, ret);
576 return -1;
577 }
578 }
579
580 /*
581 * Setup Interrupt-remapping for all the DRHD's now.
582 */
583 for_each_drhd_unit(drhd) {
584 struct intel_iommu *iommu = drhd->iommu;
585
586 if (!ecap_ir_support(iommu->ecap))
587 continue;
588
589 if (setup_intr_remapping(iommu, eim))
590 goto error;
591
592 setup = 1;
593 }
594
595 if (!setup)
596 goto error;
597
598 intr_remapping_enabled = 1;
599
600 return 0;
601
602error:
603 /*
604 * handle error condition gracefully here!
605 */
606 return -1;
607}
Suresh Siddhaad3ad3f2008-07-10 11:16:40 -0700608
609static int ir_parse_ioapic_scope(struct acpi_dmar_header *header,
610 struct intel_iommu *iommu)
611{
612 struct acpi_dmar_hardware_unit *drhd;
613 struct acpi_dmar_device_scope *scope;
614 void *start, *end;
615
616 drhd = (struct acpi_dmar_hardware_unit *)header;
617
618 start = (void *)(drhd + 1);
619 end = ((void *)drhd) + header->length;
620
621 while (start < end) {
622 scope = start;
623 if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_IOAPIC) {
624 if (ir_ioapic_num == MAX_IO_APICS) {
625 printk(KERN_WARNING "Exceeded Max IO APICS\n");
626 return -1;
627 }
628
629 printk(KERN_INFO "IOAPIC id %d under DRHD base"
630 " 0x%Lx\n", scope->enumeration_id,
631 drhd->address);
632
633 ir_ioapic[ir_ioapic_num].iommu = iommu;
634 ir_ioapic[ir_ioapic_num].id = scope->enumeration_id;
635 ir_ioapic_num++;
636 }
637 start += scope->length;
638 }
639
640 return 0;
641}
642
643/*
644 * Finds the assocaition between IOAPIC's and its Interrupt-remapping
645 * hardware unit.
646 */
647int __init parse_ioapics_under_ir(void)
648{
649 struct dmar_drhd_unit *drhd;
650 int ir_supported = 0;
651
652 for_each_drhd_unit(drhd) {
653 struct intel_iommu *iommu = drhd->iommu;
654
655 if (ecap_ir_support(iommu->ecap)) {
656 if (ir_parse_ioapic_scope(drhd->hdr, iommu))
657 return -1;
658
659 ir_supported = 1;
660 }
661 }
662
663 if (ir_supported && ir_ioapic_num != nr_ioapics) {
664 printk(KERN_WARNING
665 "Not all IO-APIC's listed under remapping hardware\n");
666 return -1;
667 }
668
669 return ir_supported;
670}
Fenghua Yub24696b2009-03-27 14:22:44 -0700671
672void disable_intr_remapping(void)
673{
674 struct dmar_drhd_unit *drhd;
675 struct intel_iommu *iommu = NULL;
676
677 /*
678 * Disable Interrupt-remapping for all the DRHD's now.
679 */
680 for_each_iommu(iommu, drhd) {
681 if (!ecap_ir_support(iommu->ecap))
682 continue;
683
684 iommu_disable_intr_remapping(iommu);
685 }
686}
687
688int reenable_intr_remapping(int eim)
689{
690 struct dmar_drhd_unit *drhd;
691 int setup = 0;
692 struct intel_iommu *iommu = NULL;
693
694 for_each_iommu(iommu, drhd)
695 if (iommu->qi)
696 dmar_reenable_qi(iommu);
697
698 /*
699 * Setup Interrupt-remapping for all the DRHD's now.
700 */
701 for_each_iommu(iommu, drhd) {
702 if (!ecap_ir_support(iommu->ecap))
703 continue;
704
705 /* Set up interrupt remapping for iommu.*/
706 iommu_set_intr_remapping(iommu, eim);
707 setup = 1;
708 }
709
710 if (!setup)
711 goto error;
712
713 return 0;
714
715error:
716 /*
717 * handle error condition gracefully here!
718 */
719 return -1;
720}
721