blob: 4b33acd8ed4ef262bfa43940eab2be7a3deb66f5 [file] [log] [blame]
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001/*
2 * Xen event channels
3 *
4 * Xen models interrupts with abstract event channels. Because each
5 * domain gets 1024 event channels, but NR_IRQ is not that large, we
6 * must dynamically map irqs<->event channels. The event channels
7 * interface with the rest of the kernel by defining a xen interrupt
Lucas De Marchi25985ed2011-03-30 22:57:33 -03008 * chip. When an event is received, it is mapped to an irq and sent
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07009 * through the normal interrupt processing path.
10 *
11 * There are four kinds of events which can be mapped to an event
12 * channel:
13 *
14 * 1. Inter-domain notifications. This includes all the virtual
15 * device events, since they're driven by front-ends in another domain
16 * (typically dom0).
17 * 2. VIRQs, typically used for timers. These are per-cpu events.
18 * 3. IPIs.
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -040019 * 4. PIRQs - Hardware interrupts.
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -070020 *
21 * Jeremy Fitzhardinge <jeremy@xensource.com>, XenSource Inc, 2007
22 */
23
24#include <linux/linkage.h>
25#include <linux/interrupt.h>
26#include <linux/irq.h>
27#include <linux/module.h>
28#include <linux/string.h>
Christophe Saout28e08862009-01-11 11:46:23 -080029#include <linux/bootmem.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090030#include <linux/slab.h>
Jeremy Fitzhardingeb21ddbf2010-06-07 16:28:49 -040031#include <linux/irqnr.h>
Qing Hef731e3ef2010-10-11 15:30:09 +010032#include <linux/pci.h>
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -070033
Sheng Yang38e20b02010-05-14 12:40:51 +010034#include <asm/desc.h>
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -070035#include <asm/ptrace.h>
36#include <asm/irq.h>
Jeremy Fitzhardinge792dc4f2009-02-06 14:09:43 -080037#include <asm/idle.h>
Konrad Rzeszutek Wilk0794bfc2010-10-18 10:41:08 -040038#include <asm/io_apic.h>
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -070039#include <asm/sync_bitops.h>
Stefano Stabellini9846ff12012-01-30 16:21:48 +000040#include <asm/xen/page.h>
Stefano Stabellini42a1de52010-06-24 16:42:04 +010041#include <asm/xen/pci.h>
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -070042#include <asm/xen/hypercall.h>
Adrian Bunk8d1b8752007-07-20 00:31:44 -070043#include <asm/xen/hypervisor.h>
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -070044
Sheng Yang38e20b02010-05-14 12:40:51 +010045#include <xen/xen.h>
46#include <xen/hvm.h>
Isaku Yamahatae04d0d02008-04-02 10:53:55 -070047#include <xen/xen-ops.h>
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -070048#include <xen/events.h>
49#include <xen/interface/xen.h>
50#include <xen/interface/event_channel.h>
Sheng Yang38e20b02010-05-14 12:40:51 +010051#include <xen/interface/hvm/hvm_op.h>
52#include <xen/interface/hvm/params.h>
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -070053
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -070054/*
55 * This lock protects updates to the following mapping and reference-count
56 * arrays. The lock does not need to be acquired to read the mapping tables.
57 */
Konrad Rzeszutek Wilk77365942011-09-14 05:10:00 -040058static DEFINE_MUTEX(irq_mapping_update_lock);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -070059
Ian Campbell6cb65372011-03-10 16:08:11 +000060static LIST_HEAD(xen_irq_list_head);
61
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -070062/* IRQ <-> VIRQ mapping. */
Tejun Heo204fba42009-06-24 15:13:45 +090063static DEFINE_PER_CPU(int [NR_VIRQS], virq_to_irq) = {[0 ... NR_VIRQS-1] = -1};
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -070064
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -070065/* IRQ <-> IPI mapping */
Tejun Heo204fba42009-06-24 15:13:45 +090066static DEFINE_PER_CPU(int [XEN_NR_IPIS], ipi_to_irq) = {[0 ... XEN_NR_IPIS-1] = -1};
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -070067
Jeremy Fitzhardingeced40d02009-02-06 14:09:44 -080068/* Interrupt types. */
69enum xen_irq_type {
Jeremy Fitzhardinged77bbd42009-02-06 14:09:45 -080070 IRQT_UNBOUND = 0,
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -070071 IRQT_PIRQ,
72 IRQT_VIRQ,
73 IRQT_IPI,
74 IRQT_EVTCHN
75};
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -070076
Jeremy Fitzhardingeced40d02009-02-06 14:09:44 -080077/*
78 * Packed IRQ information:
79 * type - enum xen_irq_type
80 * event channel - irq->event channel mapping
81 * cpu - cpu this event channel is bound to
82 * index - type-specific information:
Stefano Stabellini42a1de52010-06-24 16:42:04 +010083 * PIRQ - vector, with MSB being "needs EIO", or physical IRQ of the HVM
84 * guest, or GSI (real passthrough IRQ) of the device.
Jeremy Fitzhardingeced40d02009-02-06 14:09:44 -080085 * VIRQ - virq number
86 * IPI - IPI vector
87 * EVTCHN -
88 */
Ruslan Pisarev088c05a2011-07-26 14:16:13 +030089struct irq_info {
Ian Campbell6cb65372011-03-10 16:08:11 +000090 struct list_head list;
Daniel De Graaf420eb552011-10-27 17:58:47 -040091 int refcnt;
Jeremy Fitzhardingeced40d02009-02-06 14:09:44 -080092 enum xen_irq_type type; /* type */
Ian Campbell6cb65372011-03-10 16:08:11 +000093 unsigned irq;
Jeremy Fitzhardingeced40d02009-02-06 14:09:44 -080094 unsigned short evtchn; /* event channel */
95 unsigned short cpu; /* cpu bound */
96
97 union {
98 unsigned short virq;
99 enum ipi_vector ipi;
100 struct {
Stefano Stabellini7a043f12010-07-01 17:08:14 +0100101 unsigned short pirq;
Jeremy Fitzhardingeced40d02009-02-06 14:09:44 -0800102 unsigned short gsi;
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400103 unsigned char vector;
104 unsigned char flags;
Konrad Rzeszutek Wilkbeafbdc2011-04-14 11:17:36 -0400105 uint16_t domid;
Jeremy Fitzhardingeced40d02009-02-06 14:09:44 -0800106 } pirq;
107 } u;
108};
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400109#define PIRQ_NEEDS_EOI (1 << 0)
Konrad Rzeszutek Wilk15ebbb82010-10-04 13:43:27 -0400110#define PIRQ_SHAREABLE (1 << 1)
Jeremy Fitzhardingeced40d02009-02-06 14:09:44 -0800111
Jeremy Fitzhardingeb21ddbf2010-06-07 16:28:49 -0400112static int *evtchn_to_irq;
Stefano Stabellini9846ff12012-01-30 16:21:48 +0000113static unsigned long *pirq_eoi_map;
114static bool (*pirq_needs_eoi)(unsigned irq);
Jeremy Fitzhardinge3b32f572009-08-13 12:50:37 -0700115
Ian Campbellcb60d112011-03-10 16:08:08 +0000116static DEFINE_PER_CPU(unsigned long [NR_EVENT_CHANNELS/BITS_PER_LONG],
117 cpu_evtchn_mask);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700118
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700119/* Xen will never allocate port zero for any purpose. */
120#define VALID_EVTCHN(chn) ((chn) != 0)
121
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700122static struct irq_chip xen_dynamic_chip;
Jeremy Fitzhardingeaaca4962010-08-20 18:57:53 -0700123static struct irq_chip xen_percpu_chip;
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400124static struct irq_chip xen_pirq_chip;
Stefano Stabellini7e186bd2011-05-06 12:27:50 +0100125static void enable_dynirq(struct irq_data *data);
126static void disable_dynirq(struct irq_data *data);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700127
Ian Campbell9158c352011-03-10 16:08:09 +0000128/* Get info for IRQ */
Jeremy Fitzhardingeced40d02009-02-06 14:09:44 -0800129static struct irq_info *info_for_irq(unsigned irq)
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700130{
Thomas Gleixnerc442b802011-03-25 10:58:06 +0100131 return irq_get_handler_data(irq);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700132}
133
Ian Campbell9158c352011-03-10 16:08:09 +0000134/* Constructors for packed IRQ information. */
135static void xen_irq_info_common_init(struct irq_info *info,
Ian Campbell3d4cfa32011-03-10 16:08:10 +0000136 unsigned irq,
Ian Campbell9158c352011-03-10 16:08:09 +0000137 enum xen_irq_type type,
138 unsigned short evtchn,
139 unsigned short cpu)
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700140{
Ian Campbell9158c352011-03-10 16:08:09 +0000141
142 BUG_ON(info->type != IRQT_UNBOUND && info->type != type);
143
144 info->type = type;
Ian Campbell6cb65372011-03-10 16:08:11 +0000145 info->irq = irq;
Ian Campbell9158c352011-03-10 16:08:09 +0000146 info->evtchn = evtchn;
147 info->cpu = cpu;
Ian Campbell3d4cfa32011-03-10 16:08:10 +0000148
149 evtchn_to_irq[evtchn] = irq;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700150}
151
Ian Campbell9158c352011-03-10 16:08:09 +0000152static void xen_irq_info_evtchn_init(unsigned irq,
153 unsigned short evtchn)
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700154{
Ian Campbell9158c352011-03-10 16:08:09 +0000155 struct irq_info *info = info_for_irq(irq);
156
Ian Campbell3d4cfa32011-03-10 16:08:10 +0000157 xen_irq_info_common_init(info, irq, IRQT_EVTCHN, evtchn, 0);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700158}
159
Ian Campbell3d4cfa32011-03-10 16:08:10 +0000160static void xen_irq_info_ipi_init(unsigned cpu,
161 unsigned irq,
Ian Campbell9158c352011-03-10 16:08:09 +0000162 unsigned short evtchn,
163 enum ipi_vector ipi)
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700164{
Ian Campbell9158c352011-03-10 16:08:09 +0000165 struct irq_info *info = info_for_irq(irq);
166
Ian Campbell3d4cfa32011-03-10 16:08:10 +0000167 xen_irq_info_common_init(info, irq, IRQT_IPI, evtchn, 0);
Ian Campbell9158c352011-03-10 16:08:09 +0000168
169 info->u.ipi = ipi;
Ian Campbell3d4cfa32011-03-10 16:08:10 +0000170
171 per_cpu(ipi_to_irq, cpu)[ipi] = irq;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700172}
173
Ian Campbell3d4cfa32011-03-10 16:08:10 +0000174static void xen_irq_info_virq_init(unsigned cpu,
175 unsigned irq,
Ian Campbell9158c352011-03-10 16:08:09 +0000176 unsigned short evtchn,
177 unsigned short virq)
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700178{
Ian Campbell9158c352011-03-10 16:08:09 +0000179 struct irq_info *info = info_for_irq(irq);
180
Ian Campbell3d4cfa32011-03-10 16:08:10 +0000181 xen_irq_info_common_init(info, irq, IRQT_VIRQ, evtchn, 0);
Ian Campbell9158c352011-03-10 16:08:09 +0000182
183 info->u.virq = virq;
Ian Campbell3d4cfa32011-03-10 16:08:10 +0000184
185 per_cpu(virq_to_irq, cpu)[virq] = irq;
Ian Campbell9158c352011-03-10 16:08:09 +0000186}
187
188static void xen_irq_info_pirq_init(unsigned irq,
189 unsigned short evtchn,
190 unsigned short pirq,
191 unsigned short gsi,
192 unsigned short vector,
Konrad Rzeszutek Wilkbeafbdc2011-04-14 11:17:36 -0400193 uint16_t domid,
Ian Campbell9158c352011-03-10 16:08:09 +0000194 unsigned char flags)
195{
196 struct irq_info *info = info_for_irq(irq);
197
Ian Campbell3d4cfa32011-03-10 16:08:10 +0000198 xen_irq_info_common_init(info, irq, IRQT_PIRQ, evtchn, 0);
Ian Campbell9158c352011-03-10 16:08:09 +0000199
200 info->u.pirq.pirq = pirq;
201 info->u.pirq.gsi = gsi;
202 info->u.pirq.vector = vector;
Konrad Rzeszutek Wilkbeafbdc2011-04-14 11:17:36 -0400203 info->u.pirq.domid = domid;
Ian Campbell9158c352011-03-10 16:08:09 +0000204 info->u.pirq.flags = flags;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700205}
206
207/*
208 * Accessors for packed IRQ information.
209 */
Jeremy Fitzhardingeced40d02009-02-06 14:09:44 -0800210static unsigned int evtchn_from_irq(unsigned irq)
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700211{
Joe Jin110e7c72011-01-07 14:50:12 +0800212 if (unlikely(WARN(irq < 0 || irq >= nr_irqs, "Invalid irq %d!\n", irq)))
213 return 0;
214
Jeremy Fitzhardingeced40d02009-02-06 14:09:44 -0800215 return info_for_irq(irq)->evtchn;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700216}
217
Ian Campbelld4c04532009-02-06 19:20:31 -0800218unsigned irq_from_evtchn(unsigned int evtchn)
219{
220 return evtchn_to_irq[evtchn];
221}
222EXPORT_SYMBOL_GPL(irq_from_evtchn);
223
Jeremy Fitzhardingeced40d02009-02-06 14:09:44 -0800224static enum ipi_vector ipi_from_irq(unsigned irq)
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700225{
Jeremy Fitzhardingeced40d02009-02-06 14:09:44 -0800226 struct irq_info *info = info_for_irq(irq);
227
228 BUG_ON(info == NULL);
229 BUG_ON(info->type != IRQT_IPI);
230
231 return info->u.ipi;
232}
233
234static unsigned virq_from_irq(unsigned irq)
235{
236 struct irq_info *info = info_for_irq(irq);
237
238 BUG_ON(info == NULL);
239 BUG_ON(info->type != IRQT_VIRQ);
240
241 return info->u.virq;
242}
243
Stefano Stabellini7a043f12010-07-01 17:08:14 +0100244static unsigned pirq_from_irq(unsigned irq)
245{
246 struct irq_info *info = info_for_irq(irq);
247
248 BUG_ON(info == NULL);
249 BUG_ON(info->type != IRQT_PIRQ);
250
251 return info->u.pirq.pirq;
252}
253
Jeremy Fitzhardingeced40d02009-02-06 14:09:44 -0800254static enum xen_irq_type type_from_irq(unsigned irq)
255{
256 return info_for_irq(irq)->type;
257}
258
259static unsigned cpu_from_irq(unsigned irq)
260{
261 return info_for_irq(irq)->cpu;
262}
263
264static unsigned int cpu_from_evtchn(unsigned int evtchn)
265{
266 int irq = evtchn_to_irq[evtchn];
267 unsigned ret = 0;
268
269 if (irq != -1)
270 ret = cpu_from_irq(irq);
271
272 return ret;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700273}
274
Stefano Stabellini9846ff12012-01-30 16:21:48 +0000275static bool pirq_check_eoi_map(unsigned irq)
276{
277 return test_bit(irq, pirq_eoi_map);
278}
279
280static bool pirq_needs_eoi_flag(unsigned irq)
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400281{
282 struct irq_info *info = info_for_irq(irq);
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400283 BUG_ON(info->type != IRQT_PIRQ);
284
285 return info->u.pirq.flags & PIRQ_NEEDS_EOI;
286}
287
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700288static inline unsigned long active_evtchns(unsigned int cpu,
289 struct shared_info *sh,
290 unsigned int idx)
291{
Ruslan Pisarev088c05a2011-07-26 14:16:13 +0300292 return sh->evtchn_pending[idx] &
Ian Campbellcb60d112011-03-10 16:08:08 +0000293 per_cpu(cpu_evtchn_mask, cpu)[idx] &
Ruslan Pisarev088c05a2011-07-26 14:16:13 +0300294 ~sh->evtchn_mask[idx];
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700295}
296
297static void bind_evtchn_to_cpu(unsigned int chn, unsigned int cpu)
298{
299 int irq = evtchn_to_irq[chn];
300
301 BUG_ON(irq == -1);
302#ifdef CONFIG_SMP
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +0000303 cpumask_copy(irq_to_desc(irq)->irq_data.affinity, cpumask_of(cpu));
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700304#endif
305
Ian Campbellcb60d112011-03-10 16:08:08 +0000306 clear_bit(chn, per_cpu(cpu_evtchn_mask, cpu_from_irq(irq)));
307 set_bit(chn, per_cpu(cpu_evtchn_mask, cpu));
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700308
Ian Campbellca62ce82011-03-10 16:08:12 +0000309 info_for_irq(irq)->cpu = cpu;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700310}
311
312static void init_evtchn_cpu_bindings(void)
313{
Jan Beulich1c6969e2010-11-16 14:55:33 -0800314 int i;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700315#ifdef CONFIG_SMP
Ian Campbell6cb65372011-03-10 16:08:11 +0000316 struct irq_info *info;
Thomas Gleixner10e58082008-10-16 14:19:04 +0200317
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700318 /* By default all event channels notify CPU#0. */
Ian Campbell6cb65372011-03-10 16:08:11 +0000319 list_for_each_entry(info, &xen_irq_list_head, list) {
320 struct irq_desc *desc = irq_to_desc(info->irq);
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +0000321 cpumask_copy(desc->irq_data.affinity, cpumask_of(0));
Yinghai Lu0b8f1ef2008-12-05 18:58:31 -0800322 }
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700323#endif
324
Jan Beulich1c6969e2010-11-16 14:55:33 -0800325 for_each_possible_cpu(i)
Ian Campbellcb60d112011-03-10 16:08:08 +0000326 memset(per_cpu(cpu_evtchn_mask, i),
327 (i == 0) ? ~0 : 0, sizeof(*per_cpu(cpu_evtchn_mask, i)));
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700328}
329
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700330static inline void clear_evtchn(int port)
331{
332 struct shared_info *s = HYPERVISOR_shared_info;
333 sync_clear_bit(port, &s->evtchn_pending[0]);
334}
335
336static inline void set_evtchn(int port)
337{
338 struct shared_info *s = HYPERVISOR_shared_info;
339 sync_set_bit(port, &s->evtchn_pending[0]);
340}
341
Jeremy Fitzhardinge168d2f42008-08-20 17:02:18 -0700342static inline int test_evtchn(int port)
343{
344 struct shared_info *s = HYPERVISOR_shared_info;
345 return sync_test_bit(port, &s->evtchn_pending[0]);
346}
347
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700348
349/**
350 * notify_remote_via_irq - send event to remote end of event channel via irq
351 * @irq: irq of event channel to send event to
352 *
353 * Unlike notify_remote_via_evtchn(), this is safe to use across
354 * save/restore. Notifications on a broken connection are silently
355 * dropped.
356 */
357void notify_remote_via_irq(int irq)
358{
359 int evtchn = evtchn_from_irq(irq);
360
361 if (VALID_EVTCHN(evtchn))
362 notify_remote_via_evtchn(evtchn);
363}
364EXPORT_SYMBOL_GPL(notify_remote_via_irq);
365
366static void mask_evtchn(int port)
367{
368 struct shared_info *s = HYPERVISOR_shared_info;
369 sync_set_bit(port, &s->evtchn_mask[0]);
370}
371
372static void unmask_evtchn(int port)
373{
374 struct shared_info *s = HYPERVISOR_shared_info;
375 unsigned int cpu = get_cpu();
376
377 BUG_ON(!irqs_disabled());
378
379 /* Slow path (hypercall) if this is a non-local port. */
380 if (unlikely(cpu != cpu_from_evtchn(port))) {
381 struct evtchn_unmask unmask = { .port = port };
382 (void)HYPERVISOR_event_channel_op(EVTCHNOP_unmask, &unmask);
383 } else {
Christoph Lameter780f36d2010-12-06 11:16:29 -0600384 struct vcpu_info *vcpu_info = __this_cpu_read(xen_vcpu);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700385
386 sync_clear_bit(port, &s->evtchn_mask[0]);
387
388 /*
389 * The following is basically the equivalent of
390 * 'hw_resend_irq'. Just like a real IO-APIC we 'lose
391 * the interrupt edge' if the channel is masked.
392 */
393 if (sync_test_bit(port, &s->evtchn_pending[0]) &&
394 !sync_test_and_set_bit(port / BITS_PER_LONG,
395 &vcpu_info->evtchn_pending_sel))
396 vcpu_info->evtchn_upcall_pending = 1;
397 }
398
399 put_cpu();
400}
401
Ian Campbell6cb65372011-03-10 16:08:11 +0000402static void xen_irq_init(unsigned irq)
403{
404 struct irq_info *info;
Konrad Rzeszutek Wilkb5328cd2011-06-15 14:24:29 -0400405#ifdef CONFIG_SMP
Ian Campbell6cb65372011-03-10 16:08:11 +0000406 struct irq_desc *desc = irq_to_desc(irq);
407
408 /* By default all event channels notify CPU#0. */
409 cpumask_copy(desc->irq_data.affinity, cpumask_of(0));
Konrad Rzeszutek Wilk44626e42011-03-15 16:40:33 -0400410#endif
Ian Campbell6cb65372011-03-10 16:08:11 +0000411
Ian Campbellca62ce82011-03-10 16:08:12 +0000412 info = kzalloc(sizeof(*info), GFP_KERNEL);
413 if (info == NULL)
414 panic("Unable to allocate metadata for IRQ%d\n", irq);
Ian Campbell6cb65372011-03-10 16:08:11 +0000415
416 info->type = IRQT_UNBOUND;
Daniel De Graaf420eb552011-10-27 17:58:47 -0400417 info->refcnt = -1;
Ian Campbell6cb65372011-03-10 16:08:11 +0000418
Thomas Gleixnerc442b802011-03-25 10:58:06 +0100419 irq_set_handler_data(irq, info);
Ian Campbellca62ce82011-03-10 16:08:12 +0000420
Ian Campbell6cb65372011-03-10 16:08:11 +0000421 list_add_tail(&info->list, &xen_irq_list_head);
422}
423
Ian Campbell7bee9762011-03-10 16:08:15 +0000424static int __must_check xen_allocate_irq_dynamic(void)
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700425{
Ian Campbell89911502011-03-03 11:57:44 -0500426 int first = 0;
427 int irq;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700428
Ian Campbell89911502011-03-03 11:57:44 -0500429#ifdef CONFIG_X86_IO_APIC
430 /*
431 * For an HVM guest or domain 0 which see "real" (emulated or
Lucas De Marchi25985ed2011-03-30 22:57:33 -0300432 * actual respectively) GSIs we allocate dynamic IRQs
Ian Campbell89911502011-03-03 11:57:44 -0500433 * e.g. those corresponding to event channels or MSIs
434 * etc. from the range above those "real" GSIs to avoid
435 * collisions.
Konrad Rzeszutek Wilkd1b758e2010-12-09 14:53:29 -0500436 */
Ian Campbell89911502011-03-03 11:57:44 -0500437 if (xen_initial_domain() || xen_hvm_domain())
438 first = get_nr_irqs_gsi();
439#endif
440
Ian Campbell89911502011-03-03 11:57:44 -0500441 irq = irq_alloc_desc_from(first, -1);
442
Konrad Rzeszutek Wilke6599222011-09-29 13:26:45 -0400443 if (irq >= 0)
444 xen_irq_init(irq);
Jeremy Fitzhardingeced40d02009-02-06 14:09:44 -0800445
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700446 return irq;
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400447}
448
Ian Campbell7bee9762011-03-10 16:08:15 +0000449static int __must_check xen_allocate_irq_gsi(unsigned gsi)
Ian Campbellc9df1ce2011-01-11 17:20:15 +0000450{
451 int irq;
452
Ian Campbell89911502011-03-03 11:57:44 -0500453 /*
454 * A PV guest has no concept of a GSI (since it has no ACPI
455 * nor access to/knowledge of the physical APICs). Therefore
456 * all IRQs are dynamically allocated from the entire IRQ
457 * space.
458 */
459 if (xen_pv_domain() && !xen_initial_domain())
Ian Campbellc9df1ce2011-01-11 17:20:15 +0000460 return xen_allocate_irq_dynamic();
461
462 /* Legacy IRQ descriptors are already allocated by the arch. */
463 if (gsi < NR_IRQS_LEGACY)
Ian Campbell6cb65372011-03-10 16:08:11 +0000464 irq = gsi;
465 else
466 irq = irq_alloc_desc_at(gsi, -1);
Ian Campbellc9df1ce2011-01-11 17:20:15 +0000467
Ian Campbell6cb65372011-03-10 16:08:11 +0000468 xen_irq_init(irq);
Ian Campbellc9df1ce2011-01-11 17:20:15 +0000469
470 return irq;
471}
472
473static void xen_free_irq(unsigned irq)
474{
Thomas Gleixnerc442b802011-03-25 10:58:06 +0100475 struct irq_info *info = irq_get_handler_data(irq);
Ian Campbell6cb65372011-03-10 16:08:11 +0000476
477 list_del(&info->list);
Ian Campbell9158c352011-03-10 16:08:09 +0000478
Thomas Gleixnerc442b802011-03-25 10:58:06 +0100479 irq_set_handler_data(irq, NULL);
Ian Campbellca62ce82011-03-10 16:08:12 +0000480
Daniel De Graaf420eb552011-10-27 17:58:47 -0400481 WARN_ON(info->refcnt > 0);
482
Ian Campbellca62ce82011-03-10 16:08:12 +0000483 kfree(info);
484
Ian Campbell72146102011-02-03 09:49:35 +0000485 /* Legacy IRQ descriptors are managed by the arch. */
486 if (irq < NR_IRQS_LEGACY)
487 return;
488
Ian Campbellc9df1ce2011-01-11 17:20:15 +0000489 irq_free_desc(irq);
490}
491
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400492static void pirq_query_unmask(int irq)
493{
494 struct physdev_irq_status_query irq_status;
495 struct irq_info *info = info_for_irq(irq);
496
497 BUG_ON(info->type != IRQT_PIRQ);
498
Stefano Stabellini7a043f12010-07-01 17:08:14 +0100499 irq_status.irq = pirq_from_irq(irq);
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400500 if (HYPERVISOR_physdev_op(PHYSDEVOP_irq_status_query, &irq_status))
501 irq_status.flags = 0;
502
503 info->u.pirq.flags &= ~PIRQ_NEEDS_EOI;
504 if (irq_status.flags & XENIRQSTAT_needs_eoi)
505 info->u.pirq.flags |= PIRQ_NEEDS_EOI;
506}
507
508static bool probing_irq(int irq)
509{
510 struct irq_desc *desc = irq_to_desc(irq);
511
512 return desc && desc->action == NULL;
513}
514
Stefano Stabellini7e186bd2011-05-06 12:27:50 +0100515static void eoi_pirq(struct irq_data *data)
516{
517 int evtchn = evtchn_from_irq(data->irq);
518 struct physdev_eoi eoi = { .irq = pirq_from_irq(data->irq) };
519 int rc = 0;
520
521 irq_move_irq(data);
522
523 if (VALID_EVTCHN(evtchn))
524 clear_evtchn(evtchn);
525
526 if (pirq_needs_eoi(data->irq)) {
527 rc = HYPERVISOR_physdev_op(PHYSDEVOP_eoi, &eoi);
528 WARN_ON(rc);
529 }
530}
531
532static void mask_ack_pirq(struct irq_data *data)
533{
534 disable_dynirq(data);
535 eoi_pirq(data);
536}
537
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +0000538static unsigned int __startup_pirq(unsigned int irq)
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400539{
540 struct evtchn_bind_pirq bind_pirq;
541 struct irq_info *info = info_for_irq(irq);
542 int evtchn = evtchn_from_irq(irq);
Konrad Rzeszutek Wilk15ebbb82010-10-04 13:43:27 -0400543 int rc;
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400544
545 BUG_ON(info->type != IRQT_PIRQ);
546
547 if (VALID_EVTCHN(evtchn))
548 goto out;
549
Stefano Stabellini7a043f12010-07-01 17:08:14 +0100550 bind_pirq.pirq = pirq_from_irq(irq);
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400551 /* NB. We are happy to share unless we are probing. */
Konrad Rzeszutek Wilk15ebbb82010-10-04 13:43:27 -0400552 bind_pirq.flags = info->u.pirq.flags & PIRQ_SHAREABLE ?
553 BIND_PIRQ__WILL_SHARE : 0;
554 rc = HYPERVISOR_event_channel_op(EVTCHNOP_bind_pirq, &bind_pirq);
555 if (rc != 0) {
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400556 if (!probing_irq(irq))
557 printk(KERN_INFO "Failed to obtain physical IRQ %d\n",
558 irq);
559 return 0;
560 }
561 evtchn = bind_pirq.port;
562
563 pirq_query_unmask(irq);
564
565 evtchn_to_irq[evtchn] = irq;
566 bind_evtchn_to_cpu(evtchn, 0);
567 info->evtchn = evtchn;
568
569out:
570 unmask_evtchn(evtchn);
Stefano Stabellini7e186bd2011-05-06 12:27:50 +0100571 eoi_pirq(irq_get_irq_data(irq));
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400572
573 return 0;
574}
575
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +0000576static unsigned int startup_pirq(struct irq_data *data)
577{
578 return __startup_pirq(data->irq);
579}
580
581static void shutdown_pirq(struct irq_data *data)
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400582{
583 struct evtchn_close close;
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +0000584 unsigned int irq = data->irq;
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400585 struct irq_info *info = info_for_irq(irq);
586 int evtchn = evtchn_from_irq(irq);
587
588 BUG_ON(info->type != IRQT_PIRQ);
589
590 if (!VALID_EVTCHN(evtchn))
591 return;
592
593 mask_evtchn(evtchn);
594
595 close.port = evtchn;
596 if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close) != 0)
597 BUG();
598
599 bind_evtchn_to_cpu(evtchn, 0);
600 evtchn_to_irq[evtchn] = -1;
601 info->evtchn = 0;
602}
603
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +0000604static void enable_pirq(struct irq_data *data)
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400605{
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +0000606 startup_pirq(data);
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400607}
608
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +0000609static void disable_pirq(struct irq_data *data)
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400610{
Stefano Stabellini7e186bd2011-05-06 12:27:50 +0100611 disable_dynirq(data);
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400612}
613
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400614static int find_irq_by_gsi(unsigned gsi)
615{
Ian Campbell6cb65372011-03-10 16:08:11 +0000616 struct irq_info *info;
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400617
Ian Campbell6cb65372011-03-10 16:08:11 +0000618 list_for_each_entry(info, &xen_irq_list_head, list) {
619 if (info->type != IRQT_PIRQ)
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400620 continue;
621
Ian Campbell6cb65372011-03-10 16:08:11 +0000622 if (info->u.pirq.gsi == gsi)
623 return info->irq;
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400624 }
625
626 return -1;
627}
628
Ian Campbell653378a2011-03-10 16:08:04 +0000629/*
630 * Do not make any assumptions regarding the relationship between the
631 * IRQ number returned here and the Xen pirq argument.
Stefano Stabellini7a043f12010-07-01 17:08:14 +0100632 *
633 * Note: We don't assign an event channel until the irq actually started
634 * up. Return an existing irq if we've already got one for the gsi.
Stefano Stabellinie5ac0bd2011-05-25 12:33:23 +0100635 *
636 * Shareable implies level triggered, not shareable implies edge
637 * triggered here.
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400638 */
Ian Campbellf4d06352011-03-10 16:08:07 +0000639int xen_bind_pirq_gsi_to_irq(unsigned gsi,
640 unsigned pirq, int shareable, char *name)
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400641{
Ian Campbella0e18112011-03-10 16:08:03 +0000642 int irq = -1;
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400643 struct physdev_irq irq_op;
644
Konrad Rzeszutek Wilk77365942011-09-14 05:10:00 -0400645 mutex_lock(&irq_mapping_update_lock);
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400646
647 irq = find_irq_by_gsi(gsi);
648 if (irq != -1) {
Stefano Stabellini7a043f12010-07-01 17:08:14 +0100649 printk(KERN_INFO "xen_map_pirq_gsi: returning irq %d for gsi %u\n",
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400650 irq, gsi);
Daniel De Graaf420eb552011-10-27 17:58:47 -0400651 goto out;
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400652 }
653
Ian Campbellc9df1ce2011-01-11 17:20:15 +0000654 irq = xen_allocate_irq_gsi(gsi);
Ian Campbell7bee9762011-03-10 16:08:15 +0000655 if (irq < 0)
656 goto out;
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400657
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400658 irq_op.irq = irq;
Alex Nixonb5401a92010-03-18 16:31:34 -0400659 irq_op.vector = 0;
660
661 /* Only the privileged domain can do this. For non-priv, the pcifront
662 * driver provides a PCI bus that does the call to do exactly
663 * this in the priv domain. */
664 if (xen_initial_domain() &&
665 HYPERVISOR_physdev_op(PHYSDEVOP_alloc_irq_vector, &irq_op)) {
Ian Campbellc9df1ce2011-01-11 17:20:15 +0000666 xen_free_irq(irq);
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400667 irq = -ENOSPC;
668 goto out;
669 }
670
Konrad Rzeszutek Wilkbeafbdc2011-04-14 11:17:36 -0400671 xen_irq_info_pirq_init(irq, 0, pirq, gsi, irq_op.vector, DOMID_SELF,
Ian Campbell9158c352011-03-10 16:08:09 +0000672 shareable ? PIRQ_SHAREABLE : 0);
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400673
Stefano Stabellini7e186bd2011-05-06 12:27:50 +0100674 pirq_query_unmask(irq);
675 /* We try to use the handler with the appropriate semantic for the
Stefano Stabellinie5ac0bd2011-05-25 12:33:23 +0100676 * type of interrupt: if the interrupt is an edge triggered
677 * interrupt we use handle_edge_irq.
Stefano Stabellini7e186bd2011-05-06 12:27:50 +0100678 *
Stefano Stabellinie5ac0bd2011-05-25 12:33:23 +0100679 * On the other hand if the interrupt is level triggered we use
680 * handle_fasteoi_irq like the native code does for this kind of
Stefano Stabellini7e186bd2011-05-06 12:27:50 +0100681 * interrupts.
Stefano Stabellinie5ac0bd2011-05-25 12:33:23 +0100682 *
Stefano Stabellini7e186bd2011-05-06 12:27:50 +0100683 * Depending on the Xen version, pirq_needs_eoi might return true
684 * not only for level triggered interrupts but for edge triggered
685 * interrupts too. In any case Xen always honors the eoi mechanism,
686 * not injecting any more pirqs of the same kind if the first one
687 * hasn't received an eoi yet. Therefore using the fasteoi handler
688 * is the right choice either way.
689 */
Stefano Stabellinie5ac0bd2011-05-25 12:33:23 +0100690 if (shareable)
Stefano Stabellini7e186bd2011-05-06 12:27:50 +0100691 irq_set_chip_and_handler_name(irq, &xen_pirq_chip,
692 handle_fasteoi_irq, name);
693 else
694 irq_set_chip_and_handler_name(irq, &xen_pirq_chip,
695 handle_edge_irq, name);
696
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400697out:
Konrad Rzeszutek Wilk77365942011-09-14 05:10:00 -0400698 mutex_unlock(&irq_mapping_update_lock);
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400699
700 return irq;
701}
702
Qing Hef731e3ef2010-10-11 15:30:09 +0100703#ifdef CONFIG_PCI_MSI
Ian Campbellbf480d92011-02-18 16:43:32 +0000704int xen_allocate_pirq_msi(struct pci_dev *dev, struct msi_desc *msidesc)
Ian Campbellcbf6aa82011-01-11 17:20:14 +0000705{
Ian Campbell5cad61a2011-02-18 16:43:31 +0000706 int rc;
Ian Campbellcbf6aa82011-01-11 17:20:14 +0000707 struct physdev_get_free_pirq op_get_free_pirq;
Ian Campbell5cad61a2011-02-18 16:43:31 +0000708
Ian Campbellbf480d92011-02-18 16:43:32 +0000709 op_get_free_pirq.type = MAP_PIRQ_TYPE_MSI;
Ian Campbellcbf6aa82011-01-11 17:20:14 +0000710 rc = HYPERVISOR_physdev_op(PHYSDEVOP_get_free_pirq, &op_get_free_pirq);
Ian Campbellcbf6aa82011-01-11 17:20:14 +0000711
Ian Campbell5cad61a2011-02-18 16:43:31 +0000712 WARN_ONCE(rc == -ENOSYS,
713 "hypervisor does not support the PHYSDEVOP_get_free_pirq interface\n");
714
715 return rc ? -1 : op_get_free_pirq.pirq;
Ian Campbellcbf6aa82011-01-11 17:20:14 +0000716}
717
Ian Campbellbf480d92011-02-18 16:43:32 +0000718int xen_bind_pirq_msi_to_irq(struct pci_dev *dev, struct msi_desc *msidesc,
Konrad Rzeszutek Wilkbeafbdc2011-04-14 11:17:36 -0400719 int pirq, int vector, const char *name,
720 domid_t domid)
Stefano Stabellini809f9262010-07-01 17:10:39 +0100721{
Ian Campbellbf480d92011-02-18 16:43:32 +0000722 int irq, ret;
Ian Campbell4b41df72011-02-18 16:43:29 +0000723
Konrad Rzeszutek Wilk77365942011-09-14 05:10:00 -0400724 mutex_lock(&irq_mapping_update_lock);
Stefano Stabellini809f9262010-07-01 17:10:39 +0100725
Ian Campbell4b41df72011-02-18 16:43:29 +0000726 irq = xen_allocate_irq_dynamic();
Konrad Rzeszutek Wilke6599222011-09-29 13:26:45 -0400727 if (irq < 0)
Ian Campbellbb5d0792011-02-18 16:43:28 +0000728 goto out;
Stefano Stabellini809f9262010-07-01 17:10:39 +0100729
Stefano Stabellini7e186bd2011-05-06 12:27:50 +0100730 irq_set_chip_and_handler_name(irq, &xen_pirq_chip, handle_edge_irq,
731 name);
Stefano Stabellini809f9262010-07-01 17:10:39 +0100732
Konrad Rzeszutek Wilkbeafbdc2011-04-14 11:17:36 -0400733 xen_irq_info_pirq_init(irq, 0, pirq, 0, vector, domid, 0);
Linus Torvalds5f6fb452011-03-15 19:23:40 -0700734 ret = irq_set_msi_desc(irq, msidesc);
Ian Campbellbf480d92011-02-18 16:43:32 +0000735 if (ret < 0)
736 goto error_irq;
Stefano Stabellini809f9262010-07-01 17:10:39 +0100737out:
Konrad Rzeszutek Wilk77365942011-09-14 05:10:00 -0400738 mutex_unlock(&irq_mapping_update_lock);
Ian Campbell4b41df72011-02-18 16:43:29 +0000739 return irq;
Ian Campbellbf480d92011-02-18 16:43:32 +0000740error_irq:
Konrad Rzeszutek Wilk77365942011-09-14 05:10:00 -0400741 mutex_unlock(&irq_mapping_update_lock);
Ian Campbellbf480d92011-02-18 16:43:32 +0000742 xen_free_irq(irq);
Konrad Rzeszutek Wilke6599222011-09-29 13:26:45 -0400743 return ret;
Stefano Stabellini809f9262010-07-01 17:10:39 +0100744}
Qing Hef731e3ef2010-10-11 15:30:09 +0100745#endif
746
Alex Nixonb5401a92010-03-18 16:31:34 -0400747int xen_destroy_irq(int irq)
748{
749 struct irq_desc *desc;
Jeremy Fitzhardinge38aa66f2010-09-02 14:51:39 +0100750 struct physdev_unmap_pirq unmap_irq;
751 struct irq_info *info = info_for_irq(irq);
Alex Nixonb5401a92010-03-18 16:31:34 -0400752 int rc = -ENOENT;
753
Konrad Rzeszutek Wilk77365942011-09-14 05:10:00 -0400754 mutex_lock(&irq_mapping_update_lock);
Alex Nixonb5401a92010-03-18 16:31:34 -0400755
756 desc = irq_to_desc(irq);
757 if (!desc)
758 goto out;
759
Jeremy Fitzhardinge38aa66f2010-09-02 14:51:39 +0100760 if (xen_initial_domain()) {
Konrad Rzeszutek Wilk12334712010-11-19 11:27:09 -0500761 unmap_irq.pirq = info->u.pirq.pirq;
Konrad Rzeszutek Wilkbeafbdc2011-04-14 11:17:36 -0400762 unmap_irq.domid = info->u.pirq.domid;
Jeremy Fitzhardinge38aa66f2010-09-02 14:51:39 +0100763 rc = HYPERVISOR_physdev_op(PHYSDEVOP_unmap_pirq, &unmap_irq);
Konrad Rzeszutek Wilk1eff1ad2011-02-16 16:26:44 -0500764 /* If another domain quits without making the pci_disable_msix
765 * call, the Xen hypervisor takes care of freeing the PIRQs
766 * (free_domain_pirqs).
767 */
768 if ((rc == -ESRCH && info->u.pirq.domid != DOMID_SELF))
769 printk(KERN_INFO "domain %d does not have %d anymore\n",
770 info->u.pirq.domid, info->u.pirq.pirq);
771 else if (rc) {
Jeremy Fitzhardinge38aa66f2010-09-02 14:51:39 +0100772 printk(KERN_WARNING "unmap irq failed %d\n", rc);
773 goto out;
774 }
775 }
Alex Nixonb5401a92010-03-18 16:31:34 -0400776
Ian Campbellc9df1ce2011-01-11 17:20:15 +0000777 xen_free_irq(irq);
Alex Nixonb5401a92010-03-18 16:31:34 -0400778
779out:
Konrad Rzeszutek Wilk77365942011-09-14 05:10:00 -0400780 mutex_unlock(&irq_mapping_update_lock);
Alex Nixonb5401a92010-03-18 16:31:34 -0400781 return rc;
782}
783
Stefano Stabelliniaf42b8d2010-12-01 14:51:44 +0000784int xen_irq_from_pirq(unsigned pirq)
785{
Ian Campbell69c358c2011-03-10 16:08:13 +0000786 int irq;
787
788 struct irq_info *info;
789
Konrad Rzeszutek Wilk77365942011-09-14 05:10:00 -0400790 mutex_lock(&irq_mapping_update_lock);
Ian Campbell69c358c2011-03-10 16:08:13 +0000791
792 list_for_each_entry(info, &xen_irq_list_head, list) {
Konrad Rzeszutek Wilk9bb9efe2011-09-29 13:13:30 -0400793 if (info->type != IRQT_PIRQ)
Ian Campbell69c358c2011-03-10 16:08:13 +0000794 continue;
795 irq = info->irq;
796 if (info->u.pirq.pirq == pirq)
797 goto out;
798 }
799 irq = -1;
800out:
Konrad Rzeszutek Wilk77365942011-09-14 05:10:00 -0400801 mutex_unlock(&irq_mapping_update_lock);
Ian Campbell69c358c2011-03-10 16:08:13 +0000802
803 return irq;
Stefano Stabelliniaf42b8d2010-12-01 14:51:44 +0000804}
805
Konrad Rzeszutek Wilke6197ac2011-02-24 14:20:12 -0500806
807int xen_pirq_from_irq(unsigned irq)
808{
809 return pirq_from_irq(irq);
810}
811EXPORT_SYMBOL_GPL(xen_pirq_from_irq);
Jeremy Fitzhardingeb536b4b2007-07-17 18:37:06 -0700812int bind_evtchn_to_irq(unsigned int evtchn)
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700813{
814 int irq;
815
Konrad Rzeszutek Wilk77365942011-09-14 05:10:00 -0400816 mutex_lock(&irq_mapping_update_lock);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700817
818 irq = evtchn_to_irq[evtchn];
819
820 if (irq == -1) {
Ian Campbellc9df1ce2011-01-11 17:20:15 +0000821 irq = xen_allocate_irq_dynamic();
Ian Campbell7bee9762011-03-10 16:08:15 +0000822 if (irq == -1)
823 goto out;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700824
Thomas Gleixnerc442b802011-03-25 10:58:06 +0100825 irq_set_chip_and_handler_name(irq, &xen_dynamic_chip,
Stefano Stabellini7e186bd2011-05-06 12:27:50 +0100826 handle_edge_irq, "event");
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700827
Ian Campbell9158c352011-03-10 16:08:09 +0000828 xen_irq_info_evtchn_init(irq, evtchn);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700829 }
830
Ian Campbell7bee9762011-03-10 16:08:15 +0000831out:
Konrad Rzeszutek Wilk77365942011-09-14 05:10:00 -0400832 mutex_unlock(&irq_mapping_update_lock);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700833
834 return irq;
835}
Jeremy Fitzhardingeb536b4b2007-07-17 18:37:06 -0700836EXPORT_SYMBOL_GPL(bind_evtchn_to_irq);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700837
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -0700838static int bind_ipi_to_irq(unsigned int ipi, unsigned int cpu)
839{
840 struct evtchn_bind_ipi bind_ipi;
841 int evtchn, irq;
842
Konrad Rzeszutek Wilk77365942011-09-14 05:10:00 -0400843 mutex_lock(&irq_mapping_update_lock);
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -0700844
845 irq = per_cpu(ipi_to_irq, cpu)[ipi];
Ian Campbell90af9512009-02-06 16:55:58 -0800846
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -0700847 if (irq == -1) {
Ian Campbellc9df1ce2011-01-11 17:20:15 +0000848 irq = xen_allocate_irq_dynamic();
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -0700849 if (irq < 0)
850 goto out;
851
Thomas Gleixnerc442b802011-03-25 10:58:06 +0100852 irq_set_chip_and_handler_name(irq, &xen_percpu_chip,
Jeremy Fitzhardingeaaca4962010-08-20 18:57:53 -0700853 handle_percpu_irq, "ipi");
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -0700854
855 bind_ipi.vcpu = cpu;
856 if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_ipi,
857 &bind_ipi) != 0)
858 BUG();
859 evtchn = bind_ipi.port;
860
Ian Campbell3d4cfa32011-03-10 16:08:10 +0000861 xen_irq_info_ipi_init(cpu, irq, evtchn, ipi);
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -0700862
863 bind_evtchn_to_cpu(evtchn, cpu);
864 }
865
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -0700866 out:
Konrad Rzeszutek Wilk77365942011-09-14 05:10:00 -0400867 mutex_unlock(&irq_mapping_update_lock);
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -0700868 return irq;
869}
870
Ian Campbell2e820f52009-02-09 12:05:50 -0800871static int bind_interdomain_evtchn_to_irq(unsigned int remote_domain,
872 unsigned int remote_port)
873{
874 struct evtchn_bind_interdomain bind_interdomain;
875 int err;
876
877 bind_interdomain.remote_dom = remote_domain;
878 bind_interdomain.remote_port = remote_port;
879
880 err = HYPERVISOR_event_channel_op(EVTCHNOP_bind_interdomain,
881 &bind_interdomain);
882
883 return err ? : bind_evtchn_to_irq(bind_interdomain.local_port);
884}
885
Olaf Hering62cc5fc2011-08-25 18:30:48 +0200886static int find_virq(unsigned int virq, unsigned int cpu)
887{
888 struct evtchn_status status;
889 int port, rc = -ENOENT;
890
891 memset(&status, 0, sizeof(status));
892 for (port = 0; port <= NR_EVENT_CHANNELS; port++) {
893 status.dom = DOMID_SELF;
894 status.port = port;
895 rc = HYPERVISOR_event_channel_op(EVTCHNOP_status, &status);
896 if (rc < 0)
897 continue;
898 if (status.status != EVTCHNSTAT_virq)
899 continue;
900 if (status.u.virq == virq && status.vcpu == cpu) {
901 rc = port;
902 break;
903 }
904 }
905 return rc;
906}
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -0700907
Jeremy Fitzhardinge4fe7d5a2010-09-02 16:17:06 +0100908int bind_virq_to_irq(unsigned int virq, unsigned int cpu)
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700909{
910 struct evtchn_bind_virq bind_virq;
Olaf Hering62cc5fc2011-08-25 18:30:48 +0200911 int evtchn, irq, ret;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700912
Konrad Rzeszutek Wilk77365942011-09-14 05:10:00 -0400913 mutex_lock(&irq_mapping_update_lock);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700914
915 irq = per_cpu(virq_to_irq, cpu)[virq];
916
917 if (irq == -1) {
Ian Campbellc9df1ce2011-01-11 17:20:15 +0000918 irq = xen_allocate_irq_dynamic();
Ian Campbell7bee9762011-03-10 16:08:15 +0000919 if (irq == -1)
920 goto out;
Jeremy Fitzhardingea52521f2010-09-22 15:28:52 -0700921
Thomas Gleixnerc442b802011-03-25 10:58:06 +0100922 irq_set_chip_and_handler_name(irq, &xen_percpu_chip,
Jeremy Fitzhardingea52521f2010-09-22 15:28:52 -0700923 handle_percpu_irq, "virq");
924
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700925 bind_virq.virq = virq;
926 bind_virq.vcpu = cpu;
Olaf Hering62cc5fc2011-08-25 18:30:48 +0200927 ret = HYPERVISOR_event_channel_op(EVTCHNOP_bind_virq,
928 &bind_virq);
929 if (ret == 0)
930 evtchn = bind_virq.port;
931 else {
932 if (ret == -EEXIST)
933 ret = find_virq(virq, cpu);
934 BUG_ON(ret < 0);
935 evtchn = ret;
936 }
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700937
Ian Campbell3d4cfa32011-03-10 16:08:10 +0000938 xen_irq_info_virq_init(cpu, irq, evtchn, virq);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700939
940 bind_evtchn_to_cpu(evtchn, cpu);
941 }
942
Ian Campbell7bee9762011-03-10 16:08:15 +0000943out:
Konrad Rzeszutek Wilk77365942011-09-14 05:10:00 -0400944 mutex_unlock(&irq_mapping_update_lock);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700945
946 return irq;
947}
948
949static void unbind_from_irq(unsigned int irq)
950{
951 struct evtchn_close close;
952 int evtchn = evtchn_from_irq(irq);
Daniel De Graaf420eb552011-10-27 17:58:47 -0400953 struct irq_info *info = irq_get_handler_data(irq);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700954
Konrad Rzeszutek Wilk77365942011-09-14 05:10:00 -0400955 mutex_lock(&irq_mapping_update_lock);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700956
Daniel De Graaf420eb552011-10-27 17:58:47 -0400957 if (info->refcnt > 0) {
958 info->refcnt--;
959 if (info->refcnt != 0)
960 goto done;
961 }
962
Jeremy Fitzhardinged77bbd42009-02-06 14:09:45 -0800963 if (VALID_EVTCHN(evtchn)) {
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700964 close.port = evtchn;
965 if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close) != 0)
966 BUG();
967
968 switch (type_from_irq(irq)) {
969 case IRQT_VIRQ:
970 per_cpu(virq_to_irq, cpu_from_evtchn(evtchn))
Jeremy Fitzhardingeced40d02009-02-06 14:09:44 -0800971 [virq_from_irq(irq)] = -1;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700972 break;
Alex Nixond68d82a2008-08-22 11:52:15 +0100973 case IRQT_IPI:
974 per_cpu(ipi_to_irq, cpu_from_evtchn(evtchn))
Jeremy Fitzhardingeced40d02009-02-06 14:09:44 -0800975 [ipi_from_irq(irq)] = -1;
Alex Nixond68d82a2008-08-22 11:52:15 +0100976 break;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700977 default:
978 break;
979 }
980
981 /* Closed ports are implicitly re-bound to VCPU0. */
982 bind_evtchn_to_cpu(evtchn, 0);
983
984 evtchn_to_irq[evtchn] = -1;
Ian Campbellfed5ea82009-12-01 16:15:30 +0000985 }
986
Ian Campbellca62ce82011-03-10 16:08:12 +0000987 BUG_ON(info_for_irq(irq)->type == IRQT_UNBOUND);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700988
Ian Campbell9158c352011-03-10 16:08:09 +0000989 xen_free_irq(irq);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700990
Daniel De Graaf420eb552011-10-27 17:58:47 -0400991 done:
Konrad Rzeszutek Wilk77365942011-09-14 05:10:00 -0400992 mutex_unlock(&irq_mapping_update_lock);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700993}
994
995int bind_evtchn_to_irqhandler(unsigned int evtchn,
Jeff Garzik7c239972007-10-19 03:12:20 -0400996 irq_handler_t handler,
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700997 unsigned long irqflags,
998 const char *devname, void *dev_id)
999{
Nicolas Kaiser361ae8c2011-03-30 21:14:26 +02001000 int irq, retval;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001001
1002 irq = bind_evtchn_to_irq(evtchn);
Ian Campbell7bee9762011-03-10 16:08:15 +00001003 if (irq < 0)
1004 return irq;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001005 retval = request_irq(irq, handler, irqflags, devname, dev_id);
1006 if (retval != 0) {
1007 unbind_from_irq(irq);
1008 return retval;
1009 }
1010
1011 return irq;
1012}
1013EXPORT_SYMBOL_GPL(bind_evtchn_to_irqhandler);
1014
Ian Campbell2e820f52009-02-09 12:05:50 -08001015int bind_interdomain_evtchn_to_irqhandler(unsigned int remote_domain,
1016 unsigned int remote_port,
1017 irq_handler_t handler,
1018 unsigned long irqflags,
1019 const char *devname,
1020 void *dev_id)
1021{
1022 int irq, retval;
1023
1024 irq = bind_interdomain_evtchn_to_irq(remote_domain, remote_port);
1025 if (irq < 0)
1026 return irq;
1027
1028 retval = request_irq(irq, handler, irqflags, devname, dev_id);
1029 if (retval != 0) {
1030 unbind_from_irq(irq);
1031 return retval;
1032 }
1033
1034 return irq;
1035}
1036EXPORT_SYMBOL_GPL(bind_interdomain_evtchn_to_irqhandler);
1037
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001038int bind_virq_to_irqhandler(unsigned int virq, unsigned int cpu,
Jeff Garzik7c239972007-10-19 03:12:20 -04001039 irq_handler_t handler,
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001040 unsigned long irqflags, const char *devname, void *dev_id)
1041{
Nicolas Kaiser361ae8c2011-03-30 21:14:26 +02001042 int irq, retval;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001043
1044 irq = bind_virq_to_irq(virq, cpu);
Ian Campbell7bee9762011-03-10 16:08:15 +00001045 if (irq < 0)
1046 return irq;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001047 retval = request_irq(irq, handler, irqflags, devname, dev_id);
1048 if (retval != 0) {
1049 unbind_from_irq(irq);
1050 return retval;
1051 }
1052
1053 return irq;
1054}
1055EXPORT_SYMBOL_GPL(bind_virq_to_irqhandler);
1056
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -07001057int bind_ipi_to_irqhandler(enum ipi_vector ipi,
1058 unsigned int cpu,
1059 irq_handler_t handler,
1060 unsigned long irqflags,
1061 const char *devname,
1062 void *dev_id)
1063{
1064 int irq, retval;
1065
1066 irq = bind_ipi_to_irq(ipi, cpu);
1067 if (irq < 0)
1068 return irq;
1069
Ian Campbell9bab0b72011-10-03 15:37:00 +01001070 irqflags |= IRQF_NO_SUSPEND | IRQF_FORCE_RESUME | IRQF_EARLY_RESUME;
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -07001071 retval = request_irq(irq, handler, irqflags, devname, dev_id);
1072 if (retval != 0) {
1073 unbind_from_irq(irq);
1074 return retval;
1075 }
1076
1077 return irq;
1078}
1079
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001080void unbind_from_irqhandler(unsigned int irq, void *dev_id)
1081{
1082 free_irq(irq, dev_id);
1083 unbind_from_irq(irq);
1084}
1085EXPORT_SYMBOL_GPL(unbind_from_irqhandler);
1086
Daniel De Graaf420eb552011-10-27 17:58:47 -04001087int evtchn_make_refcounted(unsigned int evtchn)
1088{
1089 int irq = evtchn_to_irq[evtchn];
1090 struct irq_info *info;
1091
1092 if (irq == -1)
1093 return -ENOENT;
1094
1095 info = irq_get_handler_data(irq);
1096
1097 if (!info)
1098 return -ENOENT;
1099
1100 WARN_ON(info->refcnt != -1);
1101
1102 info->refcnt = 1;
1103
1104 return 0;
1105}
1106EXPORT_SYMBOL_GPL(evtchn_make_refcounted);
1107
1108int evtchn_get(unsigned int evtchn)
1109{
1110 int irq;
1111 struct irq_info *info;
1112 int err = -ENOENT;
1113
Daniel De Graafc3b3f162011-11-28 11:49:09 -05001114 if (evtchn >= NR_EVENT_CHANNELS)
1115 return -EINVAL;
1116
Daniel De Graaf420eb552011-10-27 17:58:47 -04001117 mutex_lock(&irq_mapping_update_lock);
1118
1119 irq = evtchn_to_irq[evtchn];
1120 if (irq == -1)
1121 goto done;
1122
1123 info = irq_get_handler_data(irq);
1124
1125 if (!info)
1126 goto done;
1127
1128 err = -EINVAL;
1129 if (info->refcnt <= 0)
1130 goto done;
1131
1132 info->refcnt++;
1133 err = 0;
1134 done:
1135 mutex_unlock(&irq_mapping_update_lock);
1136
1137 return err;
1138}
1139EXPORT_SYMBOL_GPL(evtchn_get);
1140
1141void evtchn_put(unsigned int evtchn)
1142{
1143 int irq = evtchn_to_irq[evtchn];
1144 if (WARN_ON(irq == -1))
1145 return;
1146 unbind_from_irq(irq);
1147}
1148EXPORT_SYMBOL_GPL(evtchn_put);
1149
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -07001150void xen_send_IPI_one(unsigned int cpu, enum ipi_vector vector)
1151{
1152 int irq = per_cpu(ipi_to_irq, cpu)[vector];
1153 BUG_ON(irq < 0);
1154 notify_remote_via_irq(irq);
1155}
1156
Jeremy Fitzhardingeee523ca2008-03-17 16:37:18 -07001157irqreturn_t xen_debug_interrupt(int irq, void *dev_id)
1158{
1159 struct shared_info *sh = HYPERVISOR_shared_info;
1160 int cpu = smp_processor_id();
Ian Campbellcb60d112011-03-10 16:08:08 +00001161 unsigned long *cpu_evtchn = per_cpu(cpu_evtchn_mask, cpu);
Jeremy Fitzhardingeee523ca2008-03-17 16:37:18 -07001162 int i;
1163 unsigned long flags;
1164 static DEFINE_SPINLOCK(debug_lock);
Ian Campbellcb52e6d2010-10-15 11:52:46 +01001165 struct vcpu_info *v;
Jeremy Fitzhardingeee523ca2008-03-17 16:37:18 -07001166
1167 spin_lock_irqsave(&debug_lock, flags);
1168
Ian Campbellcb52e6d2010-10-15 11:52:46 +01001169 printk("\nvcpu %d\n ", cpu);
Jeremy Fitzhardingeee523ca2008-03-17 16:37:18 -07001170
1171 for_each_online_cpu(i) {
Ian Campbellcb52e6d2010-10-15 11:52:46 +01001172 int pending;
1173 v = per_cpu(xen_vcpu, i);
1174 pending = (get_irq_regs() && i == cpu)
1175 ? xen_irqs_disabled(get_irq_regs())
1176 : v->evtchn_upcall_mask;
1177 printk("%d: masked=%d pending=%d event_sel %0*lx\n ", i,
1178 pending, v->evtchn_upcall_pending,
1179 (int)(sizeof(v->evtchn_pending_sel)*2),
1180 v->evtchn_pending_sel);
Jeremy Fitzhardingeee523ca2008-03-17 16:37:18 -07001181 }
Ian Campbellcb52e6d2010-10-15 11:52:46 +01001182 v = per_cpu(xen_vcpu, cpu);
Jeremy Fitzhardingeee523ca2008-03-17 16:37:18 -07001183
Ian Campbellcb52e6d2010-10-15 11:52:46 +01001184 printk("\npending:\n ");
1185 for (i = ARRAY_SIZE(sh->evtchn_pending)-1; i >= 0; i--)
1186 printk("%0*lx%s", (int)sizeof(sh->evtchn_pending[0])*2,
1187 sh->evtchn_pending[i],
1188 i % 8 == 0 ? "\n " : " ");
1189 printk("\nglobal mask:\n ");
1190 for (i = ARRAY_SIZE(sh->evtchn_mask)-1; i >= 0; i--)
1191 printk("%0*lx%s",
1192 (int)(sizeof(sh->evtchn_mask[0])*2),
1193 sh->evtchn_mask[i],
1194 i % 8 == 0 ? "\n " : " ");
1195
1196 printk("\nglobally unmasked:\n ");
1197 for (i = ARRAY_SIZE(sh->evtchn_mask)-1; i >= 0; i--)
1198 printk("%0*lx%s", (int)(sizeof(sh->evtchn_mask[0])*2),
1199 sh->evtchn_pending[i] & ~sh->evtchn_mask[i],
1200 i % 8 == 0 ? "\n " : " ");
1201
1202 printk("\nlocal cpu%d mask:\n ", cpu);
1203 for (i = (NR_EVENT_CHANNELS/BITS_PER_LONG)-1; i >= 0; i--)
1204 printk("%0*lx%s", (int)(sizeof(cpu_evtchn[0])*2),
1205 cpu_evtchn[i],
1206 i % 8 == 0 ? "\n " : " ");
1207
1208 printk("\nlocally unmasked:\n ");
1209 for (i = ARRAY_SIZE(sh->evtchn_mask)-1; i >= 0; i--) {
1210 unsigned long pending = sh->evtchn_pending[i]
1211 & ~sh->evtchn_mask[i]
1212 & cpu_evtchn[i];
1213 printk("%0*lx%s", (int)(sizeof(sh->evtchn_mask[0])*2),
1214 pending, i % 8 == 0 ? "\n " : " ");
1215 }
Jeremy Fitzhardingeee523ca2008-03-17 16:37:18 -07001216
1217 printk("\npending list:\n");
Ian Campbellcb52e6d2010-10-15 11:52:46 +01001218 for (i = 0; i < NR_EVENT_CHANNELS; i++) {
Jeremy Fitzhardingeee523ca2008-03-17 16:37:18 -07001219 if (sync_test_bit(i, sh->evtchn_pending)) {
Ian Campbellcb52e6d2010-10-15 11:52:46 +01001220 int word_idx = i / BITS_PER_LONG;
1221 printk(" %d: event %d -> irq %d%s%s%s\n",
Jeremy Fitzhardingeced40d02009-02-06 14:09:44 -08001222 cpu_from_evtchn(i), i,
Ian Campbellcb52e6d2010-10-15 11:52:46 +01001223 evtchn_to_irq[i],
1224 sync_test_bit(word_idx, &v->evtchn_pending_sel)
1225 ? "" : " l2-clear",
1226 !sync_test_bit(i, sh->evtchn_mask)
1227 ? "" : " globally-masked",
1228 sync_test_bit(i, cpu_evtchn)
1229 ? "" : " locally-masked");
Jeremy Fitzhardingeee523ca2008-03-17 16:37:18 -07001230 }
1231 }
1232
1233 spin_unlock_irqrestore(&debug_lock, flags);
1234
1235 return IRQ_HANDLED;
1236}
1237
Tejun Heo245b2e72009-06-24 15:13:48 +09001238static DEFINE_PER_CPU(unsigned, xed_nesting_count);
Keir Fraserada68142011-03-03 10:01:11 +00001239static DEFINE_PER_CPU(unsigned int, current_word_idx);
1240static DEFINE_PER_CPU(unsigned int, current_bit_idx);
Tejun Heo245b2e72009-06-24 15:13:48 +09001241
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001242/*
Scott Rixnerab7f8632011-03-03 09:30:08 +00001243 * Mask out the i least significant bits of w
1244 */
1245#define MASK_LSBS(w, i) (w & ((~0UL) << i))
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001246
1247/*
1248 * Search the CPUs pending events bitmasks. For each one found, map
1249 * the event number to an irq, and feed it into do_IRQ() for
1250 * handling.
1251 *
1252 * Xen uses a two-level bitmap to speed searching. The first level is
1253 * a bitset of words which contain pending event bits. The second
1254 * level is a bitset of pending events themselves.
1255 */
Sheng Yang38e20b02010-05-14 12:40:51 +01001256static void __xen_evtchn_do_upcall(void)
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001257{
Keir Fraser24b51c22011-03-03 11:06:28 +00001258 int start_word_idx, start_bit_idx;
Scott Rixnerab7f8632011-03-03 09:30:08 +00001259 int word_idx, bit_idx;
Keir Fraser24b51c22011-03-03 11:06:28 +00001260 int i;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001261 int cpu = get_cpu();
1262 struct shared_info *s = HYPERVISOR_shared_info;
Christoph Lameter780f36d2010-12-06 11:16:29 -06001263 struct vcpu_info *vcpu_info = __this_cpu_read(xen_vcpu);
Ruslan Pisarev088c05a2011-07-26 14:16:13 +03001264 unsigned count;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001265
Jeremy Fitzhardinge229664b2008-03-17 16:37:20 -07001266 do {
1267 unsigned long pending_words;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001268
Jeremy Fitzhardinge229664b2008-03-17 16:37:20 -07001269 vcpu_info->evtchn_upcall_pending = 0;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001270
Christoph Lameterb2e4ae62010-12-06 11:40:07 -06001271 if (__this_cpu_inc_return(xed_nesting_count) - 1)
Jeremy Fitzhardinge229664b2008-03-17 16:37:20 -07001272 goto out;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001273
Isaku Yamahatae849c3e2008-04-02 10:53:56 -07001274#ifndef CONFIG_X86 /* No need for a barrier -- XCHG is a barrier on x86. */
1275 /* Clear master flag /before/ clearing selector flag. */
Isaku Yamahata6673cf62008-06-16 14:58:13 -07001276 wmb();
Isaku Yamahatae849c3e2008-04-02 10:53:56 -07001277#endif
Jeremy Fitzhardinge229664b2008-03-17 16:37:20 -07001278 pending_words = xchg(&vcpu_info->evtchn_pending_sel, 0);
Jeremy Fitzhardinge229664b2008-03-17 16:37:20 -07001279
Keir Fraser24b51c22011-03-03 11:06:28 +00001280 start_word_idx = __this_cpu_read(current_word_idx);
1281 start_bit_idx = __this_cpu_read(current_bit_idx);
Scott Rixnerab7f8632011-03-03 09:30:08 +00001282
Keir Fraser24b51c22011-03-03 11:06:28 +00001283 word_idx = start_word_idx;
1284
1285 for (i = 0; pending_words != 0; i++) {
Jeremy Fitzhardinge229664b2008-03-17 16:37:20 -07001286 unsigned long pending_bits;
Scott Rixnerab7f8632011-03-03 09:30:08 +00001287 unsigned long words;
Jeremy Fitzhardinge229664b2008-03-17 16:37:20 -07001288
Scott Rixnerab7f8632011-03-03 09:30:08 +00001289 words = MASK_LSBS(pending_words, word_idx);
1290
1291 /*
Keir Fraserada68142011-03-03 10:01:11 +00001292 * If we masked out all events, wrap to beginning.
Scott Rixnerab7f8632011-03-03 09:30:08 +00001293 */
1294 if (words == 0) {
Keir Fraserada68142011-03-03 10:01:11 +00001295 word_idx = 0;
1296 bit_idx = 0;
Scott Rixnerab7f8632011-03-03 09:30:08 +00001297 continue;
1298 }
1299 word_idx = __ffs(words);
1300
Keir Fraser24b51c22011-03-03 11:06:28 +00001301 pending_bits = active_evtchns(cpu, s, word_idx);
1302 bit_idx = 0; /* usually scan entire word from start */
1303 if (word_idx == start_word_idx) {
1304 /* We scan the starting word in two parts */
1305 if (i == 0)
1306 /* 1st time: start in the middle */
1307 bit_idx = start_bit_idx;
1308 else
1309 /* 2nd time: mask bits done already */
1310 bit_idx &= (1UL << start_bit_idx) - 1;
1311 }
1312
Scott Rixnerab7f8632011-03-03 09:30:08 +00001313 do {
1314 unsigned long bits;
1315 int port, irq;
Eric W. Biedermanca4dbc62010-02-17 18:49:54 -08001316 struct irq_desc *desc;
Jeremy Fitzhardinge229664b2008-03-17 16:37:20 -07001317
Scott Rixnerab7f8632011-03-03 09:30:08 +00001318 bits = MASK_LSBS(pending_bits, bit_idx);
1319
1320 /* If we masked out all events, move on. */
Keir Fraserada68142011-03-03 10:01:11 +00001321 if (bits == 0)
Scott Rixnerab7f8632011-03-03 09:30:08 +00001322 break;
Scott Rixnerab7f8632011-03-03 09:30:08 +00001323
1324 bit_idx = __ffs(bits);
1325
1326 /* Process port. */
1327 port = (word_idx * BITS_PER_LONG) + bit_idx;
1328 irq = evtchn_to_irq[port];
1329
Eric W. Biedermanca4dbc62010-02-17 18:49:54 -08001330 if (irq != -1) {
1331 desc = irq_to_desc(irq);
1332 if (desc)
1333 generic_handle_irq_desc(irq, desc);
1334 }
Scott Rixnerab7f8632011-03-03 09:30:08 +00001335
Keir Fraserada68142011-03-03 10:01:11 +00001336 bit_idx = (bit_idx + 1) % BITS_PER_LONG;
1337
1338 /* Next caller starts at last processed + 1 */
1339 __this_cpu_write(current_word_idx,
1340 bit_idx ? word_idx :
1341 (word_idx+1) % BITS_PER_LONG);
1342 __this_cpu_write(current_bit_idx, bit_idx);
1343 } while (bit_idx != 0);
Scott Rixnerab7f8632011-03-03 09:30:08 +00001344
Keir Fraser24b51c22011-03-03 11:06:28 +00001345 /* Scan start_l1i twice; all others once. */
1346 if ((word_idx != start_word_idx) || (i != 0))
Scott Rixnerab7f8632011-03-03 09:30:08 +00001347 pending_words &= ~(1UL << word_idx);
Keir Fraserada68142011-03-03 10:01:11 +00001348
1349 word_idx = (word_idx + 1) % BITS_PER_LONG;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001350 }
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001351
Jeremy Fitzhardinge229664b2008-03-17 16:37:20 -07001352 BUG_ON(!irqs_disabled());
1353
Christoph Lameter780f36d2010-12-06 11:16:29 -06001354 count = __this_cpu_read(xed_nesting_count);
1355 __this_cpu_write(xed_nesting_count, 0);
Stefano Stabellini183d03c2010-05-17 17:08:21 +01001356 } while (count != 1 || vcpu_info->evtchn_upcall_pending);
Jeremy Fitzhardinge229664b2008-03-17 16:37:20 -07001357
1358out:
Jeremy Fitzhardinge3445a8f2009-02-06 14:09:46 -08001359
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001360 put_cpu();
1361}
1362
Sheng Yang38e20b02010-05-14 12:40:51 +01001363void xen_evtchn_do_upcall(struct pt_regs *regs)
1364{
1365 struct pt_regs *old_regs = set_irq_regs(regs);
1366
1367 exit_idle();
1368 irq_enter();
1369
1370 __xen_evtchn_do_upcall();
1371
1372 irq_exit();
1373 set_irq_regs(old_regs);
1374}
1375
1376void xen_hvm_evtchn_do_upcall(void)
1377{
1378 __xen_evtchn_do_upcall();
1379}
Stefano Stabellini183d03c2010-05-17 17:08:21 +01001380EXPORT_SYMBOL_GPL(xen_hvm_evtchn_do_upcall);
Sheng Yang38e20b02010-05-14 12:40:51 +01001381
Jeremy Fitzhardingeeb1e3052008-05-26 23:31:23 +01001382/* Rebind a new event channel to an existing irq. */
1383void rebind_evtchn_irq(int evtchn, int irq)
1384{
Jeremy Fitzhardinged77bbd42009-02-06 14:09:45 -08001385 struct irq_info *info = info_for_irq(irq);
1386
Jeremy Fitzhardingeeb1e3052008-05-26 23:31:23 +01001387 /* Make sure the irq is masked, since the new event channel
1388 will also be masked. */
1389 disable_irq(irq);
1390
Konrad Rzeszutek Wilk77365942011-09-14 05:10:00 -04001391 mutex_lock(&irq_mapping_update_lock);
Jeremy Fitzhardingeeb1e3052008-05-26 23:31:23 +01001392
1393 /* After resume the irq<->evtchn mappings are all cleared out */
1394 BUG_ON(evtchn_to_irq[evtchn] != -1);
1395 /* Expect irq to have been bound before,
Jeremy Fitzhardinged77bbd42009-02-06 14:09:45 -08001396 so there should be a proper type */
1397 BUG_ON(info->type == IRQT_UNBOUND);
Jeremy Fitzhardingeeb1e3052008-05-26 23:31:23 +01001398
Ian Campbell9158c352011-03-10 16:08:09 +00001399 xen_irq_info_evtchn_init(irq, evtchn);
Jeremy Fitzhardingeeb1e3052008-05-26 23:31:23 +01001400
Konrad Rzeszutek Wilk77365942011-09-14 05:10:00 -04001401 mutex_unlock(&irq_mapping_update_lock);
Jeremy Fitzhardingeeb1e3052008-05-26 23:31:23 +01001402
1403 /* new event channels are always bound to cpu 0 */
Rusty Russell0de26522008-12-13 21:20:26 +10301404 irq_set_affinity(irq, cpumask_of(0));
Jeremy Fitzhardingeeb1e3052008-05-26 23:31:23 +01001405
1406 /* Unmask the event channel. */
1407 enable_irq(irq);
1408}
1409
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001410/* Rebind an evtchn so that it gets delivered to a specific cpu */
Yinghai Lud5dedd42009-04-27 17:59:21 -07001411static int rebind_irq_to_cpu(unsigned irq, unsigned tcpu)
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001412{
1413 struct evtchn_bind_vcpu bind_vcpu;
1414 int evtchn = evtchn_from_irq(irq);
1415
Ian Campbellbe494722011-03-10 16:08:02 +00001416 if (!VALID_EVTCHN(evtchn))
1417 return -1;
1418
1419 /*
1420 * Events delivered via platform PCI interrupts are always
1421 * routed to vcpu 0 and hence cannot be rebound.
1422 */
1423 if (xen_hvm_domain() && !xen_have_vector_callback)
Yinghai Lud5dedd42009-04-27 17:59:21 -07001424 return -1;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001425
1426 /* Send future instances of this interrupt to other vcpu. */
1427 bind_vcpu.port = evtchn;
1428 bind_vcpu.vcpu = tcpu;
1429
1430 /*
1431 * If this fails, it usually just indicates that we're dealing with a
1432 * virq or IPI channel, which don't actually need to be rebound. Ignore
1433 * it, but don't do the xenlinux-level rebind in that case.
1434 */
1435 if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_vcpu, &bind_vcpu) >= 0)
1436 bind_evtchn_to_cpu(evtchn, tcpu);
Yinghai Lud5dedd42009-04-27 17:59:21 -07001437
1438 return 0;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001439}
1440
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +00001441static int set_affinity_irq(struct irq_data *data, const struct cpumask *dest,
1442 bool force)
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001443{
Rusty Russell0de26522008-12-13 21:20:26 +10301444 unsigned tcpu = cpumask_first(dest);
Yinghai Lud5dedd42009-04-27 17:59:21 -07001445
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +00001446 return rebind_irq_to_cpu(data->irq, tcpu);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001447}
1448
Isaku Yamahata642e0c82008-04-02 10:53:57 -07001449int resend_irq_on_evtchn(unsigned int irq)
1450{
1451 int masked, evtchn = evtchn_from_irq(irq);
1452 struct shared_info *s = HYPERVISOR_shared_info;
1453
1454 if (!VALID_EVTCHN(evtchn))
1455 return 1;
1456
1457 masked = sync_test_and_set_bit(evtchn, s->evtchn_mask);
1458 sync_set_bit(evtchn, s->evtchn_pending);
1459 if (!masked)
1460 unmask_evtchn(evtchn);
1461
1462 return 1;
1463}
1464
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +00001465static void enable_dynirq(struct irq_data *data)
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001466{
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +00001467 int evtchn = evtchn_from_irq(data->irq);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001468
1469 if (VALID_EVTCHN(evtchn))
1470 unmask_evtchn(evtchn);
1471}
1472
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +00001473static void disable_dynirq(struct irq_data *data)
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001474{
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +00001475 int evtchn = evtchn_from_irq(data->irq);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001476
1477 if (VALID_EVTCHN(evtchn))
1478 mask_evtchn(evtchn);
1479}
1480
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +00001481static void ack_dynirq(struct irq_data *data)
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001482{
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +00001483 int evtchn = evtchn_from_irq(data->irq);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001484
Stefano Stabellini7e186bd2011-05-06 12:27:50 +01001485 irq_move_irq(data);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001486
1487 if (VALID_EVTCHN(evtchn))
Stefano Stabellini7e186bd2011-05-06 12:27:50 +01001488 clear_evtchn(evtchn);
1489}
1490
1491static void mask_ack_dynirq(struct irq_data *data)
1492{
1493 disable_dynirq(data);
1494 ack_dynirq(data);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001495}
1496
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +00001497static int retrigger_dynirq(struct irq_data *data)
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001498{
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +00001499 int evtchn = evtchn_from_irq(data->irq);
Jeremy Fitzhardingeee8fa1c2008-03-17 16:37:19 -07001500 struct shared_info *sh = HYPERVISOR_shared_info;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001501 int ret = 0;
1502
1503 if (VALID_EVTCHN(evtchn)) {
Jeremy Fitzhardingeee8fa1c2008-03-17 16:37:19 -07001504 int masked;
1505
1506 masked = sync_test_and_set_bit(evtchn, sh->evtchn_mask);
1507 sync_set_bit(evtchn, sh->evtchn_pending);
1508 if (!masked)
1509 unmask_evtchn(evtchn);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001510 ret = 1;
1511 }
1512
1513 return ret;
1514}
1515
Ian Campbell0a852262011-03-10 16:08:06 +00001516static void restore_pirqs(void)
Stefano Stabellini9a069c32010-12-01 14:51:44 +00001517{
1518 int pirq, rc, irq, gsi;
1519 struct physdev_map_pirq map_irq;
Ian Campbell69c358c2011-03-10 16:08:13 +00001520 struct irq_info *info;
Stefano Stabellini9a069c32010-12-01 14:51:44 +00001521
Ian Campbell69c358c2011-03-10 16:08:13 +00001522 list_for_each_entry(info, &xen_irq_list_head, list) {
1523 if (info->type != IRQT_PIRQ)
Stefano Stabellini9a069c32010-12-01 14:51:44 +00001524 continue;
1525
Ian Campbell69c358c2011-03-10 16:08:13 +00001526 pirq = info->u.pirq.pirq;
1527 gsi = info->u.pirq.gsi;
1528 irq = info->irq;
1529
Stefano Stabellini9a069c32010-12-01 14:51:44 +00001530 /* save/restore of PT devices doesn't work, so at this point the
1531 * only devices present are GSI based emulated devices */
Stefano Stabellini9a069c32010-12-01 14:51:44 +00001532 if (!gsi)
1533 continue;
1534
1535 map_irq.domid = DOMID_SELF;
1536 map_irq.type = MAP_PIRQ_TYPE_GSI;
1537 map_irq.index = gsi;
1538 map_irq.pirq = pirq;
1539
1540 rc = HYPERVISOR_physdev_op(PHYSDEVOP_map_pirq, &map_irq);
1541 if (rc) {
1542 printk(KERN_WARNING "xen map irq failed gsi=%d irq=%d pirq=%d rc=%d\n",
1543 gsi, irq, pirq, rc);
Ian Campbell9158c352011-03-10 16:08:09 +00001544 xen_free_irq(irq);
Stefano Stabellini9a069c32010-12-01 14:51:44 +00001545 continue;
1546 }
1547
1548 printk(KERN_DEBUG "xen: --> irq=%d, pirq=%d\n", irq, map_irq.pirq);
1549
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +00001550 __startup_pirq(irq);
Stefano Stabellini9a069c32010-12-01 14:51:44 +00001551 }
1552}
1553
Jeremy Fitzhardinge0e913982008-05-26 23:31:27 +01001554static void restore_cpu_virqs(unsigned int cpu)
1555{
1556 struct evtchn_bind_virq bind_virq;
1557 int virq, irq, evtchn;
1558
1559 for (virq = 0; virq < NR_VIRQS; virq++) {
1560 if ((irq = per_cpu(virq_to_irq, cpu)[virq]) == -1)
1561 continue;
1562
Jeremy Fitzhardingeced40d02009-02-06 14:09:44 -08001563 BUG_ON(virq_from_irq(irq) != virq);
Jeremy Fitzhardinge0e913982008-05-26 23:31:27 +01001564
1565 /* Get a new binding from Xen. */
1566 bind_virq.virq = virq;
1567 bind_virq.vcpu = cpu;
1568 if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_virq,
1569 &bind_virq) != 0)
1570 BUG();
1571 evtchn = bind_virq.port;
1572
1573 /* Record the new mapping. */
Ian Campbell3d4cfa32011-03-10 16:08:10 +00001574 xen_irq_info_virq_init(cpu, irq, evtchn, virq);
Jeremy Fitzhardinge0e913982008-05-26 23:31:27 +01001575 bind_evtchn_to_cpu(evtchn, cpu);
Jeremy Fitzhardinge0e913982008-05-26 23:31:27 +01001576 }
1577}
1578
1579static void restore_cpu_ipis(unsigned int cpu)
1580{
1581 struct evtchn_bind_ipi bind_ipi;
1582 int ipi, irq, evtchn;
1583
1584 for (ipi = 0; ipi < XEN_NR_IPIS; ipi++) {
1585 if ((irq = per_cpu(ipi_to_irq, cpu)[ipi]) == -1)
1586 continue;
1587
Jeremy Fitzhardingeced40d02009-02-06 14:09:44 -08001588 BUG_ON(ipi_from_irq(irq) != ipi);
Jeremy Fitzhardinge0e913982008-05-26 23:31:27 +01001589
1590 /* Get a new binding from Xen. */
1591 bind_ipi.vcpu = cpu;
1592 if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_ipi,
1593 &bind_ipi) != 0)
1594 BUG();
1595 evtchn = bind_ipi.port;
1596
1597 /* Record the new mapping. */
Ian Campbell3d4cfa32011-03-10 16:08:10 +00001598 xen_irq_info_ipi_init(cpu, irq, evtchn, ipi);
Jeremy Fitzhardinge0e913982008-05-26 23:31:27 +01001599 bind_evtchn_to_cpu(evtchn, cpu);
Jeremy Fitzhardinge0e913982008-05-26 23:31:27 +01001600 }
1601}
1602
Jeremy Fitzhardinge2d9e1e22008-07-07 12:07:53 -07001603/* Clear an irq's pending state, in preparation for polling on it */
1604void xen_clear_irq_pending(int irq)
1605{
1606 int evtchn = evtchn_from_irq(irq);
1607
1608 if (VALID_EVTCHN(evtchn))
1609 clear_evtchn(evtchn);
1610}
Konrad Rzeszutek Wilkd9a88142009-11-05 16:33:09 -05001611EXPORT_SYMBOL(xen_clear_irq_pending);
Jeremy Fitzhardinge168d2f42008-08-20 17:02:18 -07001612void xen_set_irq_pending(int irq)
1613{
1614 int evtchn = evtchn_from_irq(irq);
1615
1616 if (VALID_EVTCHN(evtchn))
1617 set_evtchn(evtchn);
1618}
1619
1620bool xen_test_irq_pending(int irq)
1621{
1622 int evtchn = evtchn_from_irq(irq);
1623 bool ret = false;
1624
1625 if (VALID_EVTCHN(evtchn))
1626 ret = test_evtchn(evtchn);
1627
1628 return ret;
1629}
1630
Konrad Rzeszutek Wilkd9a88142009-11-05 16:33:09 -05001631/* Poll waiting for an irq to become pending with timeout. In the usual case,
1632 * the irq will be disabled so it won't deliver an interrupt. */
1633void xen_poll_irq_timeout(int irq, u64 timeout)
Jeremy Fitzhardinge2d9e1e22008-07-07 12:07:53 -07001634{
1635 evtchn_port_t evtchn = evtchn_from_irq(irq);
1636
1637 if (VALID_EVTCHN(evtchn)) {
1638 struct sched_poll poll;
1639
1640 poll.nr_ports = 1;
Konrad Rzeszutek Wilkd9a88142009-11-05 16:33:09 -05001641 poll.timeout = timeout;
Isaku Yamahataff3c5362008-10-14 17:50:44 -07001642 set_xen_guest_handle(poll.ports, &evtchn);
Jeremy Fitzhardinge2d9e1e22008-07-07 12:07:53 -07001643
1644 if (HYPERVISOR_sched_op(SCHEDOP_poll, &poll) != 0)
1645 BUG();
1646 }
1647}
Konrad Rzeszutek Wilkd9a88142009-11-05 16:33:09 -05001648EXPORT_SYMBOL(xen_poll_irq_timeout);
1649/* Poll waiting for an irq to become pending. In the usual case, the
1650 * irq will be disabled so it won't deliver an interrupt. */
1651void xen_poll_irq(int irq)
1652{
1653 xen_poll_irq_timeout(irq, 0 /* no timeout */);
1654}
Jeremy Fitzhardinge2d9e1e22008-07-07 12:07:53 -07001655
Konrad Rzeszutek Wilkc7c2c3a2010-11-08 14:26:36 -05001656/* Check whether the IRQ line is shared with other guests. */
1657int xen_test_irq_shared(int irq)
1658{
1659 struct irq_info *info = info_for_irq(irq);
1660 struct physdev_irq_status_query irq_status = { .irq = info->u.pirq.pirq };
1661
1662 if (HYPERVISOR_physdev_op(PHYSDEVOP_irq_status_query, &irq_status))
1663 return 0;
1664 return !(irq_status.flags & XENIRQSTAT_shared);
1665}
1666EXPORT_SYMBOL_GPL(xen_test_irq_shared);
1667
Jeremy Fitzhardinge0e913982008-05-26 23:31:27 +01001668void xen_irq_resume(void)
1669{
Ian Campbell6cb65372011-03-10 16:08:11 +00001670 unsigned int cpu, evtchn;
1671 struct irq_info *info;
Jeremy Fitzhardinge0e913982008-05-26 23:31:27 +01001672
1673 init_evtchn_cpu_bindings();
1674
1675 /* New event-channel space is not 'live' yet. */
1676 for (evtchn = 0; evtchn < NR_EVENT_CHANNELS; evtchn++)
1677 mask_evtchn(evtchn);
1678
1679 /* No IRQ <-> event-channel mappings. */
Ian Campbell6cb65372011-03-10 16:08:11 +00001680 list_for_each_entry(info, &xen_irq_list_head, list)
1681 info->evtchn = 0; /* zap event-channel binding */
Jeremy Fitzhardinge0e913982008-05-26 23:31:27 +01001682
1683 for (evtchn = 0; evtchn < NR_EVENT_CHANNELS; evtchn++)
1684 evtchn_to_irq[evtchn] = -1;
1685
1686 for_each_possible_cpu(cpu) {
1687 restore_cpu_virqs(cpu);
1688 restore_cpu_ipis(cpu);
1689 }
Ian Campbell69035912010-11-01 16:30:09 +00001690
Ian Campbell0a852262011-03-10 16:08:06 +00001691 restore_pirqs();
Jeremy Fitzhardinge0e913982008-05-26 23:31:27 +01001692}
1693
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001694static struct irq_chip xen_dynamic_chip __read_mostly = {
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +00001695 .name = "xen-dyn",
Jeremy Fitzhardinge54a353a2009-02-06 14:09:42 -08001696
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +00001697 .irq_disable = disable_dynirq,
1698 .irq_mask = disable_dynirq,
1699 .irq_unmask = enable_dynirq,
Jeremy Fitzhardinge54a353a2009-02-06 14:09:42 -08001700
Stefano Stabellini7e186bd2011-05-06 12:27:50 +01001701 .irq_ack = ack_dynirq,
1702 .irq_mask_ack = mask_ack_dynirq,
1703
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +00001704 .irq_set_affinity = set_affinity_irq,
1705 .irq_retrigger = retrigger_dynirq,
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001706};
1707
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -04001708static struct irq_chip xen_pirq_chip __read_mostly = {
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +00001709 .name = "xen-pirq",
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -04001710
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +00001711 .irq_startup = startup_pirq,
1712 .irq_shutdown = shutdown_pirq,
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +00001713 .irq_enable = enable_pirq,
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +00001714 .irq_disable = disable_pirq,
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -04001715
Stefano Stabellini7e186bd2011-05-06 12:27:50 +01001716 .irq_mask = disable_dynirq,
1717 .irq_unmask = enable_dynirq,
1718
1719 .irq_ack = eoi_pirq,
1720 .irq_eoi = eoi_pirq,
1721 .irq_mask_ack = mask_ack_pirq,
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -04001722
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +00001723 .irq_set_affinity = set_affinity_irq,
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -04001724
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +00001725 .irq_retrigger = retrigger_dynirq,
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -04001726};
1727
Jeremy Fitzhardingeaaca4962010-08-20 18:57:53 -07001728static struct irq_chip xen_percpu_chip __read_mostly = {
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +00001729 .name = "xen-percpu",
Jeremy Fitzhardingeaaca4962010-08-20 18:57:53 -07001730
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +00001731 .irq_disable = disable_dynirq,
1732 .irq_mask = disable_dynirq,
1733 .irq_unmask = enable_dynirq,
Jeremy Fitzhardingeaaca4962010-08-20 18:57:53 -07001734
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +00001735 .irq_ack = ack_dynirq,
Jeremy Fitzhardingeaaca4962010-08-20 18:57:53 -07001736};
1737
Sheng Yang38e20b02010-05-14 12:40:51 +01001738int xen_set_callback_via(uint64_t via)
1739{
1740 struct xen_hvm_param a;
1741 a.domid = DOMID_SELF;
1742 a.index = HVM_PARAM_CALLBACK_IRQ;
1743 a.value = via;
1744 return HYPERVISOR_hvm_op(HVMOP_set_param, &a);
1745}
1746EXPORT_SYMBOL_GPL(xen_set_callback_via);
1747
Stefano Stabellinica65f9f2010-07-29 14:37:48 +01001748#ifdef CONFIG_XEN_PVHVM
Sheng Yang38e20b02010-05-14 12:40:51 +01001749/* Vector callbacks are better than PCI interrupts to receive event
1750 * channel notifications because we can receive vector callbacks on any
1751 * vcpu and we don't need PCI support or APIC interactions. */
1752void xen_callback_vector(void)
1753{
1754 int rc;
1755 uint64_t callback_via;
1756 if (xen_have_vector_callback) {
1757 callback_via = HVM_CALLBACK_VECTOR(XEN_HVM_EVTCHN_CALLBACK);
1758 rc = xen_set_callback_via(callback_via);
1759 if (rc) {
1760 printk(KERN_ERR "Request for Xen HVM callback vector"
1761 " failed.\n");
1762 xen_have_vector_callback = 0;
1763 return;
1764 }
1765 printk(KERN_INFO "Xen HVM callback vector for event delivery is "
1766 "enabled\n");
1767 /* in the restore case the vector has already been allocated */
1768 if (!test_bit(XEN_HVM_EVTCHN_CALLBACK, used_vectors))
1769 alloc_intr_gate(XEN_HVM_EVTCHN_CALLBACK, xen_hvm_callback_vector);
1770 }
1771}
Stefano Stabellinica65f9f2010-07-29 14:37:48 +01001772#else
1773void xen_callback_vector(void) {}
1774#endif
Sheng Yang38e20b02010-05-14 12:40:51 +01001775
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001776void __init xen_init_IRQ(void)
1777{
Stefano Stabellini9846ff12012-01-30 16:21:48 +00001778 int i, rc;
Mike Travisc7a35892009-01-10 21:58:11 -08001779
Jeremy Fitzhardingeb21ddbf2010-06-07 16:28:49 -04001780 evtchn_to_irq = kcalloc(NR_EVENT_CHANNELS, sizeof(*evtchn_to_irq),
1781 GFP_KERNEL);
Konrad Rzeszutek Wilk9d093e22011-09-29 13:31:21 -04001782 BUG_ON(!evtchn_to_irq);
Jeremy Fitzhardingeb21ddbf2010-06-07 16:28:49 -04001783 for (i = 0; i < NR_EVENT_CHANNELS; i++)
1784 evtchn_to_irq[i] = -1;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001785
1786 init_evtchn_cpu_bindings();
1787
1788 /* No event channels are 'live' right now. */
1789 for (i = 0; i < NR_EVENT_CHANNELS; i++)
1790 mask_evtchn(i);
1791
Stefano Stabellini9846ff12012-01-30 16:21:48 +00001792 pirq_needs_eoi = pirq_needs_eoi_flag;
1793
Sheng Yang38e20b02010-05-14 12:40:51 +01001794 if (xen_hvm_domain()) {
1795 xen_callback_vector();
1796 native_init_IRQ();
Stefano Stabellini3942b742010-06-24 17:50:18 +01001797 /* pci_xen_hvm_init must be called after native_init_IRQ so that
1798 * __acpi_register_gsi can point at the right function */
1799 pci_xen_hvm_init();
Sheng Yang38e20b02010-05-14 12:40:51 +01001800 } else {
Stefano Stabellini9846ff12012-01-30 16:21:48 +00001801 struct physdev_pirq_eoi_gmfn eoi_gmfn;
1802
Sheng Yang38e20b02010-05-14 12:40:51 +01001803 irq_ctx_init(smp_processor_id());
Jeremy Fitzhardinge38aa66f2010-09-02 14:51:39 +01001804 if (xen_initial_domain())
Konrad Rzeszutek Wilka0ee0562011-06-09 09:49:13 -04001805 pci_xen_initial_domain();
Stefano Stabellini9846ff12012-01-30 16:21:48 +00001806
1807 pirq_eoi_map = (void *)__get_free_page(GFP_KERNEL|__GFP_ZERO);
1808 eoi_gmfn.gmfn = virt_to_mfn(pirq_eoi_map);
1809 rc = HYPERVISOR_physdev_op(PHYSDEVOP_pirq_eoi_gmfn_v2, &eoi_gmfn);
1810 if (rc != 0) {
1811 free_page((unsigned long) pirq_eoi_map);
1812 pirq_eoi_map = NULL;
1813 } else
1814 pirq_needs_eoi = pirq_check_eoi_map;
Sheng Yang38e20b02010-05-14 12:40:51 +01001815 }
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001816}