blob: 1e3b934a4cf707cbe06084f21efb39874d4802f4 [file] [log] [blame]
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001/*
2 * Xen event channels
3 *
4 * Xen models interrupts with abstract event channels. Because each
5 * domain gets 1024 event channels, but NR_IRQ is not that large, we
6 * must dynamically map irqs<->event channels. The event channels
7 * interface with the rest of the kernel by defining a xen interrupt
8 * chip. When an event is recieved, it is mapped to an irq and sent
9 * through the normal interrupt processing path.
10 *
11 * There are four kinds of events which can be mapped to an event
12 * channel:
13 *
14 * 1. Inter-domain notifications. This includes all the virtual
15 * device events, since they're driven by front-ends in another domain
16 * (typically dom0).
17 * 2. VIRQs, typically used for timers. These are per-cpu events.
18 * 3. IPIs.
19 * 4. Hardware interrupts. Not supported at present.
20 *
21 * Jeremy Fitzhardinge <jeremy@xensource.com>, XenSource Inc, 2007
22 */
23
24#include <linux/linkage.h>
25#include <linux/interrupt.h>
26#include <linux/irq.h>
27#include <linux/module.h>
28#include <linux/string.h>
29
30#include <asm/ptrace.h>
31#include <asm/irq.h>
32#include <asm/sync_bitops.h>
33#include <asm/xen/hypercall.h>
Adrian Bunk8d1b8752007-07-20 00:31:44 -070034#include <asm/xen/hypervisor.h>
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -070035
Isaku Yamahatae04d0d02008-04-02 10:53:55 -070036#include <xen/xen-ops.h>
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -070037#include <xen/events.h>
38#include <xen/interface/xen.h>
39#include <xen/interface/event_channel.h>
40
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -070041/*
42 * This lock protects updates to the following mapping and reference-count
43 * arrays. The lock does not need to be acquired to read the mapping tables.
44 */
45static DEFINE_SPINLOCK(irq_mapping_update_lock);
46
47/* IRQ <-> VIRQ mapping. */
48static DEFINE_PER_CPU(int, virq_to_irq[NR_VIRQS]) = {[0 ... NR_VIRQS-1] = -1};
49
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -070050/* IRQ <-> IPI mapping */
51static DEFINE_PER_CPU(int, ipi_to_irq[XEN_NR_IPIS]) = {[0 ... XEN_NR_IPIS-1] = -1};
52
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -070053/* Packed IRQ information: binding type, sub-type index, and event channel. */
54struct packed_irq
55{
56 unsigned short evtchn;
57 unsigned char index;
58 unsigned char type;
59};
60
61static struct packed_irq irq_info[NR_IRQS];
62
63/* Binding types. */
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -070064enum {
65 IRQT_UNBOUND,
66 IRQT_PIRQ,
67 IRQT_VIRQ,
68 IRQT_IPI,
69 IRQT_EVTCHN
70};
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -070071
72/* Convenient shorthand for packed representation of an unbound IRQ. */
73#define IRQ_UNBOUND mk_irq_info(IRQT_UNBOUND, 0, 0)
74
75static int evtchn_to_irq[NR_EVENT_CHANNELS] = {
76 [0 ... NR_EVENT_CHANNELS-1] = -1
77};
78static unsigned long cpu_evtchn_mask[NR_CPUS][NR_EVENT_CHANNELS/BITS_PER_LONG];
79static u8 cpu_evtchn[NR_EVENT_CHANNELS];
80
81/* Reference counts for bindings to IRQs. */
82static int irq_bindcount[NR_IRQS];
83
84/* Xen will never allocate port zero for any purpose. */
85#define VALID_EVTCHN(chn) ((chn) != 0)
86
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -070087static struct irq_chip xen_dynamic_chip;
88
89/* Constructor for packed IRQ information. */
90static inline struct packed_irq mk_irq_info(u32 type, u32 index, u32 evtchn)
91{
92 return (struct packed_irq) { evtchn, index, type };
93}
94
95/*
96 * Accessors for packed IRQ information.
97 */
98static inline unsigned int evtchn_from_irq(int irq)
99{
100 return irq_info[irq].evtchn;
101}
102
103static inline unsigned int index_from_irq(int irq)
104{
105 return irq_info[irq].index;
106}
107
108static inline unsigned int type_from_irq(int irq)
109{
110 return irq_info[irq].type;
111}
112
113static inline unsigned long active_evtchns(unsigned int cpu,
114 struct shared_info *sh,
115 unsigned int idx)
116{
117 return (sh->evtchn_pending[idx] &
118 cpu_evtchn_mask[cpu][idx] &
119 ~sh->evtchn_mask[idx]);
120}
121
122static void bind_evtchn_to_cpu(unsigned int chn, unsigned int cpu)
123{
124 int irq = evtchn_to_irq[chn];
125
126 BUG_ON(irq == -1);
127#ifdef CONFIG_SMP
Yinghai Lu08678b02008-08-19 20:50:05 -0700128 irq_to_desc(irq)->affinity = cpumask_of_cpu(cpu);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700129#endif
130
131 __clear_bit(chn, cpu_evtchn_mask[cpu_evtchn[chn]]);
132 __set_bit(chn, cpu_evtchn_mask[cpu]);
133
134 cpu_evtchn[chn] = cpu;
135}
136
137static void init_evtchn_cpu_bindings(void)
138{
139#ifdef CONFIG_SMP
Thomas Gleixner10e58082008-10-16 14:19:04 +0200140 struct irq_desc *desc;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700141 int i;
Thomas Gleixner10e58082008-10-16 14:19:04 +0200142
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700143 /* By default all event channels notify CPU#0. */
Thomas Gleixner10e58082008-10-16 14:19:04 +0200144 for_each_irq_desc(i, desc)
Yinghai Lu08678b02008-08-19 20:50:05 -0700145 desc->affinity = cpumask_of_cpu(0);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700146#endif
147
148 memset(cpu_evtchn, 0, sizeof(cpu_evtchn));
149 memset(cpu_evtchn_mask[0], ~0, sizeof(cpu_evtchn_mask[0]));
150}
151
152static inline unsigned int cpu_from_evtchn(unsigned int evtchn)
153{
154 return cpu_evtchn[evtchn];
155}
156
157static inline void clear_evtchn(int port)
158{
159 struct shared_info *s = HYPERVISOR_shared_info;
160 sync_clear_bit(port, &s->evtchn_pending[0]);
161}
162
163static inline void set_evtchn(int port)
164{
165 struct shared_info *s = HYPERVISOR_shared_info;
166 sync_set_bit(port, &s->evtchn_pending[0]);
167}
168
Jeremy Fitzhardinge168d2f42008-08-20 17:02:18 -0700169static inline int test_evtchn(int port)
170{
171 struct shared_info *s = HYPERVISOR_shared_info;
172 return sync_test_bit(port, &s->evtchn_pending[0]);
173}
174
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700175
176/**
177 * notify_remote_via_irq - send event to remote end of event channel via irq
178 * @irq: irq of event channel to send event to
179 *
180 * Unlike notify_remote_via_evtchn(), this is safe to use across
181 * save/restore. Notifications on a broken connection are silently
182 * dropped.
183 */
184void notify_remote_via_irq(int irq)
185{
186 int evtchn = evtchn_from_irq(irq);
187
188 if (VALID_EVTCHN(evtchn))
189 notify_remote_via_evtchn(evtchn);
190}
191EXPORT_SYMBOL_GPL(notify_remote_via_irq);
192
193static void mask_evtchn(int port)
194{
195 struct shared_info *s = HYPERVISOR_shared_info;
196 sync_set_bit(port, &s->evtchn_mask[0]);
197}
198
199static void unmask_evtchn(int port)
200{
201 struct shared_info *s = HYPERVISOR_shared_info;
202 unsigned int cpu = get_cpu();
203
204 BUG_ON(!irqs_disabled());
205
206 /* Slow path (hypercall) if this is a non-local port. */
207 if (unlikely(cpu != cpu_from_evtchn(port))) {
208 struct evtchn_unmask unmask = { .port = port };
209 (void)HYPERVISOR_event_channel_op(EVTCHNOP_unmask, &unmask);
210 } else {
211 struct vcpu_info *vcpu_info = __get_cpu_var(xen_vcpu);
212
213 sync_clear_bit(port, &s->evtchn_mask[0]);
214
215 /*
216 * The following is basically the equivalent of
217 * 'hw_resend_irq'. Just like a real IO-APIC we 'lose
218 * the interrupt edge' if the channel is masked.
219 */
220 if (sync_test_bit(port, &s->evtchn_pending[0]) &&
221 !sync_test_and_set_bit(port / BITS_PER_LONG,
222 &vcpu_info->evtchn_pending_sel))
223 vcpu_info->evtchn_upcall_pending = 1;
224 }
225
226 put_cpu();
227}
228
229static int find_unbound_irq(void)
230{
231 int irq;
232
233 /* Only allocate from dynirq range */
Thomas Gleixner10e58082008-10-16 14:19:04 +0200234 for_each_irq_nr(irq)
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700235 if (irq_bindcount[irq] == 0)
236 break;
237
Yinghai Lu5a15d7e2008-08-19 20:49:57 -0700238 if (irq == nr_irqs)
239 panic("No available IRQ to bind to: increase nr_irqs!\n");
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700240
241 return irq;
242}
243
Jeremy Fitzhardingeb536b4b2007-07-17 18:37:06 -0700244int bind_evtchn_to_irq(unsigned int evtchn)
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700245{
246 int irq;
247
248 spin_lock(&irq_mapping_update_lock);
249
250 irq = evtchn_to_irq[evtchn];
251
252 if (irq == -1) {
253 irq = find_unbound_irq();
254
255 dynamic_irq_init(irq);
256 set_irq_chip_and_handler_name(irq, &xen_dynamic_chip,
257 handle_level_irq, "event");
258
259 evtchn_to_irq[evtchn] = irq;
260 irq_info[irq] = mk_irq_info(IRQT_EVTCHN, 0, evtchn);
261 }
262
263 irq_bindcount[irq]++;
264
265 spin_unlock(&irq_mapping_update_lock);
266
267 return irq;
268}
Jeremy Fitzhardingeb536b4b2007-07-17 18:37:06 -0700269EXPORT_SYMBOL_GPL(bind_evtchn_to_irq);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700270
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -0700271static int bind_ipi_to_irq(unsigned int ipi, unsigned int cpu)
272{
273 struct evtchn_bind_ipi bind_ipi;
274 int evtchn, irq;
275
276 spin_lock(&irq_mapping_update_lock);
277
278 irq = per_cpu(ipi_to_irq, cpu)[ipi];
279 if (irq == -1) {
280 irq = find_unbound_irq();
281 if (irq < 0)
282 goto out;
283
284 dynamic_irq_init(irq);
285 set_irq_chip_and_handler_name(irq, &xen_dynamic_chip,
286 handle_level_irq, "ipi");
287
288 bind_ipi.vcpu = cpu;
289 if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_ipi,
290 &bind_ipi) != 0)
291 BUG();
292 evtchn = bind_ipi.port;
293
294 evtchn_to_irq[evtchn] = irq;
295 irq_info[irq] = mk_irq_info(IRQT_IPI, ipi, evtchn);
296
297 per_cpu(ipi_to_irq, cpu)[ipi] = irq;
298
299 bind_evtchn_to_cpu(evtchn, cpu);
300 }
301
302 irq_bindcount[irq]++;
303
304 out:
305 spin_unlock(&irq_mapping_update_lock);
306 return irq;
307}
308
309
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700310static int bind_virq_to_irq(unsigned int virq, unsigned int cpu)
311{
312 struct evtchn_bind_virq bind_virq;
313 int evtchn, irq;
314
315 spin_lock(&irq_mapping_update_lock);
316
317 irq = per_cpu(virq_to_irq, cpu)[virq];
318
319 if (irq == -1) {
320 bind_virq.virq = virq;
321 bind_virq.vcpu = cpu;
322 if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_virq,
323 &bind_virq) != 0)
324 BUG();
325 evtchn = bind_virq.port;
326
327 irq = find_unbound_irq();
328
329 dynamic_irq_init(irq);
330 set_irq_chip_and_handler_name(irq, &xen_dynamic_chip,
331 handle_level_irq, "virq");
332
333 evtchn_to_irq[evtchn] = irq;
334 irq_info[irq] = mk_irq_info(IRQT_VIRQ, virq, evtchn);
335
336 per_cpu(virq_to_irq, cpu)[virq] = irq;
337
338 bind_evtchn_to_cpu(evtchn, cpu);
339 }
340
341 irq_bindcount[irq]++;
342
343 spin_unlock(&irq_mapping_update_lock);
344
345 return irq;
346}
347
348static void unbind_from_irq(unsigned int irq)
349{
350 struct evtchn_close close;
351 int evtchn = evtchn_from_irq(irq);
352
353 spin_lock(&irq_mapping_update_lock);
354
Jeremy Fitzhardinge0f2287a2008-05-26 23:31:24 +0100355 if ((--irq_bindcount[irq] == 0) && VALID_EVTCHN(evtchn)) {
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700356 close.port = evtchn;
357 if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close) != 0)
358 BUG();
359
360 switch (type_from_irq(irq)) {
361 case IRQT_VIRQ:
362 per_cpu(virq_to_irq, cpu_from_evtchn(evtchn))
363 [index_from_irq(irq)] = -1;
364 break;
Alex Nixond68d82a2008-08-22 11:52:15 +0100365 case IRQT_IPI:
366 per_cpu(ipi_to_irq, cpu_from_evtchn(evtchn))
367 [index_from_irq(irq)] = -1;
368 break;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700369 default:
370 break;
371 }
372
373 /* Closed ports are implicitly re-bound to VCPU0. */
374 bind_evtchn_to_cpu(evtchn, 0);
375
376 evtchn_to_irq[evtchn] = -1;
377 irq_info[irq] = IRQ_UNBOUND;
378
Jeremy Fitzhardinge0f2287a2008-05-26 23:31:24 +0100379 dynamic_irq_cleanup(irq);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700380 }
381
382 spin_unlock(&irq_mapping_update_lock);
383}
384
385int bind_evtchn_to_irqhandler(unsigned int evtchn,
Jeff Garzik7c239972007-10-19 03:12:20 -0400386 irq_handler_t handler,
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700387 unsigned long irqflags,
388 const char *devname, void *dev_id)
389{
390 unsigned int irq;
391 int retval;
392
393 irq = bind_evtchn_to_irq(evtchn);
394 retval = request_irq(irq, handler, irqflags, devname, dev_id);
395 if (retval != 0) {
396 unbind_from_irq(irq);
397 return retval;
398 }
399
400 return irq;
401}
402EXPORT_SYMBOL_GPL(bind_evtchn_to_irqhandler);
403
404int bind_virq_to_irqhandler(unsigned int virq, unsigned int cpu,
Jeff Garzik7c239972007-10-19 03:12:20 -0400405 irq_handler_t handler,
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700406 unsigned long irqflags, const char *devname, void *dev_id)
407{
408 unsigned int irq;
409 int retval;
410
411 irq = bind_virq_to_irq(virq, cpu);
412 retval = request_irq(irq, handler, irqflags, devname, dev_id);
413 if (retval != 0) {
414 unbind_from_irq(irq);
415 return retval;
416 }
417
418 return irq;
419}
420EXPORT_SYMBOL_GPL(bind_virq_to_irqhandler);
421
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -0700422int bind_ipi_to_irqhandler(enum ipi_vector ipi,
423 unsigned int cpu,
424 irq_handler_t handler,
425 unsigned long irqflags,
426 const char *devname,
427 void *dev_id)
428{
429 int irq, retval;
430
431 irq = bind_ipi_to_irq(ipi, cpu);
432 if (irq < 0)
433 return irq;
434
435 retval = request_irq(irq, handler, irqflags, devname, dev_id);
436 if (retval != 0) {
437 unbind_from_irq(irq);
438 return retval;
439 }
440
441 return irq;
442}
443
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700444void unbind_from_irqhandler(unsigned int irq, void *dev_id)
445{
446 free_irq(irq, dev_id);
447 unbind_from_irq(irq);
448}
449EXPORT_SYMBOL_GPL(unbind_from_irqhandler);
450
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -0700451void xen_send_IPI_one(unsigned int cpu, enum ipi_vector vector)
452{
453 int irq = per_cpu(ipi_to_irq, cpu)[vector];
454 BUG_ON(irq < 0);
455 notify_remote_via_irq(irq);
456}
457
Jeremy Fitzhardingeee523ca2008-03-17 16:37:18 -0700458irqreturn_t xen_debug_interrupt(int irq, void *dev_id)
459{
460 struct shared_info *sh = HYPERVISOR_shared_info;
461 int cpu = smp_processor_id();
462 int i;
463 unsigned long flags;
464 static DEFINE_SPINLOCK(debug_lock);
465
466 spin_lock_irqsave(&debug_lock, flags);
467
468 printk("vcpu %d\n ", cpu);
469
470 for_each_online_cpu(i) {
471 struct vcpu_info *v = per_cpu(xen_vcpu, i);
472 printk("%d: masked=%d pending=%d event_sel %08lx\n ", i,
Isaku Yamahatae849c3e2008-04-02 10:53:56 -0700473 (get_irq_regs() && i == cpu) ? xen_irqs_disabled(get_irq_regs()) : v->evtchn_upcall_mask,
Jeremy Fitzhardingeee523ca2008-03-17 16:37:18 -0700474 v->evtchn_upcall_pending,
475 v->evtchn_pending_sel);
476 }
477 printk("pending:\n ");
478 for(i = ARRAY_SIZE(sh->evtchn_pending)-1; i >= 0; i--)
479 printk("%08lx%s", sh->evtchn_pending[i],
480 i % 8 == 0 ? "\n " : " ");
481 printk("\nmasks:\n ");
482 for(i = ARRAY_SIZE(sh->evtchn_mask)-1; i >= 0; i--)
483 printk("%08lx%s", sh->evtchn_mask[i],
484 i % 8 == 0 ? "\n " : " ");
485
486 printk("\nunmasked:\n ");
487 for(i = ARRAY_SIZE(sh->evtchn_mask)-1; i >= 0; i--)
488 printk("%08lx%s", sh->evtchn_pending[i] & ~sh->evtchn_mask[i],
489 i % 8 == 0 ? "\n " : " ");
490
491 printk("\npending list:\n");
492 for(i = 0; i < NR_EVENT_CHANNELS; i++) {
493 if (sync_test_bit(i, sh->evtchn_pending)) {
494 printk(" %d: event %d -> irq %d\n",
495 cpu_evtchn[i], i,
496 evtchn_to_irq[i]);
497 }
498 }
499
500 spin_unlock_irqrestore(&debug_lock, flags);
501
502 return IRQ_HANDLED;
503}
504
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -0700505
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700506/*
507 * Search the CPUs pending events bitmasks. For each one found, map
508 * the event number to an irq, and feed it into do_IRQ() for
509 * handling.
510 *
511 * Xen uses a two-level bitmap to speed searching. The first level is
512 * a bitset of words which contain pending event bits. The second
513 * level is a bitset of pending events themselves.
514 */
Harvey Harrison75604d72008-01-30 13:31:17 +0100515void xen_evtchn_do_upcall(struct pt_regs *regs)
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700516{
517 int cpu = get_cpu();
518 struct shared_info *s = HYPERVISOR_shared_info;
519 struct vcpu_info *vcpu_info = __get_cpu_var(xen_vcpu);
Jeremy Fitzhardinge229664b2008-03-17 16:37:20 -0700520 static DEFINE_PER_CPU(unsigned, nesting_count);
521 unsigned count;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700522
Jeremy Fitzhardinge229664b2008-03-17 16:37:20 -0700523 do {
524 unsigned long pending_words;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700525
Jeremy Fitzhardinge229664b2008-03-17 16:37:20 -0700526 vcpu_info->evtchn_upcall_pending = 0;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700527
Jeremy Fitzhardinge229664b2008-03-17 16:37:20 -0700528 if (__get_cpu_var(nesting_count)++)
529 goto out;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700530
Isaku Yamahatae849c3e2008-04-02 10:53:56 -0700531#ifndef CONFIG_X86 /* No need for a barrier -- XCHG is a barrier on x86. */
532 /* Clear master flag /before/ clearing selector flag. */
Isaku Yamahata6673cf62008-06-16 14:58:13 -0700533 wmb();
Isaku Yamahatae849c3e2008-04-02 10:53:56 -0700534#endif
Jeremy Fitzhardinge229664b2008-03-17 16:37:20 -0700535 pending_words = xchg(&vcpu_info->evtchn_pending_sel, 0);
536 while (pending_words != 0) {
537 unsigned long pending_bits;
538 int word_idx = __ffs(pending_words);
539 pending_words &= ~(1UL << word_idx);
540
541 while ((pending_bits = active_evtchns(cpu, s, word_idx)) != 0) {
542 int bit_idx = __ffs(pending_bits);
543 int port = (word_idx * BITS_PER_LONG) + bit_idx;
544 int irq = evtchn_to_irq[port];
545
Isaku Yamahatae849c3e2008-04-02 10:53:56 -0700546 if (irq != -1)
547 xen_do_IRQ(irq, regs);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700548 }
549 }
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700550
Jeremy Fitzhardinge229664b2008-03-17 16:37:20 -0700551 BUG_ON(!irqs_disabled());
552
553 count = __get_cpu_var(nesting_count);
554 __get_cpu_var(nesting_count) = 0;
555 } while(count != 1);
556
557out:
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700558 put_cpu();
559}
560
Jeremy Fitzhardingeeb1e3052008-05-26 23:31:23 +0100561/* Rebind a new event channel to an existing irq. */
562void rebind_evtchn_irq(int evtchn, int irq)
563{
564 /* Make sure the irq is masked, since the new event channel
565 will also be masked. */
566 disable_irq(irq);
567
568 spin_lock(&irq_mapping_update_lock);
569
570 /* After resume the irq<->evtchn mappings are all cleared out */
571 BUG_ON(evtchn_to_irq[evtchn] != -1);
572 /* Expect irq to have been bound before,
573 so the bindcount should be non-0 */
574 BUG_ON(irq_bindcount[irq] == 0);
575
576 evtchn_to_irq[evtchn] = irq;
577 irq_info[irq] = mk_irq_info(IRQT_EVTCHN, 0, evtchn);
578
579 spin_unlock(&irq_mapping_update_lock);
580
581 /* new event channels are always bound to cpu 0 */
582 irq_set_affinity(irq, cpumask_of_cpu(0));
583
584 /* Unmask the event channel. */
585 enable_irq(irq);
586}
587
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700588/* Rebind an evtchn so that it gets delivered to a specific cpu */
589static void rebind_irq_to_cpu(unsigned irq, unsigned tcpu)
590{
591 struct evtchn_bind_vcpu bind_vcpu;
592 int evtchn = evtchn_from_irq(irq);
593
594 if (!VALID_EVTCHN(evtchn))
595 return;
596
597 /* Send future instances of this interrupt to other vcpu. */
598 bind_vcpu.port = evtchn;
599 bind_vcpu.vcpu = tcpu;
600
601 /*
602 * If this fails, it usually just indicates that we're dealing with a
603 * virq or IPI channel, which don't actually need to be rebound. Ignore
604 * it, but don't do the xenlinux-level rebind in that case.
605 */
606 if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_vcpu, &bind_vcpu) >= 0)
607 bind_evtchn_to_cpu(evtchn, tcpu);
608}
609
610
611static void set_affinity_irq(unsigned irq, cpumask_t dest)
612{
613 unsigned tcpu = first_cpu(dest);
614 rebind_irq_to_cpu(irq, tcpu);
615}
616
Isaku Yamahata642e0c82008-04-02 10:53:57 -0700617int resend_irq_on_evtchn(unsigned int irq)
618{
619 int masked, evtchn = evtchn_from_irq(irq);
620 struct shared_info *s = HYPERVISOR_shared_info;
621
622 if (!VALID_EVTCHN(evtchn))
623 return 1;
624
625 masked = sync_test_and_set_bit(evtchn, s->evtchn_mask);
626 sync_set_bit(evtchn, s->evtchn_pending);
627 if (!masked)
628 unmask_evtchn(evtchn);
629
630 return 1;
631}
632
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700633static void enable_dynirq(unsigned int irq)
634{
635 int evtchn = evtchn_from_irq(irq);
636
637 if (VALID_EVTCHN(evtchn))
638 unmask_evtchn(evtchn);
639}
640
641static void disable_dynirq(unsigned int irq)
642{
643 int evtchn = evtchn_from_irq(irq);
644
645 if (VALID_EVTCHN(evtchn))
646 mask_evtchn(evtchn);
647}
648
649static void ack_dynirq(unsigned int irq)
650{
651 int evtchn = evtchn_from_irq(irq);
652
653 move_native_irq(irq);
654
655 if (VALID_EVTCHN(evtchn))
656 clear_evtchn(evtchn);
657}
658
659static int retrigger_dynirq(unsigned int irq)
660{
661 int evtchn = evtchn_from_irq(irq);
Jeremy Fitzhardingeee8fa1c2008-03-17 16:37:19 -0700662 struct shared_info *sh = HYPERVISOR_shared_info;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700663 int ret = 0;
664
665 if (VALID_EVTCHN(evtchn)) {
Jeremy Fitzhardingeee8fa1c2008-03-17 16:37:19 -0700666 int masked;
667
668 masked = sync_test_and_set_bit(evtchn, sh->evtchn_mask);
669 sync_set_bit(evtchn, sh->evtchn_pending);
670 if (!masked)
671 unmask_evtchn(evtchn);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700672 ret = 1;
673 }
674
675 return ret;
676}
677
Jeremy Fitzhardinge0e913982008-05-26 23:31:27 +0100678static void restore_cpu_virqs(unsigned int cpu)
679{
680 struct evtchn_bind_virq bind_virq;
681 int virq, irq, evtchn;
682
683 for (virq = 0; virq < NR_VIRQS; virq++) {
684 if ((irq = per_cpu(virq_to_irq, cpu)[virq]) == -1)
685 continue;
686
687 BUG_ON(irq_info[irq].type != IRQT_VIRQ);
688 BUG_ON(irq_info[irq].index != virq);
689
690 /* Get a new binding from Xen. */
691 bind_virq.virq = virq;
692 bind_virq.vcpu = cpu;
693 if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_virq,
694 &bind_virq) != 0)
695 BUG();
696 evtchn = bind_virq.port;
697
698 /* Record the new mapping. */
699 evtchn_to_irq[evtchn] = irq;
700 irq_info[irq] = mk_irq_info(IRQT_VIRQ, virq, evtchn);
701 bind_evtchn_to_cpu(evtchn, cpu);
702
703 /* Ready for use. */
704 unmask_evtchn(evtchn);
705 }
706}
707
708static void restore_cpu_ipis(unsigned int cpu)
709{
710 struct evtchn_bind_ipi bind_ipi;
711 int ipi, irq, evtchn;
712
713 for (ipi = 0; ipi < XEN_NR_IPIS; ipi++) {
714 if ((irq = per_cpu(ipi_to_irq, cpu)[ipi]) == -1)
715 continue;
716
717 BUG_ON(irq_info[irq].type != IRQT_IPI);
718 BUG_ON(irq_info[irq].index != ipi);
719
720 /* Get a new binding from Xen. */
721 bind_ipi.vcpu = cpu;
722 if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_ipi,
723 &bind_ipi) != 0)
724 BUG();
725 evtchn = bind_ipi.port;
726
727 /* Record the new mapping. */
728 evtchn_to_irq[evtchn] = irq;
729 irq_info[irq] = mk_irq_info(IRQT_IPI, ipi, evtchn);
730 bind_evtchn_to_cpu(evtchn, cpu);
731
732 /* Ready for use. */
733 unmask_evtchn(evtchn);
734
735 }
736}
737
Jeremy Fitzhardinge2d9e1e22008-07-07 12:07:53 -0700738/* Clear an irq's pending state, in preparation for polling on it */
739void xen_clear_irq_pending(int irq)
740{
741 int evtchn = evtchn_from_irq(irq);
742
743 if (VALID_EVTCHN(evtchn))
744 clear_evtchn(evtchn);
745}
746
Jeremy Fitzhardinge168d2f42008-08-20 17:02:18 -0700747void xen_set_irq_pending(int irq)
748{
749 int evtchn = evtchn_from_irq(irq);
750
751 if (VALID_EVTCHN(evtchn))
752 set_evtchn(evtchn);
753}
754
755bool xen_test_irq_pending(int irq)
756{
757 int evtchn = evtchn_from_irq(irq);
758 bool ret = false;
759
760 if (VALID_EVTCHN(evtchn))
761 ret = test_evtchn(evtchn);
762
763 return ret;
764}
765
Jeremy Fitzhardinge2d9e1e22008-07-07 12:07:53 -0700766/* Poll waiting for an irq to become pending. In the usual case, the
767 irq will be disabled so it won't deliver an interrupt. */
768void xen_poll_irq(int irq)
769{
770 evtchn_port_t evtchn = evtchn_from_irq(irq);
771
772 if (VALID_EVTCHN(evtchn)) {
773 struct sched_poll poll;
774
775 poll.nr_ports = 1;
776 poll.timeout = 0;
Isaku Yamahataff3c5362008-10-14 17:50:44 -0700777 set_xen_guest_handle(poll.ports, &evtchn);
Jeremy Fitzhardinge2d9e1e22008-07-07 12:07:53 -0700778
779 if (HYPERVISOR_sched_op(SCHEDOP_poll, &poll) != 0)
780 BUG();
781 }
782}
783
Jeremy Fitzhardinge0e913982008-05-26 23:31:27 +0100784void xen_irq_resume(void)
785{
786 unsigned int cpu, irq, evtchn;
787
788 init_evtchn_cpu_bindings();
789
790 /* New event-channel space is not 'live' yet. */
791 for (evtchn = 0; evtchn < NR_EVENT_CHANNELS; evtchn++)
792 mask_evtchn(evtchn);
793
794 /* No IRQ <-> event-channel mappings. */
Thomas Gleixner10e58082008-10-16 14:19:04 +0200795 for_each_irq_nr(irq)
Jeremy Fitzhardinge0e913982008-05-26 23:31:27 +0100796 irq_info[irq].evtchn = 0; /* zap event-channel binding */
797
798 for (evtchn = 0; evtchn < NR_EVENT_CHANNELS; evtchn++)
799 evtchn_to_irq[evtchn] = -1;
800
801 for_each_possible_cpu(cpu) {
802 restore_cpu_virqs(cpu);
803 restore_cpu_ipis(cpu);
804 }
805}
806
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700807static struct irq_chip xen_dynamic_chip __read_mostly = {
808 .name = "xen-dyn",
809 .mask = disable_dynirq,
810 .unmask = enable_dynirq,
811 .ack = ack_dynirq,
812 .set_affinity = set_affinity_irq,
813 .retrigger = retrigger_dynirq,
814};
815
816void __init xen_init_IRQ(void)
817{
818 int i;
819
820 init_evtchn_cpu_bindings();
821
822 /* No event channels are 'live' right now. */
823 for (i = 0; i < NR_EVENT_CHANNELS; i++)
824 mask_evtchn(i);
825
826 /* Dynamic IRQ space is currently unbound. Zero the refcnts. */
Thomas Gleixner10e58082008-10-16 14:19:04 +0200827 for_each_irq_nr(i)
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700828 irq_bindcount[i] = 0;
829
830 irq_ctx_init(smp_processor_id());
831}