blob: 849500e4e14d14af7dd29fe7737d77d62f87a36d [file] [log] [blame]
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001/*
2 * Xen event channels
3 *
4 * Xen models interrupts with abstract event channels. Because each
5 * domain gets 1024 event channels, but NR_IRQ is not that large, we
6 * must dynamically map irqs<->event channels. The event channels
7 * interface with the rest of the kernel by defining a xen interrupt
Lucas De Marchi25985ed2011-03-30 22:57:33 -03008 * chip. When an event is received, it is mapped to an irq and sent
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07009 * through the normal interrupt processing path.
10 *
11 * There are four kinds of events which can be mapped to an event
12 * channel:
13 *
14 * 1. Inter-domain notifications. This includes all the virtual
15 * device events, since they're driven by front-ends in another domain
16 * (typically dom0).
17 * 2. VIRQs, typically used for timers. These are per-cpu events.
18 * 3. IPIs.
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -040019 * 4. PIRQs - Hardware interrupts.
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -070020 *
21 * Jeremy Fitzhardinge <jeremy@xensource.com>, XenSource Inc, 2007
22 */
23
Joe Perches283c0972013-06-28 03:21:41 -070024#define pr_fmt(fmt) "xen:" KBUILD_MODNAME ": " fmt
25
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -070026#include <linux/linkage.h>
27#include <linux/interrupt.h>
28#include <linux/irq.h>
29#include <linux/module.h>
30#include <linux/string.h>
Christophe Saout28e08862009-01-11 11:46:23 -080031#include <linux/bootmem.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090032#include <linux/slab.h>
Jeremy Fitzhardingeb21ddbf2010-06-07 16:28:49 -040033#include <linux/irqnr.h>
Qing Hef731e3ef2010-10-11 15:30:09 +010034#include <linux/pci.h>
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -070035
Stefano Stabellini0ec53ec2012-09-14 13:37:32 +000036#ifdef CONFIG_X86
Sheng Yang38e20b02010-05-14 12:40:51 +010037#include <asm/desc.h>
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -070038#include <asm/ptrace.h>
39#include <asm/irq.h>
Jeremy Fitzhardinge792dc4f2009-02-06 14:09:43 -080040#include <asm/idle.h>
Konrad Rzeszutek Wilk0794bfc2010-10-18 10:41:08 -040041#include <asm/io_apic.h>
Stefano Stabellini42a1de52010-06-24 16:42:04 +010042#include <asm/xen/pci.h>
Stefano Stabellini0ec53ec2012-09-14 13:37:32 +000043#endif
44#include <asm/sync_bitops.h>
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -070045#include <asm/xen/hypercall.h>
Adrian Bunk8d1b8752007-07-20 00:31:44 -070046#include <asm/xen/hypervisor.h>
Julien Gralla001c9d2015-05-05 16:37:30 +010047#include <xen/page.h>
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -070048
Sheng Yang38e20b02010-05-14 12:40:51 +010049#include <xen/xen.h>
50#include <xen/hvm.h>
Isaku Yamahatae04d0d02008-04-02 10:53:55 -070051#include <xen/xen-ops.h>
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -070052#include <xen/events.h>
53#include <xen/interface/xen.h>
54#include <xen/interface/event_channel.h>
Sheng Yang38e20b02010-05-14 12:40:51 +010055#include <xen/interface/hvm/hvm_op.h>
56#include <xen/interface/hvm/params.h>
Stefano Stabellini0ec53ec2012-09-14 13:37:32 +000057#include <xen/interface/physdev.h>
58#include <xen/interface/sched.h>
Konrad Rzeszutek Wilk6efa20e2013-07-19 11:51:31 -040059#include <xen/interface/vcpu.h>
Stefano Stabellini0ec53ec2012-09-14 13:37:32 +000060#include <asm/hw_irq.h>
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -070061
David Vrabel9a489f42013-03-13 15:29:25 +000062#include "events_internal.h"
63
David Vrabelab9a1cc2013-03-14 12:49:19 +000064const struct evtchn_ops *evtchn_ops;
65
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -070066/*
67 * This lock protects updates to the following mapping and reference-count
68 * arrays. The lock does not need to be acquired to read the mapping tables.
69 */
Konrad Rzeszutek Wilk77365942011-09-14 05:10:00 -040070static DEFINE_MUTEX(irq_mapping_update_lock);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -070071
Ian Campbell6cb65372011-03-10 16:08:11 +000072static LIST_HEAD(xen_irq_list_head);
73
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -070074/* IRQ <-> VIRQ mapping. */
Tejun Heo204fba42009-06-24 15:13:45 +090075static DEFINE_PER_CPU(int [NR_VIRQS], virq_to_irq) = {[0 ... NR_VIRQS-1] = -1};
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -070076
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -070077/* IRQ <-> IPI mapping */
Tejun Heo204fba42009-06-24 15:13:45 +090078static DEFINE_PER_CPU(int [XEN_NR_IPIS], ipi_to_irq) = {[0 ... XEN_NR_IPIS-1] = -1};
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -070079
David Vrabeld0b075f2013-10-17 15:23:15 +010080int **evtchn_to_irq;
Ian Campbellbf86ad82012-10-17 09:39:12 +010081#ifdef CONFIG_X86
Stefano Stabellini9846ff12012-01-30 16:21:48 +000082static unsigned long *pirq_eoi_map;
Ian Campbellbf86ad82012-10-17 09:39:12 +010083#endif
Stefano Stabellini9846ff12012-01-30 16:21:48 +000084static bool (*pirq_needs_eoi)(unsigned irq);
Jeremy Fitzhardinge3b32f572009-08-13 12:50:37 -070085
David Vrabeld0b075f2013-10-17 15:23:15 +010086#define EVTCHN_ROW(e) (e / (PAGE_SIZE/sizeof(**evtchn_to_irq)))
87#define EVTCHN_COL(e) (e % (PAGE_SIZE/sizeof(**evtchn_to_irq)))
88#define EVTCHN_PER_ROW (PAGE_SIZE / sizeof(**evtchn_to_irq))
89
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -070090/* Xen will never allocate port zero for any purpose. */
91#define VALID_EVTCHN(chn) ((chn) != 0)
92
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -070093static struct irq_chip xen_dynamic_chip;
Jeremy Fitzhardingeaaca4962010-08-20 18:57:53 -070094static struct irq_chip xen_percpu_chip;
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -040095static struct irq_chip xen_pirq_chip;
Stefano Stabellini7e186bd2011-05-06 12:27:50 +010096static void enable_dynirq(struct irq_data *data);
97static void disable_dynirq(struct irq_data *data);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -070098
David Vrabeld0b075f2013-10-17 15:23:15 +010099static void clear_evtchn_to_irq_row(unsigned row)
100{
101 unsigned col;
102
103 for (col = 0; col < EVTCHN_PER_ROW; col++)
104 evtchn_to_irq[row][col] = -1;
105}
106
107static void clear_evtchn_to_irq_all(void)
108{
109 unsigned row;
110
111 for (row = 0; row < EVTCHN_ROW(xen_evtchn_max_channels()); row++) {
112 if (evtchn_to_irq[row] == NULL)
113 continue;
114 clear_evtchn_to_irq_row(row);
115 }
116}
117
118static int set_evtchn_to_irq(unsigned evtchn, unsigned irq)
119{
120 unsigned row;
121 unsigned col;
122
123 if (evtchn >= xen_evtchn_max_channels())
124 return -EINVAL;
125
126 row = EVTCHN_ROW(evtchn);
127 col = EVTCHN_COL(evtchn);
128
129 if (evtchn_to_irq[row] == NULL) {
130 /* Unallocated irq entries return -1 anyway */
131 if (irq == -1)
132 return 0;
133
134 evtchn_to_irq[row] = (int *)get_zeroed_page(GFP_KERNEL);
135 if (evtchn_to_irq[row] == NULL)
136 return -ENOMEM;
137
138 clear_evtchn_to_irq_row(row);
139 }
140
141 evtchn_to_irq[EVTCHN_ROW(evtchn)][EVTCHN_COL(evtchn)] = irq;
142 return 0;
143}
144
145int get_evtchn_to_irq(unsigned evtchn)
146{
147 if (evtchn >= xen_evtchn_max_channels())
148 return -1;
149 if (evtchn_to_irq[EVTCHN_ROW(evtchn)] == NULL)
150 return -1;
151 return evtchn_to_irq[EVTCHN_ROW(evtchn)][EVTCHN_COL(evtchn)];
152}
153
Ian Campbell9158c352011-03-10 16:08:09 +0000154/* Get info for IRQ */
David Vrabel9a489f42013-03-13 15:29:25 +0000155struct irq_info *info_for_irq(unsigned irq)
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700156{
Thomas Gleixnerc442b802011-03-25 10:58:06 +0100157 return irq_get_handler_data(irq);
Jeremy Fitzhardingeced40d02009-02-06 14:09:44 -0800158}
159
Ian Campbell9158c352011-03-10 16:08:09 +0000160/* Constructors for packed IRQ information. */
David Vrabel96d4c5882013-03-18 15:50:17 +0000161static int xen_irq_info_common_setup(struct irq_info *info,
Ian Campbell3d4cfa32011-03-10 16:08:10 +0000162 unsigned irq,
Ian Campbell9158c352011-03-10 16:08:09 +0000163 enum xen_irq_type type,
David Vrabeld0b075f2013-10-17 15:23:15 +0100164 unsigned evtchn,
Ian Campbell9158c352011-03-10 16:08:09 +0000165 unsigned short cpu)
Jeremy Fitzhardingeced40d02009-02-06 14:09:44 -0800166{
David Vrabeld0b075f2013-10-17 15:23:15 +0100167 int ret;
Ian Campbell9158c352011-03-10 16:08:09 +0000168
169 BUG_ON(info->type != IRQT_UNBOUND && info->type != type);
170
171 info->type = type;
Ian Campbell6cb65372011-03-10 16:08:11 +0000172 info->irq = irq;
Ian Campbell9158c352011-03-10 16:08:09 +0000173 info->evtchn = evtchn;
174 info->cpu = cpu;
Ian Campbell3d4cfa32011-03-10 16:08:10 +0000175
David Vrabeld0b075f2013-10-17 15:23:15 +0100176 ret = set_evtchn_to_irq(evtchn, irq);
177 if (ret < 0)
178 return ret;
Julien Grall934f5852013-04-30 18:29:13 +0100179
180 irq_clear_status_flags(irq, IRQ_NOREQUEST|IRQ_NOAUTOEN);
David Vrabel96d4c5882013-03-18 15:50:17 +0000181
David Vrabel08385872013-03-18 16:54:57 +0000182 return xen_evtchn_port_setup(info);
Jeremy Fitzhardingeced40d02009-02-06 14:09:44 -0800183}
184
David Vrabel96d4c5882013-03-18 15:50:17 +0000185static int xen_irq_info_evtchn_setup(unsigned irq,
David Vrabeld0b075f2013-10-17 15:23:15 +0100186 unsigned evtchn)
Jeremy Fitzhardingeced40d02009-02-06 14:09:44 -0800187{
Ian Campbell9158c352011-03-10 16:08:09 +0000188 struct irq_info *info = info_for_irq(irq);
189
David Vrabel96d4c5882013-03-18 15:50:17 +0000190 return xen_irq_info_common_setup(info, irq, IRQT_EVTCHN, evtchn, 0);
Jeremy Fitzhardingeced40d02009-02-06 14:09:44 -0800191}
192
David Vrabel96d4c5882013-03-18 15:50:17 +0000193static int xen_irq_info_ipi_setup(unsigned cpu,
Ian Campbell3d4cfa32011-03-10 16:08:10 +0000194 unsigned irq,
David Vrabeld0b075f2013-10-17 15:23:15 +0100195 unsigned evtchn,
Ian Campbell9158c352011-03-10 16:08:09 +0000196 enum ipi_vector ipi)
Jeremy Fitzhardingeced40d02009-02-06 14:09:44 -0800197{
Ian Campbell9158c352011-03-10 16:08:09 +0000198 struct irq_info *info = info_for_irq(irq);
199
Ian Campbell9158c352011-03-10 16:08:09 +0000200 info->u.ipi = ipi;
Ian Campbell3d4cfa32011-03-10 16:08:10 +0000201
202 per_cpu(ipi_to_irq, cpu)[ipi] = irq;
David Vrabel96d4c5882013-03-18 15:50:17 +0000203
204 return xen_irq_info_common_setup(info, irq, IRQT_IPI, evtchn, 0);
Jeremy Fitzhardingeced40d02009-02-06 14:09:44 -0800205}
206
David Vrabel96d4c5882013-03-18 15:50:17 +0000207static int xen_irq_info_virq_setup(unsigned cpu,
Ian Campbell3d4cfa32011-03-10 16:08:10 +0000208 unsigned irq,
David Vrabeld0b075f2013-10-17 15:23:15 +0100209 unsigned evtchn,
210 unsigned virq)
Jeremy Fitzhardingeced40d02009-02-06 14:09:44 -0800211{
Ian Campbell9158c352011-03-10 16:08:09 +0000212 struct irq_info *info = info_for_irq(irq);
213
Ian Campbell9158c352011-03-10 16:08:09 +0000214 info->u.virq = virq;
Ian Campbell3d4cfa32011-03-10 16:08:10 +0000215
216 per_cpu(virq_to_irq, cpu)[virq] = irq;
David Vrabel96d4c5882013-03-18 15:50:17 +0000217
218 return xen_irq_info_common_setup(info, irq, IRQT_VIRQ, evtchn, 0);
Ian Campbell9158c352011-03-10 16:08:09 +0000219}
220
David Vrabel96d4c5882013-03-18 15:50:17 +0000221static int xen_irq_info_pirq_setup(unsigned irq,
David Vrabeld0b075f2013-10-17 15:23:15 +0100222 unsigned evtchn,
223 unsigned pirq,
224 unsigned gsi,
Konrad Rzeszutek Wilkbeafbdc2011-04-14 11:17:36 -0400225 uint16_t domid,
Ian Campbell9158c352011-03-10 16:08:09 +0000226 unsigned char flags)
227{
228 struct irq_info *info = info_for_irq(irq);
229
Ian Campbell9158c352011-03-10 16:08:09 +0000230 info->u.pirq.pirq = pirq;
231 info->u.pirq.gsi = gsi;
Konrad Rzeszutek Wilkbeafbdc2011-04-14 11:17:36 -0400232 info->u.pirq.domid = domid;
Ian Campbell9158c352011-03-10 16:08:09 +0000233 info->u.pirq.flags = flags;
David Vrabel96d4c5882013-03-18 15:50:17 +0000234
235 return xen_irq_info_common_setup(info, irq, IRQT_PIRQ, evtchn, 0);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700236}
237
David Vrabeld0b075f2013-10-17 15:23:15 +0100238static void xen_irq_info_cleanup(struct irq_info *info)
239{
240 set_evtchn_to_irq(info->evtchn, -1);
241 info->evtchn = 0;
242}
243
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700244/*
245 * Accessors for packed IRQ information.
246 */
David Vrabel9a489f42013-03-13 15:29:25 +0000247unsigned int evtchn_from_irq(unsigned irq)
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700248{
Andrey Utkin474b8fe2014-07-17 19:25:50 +0300249 if (unlikely(WARN(irq >= nr_irqs, "Invalid irq %d!\n", irq)))
Joe Jin110e7c72011-01-07 14:50:12 +0800250 return 0;
251
Jeremy Fitzhardingeced40d02009-02-06 14:09:44 -0800252 return info_for_irq(irq)->evtchn;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700253}
254
Ian Campbelld4c04532009-02-06 19:20:31 -0800255unsigned irq_from_evtchn(unsigned int evtchn)
256{
David Vrabeld0b075f2013-10-17 15:23:15 +0100257 return get_evtchn_to_irq(evtchn);
Ian Campbelld4c04532009-02-06 19:20:31 -0800258}
259EXPORT_SYMBOL_GPL(irq_from_evtchn);
260
David Vrabel9a489f42013-03-13 15:29:25 +0000261int irq_from_virq(unsigned int cpu, unsigned int virq)
262{
263 return per_cpu(virq_to_irq, cpu)[virq];
264}
265
Jeremy Fitzhardingeced40d02009-02-06 14:09:44 -0800266static enum ipi_vector ipi_from_irq(unsigned irq)
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700267{
Jeremy Fitzhardingeced40d02009-02-06 14:09:44 -0800268 struct irq_info *info = info_for_irq(irq);
269
270 BUG_ON(info == NULL);
271 BUG_ON(info->type != IRQT_IPI);
272
273 return info->u.ipi;
274}
275
276static unsigned virq_from_irq(unsigned irq)
277{
278 struct irq_info *info = info_for_irq(irq);
279
280 BUG_ON(info == NULL);
281 BUG_ON(info->type != IRQT_VIRQ);
282
283 return info->u.virq;
284}
285
Stefano Stabellini7a043f12010-07-01 17:08:14 +0100286static unsigned pirq_from_irq(unsigned irq)
287{
288 struct irq_info *info = info_for_irq(irq);
289
290 BUG_ON(info == NULL);
291 BUG_ON(info->type != IRQT_PIRQ);
292
293 return info->u.pirq.pirq;
294}
295
Jeremy Fitzhardingeced40d02009-02-06 14:09:44 -0800296static enum xen_irq_type type_from_irq(unsigned irq)
297{
298 return info_for_irq(irq)->type;
299}
300
David Vrabel9a489f42013-03-13 15:29:25 +0000301unsigned cpu_from_irq(unsigned irq)
Jeremy Fitzhardingeced40d02009-02-06 14:09:44 -0800302{
303 return info_for_irq(irq)->cpu;
304}
305
David Vrabel9a489f42013-03-13 15:29:25 +0000306unsigned int cpu_from_evtchn(unsigned int evtchn)
Jeremy Fitzhardingeced40d02009-02-06 14:09:44 -0800307{
David Vrabeld0b075f2013-10-17 15:23:15 +0100308 int irq = get_evtchn_to_irq(evtchn);
Jeremy Fitzhardingeced40d02009-02-06 14:09:44 -0800309 unsigned ret = 0;
310
311 if (irq != -1)
312 ret = cpu_from_irq(irq);
313
314 return ret;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700315}
316
Ian Campbellbf86ad82012-10-17 09:39:12 +0100317#ifdef CONFIG_X86
Stefano Stabellini9846ff12012-01-30 16:21:48 +0000318static bool pirq_check_eoi_map(unsigned irq)
319{
Stefano Stabellini521394e2012-04-25 16:11:38 +0100320 return test_bit(pirq_from_irq(irq), pirq_eoi_map);
Stefano Stabellini9846ff12012-01-30 16:21:48 +0000321}
Ian Campbellbf86ad82012-10-17 09:39:12 +0100322#endif
Stefano Stabellini9846ff12012-01-30 16:21:48 +0000323
324static bool pirq_needs_eoi_flag(unsigned irq)
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400325{
326 struct irq_info *info = info_for_irq(irq);
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400327 BUG_ON(info->type != IRQT_PIRQ);
328
329 return info->u.pirq.flags & PIRQ_NEEDS_EOI;
330}
331
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700332static void bind_evtchn_to_cpu(unsigned int chn, unsigned int cpu)
333{
David Vrabeld0b075f2013-10-17 15:23:15 +0100334 int irq = get_evtchn_to_irq(chn);
David Vrabel9a489f42013-03-13 15:29:25 +0000335 struct irq_info *info = info_for_irq(irq);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700336
337 BUG_ON(irq == -1);
338#ifdef CONFIG_SMP
Jiang Liuc149e4c2015-06-03 11:46:22 +0800339 cpumask_copy(irq_get_affinity_mask(irq), cpumask_of(cpu));
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700340#endif
David Vrabel9a489f42013-03-13 15:29:25 +0000341 xen_evtchn_port_bind_to_cpu(info, cpu);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700342
David Vrabel9a489f42013-03-13 15:29:25 +0000343 info->cpu = cpu;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700344}
345
David Vrabelfd210692013-09-05 18:11:38 +0100346static void xen_evtchn_mask_all(void)
347{
348 unsigned int evtchn;
349
350 for (evtchn = 0; evtchn < xen_evtchn_nr_channels(); evtchn++)
351 mask_evtchn(evtchn);
352}
353
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700354/**
355 * notify_remote_via_irq - send event to remote end of event channel via irq
356 * @irq: irq of event channel to send event to
357 *
358 * Unlike notify_remote_via_evtchn(), this is safe to use across
359 * save/restore. Notifications on a broken connection are silently
360 * dropped.
361 */
362void notify_remote_via_irq(int irq)
363{
364 int evtchn = evtchn_from_irq(irq);
365
366 if (VALID_EVTCHN(evtchn))
367 notify_remote_via_evtchn(evtchn);
368}
369EXPORT_SYMBOL_GPL(notify_remote_via_irq);
370
Ian Campbell6cb65372011-03-10 16:08:11 +0000371static void xen_irq_init(unsigned irq)
372{
373 struct irq_info *info;
Konrad Rzeszutek Wilkb5328cd2011-06-15 14:24:29 -0400374#ifdef CONFIG_SMP
Ian Campbell6cb65372011-03-10 16:08:11 +0000375 /* By default all event channels notify CPU#0. */
Jiang Liuc149e4c2015-06-03 11:46:22 +0800376 cpumask_copy(irq_get_affinity_mask(irq), cpumask_of(0));
Konrad Rzeszutek Wilk44626e42011-03-15 16:40:33 -0400377#endif
Ian Campbell6cb65372011-03-10 16:08:11 +0000378
Ian Campbellca62ce82011-03-10 16:08:12 +0000379 info = kzalloc(sizeof(*info), GFP_KERNEL);
380 if (info == NULL)
381 panic("Unable to allocate metadata for IRQ%d\n", irq);
Ian Campbell6cb65372011-03-10 16:08:11 +0000382
383 info->type = IRQT_UNBOUND;
Daniel De Graaf420eb552011-10-27 17:58:47 -0400384 info->refcnt = -1;
Ian Campbell6cb65372011-03-10 16:08:11 +0000385
Thomas Gleixnerc442b802011-03-25 10:58:06 +0100386 irq_set_handler_data(irq, info);
Ian Campbellca62ce82011-03-10 16:08:12 +0000387
Ian Campbell6cb65372011-03-10 16:08:11 +0000388 list_add_tail(&info->list, &xen_irq_list_head);
389}
390
Roger Pau Monne4892c9b2014-02-27 19:15:35 +0100391static int __must_check xen_allocate_irqs_dynamic(int nvec)
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700392{
Thomas Gleixnerd07c9f12014-05-07 15:44:10 +0000393 int i, irq = irq_alloc_descs(-1, 0, nvec, -1);
Ian Campbell89911502011-03-03 11:57:44 -0500394
Roger Pau Monne4892c9b2014-02-27 19:15:35 +0100395 if (irq >= 0) {
396 for (i = 0; i < nvec; i++)
397 xen_irq_init(irq + i);
398 }
Jeremy Fitzhardingeced40d02009-02-06 14:09:44 -0800399
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700400 return irq;
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400401}
402
Roger Pau Monne4892c9b2014-02-27 19:15:35 +0100403static inline int __must_check xen_allocate_irq_dynamic(void)
404{
405
406 return xen_allocate_irqs_dynamic(1);
407}
408
Ian Campbell7bee9762011-03-10 16:08:15 +0000409static int __must_check xen_allocate_irq_gsi(unsigned gsi)
Ian Campbellc9df1ce2011-01-11 17:20:15 +0000410{
411 int irq;
412
Ian Campbell89911502011-03-03 11:57:44 -0500413 /*
414 * A PV guest has no concept of a GSI (since it has no ACPI
415 * nor access to/knowledge of the physical APICs). Therefore
416 * all IRQs are dynamically allocated from the entire IRQ
417 * space.
418 */
419 if (xen_pv_domain() && !xen_initial_domain())
Ian Campbellc9df1ce2011-01-11 17:20:15 +0000420 return xen_allocate_irq_dynamic();
421
422 /* Legacy IRQ descriptors are already allocated by the arch. */
423 if (gsi < NR_IRQS_LEGACY)
Ian Campbell6cb65372011-03-10 16:08:11 +0000424 irq = gsi;
425 else
426 irq = irq_alloc_desc_at(gsi, -1);
Ian Campbellc9df1ce2011-01-11 17:20:15 +0000427
Ian Campbell6cb65372011-03-10 16:08:11 +0000428 xen_irq_init(irq);
Ian Campbellc9df1ce2011-01-11 17:20:15 +0000429
430 return irq;
431}
432
433static void xen_free_irq(unsigned irq)
434{
Thomas Gleixnerc442b802011-03-25 10:58:06 +0100435 struct irq_info *info = irq_get_handler_data(irq);
Ian Campbell6cb65372011-03-10 16:08:11 +0000436
Konrad Rzeszutek Wilk94032c52013-04-16 10:55:18 -0400437 if (WARN_ON(!info))
438 return;
439
Ian Campbell6cb65372011-03-10 16:08:11 +0000440 list_del(&info->list);
Ian Campbell9158c352011-03-10 16:08:09 +0000441
Thomas Gleixnerc442b802011-03-25 10:58:06 +0100442 irq_set_handler_data(irq, NULL);
Ian Campbellca62ce82011-03-10 16:08:12 +0000443
Daniel De Graaf420eb552011-10-27 17:58:47 -0400444 WARN_ON(info->refcnt > 0);
445
Ian Campbellca62ce82011-03-10 16:08:12 +0000446 kfree(info);
447
Ian Campbell72146102011-02-03 09:49:35 +0000448 /* Legacy IRQ descriptors are managed by the arch. */
449 if (irq < NR_IRQS_LEGACY)
450 return;
451
Ian Campbellc9df1ce2011-01-11 17:20:15 +0000452 irq_free_desc(irq);
453}
454
David Vrabeld0b075f2013-10-17 15:23:15 +0100455static void xen_evtchn_close(unsigned int port)
456{
457 struct evtchn_close close;
458
459 close.port = port;
460 if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close) != 0)
461 BUG();
David Vrabeld0b075f2013-10-17 15:23:15 +0100462}
463
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400464static void pirq_query_unmask(int irq)
465{
466 struct physdev_irq_status_query irq_status;
467 struct irq_info *info = info_for_irq(irq);
468
469 BUG_ON(info->type != IRQT_PIRQ);
470
Stefano Stabellini7a043f12010-07-01 17:08:14 +0100471 irq_status.irq = pirq_from_irq(irq);
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400472 if (HYPERVISOR_physdev_op(PHYSDEVOP_irq_status_query, &irq_status))
473 irq_status.flags = 0;
474
475 info->u.pirq.flags &= ~PIRQ_NEEDS_EOI;
476 if (irq_status.flags & XENIRQSTAT_needs_eoi)
477 info->u.pirq.flags |= PIRQ_NEEDS_EOI;
478}
479
Stefano Stabellini7e186bd2011-05-06 12:27:50 +0100480static void eoi_pirq(struct irq_data *data)
481{
482 int evtchn = evtchn_from_irq(data->irq);
483 struct physdev_eoi eoi = { .irq = pirq_from_irq(data->irq) };
484 int rc = 0;
485
486 irq_move_irq(data);
487
488 if (VALID_EVTCHN(evtchn))
489 clear_evtchn(evtchn);
490
491 if (pirq_needs_eoi(data->irq)) {
492 rc = HYPERVISOR_physdev_op(PHYSDEVOP_eoi, &eoi);
493 WARN_ON(rc);
494 }
495}
496
497static void mask_ack_pirq(struct irq_data *data)
498{
499 disable_dynirq(data);
500 eoi_pirq(data);
501}
502
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +0000503static unsigned int __startup_pirq(unsigned int irq)
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400504{
505 struct evtchn_bind_pirq bind_pirq;
506 struct irq_info *info = info_for_irq(irq);
507 int evtchn = evtchn_from_irq(irq);
Konrad Rzeszutek Wilk15ebbb82010-10-04 13:43:27 -0400508 int rc;
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400509
510 BUG_ON(info->type != IRQT_PIRQ);
511
512 if (VALID_EVTCHN(evtchn))
513 goto out;
514
Stefano Stabellini7a043f12010-07-01 17:08:14 +0100515 bind_pirq.pirq = pirq_from_irq(irq);
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400516 /* NB. We are happy to share unless we are probing. */
Konrad Rzeszutek Wilk15ebbb82010-10-04 13:43:27 -0400517 bind_pirq.flags = info->u.pirq.flags & PIRQ_SHAREABLE ?
518 BIND_PIRQ__WILL_SHARE : 0;
519 rc = HYPERVISOR_event_channel_op(EVTCHNOP_bind_pirq, &bind_pirq);
520 if (rc != 0) {
Thomas Gleixner02893af2014-02-23 21:40:20 +0000521 pr_warn("Failed to obtain physical IRQ %d\n", irq);
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400522 return 0;
523 }
524 evtchn = bind_pirq.port;
525
526 pirq_query_unmask(irq);
527
David Vrabeld0b075f2013-10-17 15:23:15 +0100528 rc = set_evtchn_to_irq(evtchn, irq);
Juergen Gross85e40b02015-02-26 06:52:05 +0100529 if (rc)
530 goto err;
531
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400532 info->evtchn = evtchn;
Boris Ostrovsky16e6bd52015-04-29 17:10:15 -0400533 bind_evtchn_to_cpu(evtchn, 0);
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400534
Juergen Gross85e40b02015-02-26 06:52:05 +0100535 rc = xen_evtchn_port_setup(info);
536 if (rc)
537 goto err;
538
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400539out:
540 unmask_evtchn(evtchn);
Stefano Stabellini7e186bd2011-05-06 12:27:50 +0100541 eoi_pirq(irq_get_irq_data(irq));
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400542
543 return 0;
Juergen Gross85e40b02015-02-26 06:52:05 +0100544
545err:
546 pr_err("irq%d: Failed to set port to irq mapping (%d)\n", irq, rc);
547 xen_evtchn_close(evtchn);
548 return 0;
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400549}
550
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +0000551static unsigned int startup_pirq(struct irq_data *data)
552{
553 return __startup_pirq(data->irq);
554}
555
556static void shutdown_pirq(struct irq_data *data)
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400557{
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +0000558 unsigned int irq = data->irq;
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400559 struct irq_info *info = info_for_irq(irq);
David Vrabeld0b075f2013-10-17 15:23:15 +0100560 unsigned evtchn = evtchn_from_irq(irq);
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400561
562 BUG_ON(info->type != IRQT_PIRQ);
563
564 if (!VALID_EVTCHN(evtchn))
565 return;
566
567 mask_evtchn(evtchn);
David Vrabeld0b075f2013-10-17 15:23:15 +0100568 xen_evtchn_close(evtchn);
569 xen_irq_info_cleanup(info);
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400570}
571
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +0000572static void enable_pirq(struct irq_data *data)
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400573{
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +0000574 startup_pirq(data);
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400575}
576
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +0000577static void disable_pirq(struct irq_data *data)
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400578{
Stefano Stabellini7e186bd2011-05-06 12:27:50 +0100579 disable_dynirq(data);
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400580}
581
Stefano Stabellini68c2c392012-05-21 16:54:10 +0100582int xen_irq_from_gsi(unsigned gsi)
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400583{
Ian Campbell6cb65372011-03-10 16:08:11 +0000584 struct irq_info *info;
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400585
Ian Campbell6cb65372011-03-10 16:08:11 +0000586 list_for_each_entry(info, &xen_irq_list_head, list) {
587 if (info->type != IRQT_PIRQ)
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400588 continue;
589
Ian Campbell6cb65372011-03-10 16:08:11 +0000590 if (info->u.pirq.gsi == gsi)
591 return info->irq;
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400592 }
593
594 return -1;
595}
Stefano Stabellini68c2c392012-05-21 16:54:10 +0100596EXPORT_SYMBOL_GPL(xen_irq_from_gsi);
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400597
David Vrabel96d4c5882013-03-18 15:50:17 +0000598static void __unbind_from_irq(unsigned int irq)
599{
David Vrabel96d4c5882013-03-18 15:50:17 +0000600 int evtchn = evtchn_from_irq(irq);
601 struct irq_info *info = irq_get_handler_data(irq);
602
603 if (info->refcnt > 0) {
604 info->refcnt--;
605 if (info->refcnt != 0)
606 return;
607 }
608
609 if (VALID_EVTCHN(evtchn)) {
David Vrabeld0b075f2013-10-17 15:23:15 +0100610 unsigned int cpu = cpu_from_irq(irq);
611
612 xen_evtchn_close(evtchn);
David Vrabel96d4c5882013-03-18 15:50:17 +0000613
614 switch (type_from_irq(irq)) {
615 case IRQT_VIRQ:
David Vrabeld0b075f2013-10-17 15:23:15 +0100616 per_cpu(virq_to_irq, cpu)[virq_from_irq(irq)] = -1;
David Vrabel96d4c5882013-03-18 15:50:17 +0000617 break;
618 case IRQT_IPI:
David Vrabeld0b075f2013-10-17 15:23:15 +0100619 per_cpu(ipi_to_irq, cpu)[ipi_from_irq(irq)] = -1;
David Vrabel96d4c5882013-03-18 15:50:17 +0000620 break;
621 default:
622 break;
623 }
624
David Vrabeld0b075f2013-10-17 15:23:15 +0100625 xen_irq_info_cleanup(info);
David Vrabel96d4c5882013-03-18 15:50:17 +0000626 }
627
628 BUG_ON(info_for_irq(irq)->type == IRQT_UNBOUND);
629
630 xen_free_irq(irq);
631}
632
Ian Campbell653378a2011-03-10 16:08:04 +0000633/*
634 * Do not make any assumptions regarding the relationship between the
635 * IRQ number returned here and the Xen pirq argument.
Stefano Stabellini7a043f12010-07-01 17:08:14 +0100636 *
637 * Note: We don't assign an event channel until the irq actually started
638 * up. Return an existing irq if we've already got one for the gsi.
Stefano Stabellinie5ac0bd2011-05-25 12:33:23 +0100639 *
640 * Shareable implies level triggered, not shareable implies edge
641 * triggered here.
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400642 */
Ian Campbellf4d06352011-03-10 16:08:07 +0000643int xen_bind_pirq_gsi_to_irq(unsigned gsi,
644 unsigned pirq, int shareable, char *name)
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400645{
Ian Campbella0e18112011-03-10 16:08:03 +0000646 int irq = -1;
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400647 struct physdev_irq irq_op;
David Vrabel96d4c5882013-03-18 15:50:17 +0000648 int ret;
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400649
Konrad Rzeszutek Wilk77365942011-09-14 05:10:00 -0400650 mutex_lock(&irq_mapping_update_lock);
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400651
Stefano Stabellini68c2c392012-05-21 16:54:10 +0100652 irq = xen_irq_from_gsi(gsi);
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400653 if (irq != -1) {
Joe Perches283c0972013-06-28 03:21:41 -0700654 pr_info("%s: returning irq %d for gsi %u\n",
655 __func__, irq, gsi);
Daniel De Graaf420eb552011-10-27 17:58:47 -0400656 goto out;
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400657 }
658
Ian Campbellc9df1ce2011-01-11 17:20:15 +0000659 irq = xen_allocate_irq_gsi(gsi);
Ian Campbell7bee9762011-03-10 16:08:15 +0000660 if (irq < 0)
661 goto out;
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400662
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400663 irq_op.irq = irq;
Alex Nixonb5401a92010-03-18 16:31:34 -0400664 irq_op.vector = 0;
665
666 /* Only the privileged domain can do this. For non-priv, the pcifront
667 * driver provides a PCI bus that does the call to do exactly
668 * this in the priv domain. */
669 if (xen_initial_domain() &&
670 HYPERVISOR_physdev_op(PHYSDEVOP_alloc_irq_vector, &irq_op)) {
Ian Campbellc9df1ce2011-01-11 17:20:15 +0000671 xen_free_irq(irq);
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400672 irq = -ENOSPC;
673 goto out;
674 }
675
David Vrabel96d4c5882013-03-18 15:50:17 +0000676 ret = xen_irq_info_pirq_setup(irq, 0, pirq, gsi, DOMID_SELF,
Ian Campbell9158c352011-03-10 16:08:09 +0000677 shareable ? PIRQ_SHAREABLE : 0);
David Vrabel96d4c5882013-03-18 15:50:17 +0000678 if (ret < 0) {
679 __unbind_from_irq(irq);
680 irq = ret;
681 goto out;
682 }
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400683
Stefano Stabellini7e186bd2011-05-06 12:27:50 +0100684 pirq_query_unmask(irq);
685 /* We try to use the handler with the appropriate semantic for the
Stefano Stabellinie5ac0bd2011-05-25 12:33:23 +0100686 * type of interrupt: if the interrupt is an edge triggered
687 * interrupt we use handle_edge_irq.
Stefano Stabellini7e186bd2011-05-06 12:27:50 +0100688 *
Stefano Stabellinie5ac0bd2011-05-25 12:33:23 +0100689 * On the other hand if the interrupt is level triggered we use
690 * handle_fasteoi_irq like the native code does for this kind of
Stefano Stabellini7e186bd2011-05-06 12:27:50 +0100691 * interrupts.
Stefano Stabellinie5ac0bd2011-05-25 12:33:23 +0100692 *
Stefano Stabellini7e186bd2011-05-06 12:27:50 +0100693 * Depending on the Xen version, pirq_needs_eoi might return true
694 * not only for level triggered interrupts but for edge triggered
695 * interrupts too. In any case Xen always honors the eoi mechanism,
696 * not injecting any more pirqs of the same kind if the first one
697 * hasn't received an eoi yet. Therefore using the fasteoi handler
698 * is the right choice either way.
699 */
Stefano Stabellinie5ac0bd2011-05-25 12:33:23 +0100700 if (shareable)
Stefano Stabellini7e186bd2011-05-06 12:27:50 +0100701 irq_set_chip_and_handler_name(irq, &xen_pirq_chip,
702 handle_fasteoi_irq, name);
703 else
704 irq_set_chip_and_handler_name(irq, &xen_pirq_chip,
705 handle_edge_irq, name);
706
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400707out:
Konrad Rzeszutek Wilk77365942011-09-14 05:10:00 -0400708 mutex_unlock(&irq_mapping_update_lock);
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -0400709
710 return irq;
711}
712
Qing Hef731e3ef2010-10-11 15:30:09 +0100713#ifdef CONFIG_PCI_MSI
Ian Campbellbf480d92011-02-18 16:43:32 +0000714int xen_allocate_pirq_msi(struct pci_dev *dev, struct msi_desc *msidesc)
Ian Campbellcbf6aa82011-01-11 17:20:14 +0000715{
Ian Campbell5cad61a2011-02-18 16:43:31 +0000716 int rc;
Ian Campbellcbf6aa82011-01-11 17:20:14 +0000717 struct physdev_get_free_pirq op_get_free_pirq;
Ian Campbell5cad61a2011-02-18 16:43:31 +0000718
Ian Campbellbf480d92011-02-18 16:43:32 +0000719 op_get_free_pirq.type = MAP_PIRQ_TYPE_MSI;
Ian Campbellcbf6aa82011-01-11 17:20:14 +0000720 rc = HYPERVISOR_physdev_op(PHYSDEVOP_get_free_pirq, &op_get_free_pirq);
Ian Campbellcbf6aa82011-01-11 17:20:14 +0000721
Ian Campbell5cad61a2011-02-18 16:43:31 +0000722 WARN_ONCE(rc == -ENOSYS,
723 "hypervisor does not support the PHYSDEVOP_get_free_pirq interface\n");
724
725 return rc ? -1 : op_get_free_pirq.pirq;
Ian Campbellcbf6aa82011-01-11 17:20:14 +0000726}
727
Ian Campbellbf480d92011-02-18 16:43:32 +0000728int xen_bind_pirq_msi_to_irq(struct pci_dev *dev, struct msi_desc *msidesc,
Roger Pau Monne4892c9b2014-02-27 19:15:35 +0100729 int pirq, int nvec, const char *name, domid_t domid)
Stefano Stabellini809f9262010-07-01 17:10:39 +0100730{
Roger Pau Monne4892c9b2014-02-27 19:15:35 +0100731 int i, irq, ret;
Ian Campbell4b41df72011-02-18 16:43:29 +0000732
Konrad Rzeszutek Wilk77365942011-09-14 05:10:00 -0400733 mutex_lock(&irq_mapping_update_lock);
Stefano Stabellini809f9262010-07-01 17:10:39 +0100734
Roger Pau Monne4892c9b2014-02-27 19:15:35 +0100735 irq = xen_allocate_irqs_dynamic(nvec);
Konrad Rzeszutek Wilke6599222011-09-29 13:26:45 -0400736 if (irq < 0)
Ian Campbellbb5d0792011-02-18 16:43:28 +0000737 goto out;
Stefano Stabellini809f9262010-07-01 17:10:39 +0100738
Roger Pau Monne4892c9b2014-02-27 19:15:35 +0100739 for (i = 0; i < nvec; i++) {
740 irq_set_chip_and_handler_name(irq + i, &xen_pirq_chip, handle_edge_irq, name);
Stefano Stabellini809f9262010-07-01 17:10:39 +0100741
Roger Pau Monne4892c9b2014-02-27 19:15:35 +0100742 ret = xen_irq_info_pirq_setup(irq + i, 0, pirq + i, 0, domid,
743 i == 0 ? 0 : PIRQ_MSI_GROUP);
744 if (ret < 0)
745 goto error_irq;
746 }
747
Linus Torvalds5f6fb452011-03-15 19:23:40 -0700748 ret = irq_set_msi_desc(irq, msidesc);
Ian Campbellbf480d92011-02-18 16:43:32 +0000749 if (ret < 0)
750 goto error_irq;
Stefano Stabellini809f9262010-07-01 17:10:39 +0100751out:
Konrad Rzeszutek Wilk77365942011-09-14 05:10:00 -0400752 mutex_unlock(&irq_mapping_update_lock);
Ian Campbell4b41df72011-02-18 16:43:29 +0000753 return irq;
Ian Campbellbf480d92011-02-18 16:43:32 +0000754error_irq:
Roger Pau Monne4892c9b2014-02-27 19:15:35 +0100755 for (; i >= 0; i--)
756 __unbind_from_irq(irq + i);
Konrad Rzeszutek Wilk77365942011-09-14 05:10:00 -0400757 mutex_unlock(&irq_mapping_update_lock);
Konrad Rzeszutek Wilke6599222011-09-29 13:26:45 -0400758 return ret;
Stefano Stabellini809f9262010-07-01 17:10:39 +0100759}
Qing Hef731e3ef2010-10-11 15:30:09 +0100760#endif
761
Alex Nixonb5401a92010-03-18 16:31:34 -0400762int xen_destroy_irq(int irq)
763{
Jeremy Fitzhardinge38aa66f2010-09-02 14:51:39 +0100764 struct physdev_unmap_pirq unmap_irq;
765 struct irq_info *info = info_for_irq(irq);
Alex Nixonb5401a92010-03-18 16:31:34 -0400766 int rc = -ENOENT;
767
Konrad Rzeszutek Wilk77365942011-09-14 05:10:00 -0400768 mutex_lock(&irq_mapping_update_lock);
Alex Nixonb5401a92010-03-18 16:31:34 -0400769
Roger Pau Monne4892c9b2014-02-27 19:15:35 +0100770 /*
771 * If trying to remove a vector in a MSI group different
772 * than the first one skip the PIRQ unmap unless this vector
773 * is the first one in the group.
774 */
775 if (xen_initial_domain() && !(info->u.pirq.flags & PIRQ_MSI_GROUP)) {
Konrad Rzeszutek Wilk12334712010-11-19 11:27:09 -0500776 unmap_irq.pirq = info->u.pirq.pirq;
Konrad Rzeszutek Wilkbeafbdc2011-04-14 11:17:36 -0400777 unmap_irq.domid = info->u.pirq.domid;
Jeremy Fitzhardinge38aa66f2010-09-02 14:51:39 +0100778 rc = HYPERVISOR_physdev_op(PHYSDEVOP_unmap_pirq, &unmap_irq);
Konrad Rzeszutek Wilk1eff1ad2011-02-16 16:26:44 -0500779 /* If another domain quits without making the pci_disable_msix
780 * call, the Xen hypervisor takes care of freeing the PIRQs
781 * (free_domain_pirqs).
782 */
783 if ((rc == -ESRCH && info->u.pirq.domid != DOMID_SELF))
Joe Perches283c0972013-06-28 03:21:41 -0700784 pr_info("domain %d does not have %d anymore\n",
Konrad Rzeszutek Wilk1eff1ad2011-02-16 16:26:44 -0500785 info->u.pirq.domid, info->u.pirq.pirq);
786 else if (rc) {
Joe Perches283c0972013-06-28 03:21:41 -0700787 pr_warn("unmap irq failed %d\n", rc);
Jeremy Fitzhardinge38aa66f2010-09-02 14:51:39 +0100788 goto out;
789 }
790 }
Alex Nixonb5401a92010-03-18 16:31:34 -0400791
Ian Campbellc9df1ce2011-01-11 17:20:15 +0000792 xen_free_irq(irq);
Alex Nixonb5401a92010-03-18 16:31:34 -0400793
794out:
Konrad Rzeszutek Wilk77365942011-09-14 05:10:00 -0400795 mutex_unlock(&irq_mapping_update_lock);
Alex Nixonb5401a92010-03-18 16:31:34 -0400796 return rc;
797}
798
Stefano Stabelliniaf42b8d2010-12-01 14:51:44 +0000799int xen_irq_from_pirq(unsigned pirq)
800{
Ian Campbell69c358c2011-03-10 16:08:13 +0000801 int irq;
802
803 struct irq_info *info;
804
Konrad Rzeszutek Wilk77365942011-09-14 05:10:00 -0400805 mutex_lock(&irq_mapping_update_lock);
Ian Campbell69c358c2011-03-10 16:08:13 +0000806
807 list_for_each_entry(info, &xen_irq_list_head, list) {
Konrad Rzeszutek Wilk9bb9efe2011-09-29 13:13:30 -0400808 if (info->type != IRQT_PIRQ)
Ian Campbell69c358c2011-03-10 16:08:13 +0000809 continue;
810 irq = info->irq;
811 if (info->u.pirq.pirq == pirq)
812 goto out;
813 }
814 irq = -1;
815out:
Konrad Rzeszutek Wilk77365942011-09-14 05:10:00 -0400816 mutex_unlock(&irq_mapping_update_lock);
Ian Campbell69c358c2011-03-10 16:08:13 +0000817
818 return irq;
Stefano Stabelliniaf42b8d2010-12-01 14:51:44 +0000819}
820
Konrad Rzeszutek Wilke6197ac2011-02-24 14:20:12 -0500821
822int xen_pirq_from_irq(unsigned irq)
823{
824 return pirq_from_irq(irq);
825}
826EXPORT_SYMBOL_GPL(xen_pirq_from_irq);
David Vrabel96d4c5882013-03-18 15:50:17 +0000827
Jeremy Fitzhardingeb536b4b2007-07-17 18:37:06 -0700828int bind_evtchn_to_irq(unsigned int evtchn)
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700829{
830 int irq;
David Vrabel96d4c5882013-03-18 15:50:17 +0000831 int ret;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700832
David Vrabeld0b075f2013-10-17 15:23:15 +0100833 if (evtchn >= xen_evtchn_max_channels())
834 return -ENOMEM;
835
Konrad Rzeszutek Wilk77365942011-09-14 05:10:00 -0400836 mutex_lock(&irq_mapping_update_lock);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700837
David Vrabeld0b075f2013-10-17 15:23:15 +0100838 irq = get_evtchn_to_irq(evtchn);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700839
840 if (irq == -1) {
Ian Campbellc9df1ce2011-01-11 17:20:15 +0000841 irq = xen_allocate_irq_dynamic();
Wei Liu68ba45f2013-01-31 14:46:56 +0000842 if (irq < 0)
Ian Campbell7bee9762011-03-10 16:08:15 +0000843 goto out;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700844
Thomas Gleixnerc442b802011-03-25 10:58:06 +0100845 irq_set_chip_and_handler_name(irq, &xen_dynamic_chip,
Stefano Stabellini7e186bd2011-05-06 12:27:50 +0100846 handle_edge_irq, "event");
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700847
David Vrabel96d4c5882013-03-18 15:50:17 +0000848 ret = xen_irq_info_evtchn_setup(irq, evtchn);
849 if (ret < 0) {
850 __unbind_from_irq(irq);
851 irq = ret;
852 goto out;
853 }
David Vrabel97253ee2014-02-05 14:13:10 +0000854 /* New interdomain events are bound to VCPU 0. */
855 bind_evtchn_to_cpu(evtchn, 0);
Konrad Rzeszutek Wilk5e152e62012-05-23 13:28:44 -0400856 } else {
857 struct irq_info *info = info_for_irq(irq);
858 WARN_ON(info == NULL || info->type != IRQT_EVTCHN);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700859 }
860
Ian Campbell7bee9762011-03-10 16:08:15 +0000861out:
Konrad Rzeszutek Wilk77365942011-09-14 05:10:00 -0400862 mutex_unlock(&irq_mapping_update_lock);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700863
864 return irq;
865}
Jeremy Fitzhardingeb536b4b2007-07-17 18:37:06 -0700866EXPORT_SYMBOL_GPL(bind_evtchn_to_irq);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700867
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -0700868static int bind_ipi_to_irq(unsigned int ipi, unsigned int cpu)
869{
870 struct evtchn_bind_ipi bind_ipi;
871 int evtchn, irq;
David Vrabel96d4c5882013-03-18 15:50:17 +0000872 int ret;
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -0700873
Konrad Rzeszutek Wilk77365942011-09-14 05:10:00 -0400874 mutex_lock(&irq_mapping_update_lock);
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -0700875
876 irq = per_cpu(ipi_to_irq, cpu)[ipi];
Ian Campbell90af9512009-02-06 16:55:58 -0800877
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -0700878 if (irq == -1) {
Ian Campbellc9df1ce2011-01-11 17:20:15 +0000879 irq = xen_allocate_irq_dynamic();
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -0700880 if (irq < 0)
881 goto out;
882
Thomas Gleixnerc442b802011-03-25 10:58:06 +0100883 irq_set_chip_and_handler_name(irq, &xen_percpu_chip,
Jeremy Fitzhardingeaaca4962010-08-20 18:57:53 -0700884 handle_percpu_irq, "ipi");
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -0700885
886 bind_ipi.vcpu = cpu;
887 if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_ipi,
888 &bind_ipi) != 0)
889 BUG();
890 evtchn = bind_ipi.port;
891
David Vrabel96d4c5882013-03-18 15:50:17 +0000892 ret = xen_irq_info_ipi_setup(cpu, irq, evtchn, ipi);
893 if (ret < 0) {
894 __unbind_from_irq(irq);
895 irq = ret;
896 goto out;
897 }
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -0700898 bind_evtchn_to_cpu(evtchn, cpu);
Konrad Rzeszutek Wilk5e152e62012-05-23 13:28:44 -0400899 } else {
900 struct irq_info *info = info_for_irq(irq);
901 WARN_ON(info == NULL || info->type != IRQT_IPI);
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -0700902 }
903
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -0700904 out:
Konrad Rzeszutek Wilk77365942011-09-14 05:10:00 -0400905 mutex_unlock(&irq_mapping_update_lock);
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -0700906 return irq;
907}
908
Juergen Gross854072d2014-08-28 06:44:09 +0200909int bind_interdomain_evtchn_to_irq(unsigned int remote_domain,
910 unsigned int remote_port)
Ian Campbell2e820f52009-02-09 12:05:50 -0800911{
912 struct evtchn_bind_interdomain bind_interdomain;
913 int err;
914
915 bind_interdomain.remote_dom = remote_domain;
916 bind_interdomain.remote_port = remote_port;
917
918 err = HYPERVISOR_event_channel_op(EVTCHNOP_bind_interdomain,
919 &bind_interdomain);
920
921 return err ? : bind_evtchn_to_irq(bind_interdomain.local_port);
922}
Juergen Gross854072d2014-08-28 06:44:09 +0200923EXPORT_SYMBOL_GPL(bind_interdomain_evtchn_to_irq);
Ian Campbell2e820f52009-02-09 12:05:50 -0800924
Olaf Hering62cc5fc2011-08-25 18:30:48 +0200925static int find_virq(unsigned int virq, unsigned int cpu)
926{
927 struct evtchn_status status;
928 int port, rc = -ENOENT;
929
930 memset(&status, 0, sizeof(status));
David Vrabeld0b075f2013-10-17 15:23:15 +0100931 for (port = 0; port < xen_evtchn_max_channels(); port++) {
Olaf Hering62cc5fc2011-08-25 18:30:48 +0200932 status.dom = DOMID_SELF;
933 status.port = port;
934 rc = HYPERVISOR_event_channel_op(EVTCHNOP_status, &status);
935 if (rc < 0)
936 continue;
937 if (status.status != EVTCHNSTAT_virq)
938 continue;
939 if (status.u.virq == virq && status.vcpu == cpu) {
940 rc = port;
941 break;
942 }
943 }
944 return rc;
945}
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -0700946
David Vrabel0dc00642013-09-23 21:03:38 +0100947/**
948 * xen_evtchn_nr_channels - number of usable event channel ports
949 *
950 * This may be less than the maximum supported by the current
951 * hypervisor ABI. Use xen_evtchn_max_channels() for the maximum
952 * supported.
953 */
954unsigned xen_evtchn_nr_channels(void)
955{
956 return evtchn_ops->nr_channels();
957}
958EXPORT_SYMBOL_GPL(xen_evtchn_nr_channels);
959
David Vrabel77bb3df2015-05-19 18:40:49 +0100960int bind_virq_to_irq(unsigned int virq, unsigned int cpu, bool percpu)
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700961{
962 struct evtchn_bind_virq bind_virq;
Olaf Hering62cc5fc2011-08-25 18:30:48 +0200963 int evtchn, irq, ret;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700964
Konrad Rzeszutek Wilk77365942011-09-14 05:10:00 -0400965 mutex_lock(&irq_mapping_update_lock);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700966
967 irq = per_cpu(virq_to_irq, cpu)[virq];
968
969 if (irq == -1) {
Ian Campbellc9df1ce2011-01-11 17:20:15 +0000970 irq = xen_allocate_irq_dynamic();
Wei Liu68ba45f2013-01-31 14:46:56 +0000971 if (irq < 0)
Ian Campbell7bee9762011-03-10 16:08:15 +0000972 goto out;
Jeremy Fitzhardingea52521f2010-09-22 15:28:52 -0700973
David Vrabel77bb3df2015-05-19 18:40:49 +0100974 if (percpu)
975 irq_set_chip_and_handler_name(irq, &xen_percpu_chip,
976 handle_percpu_irq, "virq");
977 else
978 irq_set_chip_and_handler_name(irq, &xen_dynamic_chip,
979 handle_edge_irq, "virq");
Jeremy Fitzhardingea52521f2010-09-22 15:28:52 -0700980
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700981 bind_virq.virq = virq;
982 bind_virq.vcpu = cpu;
Olaf Hering62cc5fc2011-08-25 18:30:48 +0200983 ret = HYPERVISOR_event_channel_op(EVTCHNOP_bind_virq,
984 &bind_virq);
985 if (ret == 0)
986 evtchn = bind_virq.port;
987 else {
988 if (ret == -EEXIST)
989 ret = find_virq(virq, cpu);
990 BUG_ON(ret < 0);
991 evtchn = ret;
992 }
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -0700993
David Vrabel96d4c5882013-03-18 15:50:17 +0000994 ret = xen_irq_info_virq_setup(cpu, irq, evtchn, virq);
995 if (ret < 0) {
996 __unbind_from_irq(irq);
997 irq = ret;
998 goto out;
999 }
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001000
1001 bind_evtchn_to_cpu(evtchn, cpu);
Konrad Rzeszutek Wilk5e152e62012-05-23 13:28:44 -04001002 } else {
1003 struct irq_info *info = info_for_irq(irq);
1004 WARN_ON(info == NULL || info->type != IRQT_VIRQ);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001005 }
1006
Ian Campbell7bee9762011-03-10 16:08:15 +00001007out:
Konrad Rzeszutek Wilk77365942011-09-14 05:10:00 -04001008 mutex_unlock(&irq_mapping_update_lock);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001009
1010 return irq;
1011}
1012
1013static void unbind_from_irq(unsigned int irq)
1014{
Konrad Rzeszutek Wilk77365942011-09-14 05:10:00 -04001015 mutex_lock(&irq_mapping_update_lock);
David Vrabel96d4c5882013-03-18 15:50:17 +00001016 __unbind_from_irq(irq);
Konrad Rzeszutek Wilk77365942011-09-14 05:10:00 -04001017 mutex_unlock(&irq_mapping_update_lock);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001018}
1019
1020int bind_evtchn_to_irqhandler(unsigned int evtchn,
Jeff Garzik7c239972007-10-19 03:12:20 -04001021 irq_handler_t handler,
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001022 unsigned long irqflags,
1023 const char *devname, void *dev_id)
1024{
Nicolas Kaiser361ae8c2011-03-30 21:14:26 +02001025 int irq, retval;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001026
1027 irq = bind_evtchn_to_irq(evtchn);
Ian Campbell7bee9762011-03-10 16:08:15 +00001028 if (irq < 0)
1029 return irq;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001030 retval = request_irq(irq, handler, irqflags, devname, dev_id);
1031 if (retval != 0) {
1032 unbind_from_irq(irq);
1033 return retval;
1034 }
1035
1036 return irq;
1037}
1038EXPORT_SYMBOL_GPL(bind_evtchn_to_irqhandler);
1039
Ian Campbell2e820f52009-02-09 12:05:50 -08001040int bind_interdomain_evtchn_to_irqhandler(unsigned int remote_domain,
1041 unsigned int remote_port,
1042 irq_handler_t handler,
1043 unsigned long irqflags,
1044 const char *devname,
1045 void *dev_id)
1046{
1047 int irq, retval;
1048
1049 irq = bind_interdomain_evtchn_to_irq(remote_domain, remote_port);
1050 if (irq < 0)
1051 return irq;
1052
1053 retval = request_irq(irq, handler, irqflags, devname, dev_id);
1054 if (retval != 0) {
1055 unbind_from_irq(irq);
1056 return retval;
1057 }
1058
1059 return irq;
1060}
1061EXPORT_SYMBOL_GPL(bind_interdomain_evtchn_to_irqhandler);
1062
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001063int bind_virq_to_irqhandler(unsigned int virq, unsigned int cpu,
Jeff Garzik7c239972007-10-19 03:12:20 -04001064 irq_handler_t handler,
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001065 unsigned long irqflags, const char *devname, void *dev_id)
1066{
Nicolas Kaiser361ae8c2011-03-30 21:14:26 +02001067 int irq, retval;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001068
David Vrabel77bb3df2015-05-19 18:40:49 +01001069 irq = bind_virq_to_irq(virq, cpu, irqflags & IRQF_PERCPU);
Ian Campbell7bee9762011-03-10 16:08:15 +00001070 if (irq < 0)
1071 return irq;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001072 retval = request_irq(irq, handler, irqflags, devname, dev_id);
1073 if (retval != 0) {
1074 unbind_from_irq(irq);
1075 return retval;
1076 }
1077
1078 return irq;
1079}
1080EXPORT_SYMBOL_GPL(bind_virq_to_irqhandler);
1081
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -07001082int bind_ipi_to_irqhandler(enum ipi_vector ipi,
1083 unsigned int cpu,
1084 irq_handler_t handler,
1085 unsigned long irqflags,
1086 const char *devname,
1087 void *dev_id)
1088{
1089 int irq, retval;
1090
1091 irq = bind_ipi_to_irq(ipi, cpu);
1092 if (irq < 0)
1093 return irq;
1094
Ian Campbell9bab0b72011-10-03 15:37:00 +01001095 irqflags |= IRQF_NO_SUSPEND | IRQF_FORCE_RESUME | IRQF_EARLY_RESUME;
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -07001096 retval = request_irq(irq, handler, irqflags, devname, dev_id);
1097 if (retval != 0) {
1098 unbind_from_irq(irq);
1099 return retval;
1100 }
1101
1102 return irq;
1103}
1104
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001105void unbind_from_irqhandler(unsigned int irq, void *dev_id)
1106{
Konrad Rzeszutek Wilk94032c52013-04-16 10:55:18 -04001107 struct irq_info *info = irq_get_handler_data(irq);
1108
1109 if (WARN_ON(!info))
1110 return;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001111 free_irq(irq, dev_id);
1112 unbind_from_irq(irq);
1113}
1114EXPORT_SYMBOL_GPL(unbind_from_irqhandler);
1115
David Vrabel6ccecb02013-09-23 12:47:26 +01001116/**
1117 * xen_set_irq_priority() - set an event channel priority.
1118 * @irq:irq bound to an event channel.
1119 * @priority: priority between XEN_IRQ_PRIORITY_MAX and XEN_IRQ_PRIORITY_MIN.
1120 */
1121int xen_set_irq_priority(unsigned irq, unsigned priority)
1122{
1123 struct evtchn_set_priority set_priority;
1124
1125 set_priority.port = evtchn_from_irq(irq);
1126 set_priority.priority = priority;
1127
1128 return HYPERVISOR_event_channel_op(EVTCHNOP_set_priority,
1129 &set_priority);
1130}
1131EXPORT_SYMBOL_GPL(xen_set_irq_priority);
1132
Daniel De Graaf420eb552011-10-27 17:58:47 -04001133int evtchn_make_refcounted(unsigned int evtchn)
1134{
David Vrabeld0b075f2013-10-17 15:23:15 +01001135 int irq = get_evtchn_to_irq(evtchn);
Daniel De Graaf420eb552011-10-27 17:58:47 -04001136 struct irq_info *info;
1137
1138 if (irq == -1)
1139 return -ENOENT;
1140
1141 info = irq_get_handler_data(irq);
1142
1143 if (!info)
1144 return -ENOENT;
1145
1146 WARN_ON(info->refcnt != -1);
1147
1148 info->refcnt = 1;
1149
1150 return 0;
1151}
1152EXPORT_SYMBOL_GPL(evtchn_make_refcounted);
1153
1154int evtchn_get(unsigned int evtchn)
1155{
1156 int irq;
1157 struct irq_info *info;
1158 int err = -ENOENT;
1159
David Vrabeld0b075f2013-10-17 15:23:15 +01001160 if (evtchn >= xen_evtchn_max_channels())
Daniel De Graafc3b3f162011-11-28 11:49:09 -05001161 return -EINVAL;
1162
Daniel De Graaf420eb552011-10-27 17:58:47 -04001163 mutex_lock(&irq_mapping_update_lock);
1164
David Vrabeld0b075f2013-10-17 15:23:15 +01001165 irq = get_evtchn_to_irq(evtchn);
Daniel De Graaf420eb552011-10-27 17:58:47 -04001166 if (irq == -1)
1167 goto done;
1168
1169 info = irq_get_handler_data(irq);
1170
1171 if (!info)
1172 goto done;
1173
1174 err = -EINVAL;
1175 if (info->refcnt <= 0)
1176 goto done;
1177
1178 info->refcnt++;
1179 err = 0;
1180 done:
1181 mutex_unlock(&irq_mapping_update_lock);
1182
1183 return err;
1184}
1185EXPORT_SYMBOL_GPL(evtchn_get);
1186
1187void evtchn_put(unsigned int evtchn)
1188{
David Vrabeld0b075f2013-10-17 15:23:15 +01001189 int irq = get_evtchn_to_irq(evtchn);
Daniel De Graaf420eb552011-10-27 17:58:47 -04001190 if (WARN_ON(irq == -1))
1191 return;
1192 unbind_from_irq(irq);
1193}
1194EXPORT_SYMBOL_GPL(evtchn_put);
1195
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -07001196void xen_send_IPI_one(unsigned int cpu, enum ipi_vector vector)
1197{
Konrad Rzeszutek Wilk6efa20e2013-07-19 11:51:31 -04001198 int irq;
1199
Stefano Stabellini072b2062013-08-13 16:57:06 +00001200#ifdef CONFIG_X86
Konrad Rzeszutek Wilk6efa20e2013-07-19 11:51:31 -04001201 if (unlikely(vector == XEN_NMI_VECTOR)) {
1202 int rc = HYPERVISOR_vcpu_op(VCPUOP_send_nmi, cpu, NULL);
1203 if (rc < 0)
1204 printk(KERN_WARNING "Sending nmi to CPU%d failed (rc:%d)\n", cpu, rc);
1205 return;
1206 }
Stefano Stabellini072b2062013-08-13 16:57:06 +00001207#endif
Konrad Rzeszutek Wilk6efa20e2013-07-19 11:51:31 -04001208 irq = per_cpu(ipi_to_irq, cpu)[vector];
Jeremy Fitzhardingef87e4ca2007-07-17 18:37:06 -07001209 BUG_ON(irq < 0);
1210 notify_remote_via_irq(irq);
1211}
1212
Tejun Heo245b2e72009-06-24 15:13:48 +09001213static DEFINE_PER_CPU(unsigned, xed_nesting_count);
1214
Sheng Yang38e20b02010-05-14 12:40:51 +01001215static void __xen_evtchn_do_upcall(void)
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001216{
Christoph Lameter780f36d2010-12-06 11:16:29 -06001217 struct vcpu_info *vcpu_info = __this_cpu_read(xen_vcpu);
David Vrabel9a489f42013-03-13 15:29:25 +00001218 int cpu = get_cpu();
Ruslan Pisarev088c05a2011-07-26 14:16:13 +03001219 unsigned count;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001220
Jeremy Fitzhardinge229664b2008-03-17 16:37:20 -07001221 do {
Jeremy Fitzhardinge229664b2008-03-17 16:37:20 -07001222 vcpu_info->evtchn_upcall_pending = 0;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001223
Christoph Lameterb2e4ae62010-12-06 11:40:07 -06001224 if (__this_cpu_inc_return(xed_nesting_count) - 1)
Jeremy Fitzhardinge229664b2008-03-17 16:37:20 -07001225 goto out;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001226
David Vrabel9a489f42013-03-13 15:29:25 +00001227 xen_evtchn_handle_events(cpu);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001228
Jeremy Fitzhardinge229664b2008-03-17 16:37:20 -07001229 BUG_ON(!irqs_disabled());
1230
Christoph Lameter780f36d2010-12-06 11:16:29 -06001231 count = __this_cpu_read(xed_nesting_count);
1232 __this_cpu_write(xed_nesting_count, 0);
Stefano Stabellini183d03c2010-05-17 17:08:21 +01001233 } while (count != 1 || vcpu_info->evtchn_upcall_pending);
Jeremy Fitzhardinge229664b2008-03-17 16:37:20 -07001234
1235out:
Jeremy Fitzhardinge3445a8f2009-02-06 14:09:46 -08001236
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001237 put_cpu();
1238}
1239
Sheng Yang38e20b02010-05-14 12:40:51 +01001240void xen_evtchn_do_upcall(struct pt_regs *regs)
1241{
1242 struct pt_regs *old_regs = set_irq_regs(regs);
1243
Mojiong Qiu772aebc2012-11-06 16:08:15 +08001244 irq_enter();
Stefano Stabellini0ec53ec2012-09-14 13:37:32 +00001245#ifdef CONFIG_X86
Sheng Yang38e20b02010-05-14 12:40:51 +01001246 exit_idle();
Thomas Gleixner99c8b792014-02-23 21:40:21 +00001247 inc_irq_stat(irq_hv_callback_count);
Arnd Bergmannd06eb3e2014-03-25 16:52:25 +01001248#endif
Sheng Yang38e20b02010-05-14 12:40:51 +01001249
1250 __xen_evtchn_do_upcall();
1251
1252 irq_exit();
1253 set_irq_regs(old_regs);
1254}
1255
1256void xen_hvm_evtchn_do_upcall(void)
1257{
1258 __xen_evtchn_do_upcall();
1259}
Stefano Stabellini183d03c2010-05-17 17:08:21 +01001260EXPORT_SYMBOL_GPL(xen_hvm_evtchn_do_upcall);
Sheng Yang38e20b02010-05-14 12:40:51 +01001261
Jeremy Fitzhardingeeb1e3052008-05-26 23:31:23 +01001262/* Rebind a new event channel to an existing irq. */
1263void rebind_evtchn_irq(int evtchn, int irq)
1264{
Jeremy Fitzhardinged77bbd42009-02-06 14:09:45 -08001265 struct irq_info *info = info_for_irq(irq);
1266
Konrad Rzeszutek Wilk94032c52013-04-16 10:55:18 -04001267 if (WARN_ON(!info))
1268 return;
1269
Jeremy Fitzhardingeeb1e3052008-05-26 23:31:23 +01001270 /* Make sure the irq is masked, since the new event channel
1271 will also be masked. */
1272 disable_irq(irq);
1273
Konrad Rzeszutek Wilk77365942011-09-14 05:10:00 -04001274 mutex_lock(&irq_mapping_update_lock);
Jeremy Fitzhardingeeb1e3052008-05-26 23:31:23 +01001275
1276 /* After resume the irq<->evtchn mappings are all cleared out */
David Vrabeld0b075f2013-10-17 15:23:15 +01001277 BUG_ON(get_evtchn_to_irq(evtchn) != -1);
Jeremy Fitzhardingeeb1e3052008-05-26 23:31:23 +01001278 /* Expect irq to have been bound before,
Jeremy Fitzhardinged77bbd42009-02-06 14:09:45 -08001279 so there should be a proper type */
1280 BUG_ON(info->type == IRQT_UNBOUND);
Jeremy Fitzhardingeeb1e3052008-05-26 23:31:23 +01001281
David Vrabel96d4c5882013-03-18 15:50:17 +00001282 (void)xen_irq_info_evtchn_setup(irq, evtchn);
Jeremy Fitzhardingeeb1e3052008-05-26 23:31:23 +01001283
Konrad Rzeszutek Wilk77365942011-09-14 05:10:00 -04001284 mutex_unlock(&irq_mapping_update_lock);
Jeremy Fitzhardingeeb1e3052008-05-26 23:31:23 +01001285
Boris Ostrovsky5cec9882015-04-29 17:10:12 -04001286 bind_evtchn_to_cpu(evtchn, info->cpu);
1287 /* This will be deferred until interrupt is processed */
1288 irq_set_affinity(irq, cpumask_of(info->cpu));
Jeremy Fitzhardingeeb1e3052008-05-26 23:31:23 +01001289
1290 /* Unmask the event channel. */
1291 enable_irq(irq);
1292}
1293
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001294/* Rebind an evtchn so that it gets delivered to a specific cpu */
Yinghai Lud5dedd42009-04-27 17:59:21 -07001295static int rebind_irq_to_cpu(unsigned irq, unsigned tcpu)
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001296{
1297 struct evtchn_bind_vcpu bind_vcpu;
1298 int evtchn = evtchn_from_irq(irq);
David Vrabel4704fe42013-08-15 13:21:07 +01001299 int masked;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001300
Ian Campbellbe494722011-03-10 16:08:02 +00001301 if (!VALID_EVTCHN(evtchn))
1302 return -1;
1303
Julien Grall4a5b6942015-07-28 10:10:42 +01001304 if (!xen_support_evtchn_rebind())
Yinghai Lud5dedd42009-04-27 17:59:21 -07001305 return -1;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001306
1307 /* Send future instances of this interrupt to other vcpu. */
1308 bind_vcpu.port = evtchn;
1309 bind_vcpu.vcpu = tcpu;
1310
1311 /*
David Vrabel4704fe42013-08-15 13:21:07 +01001312 * Mask the event while changing the VCPU binding to prevent
1313 * it being delivered on an unexpected VCPU.
1314 */
Wei Liu3f70fa82013-03-07 15:50:27 +00001315 masked = test_and_set_mask(evtchn);
David Vrabel4704fe42013-08-15 13:21:07 +01001316
1317 /*
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001318 * If this fails, it usually just indicates that we're dealing with a
1319 * virq or IPI channel, which don't actually need to be rebound. Ignore
1320 * it, but don't do the xenlinux-level rebind in that case.
1321 */
1322 if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_vcpu, &bind_vcpu) >= 0)
1323 bind_evtchn_to_cpu(evtchn, tcpu);
Yinghai Lud5dedd42009-04-27 17:59:21 -07001324
David Vrabel4704fe42013-08-15 13:21:07 +01001325 if (!masked)
1326 unmask_evtchn(evtchn);
1327
Yinghai Lud5dedd42009-04-27 17:59:21 -07001328 return 0;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001329}
1330
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +00001331static int set_affinity_irq(struct irq_data *data, const struct cpumask *dest,
1332 bool force)
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001333{
Thomas Gleixner753fbd22014-03-04 20:43:40 +00001334 unsigned tcpu = cpumask_first_and(dest, cpu_online_mask);
Yinghai Lud5dedd42009-04-27 17:59:21 -07001335
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +00001336 return rebind_irq_to_cpu(data->irq, tcpu);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001337}
1338
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +00001339static void enable_dynirq(struct irq_data *data)
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001340{
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +00001341 int evtchn = evtchn_from_irq(data->irq);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001342
1343 if (VALID_EVTCHN(evtchn))
1344 unmask_evtchn(evtchn);
1345}
1346
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +00001347static void disable_dynirq(struct irq_data *data)
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001348{
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +00001349 int evtchn = evtchn_from_irq(data->irq);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001350
1351 if (VALID_EVTCHN(evtchn))
1352 mask_evtchn(evtchn);
1353}
1354
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +00001355static void ack_dynirq(struct irq_data *data)
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001356{
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +00001357 int evtchn = evtchn_from_irq(data->irq);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001358
Stefano Stabellini7e186bd2011-05-06 12:27:50 +01001359 irq_move_irq(data);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001360
1361 if (VALID_EVTCHN(evtchn))
Stefano Stabellini7e186bd2011-05-06 12:27:50 +01001362 clear_evtchn(evtchn);
1363}
1364
1365static void mask_ack_dynirq(struct irq_data *data)
1366{
1367 disable_dynirq(data);
1368 ack_dynirq(data);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001369}
1370
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +00001371static int retrigger_dynirq(struct irq_data *data)
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001372{
David Vrabel4640ddf2014-02-17 17:45:16 +00001373 unsigned int evtchn = evtchn_from_irq(data->irq);
1374 int masked;
1375
1376 if (!VALID_EVTCHN(evtchn))
1377 return 0;
1378
1379 masked = test_and_set_mask(evtchn);
1380 set_evtchn(evtchn);
1381 if (!masked)
1382 unmask_evtchn(evtchn);
1383
1384 return 1;
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001385}
1386
Ian Campbell0a852262011-03-10 16:08:06 +00001387static void restore_pirqs(void)
Stefano Stabellini9a069c32010-12-01 14:51:44 +00001388{
1389 int pirq, rc, irq, gsi;
1390 struct physdev_map_pirq map_irq;
Ian Campbell69c358c2011-03-10 16:08:13 +00001391 struct irq_info *info;
Stefano Stabellini9a069c32010-12-01 14:51:44 +00001392
Ian Campbell69c358c2011-03-10 16:08:13 +00001393 list_for_each_entry(info, &xen_irq_list_head, list) {
1394 if (info->type != IRQT_PIRQ)
Stefano Stabellini9a069c32010-12-01 14:51:44 +00001395 continue;
1396
Ian Campbell69c358c2011-03-10 16:08:13 +00001397 pirq = info->u.pirq.pirq;
1398 gsi = info->u.pirq.gsi;
1399 irq = info->irq;
1400
Stefano Stabellini9a069c32010-12-01 14:51:44 +00001401 /* save/restore of PT devices doesn't work, so at this point the
1402 * only devices present are GSI based emulated devices */
Stefano Stabellini9a069c32010-12-01 14:51:44 +00001403 if (!gsi)
1404 continue;
1405
1406 map_irq.domid = DOMID_SELF;
1407 map_irq.type = MAP_PIRQ_TYPE_GSI;
1408 map_irq.index = gsi;
1409 map_irq.pirq = pirq;
1410
1411 rc = HYPERVISOR_physdev_op(PHYSDEVOP_map_pirq, &map_irq);
1412 if (rc) {
Joe Perches283c0972013-06-28 03:21:41 -07001413 pr_warn("xen map irq failed gsi=%d irq=%d pirq=%d rc=%d\n",
1414 gsi, irq, pirq, rc);
Ian Campbell9158c352011-03-10 16:08:09 +00001415 xen_free_irq(irq);
Stefano Stabellini9a069c32010-12-01 14:51:44 +00001416 continue;
1417 }
1418
1419 printk(KERN_DEBUG "xen: --> irq=%d, pirq=%d\n", irq, map_irq.pirq);
1420
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +00001421 __startup_pirq(irq);
Stefano Stabellini9a069c32010-12-01 14:51:44 +00001422 }
1423}
1424
Jeremy Fitzhardinge0e913982008-05-26 23:31:27 +01001425static void restore_cpu_virqs(unsigned int cpu)
1426{
1427 struct evtchn_bind_virq bind_virq;
1428 int virq, irq, evtchn;
1429
1430 for (virq = 0; virq < NR_VIRQS; virq++) {
1431 if ((irq = per_cpu(virq_to_irq, cpu)[virq]) == -1)
1432 continue;
1433
Jeremy Fitzhardingeced40d02009-02-06 14:09:44 -08001434 BUG_ON(virq_from_irq(irq) != virq);
Jeremy Fitzhardinge0e913982008-05-26 23:31:27 +01001435
1436 /* Get a new binding from Xen. */
1437 bind_virq.virq = virq;
1438 bind_virq.vcpu = cpu;
1439 if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_virq,
1440 &bind_virq) != 0)
1441 BUG();
1442 evtchn = bind_virq.port;
1443
1444 /* Record the new mapping. */
David Vrabel96d4c5882013-03-18 15:50:17 +00001445 (void)xen_irq_info_virq_setup(cpu, irq, evtchn, virq);
Jeremy Fitzhardinge0e913982008-05-26 23:31:27 +01001446 bind_evtchn_to_cpu(evtchn, cpu);
Jeremy Fitzhardinge0e913982008-05-26 23:31:27 +01001447 }
1448}
1449
1450static void restore_cpu_ipis(unsigned int cpu)
1451{
1452 struct evtchn_bind_ipi bind_ipi;
1453 int ipi, irq, evtchn;
1454
1455 for (ipi = 0; ipi < XEN_NR_IPIS; ipi++) {
1456 if ((irq = per_cpu(ipi_to_irq, cpu)[ipi]) == -1)
1457 continue;
1458
Jeremy Fitzhardingeced40d02009-02-06 14:09:44 -08001459 BUG_ON(ipi_from_irq(irq) != ipi);
Jeremy Fitzhardinge0e913982008-05-26 23:31:27 +01001460
1461 /* Get a new binding from Xen. */
1462 bind_ipi.vcpu = cpu;
1463 if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_ipi,
1464 &bind_ipi) != 0)
1465 BUG();
1466 evtchn = bind_ipi.port;
1467
1468 /* Record the new mapping. */
David Vrabel96d4c5882013-03-18 15:50:17 +00001469 (void)xen_irq_info_ipi_setup(cpu, irq, evtchn, ipi);
Jeremy Fitzhardinge0e913982008-05-26 23:31:27 +01001470 bind_evtchn_to_cpu(evtchn, cpu);
Jeremy Fitzhardinge0e913982008-05-26 23:31:27 +01001471 }
1472}
1473
Jeremy Fitzhardinge2d9e1e22008-07-07 12:07:53 -07001474/* Clear an irq's pending state, in preparation for polling on it */
1475void xen_clear_irq_pending(int irq)
1476{
1477 int evtchn = evtchn_from_irq(irq);
1478
1479 if (VALID_EVTCHN(evtchn))
1480 clear_evtchn(evtchn);
1481}
Konrad Rzeszutek Wilkd9a88142009-11-05 16:33:09 -05001482EXPORT_SYMBOL(xen_clear_irq_pending);
Jeremy Fitzhardinge168d2f42008-08-20 17:02:18 -07001483void xen_set_irq_pending(int irq)
1484{
1485 int evtchn = evtchn_from_irq(irq);
1486
1487 if (VALID_EVTCHN(evtchn))
1488 set_evtchn(evtchn);
1489}
1490
1491bool xen_test_irq_pending(int irq)
1492{
1493 int evtchn = evtchn_from_irq(irq);
1494 bool ret = false;
1495
1496 if (VALID_EVTCHN(evtchn))
1497 ret = test_evtchn(evtchn);
1498
1499 return ret;
1500}
1501
Konrad Rzeszutek Wilkd9a88142009-11-05 16:33:09 -05001502/* Poll waiting for an irq to become pending with timeout. In the usual case,
1503 * the irq will be disabled so it won't deliver an interrupt. */
1504void xen_poll_irq_timeout(int irq, u64 timeout)
Jeremy Fitzhardinge2d9e1e22008-07-07 12:07:53 -07001505{
1506 evtchn_port_t evtchn = evtchn_from_irq(irq);
1507
1508 if (VALID_EVTCHN(evtchn)) {
1509 struct sched_poll poll;
1510
1511 poll.nr_ports = 1;
Konrad Rzeszutek Wilkd9a88142009-11-05 16:33:09 -05001512 poll.timeout = timeout;
Isaku Yamahataff3c5362008-10-14 17:50:44 -07001513 set_xen_guest_handle(poll.ports, &evtchn);
Jeremy Fitzhardinge2d9e1e22008-07-07 12:07:53 -07001514
1515 if (HYPERVISOR_sched_op(SCHEDOP_poll, &poll) != 0)
1516 BUG();
1517 }
1518}
Konrad Rzeszutek Wilkd9a88142009-11-05 16:33:09 -05001519EXPORT_SYMBOL(xen_poll_irq_timeout);
1520/* Poll waiting for an irq to become pending. In the usual case, the
1521 * irq will be disabled so it won't deliver an interrupt. */
1522void xen_poll_irq(int irq)
1523{
1524 xen_poll_irq_timeout(irq, 0 /* no timeout */);
1525}
Jeremy Fitzhardinge2d9e1e22008-07-07 12:07:53 -07001526
Konrad Rzeszutek Wilkc7c2c3a2010-11-08 14:26:36 -05001527/* Check whether the IRQ line is shared with other guests. */
1528int xen_test_irq_shared(int irq)
1529{
1530 struct irq_info *info = info_for_irq(irq);
Konrad Rzeszutek Wilk94032c52013-04-16 10:55:18 -04001531 struct physdev_irq_status_query irq_status;
1532
1533 if (WARN_ON(!info))
1534 return -ENOENT;
1535
1536 irq_status.irq = info->u.pirq.pirq;
Konrad Rzeszutek Wilkc7c2c3a2010-11-08 14:26:36 -05001537
1538 if (HYPERVISOR_physdev_op(PHYSDEVOP_irq_status_query, &irq_status))
1539 return 0;
1540 return !(irq_status.flags & XENIRQSTAT_shared);
1541}
1542EXPORT_SYMBOL_GPL(xen_test_irq_shared);
1543
Jeremy Fitzhardinge0e913982008-05-26 23:31:27 +01001544void xen_irq_resume(void)
1545{
David Vrabelfd210692013-09-05 18:11:38 +01001546 unsigned int cpu;
Ian Campbell6cb65372011-03-10 16:08:11 +00001547 struct irq_info *info;
Jeremy Fitzhardinge0e913982008-05-26 23:31:27 +01001548
Jeremy Fitzhardinge0e913982008-05-26 23:31:27 +01001549 /* New event-channel space is not 'live' yet. */
David Vrabelfd210692013-09-05 18:11:38 +01001550 xen_evtchn_mask_all();
David Vrabel1fe56552013-03-15 13:02:35 +00001551 xen_evtchn_resume();
Jeremy Fitzhardinge0e913982008-05-26 23:31:27 +01001552
1553 /* No IRQ <-> event-channel mappings. */
Ian Campbell6cb65372011-03-10 16:08:11 +00001554 list_for_each_entry(info, &xen_irq_list_head, list)
1555 info->evtchn = 0; /* zap event-channel binding */
Jeremy Fitzhardinge0e913982008-05-26 23:31:27 +01001556
David Vrabeld0b075f2013-10-17 15:23:15 +01001557 clear_evtchn_to_irq_all();
Jeremy Fitzhardinge0e913982008-05-26 23:31:27 +01001558
1559 for_each_possible_cpu(cpu) {
1560 restore_cpu_virqs(cpu);
1561 restore_cpu_ipis(cpu);
1562 }
Ian Campbell69035912010-11-01 16:30:09 +00001563
Ian Campbell0a852262011-03-10 16:08:06 +00001564 restore_pirqs();
Jeremy Fitzhardinge0e913982008-05-26 23:31:27 +01001565}
1566
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001567static struct irq_chip xen_dynamic_chip __read_mostly = {
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +00001568 .name = "xen-dyn",
Jeremy Fitzhardinge54a353a2009-02-06 14:09:42 -08001569
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +00001570 .irq_disable = disable_dynirq,
1571 .irq_mask = disable_dynirq,
1572 .irq_unmask = enable_dynirq,
Jeremy Fitzhardinge54a353a2009-02-06 14:09:42 -08001573
Stefano Stabellini7e186bd2011-05-06 12:27:50 +01001574 .irq_ack = ack_dynirq,
1575 .irq_mask_ack = mask_ack_dynirq,
1576
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +00001577 .irq_set_affinity = set_affinity_irq,
1578 .irq_retrigger = retrigger_dynirq,
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001579};
1580
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -04001581static struct irq_chip xen_pirq_chip __read_mostly = {
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +00001582 .name = "xen-pirq",
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -04001583
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +00001584 .irq_startup = startup_pirq,
1585 .irq_shutdown = shutdown_pirq,
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +00001586 .irq_enable = enable_pirq,
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +00001587 .irq_disable = disable_pirq,
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -04001588
Stefano Stabellini7e186bd2011-05-06 12:27:50 +01001589 .irq_mask = disable_dynirq,
1590 .irq_unmask = enable_dynirq,
1591
1592 .irq_ack = eoi_pirq,
1593 .irq_eoi = eoi_pirq,
1594 .irq_mask_ack = mask_ack_pirq,
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -04001595
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +00001596 .irq_set_affinity = set_affinity_irq,
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -04001597
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +00001598 .irq_retrigger = retrigger_dynirq,
Jeremy Fitzhardinged46a78b2010-10-01 12:20:09 -04001599};
1600
Jeremy Fitzhardingeaaca4962010-08-20 18:57:53 -07001601static struct irq_chip xen_percpu_chip __read_mostly = {
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +00001602 .name = "xen-percpu",
Jeremy Fitzhardingeaaca4962010-08-20 18:57:53 -07001603
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +00001604 .irq_disable = disable_dynirq,
1605 .irq_mask = disable_dynirq,
1606 .irq_unmask = enable_dynirq,
Jeremy Fitzhardingeaaca4962010-08-20 18:57:53 -07001607
Thomas Gleixnerc9e265e2011-02-05 20:08:54 +00001608 .irq_ack = ack_dynirq,
Jeremy Fitzhardingeaaca4962010-08-20 18:57:53 -07001609};
1610
Sheng Yang38e20b02010-05-14 12:40:51 +01001611int xen_set_callback_via(uint64_t via)
1612{
1613 struct xen_hvm_param a;
1614 a.domid = DOMID_SELF;
1615 a.index = HVM_PARAM_CALLBACK_IRQ;
1616 a.value = via;
1617 return HYPERVISOR_hvm_op(HVMOP_set_param, &a);
1618}
1619EXPORT_SYMBOL_GPL(xen_set_callback_via);
1620
Stefano Stabellinica65f9f2010-07-29 14:37:48 +01001621#ifdef CONFIG_XEN_PVHVM
Sheng Yang38e20b02010-05-14 12:40:51 +01001622/* Vector callbacks are better than PCI interrupts to receive event
1623 * channel notifications because we can receive vector callbacks on any
1624 * vcpu and we don't need PCI support or APIC interactions. */
1625void xen_callback_vector(void)
1626{
1627 int rc;
1628 uint64_t callback_via;
1629 if (xen_have_vector_callback) {
K. Y. Srinivasanbc2b0332013-02-03 17:22:39 -08001630 callback_via = HVM_CALLBACK_VECTOR(HYPERVISOR_CALLBACK_VECTOR);
Sheng Yang38e20b02010-05-14 12:40:51 +01001631 rc = xen_set_callback_via(callback_via);
1632 if (rc) {
Joe Perches283c0972013-06-28 03:21:41 -07001633 pr_err("Request for Xen HVM callback vector failed\n");
Sheng Yang38e20b02010-05-14 12:40:51 +01001634 xen_have_vector_callback = 0;
1635 return;
1636 }
Joe Perches283c0972013-06-28 03:21:41 -07001637 pr_info("Xen HVM callback vector for event delivery is enabled\n");
Sheng Yang38e20b02010-05-14 12:40:51 +01001638 /* in the restore case the vector has already been allocated */
K. Y. Srinivasanbc2b0332013-02-03 17:22:39 -08001639 if (!test_bit(HYPERVISOR_CALLBACK_VECTOR, used_vectors))
1640 alloc_intr_gate(HYPERVISOR_CALLBACK_VECTOR,
1641 xen_hvm_callback_vector);
Sheng Yang38e20b02010-05-14 12:40:51 +01001642 }
1643}
Stefano Stabellinica65f9f2010-07-29 14:37:48 +01001644#else
1645void xen_callback_vector(void) {}
1646#endif
Sheng Yang38e20b02010-05-14 12:40:51 +01001647
David Vrabel1fe56552013-03-15 13:02:35 +00001648#undef MODULE_PARAM_PREFIX
1649#define MODULE_PARAM_PREFIX "xen."
1650
1651static bool fifo_events = true;
1652module_param(fifo_events, bool, 0);
1653
Stefano Stabellini2e3d8862012-10-02 15:57:57 +01001654void __init xen_init_IRQ(void)
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001655{
David Vrabel1fe56552013-03-15 13:02:35 +00001656 int ret = -EINVAL;
1657
1658 if (fifo_events)
1659 ret = xen_evtchn_fifo_init();
1660 if (ret < 0)
1661 xen_evtchn_2l_init();
David Vrabelab9a1cc2013-03-14 12:49:19 +00001662
David Vrabeld0b075f2013-10-17 15:23:15 +01001663 evtchn_to_irq = kcalloc(EVTCHN_ROW(xen_evtchn_max_channels()),
1664 sizeof(*evtchn_to_irq), GFP_KERNEL);
Konrad Rzeszutek Wilk9d093e22011-09-29 13:31:21 -04001665 BUG_ON(!evtchn_to_irq);
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001666
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001667 /* No event channels are 'live' right now. */
David Vrabelfd210692013-09-05 18:11:38 +01001668 xen_evtchn_mask_all();
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001669
Stefano Stabellini9846ff12012-01-30 16:21:48 +00001670 pirq_needs_eoi = pirq_needs_eoi_flag;
1671
Stefano Stabellini0ec53ec2012-09-14 13:37:32 +00001672#ifdef CONFIG_X86
Mukesh Rathor27713742013-12-11 15:36:51 -05001673 if (xen_pv_domain()) {
1674 irq_ctx_init(smp_processor_id());
1675 if (xen_initial_domain())
1676 pci_xen_initial_domain();
1677 }
1678 if (xen_feature(XENFEAT_hvm_callback_vector))
Sheng Yang38e20b02010-05-14 12:40:51 +01001679 xen_callback_vector();
Mukesh Rathor27713742013-12-11 15:36:51 -05001680
1681 if (xen_hvm_domain()) {
Sheng Yang38e20b02010-05-14 12:40:51 +01001682 native_init_IRQ();
Stefano Stabellini3942b742010-06-24 17:50:18 +01001683 /* pci_xen_hvm_init must be called after native_init_IRQ so that
1684 * __acpi_register_gsi can point at the right function */
1685 pci_xen_hvm_init();
Sheng Yang38e20b02010-05-14 12:40:51 +01001686 } else {
Stefano Stabellini0ec53ec2012-09-14 13:37:32 +00001687 int rc;
Stefano Stabellini9846ff12012-01-30 16:21:48 +00001688 struct physdev_pirq_eoi_gmfn eoi_gmfn;
1689
Stefano Stabellini9846ff12012-01-30 16:21:48 +00001690 pirq_eoi_map = (void *)__get_free_page(GFP_KERNEL|__GFP_ZERO);
Julien Grall0df4f262015-08-07 17:34:37 +01001691 eoi_gmfn.gmfn = virt_to_gfn(pirq_eoi_map);
Stefano Stabellini9846ff12012-01-30 16:21:48 +00001692 rc = HYPERVISOR_physdev_op(PHYSDEVOP_pirq_eoi_gmfn_v2, &eoi_gmfn);
Mukesh Rathor27713742013-12-11 15:36:51 -05001693 /* TODO: No PVH support for PIRQ EOI */
Stefano Stabellini9846ff12012-01-30 16:21:48 +00001694 if (rc != 0) {
1695 free_page((unsigned long) pirq_eoi_map);
1696 pirq_eoi_map = NULL;
1697 } else
1698 pirq_needs_eoi = pirq_check_eoi_map;
Sheng Yang38e20b02010-05-14 12:40:51 +01001699 }
Stefano Stabellini0ec53ec2012-09-14 13:37:32 +00001700#endif
Jeremy Fitzhardingee46cdb62007-07-17 18:37:05 -07001701}