blob: 685fc72fc751fae3f898c9de4d194365521ea5a0 [file] [log] [blame]
Marc Zyngier1a89dd92013-01-21 19:36:12 -05001/*
2 * Copyright (C) 2012 ARM Ltd.
3 * Author: Marc Zyngier <marc.zyngier@arm.com>
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License version 2 as
7 * published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the Free Software
16 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17 */
18
Marc Zyngier01ac5e32013-01-21 19:36:16 -050019#include <linux/cpu.h>
Marc Zyngier1a89dd92013-01-21 19:36:12 -050020#include <linux/kvm.h>
21#include <linux/kvm_host.h>
22#include <linux/interrupt.h>
23#include <linux/io.h>
Marc Zyngier01ac5e32013-01-21 19:36:16 -050024#include <linux/of.h>
25#include <linux/of_address.h>
26#include <linux/of_irq.h>
27
28#include <linux/irqchip/arm-gic.h>
29
Marc Zyngier1a89dd92013-01-21 19:36:12 -050030#include <asm/kvm_emulate.h>
Marc Zyngier01ac5e32013-01-21 19:36:16 -050031#include <asm/kvm_arm.h>
32#include <asm/kvm_mmu.h>
Marc Zyngier1a89dd92013-01-21 19:36:12 -050033
Marc Zyngierb47ef922013-01-21 19:36:14 -050034/*
35 * How the whole thing works (courtesy of Christoffer Dall):
36 *
37 * - At any time, the dist->irq_pending_on_cpu is the oracle that knows if
38 * something is pending
39 * - VGIC pending interrupts are stored on the vgic.irq_state vgic
40 * bitmap (this bitmap is updated by both user land ioctls and guest
41 * mmio ops, and other in-kernel peripherals such as the
42 * arch. timers) and indicate the 'wire' state.
43 * - Every time the bitmap changes, the irq_pending_on_cpu oracle is
44 * recalculated
45 * - To calculate the oracle, we need info for each cpu from
46 * compute_pending_for_cpu, which considers:
47 * - PPI: dist->irq_state & dist->irq_enable
48 * - SPI: dist->irq_state & dist->irq_enable & dist->irq_spi_target
49 * - irq_spi_target is a 'formatted' version of the GICD_ICFGR
50 * registers, stored on each vcpu. We only keep one bit of
51 * information per interrupt, making sure that only one vcpu can
52 * accept the interrupt.
53 * - The same is true when injecting an interrupt, except that we only
54 * consider a single interrupt at a time. The irq_spi_cpu array
55 * contains the target CPU for each SPI.
56 *
57 * The handling of level interrupts adds some extra complexity. We
58 * need to track when the interrupt has been EOIed, so we can sample
59 * the 'line' again. This is achieved as such:
60 *
61 * - When a level interrupt is moved onto a vcpu, the corresponding
62 * bit in irq_active is set. As long as this bit is set, the line
63 * will be ignored for further interrupts. The interrupt is injected
64 * into the vcpu with the GICH_LR_EOI bit set (generate a
65 * maintenance interrupt on EOI).
66 * - When the interrupt is EOIed, the maintenance interrupt fires,
67 * and clears the corresponding bit in irq_active. This allow the
68 * interrupt line to be sampled again.
69 */
70
Christoffer Dall330690c2013-01-21 19:36:13 -050071#define VGIC_ADDR_UNDEF (-1)
72#define IS_VGIC_ADDR_UNDEF(_x) ((_x) == VGIC_ADDR_UNDEF)
73
Marc Zyngier01ac5e32013-01-21 19:36:16 -050074/* Physical address of vgic virtual cpu interface */
75static phys_addr_t vgic_vcpu_base;
76
77/* Virtual control interface base address */
78static void __iomem *vgic_vctrl_base;
79
80static struct device_node *vgic_node;
81
Marc Zyngier1a89dd92013-01-21 19:36:12 -050082#define ACCESS_READ_VALUE (1 << 0)
83#define ACCESS_READ_RAZ (0 << 0)
84#define ACCESS_READ_MASK(x) ((x) & (1 << 0))
85#define ACCESS_WRITE_IGNORED (0 << 1)
86#define ACCESS_WRITE_SETBIT (1 << 1)
87#define ACCESS_WRITE_CLEARBIT (2 << 1)
88#define ACCESS_WRITE_VALUE (3 << 1)
89#define ACCESS_WRITE_MASK(x) ((x) & (3 << 1))
90
Marc Zyngiera1fcb442013-01-21 19:36:15 -050091static void vgic_retire_disabled_irqs(struct kvm_vcpu *vcpu);
Marc Zyngierb47ef922013-01-21 19:36:14 -050092static void vgic_update_state(struct kvm *kvm);
Marc Zyngier5863c2c2013-01-21 19:36:15 -050093static void vgic_kick_vcpus(struct kvm *kvm);
Marc Zyngierb47ef922013-01-21 19:36:14 -050094static void vgic_dispatch_sgi(struct kvm_vcpu *vcpu, u32 reg);
Marc Zyngier01ac5e32013-01-21 19:36:16 -050095static u32 vgic_nr_lr;
96
97static unsigned int vgic_maint_irq;
Marc Zyngierb47ef922013-01-21 19:36:14 -050098
99static u32 *vgic_bitmap_get_reg(struct vgic_bitmap *x,
100 int cpuid, u32 offset)
101{
102 offset >>= 2;
103 if (!offset)
104 return x->percpu[cpuid].reg;
105 else
106 return x->shared.reg + offset - 1;
107}
108
109static int vgic_bitmap_get_irq_val(struct vgic_bitmap *x,
110 int cpuid, int irq)
111{
112 if (irq < VGIC_NR_PRIVATE_IRQS)
113 return test_bit(irq, x->percpu[cpuid].reg_ul);
114
115 return test_bit(irq - VGIC_NR_PRIVATE_IRQS, x->shared.reg_ul);
116}
117
118static void vgic_bitmap_set_irq_val(struct vgic_bitmap *x, int cpuid,
119 int irq, int val)
120{
121 unsigned long *reg;
122
123 if (irq < VGIC_NR_PRIVATE_IRQS) {
124 reg = x->percpu[cpuid].reg_ul;
125 } else {
126 reg = x->shared.reg_ul;
127 irq -= VGIC_NR_PRIVATE_IRQS;
128 }
129
130 if (val)
131 set_bit(irq, reg);
132 else
133 clear_bit(irq, reg);
134}
135
136static unsigned long *vgic_bitmap_get_cpu_map(struct vgic_bitmap *x, int cpuid)
137{
138 if (unlikely(cpuid >= VGIC_MAX_CPUS))
139 return NULL;
140 return x->percpu[cpuid].reg_ul;
141}
142
143static unsigned long *vgic_bitmap_get_shared_map(struct vgic_bitmap *x)
144{
145 return x->shared.reg_ul;
146}
147
148static u32 *vgic_bytemap_get_reg(struct vgic_bytemap *x, int cpuid, u32 offset)
149{
150 offset >>= 2;
151 BUG_ON(offset > (VGIC_NR_IRQS / 4));
Christoffer Dall8d989152013-08-29 11:08:24 +0100152 if (offset < 8)
Marc Zyngierb47ef922013-01-21 19:36:14 -0500153 return x->percpu[cpuid] + offset;
154 else
155 return x->shared + offset - 8;
156}
157
158#define VGIC_CFG_LEVEL 0
159#define VGIC_CFG_EDGE 1
160
161static bool vgic_irq_is_edge(struct kvm_vcpu *vcpu, int irq)
162{
163 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
164 int irq_val;
165
166 irq_val = vgic_bitmap_get_irq_val(&dist->irq_cfg, vcpu->vcpu_id, irq);
167 return irq_val == VGIC_CFG_EDGE;
168}
169
170static int vgic_irq_is_enabled(struct kvm_vcpu *vcpu, int irq)
171{
172 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
173
174 return vgic_bitmap_get_irq_val(&dist->irq_enabled, vcpu->vcpu_id, irq);
175}
176
Marc Zyngier9d949dc2013-01-21 19:36:14 -0500177static int vgic_irq_is_active(struct kvm_vcpu *vcpu, int irq)
178{
179 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
180
181 return vgic_bitmap_get_irq_val(&dist->irq_active, vcpu->vcpu_id, irq);
182}
183
184static void vgic_irq_set_active(struct kvm_vcpu *vcpu, int irq)
185{
186 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
187
188 vgic_bitmap_set_irq_val(&dist->irq_active, vcpu->vcpu_id, irq, 1);
189}
190
191static void vgic_irq_clear_active(struct kvm_vcpu *vcpu, int irq)
192{
193 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
194
195 vgic_bitmap_set_irq_val(&dist->irq_active, vcpu->vcpu_id, irq, 0);
196}
197
198static int vgic_dist_irq_is_pending(struct kvm_vcpu *vcpu, int irq)
199{
200 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
201
202 return vgic_bitmap_get_irq_val(&dist->irq_state, vcpu->vcpu_id, irq);
203}
204
Marc Zyngierb47ef922013-01-21 19:36:14 -0500205static void vgic_dist_irq_set(struct kvm_vcpu *vcpu, int irq)
206{
207 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
208
209 vgic_bitmap_set_irq_val(&dist->irq_state, vcpu->vcpu_id, irq, 1);
210}
211
212static void vgic_dist_irq_clear(struct kvm_vcpu *vcpu, int irq)
213{
214 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
215
216 vgic_bitmap_set_irq_val(&dist->irq_state, vcpu->vcpu_id, irq, 0);
217}
218
219static void vgic_cpu_irq_set(struct kvm_vcpu *vcpu, int irq)
220{
221 if (irq < VGIC_NR_PRIVATE_IRQS)
222 set_bit(irq, vcpu->arch.vgic_cpu.pending_percpu);
223 else
224 set_bit(irq - VGIC_NR_PRIVATE_IRQS,
225 vcpu->arch.vgic_cpu.pending_shared);
226}
227
228static void vgic_cpu_irq_clear(struct kvm_vcpu *vcpu, int irq)
229{
230 if (irq < VGIC_NR_PRIVATE_IRQS)
231 clear_bit(irq, vcpu->arch.vgic_cpu.pending_percpu);
232 else
233 clear_bit(irq - VGIC_NR_PRIVATE_IRQS,
234 vcpu->arch.vgic_cpu.pending_shared);
235}
236
Marc Zyngier1a89dd92013-01-21 19:36:12 -0500237static u32 mmio_data_read(struct kvm_exit_mmio *mmio, u32 mask)
238{
239 return *((u32 *)mmio->data) & mask;
240}
241
242static void mmio_data_write(struct kvm_exit_mmio *mmio, u32 mask, u32 value)
243{
244 *((u32 *)mmio->data) = value & mask;
245}
246
247/**
248 * vgic_reg_access - access vgic register
249 * @mmio: pointer to the data describing the mmio access
250 * @reg: pointer to the virtual backing of vgic distributor data
251 * @offset: least significant 2 bits used for word offset
252 * @mode: ACCESS_ mode (see defines above)
253 *
254 * Helper to make vgic register access easier using one of the access
255 * modes defined for vgic register access
256 * (read,raz,write-ignored,setbit,clearbit,write)
257 */
258static void vgic_reg_access(struct kvm_exit_mmio *mmio, u32 *reg,
259 phys_addr_t offset, int mode)
260{
261 int word_offset = (offset & 3) * 8;
262 u32 mask = (1UL << (mmio->len * 8)) - 1;
263 u32 regval;
264
265 /*
266 * Any alignment fault should have been delivered to the guest
267 * directly (ARM ARM B3.12.7 "Prioritization of aborts").
268 */
269
270 if (reg) {
271 regval = *reg;
272 } else {
273 BUG_ON(mode != (ACCESS_READ_RAZ | ACCESS_WRITE_IGNORED));
274 regval = 0;
275 }
276
277 if (mmio->is_write) {
278 u32 data = mmio_data_read(mmio, mask) << word_offset;
279 switch (ACCESS_WRITE_MASK(mode)) {
280 case ACCESS_WRITE_IGNORED:
281 return;
282
283 case ACCESS_WRITE_SETBIT:
284 regval |= data;
285 break;
286
287 case ACCESS_WRITE_CLEARBIT:
288 regval &= ~data;
289 break;
290
291 case ACCESS_WRITE_VALUE:
292 regval = (regval & ~(mask << word_offset)) | data;
293 break;
294 }
295 *reg = regval;
296 } else {
297 switch (ACCESS_READ_MASK(mode)) {
298 case ACCESS_READ_RAZ:
299 regval = 0;
300 /* fall through */
301
302 case ACCESS_READ_VALUE:
303 mmio_data_write(mmio, mask, regval >> word_offset);
304 }
305 }
306}
307
Marc Zyngierb47ef922013-01-21 19:36:14 -0500308static bool handle_mmio_misc(struct kvm_vcpu *vcpu,
309 struct kvm_exit_mmio *mmio, phys_addr_t offset)
310{
311 u32 reg;
312 u32 word_offset = offset & 3;
313
314 switch (offset & ~3) {
315 case 0: /* CTLR */
316 reg = vcpu->kvm->arch.vgic.enabled;
317 vgic_reg_access(mmio, &reg, word_offset,
318 ACCESS_READ_VALUE | ACCESS_WRITE_VALUE);
319 if (mmio->is_write) {
320 vcpu->kvm->arch.vgic.enabled = reg & 1;
321 vgic_update_state(vcpu->kvm);
322 return true;
323 }
324 break;
325
326 case 4: /* TYPER */
327 reg = (atomic_read(&vcpu->kvm->online_vcpus) - 1) << 5;
328 reg |= (VGIC_NR_IRQS >> 5) - 1;
329 vgic_reg_access(mmio, &reg, word_offset,
330 ACCESS_READ_VALUE | ACCESS_WRITE_IGNORED);
331 break;
332
333 case 8: /* IIDR */
334 reg = 0x4B00043B;
335 vgic_reg_access(mmio, &reg, word_offset,
336 ACCESS_READ_VALUE | ACCESS_WRITE_IGNORED);
337 break;
338 }
339
340 return false;
341}
342
343static bool handle_mmio_raz_wi(struct kvm_vcpu *vcpu,
344 struct kvm_exit_mmio *mmio, phys_addr_t offset)
345{
346 vgic_reg_access(mmio, NULL, offset,
347 ACCESS_READ_RAZ | ACCESS_WRITE_IGNORED);
348 return false;
349}
350
351static bool handle_mmio_set_enable_reg(struct kvm_vcpu *vcpu,
352 struct kvm_exit_mmio *mmio,
353 phys_addr_t offset)
354{
355 u32 *reg = vgic_bitmap_get_reg(&vcpu->kvm->arch.vgic.irq_enabled,
356 vcpu->vcpu_id, offset);
357 vgic_reg_access(mmio, reg, offset,
358 ACCESS_READ_VALUE | ACCESS_WRITE_SETBIT);
359 if (mmio->is_write) {
360 vgic_update_state(vcpu->kvm);
361 return true;
362 }
363
364 return false;
365}
366
367static bool handle_mmio_clear_enable_reg(struct kvm_vcpu *vcpu,
368 struct kvm_exit_mmio *mmio,
369 phys_addr_t offset)
370{
371 u32 *reg = vgic_bitmap_get_reg(&vcpu->kvm->arch.vgic.irq_enabled,
372 vcpu->vcpu_id, offset);
373 vgic_reg_access(mmio, reg, offset,
374 ACCESS_READ_VALUE | ACCESS_WRITE_CLEARBIT);
375 if (mmio->is_write) {
376 if (offset < 4) /* Force SGI enabled */
377 *reg |= 0xffff;
Marc Zyngiera1fcb442013-01-21 19:36:15 -0500378 vgic_retire_disabled_irqs(vcpu);
Marc Zyngierb47ef922013-01-21 19:36:14 -0500379 vgic_update_state(vcpu->kvm);
380 return true;
381 }
382
383 return false;
384}
385
386static bool handle_mmio_set_pending_reg(struct kvm_vcpu *vcpu,
387 struct kvm_exit_mmio *mmio,
388 phys_addr_t offset)
389{
390 u32 *reg = vgic_bitmap_get_reg(&vcpu->kvm->arch.vgic.irq_state,
391 vcpu->vcpu_id, offset);
392 vgic_reg_access(mmio, reg, offset,
393 ACCESS_READ_VALUE | ACCESS_WRITE_SETBIT);
394 if (mmio->is_write) {
395 vgic_update_state(vcpu->kvm);
396 return true;
397 }
398
399 return false;
400}
401
402static bool handle_mmio_clear_pending_reg(struct kvm_vcpu *vcpu,
403 struct kvm_exit_mmio *mmio,
404 phys_addr_t offset)
405{
406 u32 *reg = vgic_bitmap_get_reg(&vcpu->kvm->arch.vgic.irq_state,
407 vcpu->vcpu_id, offset);
408 vgic_reg_access(mmio, reg, offset,
409 ACCESS_READ_VALUE | ACCESS_WRITE_CLEARBIT);
410 if (mmio->is_write) {
411 vgic_update_state(vcpu->kvm);
412 return true;
413 }
414
415 return false;
416}
417
418static bool handle_mmio_priority_reg(struct kvm_vcpu *vcpu,
419 struct kvm_exit_mmio *mmio,
420 phys_addr_t offset)
421{
422 u32 *reg = vgic_bytemap_get_reg(&vcpu->kvm->arch.vgic.irq_priority,
423 vcpu->vcpu_id, offset);
424 vgic_reg_access(mmio, reg, offset,
425 ACCESS_READ_VALUE | ACCESS_WRITE_VALUE);
426 return false;
427}
428
429#define GICD_ITARGETSR_SIZE 32
430#define GICD_CPUTARGETS_BITS 8
431#define GICD_IRQS_PER_ITARGETSR (GICD_ITARGETSR_SIZE / GICD_CPUTARGETS_BITS)
432static u32 vgic_get_target_reg(struct kvm *kvm, int irq)
433{
434 struct vgic_dist *dist = &kvm->arch.vgic;
Marc Zyngier986af8e2013-08-29 11:08:22 +0100435 int i;
Marc Zyngierb47ef922013-01-21 19:36:14 -0500436 u32 val = 0;
437
438 irq -= VGIC_NR_PRIVATE_IRQS;
439
Marc Zyngier986af8e2013-08-29 11:08:22 +0100440 for (i = 0; i < GICD_IRQS_PER_ITARGETSR; i++)
441 val |= 1 << (dist->irq_spi_cpu[irq + i] + i * 8);
Marc Zyngierb47ef922013-01-21 19:36:14 -0500442
443 return val;
444}
445
446static void vgic_set_target_reg(struct kvm *kvm, u32 val, int irq)
447{
448 struct vgic_dist *dist = &kvm->arch.vgic;
449 struct kvm_vcpu *vcpu;
450 int i, c;
451 unsigned long *bmap;
452 u32 target;
453
454 irq -= VGIC_NR_PRIVATE_IRQS;
455
456 /*
457 * Pick the LSB in each byte. This ensures we target exactly
458 * one vcpu per IRQ. If the byte is null, assume we target
459 * CPU0.
460 */
461 for (i = 0; i < GICD_IRQS_PER_ITARGETSR; i++) {
462 int shift = i * GICD_CPUTARGETS_BITS;
463 target = ffs((val >> shift) & 0xffU);
464 target = target ? (target - 1) : 0;
465 dist->irq_spi_cpu[irq + i] = target;
466 kvm_for_each_vcpu(c, vcpu, kvm) {
467 bmap = vgic_bitmap_get_shared_map(&dist->irq_spi_target[c]);
468 if (c == target)
469 set_bit(irq + i, bmap);
470 else
471 clear_bit(irq + i, bmap);
472 }
473 }
474}
475
476static bool handle_mmio_target_reg(struct kvm_vcpu *vcpu,
477 struct kvm_exit_mmio *mmio,
478 phys_addr_t offset)
479{
480 u32 reg;
481
482 /* We treat the banked interrupts targets as read-only */
483 if (offset < 32) {
484 u32 roreg = 1 << vcpu->vcpu_id;
485 roreg |= roreg << 8;
486 roreg |= roreg << 16;
487
488 vgic_reg_access(mmio, &roreg, offset,
489 ACCESS_READ_VALUE | ACCESS_WRITE_IGNORED);
490 return false;
491 }
492
493 reg = vgic_get_target_reg(vcpu->kvm, offset & ~3U);
494 vgic_reg_access(mmio, &reg, offset,
495 ACCESS_READ_VALUE | ACCESS_WRITE_VALUE);
496 if (mmio->is_write) {
497 vgic_set_target_reg(vcpu->kvm, reg, offset & ~3U);
498 vgic_update_state(vcpu->kvm);
499 return true;
500 }
501
502 return false;
503}
504
505static u32 vgic_cfg_expand(u16 val)
506{
507 u32 res = 0;
508 int i;
509
510 /*
511 * Turn a 16bit value like abcd...mnop into a 32bit word
512 * a0b0c0d0...m0n0o0p0, which is what the HW cfg register is.
513 */
514 for (i = 0; i < 16; i++)
515 res |= ((val >> i) & VGIC_CFG_EDGE) << (2 * i + 1);
516
517 return res;
518}
519
520static u16 vgic_cfg_compress(u32 val)
521{
522 u16 res = 0;
523 int i;
524
525 /*
526 * Turn a 32bit word a0b0c0d0...m0n0o0p0 into 16bit value like
527 * abcd...mnop which is what we really care about.
528 */
529 for (i = 0; i < 16; i++)
530 res |= ((val >> (i * 2 + 1)) & VGIC_CFG_EDGE) << i;
531
532 return res;
533}
534
535/*
536 * The distributor uses 2 bits per IRQ for the CFG register, but the
537 * LSB is always 0. As such, we only keep the upper bit, and use the
538 * two above functions to compress/expand the bits
539 */
540static bool handle_mmio_cfg_reg(struct kvm_vcpu *vcpu,
541 struct kvm_exit_mmio *mmio, phys_addr_t offset)
542{
543 u32 val;
Marc Zyngier6545eae2013-08-29 11:08:23 +0100544 u32 *reg;
545
546 offset >>= 1;
547 reg = vgic_bitmap_get_reg(&vcpu->kvm->arch.vgic.irq_cfg,
548 vcpu->vcpu_id, offset);
549
Marc Zyngierb47ef922013-01-21 19:36:14 -0500550 if (offset & 2)
551 val = *reg >> 16;
552 else
553 val = *reg & 0xffff;
554
555 val = vgic_cfg_expand(val);
556 vgic_reg_access(mmio, &val, offset,
557 ACCESS_READ_VALUE | ACCESS_WRITE_VALUE);
558 if (mmio->is_write) {
559 if (offset < 4) {
560 *reg = ~0U; /* Force PPIs/SGIs to 1 */
561 return false;
562 }
563
564 val = vgic_cfg_compress(val);
565 if (offset & 2) {
566 *reg &= 0xffff;
567 *reg |= val << 16;
568 } else {
569 *reg &= 0xffff << 16;
570 *reg |= val;
571 }
572 }
573
574 return false;
575}
576
577static bool handle_mmio_sgi_reg(struct kvm_vcpu *vcpu,
578 struct kvm_exit_mmio *mmio, phys_addr_t offset)
579{
580 u32 reg;
581 vgic_reg_access(mmio, &reg, offset,
582 ACCESS_READ_RAZ | ACCESS_WRITE_VALUE);
583 if (mmio->is_write) {
584 vgic_dispatch_sgi(vcpu, reg);
585 vgic_update_state(vcpu->kvm);
586 return true;
587 }
588
589 return false;
590}
591
Marc Zyngier1a89dd92013-01-21 19:36:12 -0500592/*
593 * I would have liked to use the kvm_bus_io_*() API instead, but it
594 * cannot cope with banked registers (only the VM pointer is passed
595 * around, and we need the vcpu). One of these days, someone please
596 * fix it!
597 */
598struct mmio_range {
599 phys_addr_t base;
600 unsigned long len;
601 bool (*handle_mmio)(struct kvm_vcpu *vcpu, struct kvm_exit_mmio *mmio,
602 phys_addr_t offset);
603};
604
605static const struct mmio_range vgic_ranges[] = {
Marc Zyngierb47ef922013-01-21 19:36:14 -0500606 {
607 .base = GIC_DIST_CTRL,
608 .len = 12,
609 .handle_mmio = handle_mmio_misc,
610 },
611 {
612 .base = GIC_DIST_IGROUP,
613 .len = VGIC_NR_IRQS / 8,
614 .handle_mmio = handle_mmio_raz_wi,
615 },
616 {
617 .base = GIC_DIST_ENABLE_SET,
618 .len = VGIC_NR_IRQS / 8,
619 .handle_mmio = handle_mmio_set_enable_reg,
620 },
621 {
622 .base = GIC_DIST_ENABLE_CLEAR,
623 .len = VGIC_NR_IRQS / 8,
624 .handle_mmio = handle_mmio_clear_enable_reg,
625 },
626 {
627 .base = GIC_DIST_PENDING_SET,
628 .len = VGIC_NR_IRQS / 8,
629 .handle_mmio = handle_mmio_set_pending_reg,
630 },
631 {
632 .base = GIC_DIST_PENDING_CLEAR,
633 .len = VGIC_NR_IRQS / 8,
634 .handle_mmio = handle_mmio_clear_pending_reg,
635 },
636 {
637 .base = GIC_DIST_ACTIVE_SET,
638 .len = VGIC_NR_IRQS / 8,
639 .handle_mmio = handle_mmio_raz_wi,
640 },
641 {
642 .base = GIC_DIST_ACTIVE_CLEAR,
643 .len = VGIC_NR_IRQS / 8,
644 .handle_mmio = handle_mmio_raz_wi,
645 },
646 {
647 .base = GIC_DIST_PRI,
648 .len = VGIC_NR_IRQS,
649 .handle_mmio = handle_mmio_priority_reg,
650 },
651 {
652 .base = GIC_DIST_TARGET,
653 .len = VGIC_NR_IRQS,
654 .handle_mmio = handle_mmio_target_reg,
655 },
656 {
657 .base = GIC_DIST_CONFIG,
658 .len = VGIC_NR_IRQS / 4,
659 .handle_mmio = handle_mmio_cfg_reg,
660 },
661 {
662 .base = GIC_DIST_SOFTINT,
663 .len = 4,
664 .handle_mmio = handle_mmio_sgi_reg,
665 },
Marc Zyngier1a89dd92013-01-21 19:36:12 -0500666 {}
667};
668
669static const
670struct mmio_range *find_matching_range(const struct mmio_range *ranges,
671 struct kvm_exit_mmio *mmio,
672 phys_addr_t base)
673{
674 const struct mmio_range *r = ranges;
675 phys_addr_t addr = mmio->phys_addr - base;
676
677 while (r->len) {
678 if (addr >= r->base &&
679 (addr + mmio->len) <= (r->base + r->len))
680 return r;
681 r++;
682 }
683
684 return NULL;
685}
686
687/**
688 * vgic_handle_mmio - handle an in-kernel MMIO access
689 * @vcpu: pointer to the vcpu performing the access
690 * @run: pointer to the kvm_run structure
691 * @mmio: pointer to the data describing the access
692 *
693 * returns true if the MMIO access has been performed in kernel space,
694 * and false if it needs to be emulated in user space.
695 */
696bool vgic_handle_mmio(struct kvm_vcpu *vcpu, struct kvm_run *run,
697 struct kvm_exit_mmio *mmio)
698{
Marc Zyngierb47ef922013-01-21 19:36:14 -0500699 const struct mmio_range *range;
700 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
701 unsigned long base = dist->vgic_dist_base;
702 bool updated_state;
703 unsigned long offset;
704
705 if (!irqchip_in_kernel(vcpu->kvm) ||
706 mmio->phys_addr < base ||
707 (mmio->phys_addr + mmio->len) > (base + KVM_VGIC_V2_DIST_SIZE))
708 return false;
709
710 /* We don't support ldrd / strd or ldm / stm to the emulated vgic */
711 if (mmio->len > 4) {
712 kvm_inject_dabt(vcpu, mmio->phys_addr);
713 return true;
714 }
715
716 range = find_matching_range(vgic_ranges, mmio, base);
717 if (unlikely(!range || !range->handle_mmio)) {
718 pr_warn("Unhandled access %d %08llx %d\n",
719 mmio->is_write, mmio->phys_addr, mmio->len);
720 return false;
721 }
722
723 spin_lock(&vcpu->kvm->arch.vgic.lock);
724 offset = mmio->phys_addr - range->base - base;
725 updated_state = range->handle_mmio(vcpu, mmio, offset);
726 spin_unlock(&vcpu->kvm->arch.vgic.lock);
727 kvm_prepare_mmio(run, mmio);
728 kvm_handle_mmio_return(vcpu, run);
729
Marc Zyngier5863c2c2013-01-21 19:36:15 -0500730 if (updated_state)
731 vgic_kick_vcpus(vcpu->kvm);
732
Marc Zyngierb47ef922013-01-21 19:36:14 -0500733 return true;
734}
735
736static void vgic_dispatch_sgi(struct kvm_vcpu *vcpu, u32 reg)
737{
738 struct kvm *kvm = vcpu->kvm;
739 struct vgic_dist *dist = &kvm->arch.vgic;
740 int nrcpus = atomic_read(&kvm->online_vcpus);
741 u8 target_cpus;
742 int sgi, mode, c, vcpu_id;
743
744 vcpu_id = vcpu->vcpu_id;
745
746 sgi = reg & 0xf;
747 target_cpus = (reg >> 16) & 0xff;
748 mode = (reg >> 24) & 3;
749
750 switch (mode) {
751 case 0:
752 if (!target_cpus)
753 return;
754
755 case 1:
756 target_cpus = ((1 << nrcpus) - 1) & ~(1 << vcpu_id) & 0xff;
757 break;
758
759 case 2:
760 target_cpus = 1 << vcpu_id;
761 break;
762 }
763
764 kvm_for_each_vcpu(c, vcpu, kvm) {
765 if (target_cpus & 1) {
766 /* Flag the SGI as pending */
767 vgic_dist_irq_set(vcpu, sgi);
768 dist->irq_sgi_sources[c][sgi] |= 1 << vcpu_id;
769 kvm_debug("SGI%d from CPU%d to CPU%d\n", sgi, vcpu_id, c);
770 }
771
772 target_cpus >>= 1;
773 }
774}
775
776static int compute_pending_for_cpu(struct kvm_vcpu *vcpu)
777{
Marc Zyngier9d949dc2013-01-21 19:36:14 -0500778 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
779 unsigned long *pending, *enabled, *pend_percpu, *pend_shared;
780 unsigned long pending_private, pending_shared;
781 int vcpu_id;
782
783 vcpu_id = vcpu->vcpu_id;
784 pend_percpu = vcpu->arch.vgic_cpu.pending_percpu;
785 pend_shared = vcpu->arch.vgic_cpu.pending_shared;
786
787 pending = vgic_bitmap_get_cpu_map(&dist->irq_state, vcpu_id);
788 enabled = vgic_bitmap_get_cpu_map(&dist->irq_enabled, vcpu_id);
789 bitmap_and(pend_percpu, pending, enabled, VGIC_NR_PRIVATE_IRQS);
790
791 pending = vgic_bitmap_get_shared_map(&dist->irq_state);
792 enabled = vgic_bitmap_get_shared_map(&dist->irq_enabled);
793 bitmap_and(pend_shared, pending, enabled, VGIC_NR_SHARED_IRQS);
794 bitmap_and(pend_shared, pend_shared,
795 vgic_bitmap_get_shared_map(&dist->irq_spi_target[vcpu_id]),
796 VGIC_NR_SHARED_IRQS);
797
798 pending_private = find_first_bit(pend_percpu, VGIC_NR_PRIVATE_IRQS);
799 pending_shared = find_first_bit(pend_shared, VGIC_NR_SHARED_IRQS);
800 return (pending_private < VGIC_NR_PRIVATE_IRQS ||
801 pending_shared < VGIC_NR_SHARED_IRQS);
Marc Zyngierb47ef922013-01-21 19:36:14 -0500802}
803
804/*
805 * Update the interrupt state and determine which CPUs have pending
806 * interrupts. Must be called with distributor lock held.
807 */
808static void vgic_update_state(struct kvm *kvm)
809{
810 struct vgic_dist *dist = &kvm->arch.vgic;
811 struct kvm_vcpu *vcpu;
812 int c;
813
814 if (!dist->enabled) {
815 set_bit(0, &dist->irq_pending_on_cpu);
816 return;
817 }
818
819 kvm_for_each_vcpu(c, vcpu, kvm) {
820 if (compute_pending_for_cpu(vcpu)) {
821 pr_debug("CPU%d has pending interrupts\n", c);
822 set_bit(c, &dist->irq_pending_on_cpu);
823 }
824 }
Marc Zyngier1a89dd92013-01-21 19:36:12 -0500825}
Christoffer Dall330690c2013-01-21 19:36:13 -0500826
Marc Zyngier9d949dc2013-01-21 19:36:14 -0500827#define LR_CPUID(lr) \
828 (((lr) & GICH_LR_PHYSID_CPUID) >> GICH_LR_PHYSID_CPUID_SHIFT)
829#define MK_LR_PEND(src, irq) \
830 (GICH_LR_PENDING_BIT | ((src) << GICH_LR_PHYSID_CPUID_SHIFT) | (irq))
Marc Zyngiera1fcb442013-01-21 19:36:15 -0500831
832/*
833 * An interrupt may have been disabled after being made pending on the
834 * CPU interface (the classic case is a timer running while we're
835 * rebooting the guest - the interrupt would kick as soon as the CPU
836 * interface gets enabled, with deadly consequences).
837 *
838 * The solution is to examine already active LRs, and check the
839 * interrupt is still enabled. If not, just retire it.
840 */
841static void vgic_retire_disabled_irqs(struct kvm_vcpu *vcpu)
842{
843 struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
844 int lr;
845
846 for_each_set_bit(lr, vgic_cpu->lr_used, vgic_cpu->nr_lr) {
847 int irq = vgic_cpu->vgic_lr[lr] & GICH_LR_VIRTUALID;
848
849 if (!vgic_irq_is_enabled(vcpu, irq)) {
850 vgic_cpu->vgic_irq_lr_map[irq] = LR_EMPTY;
851 clear_bit(lr, vgic_cpu->lr_used);
852 vgic_cpu->vgic_lr[lr] &= ~GICH_LR_STATE;
853 if (vgic_irq_is_active(vcpu, irq))
854 vgic_irq_clear_active(vcpu, irq);
855 }
856 }
857}
858
Marc Zyngier9d949dc2013-01-21 19:36:14 -0500859/*
860 * Queue an interrupt to a CPU virtual interface. Return true on success,
861 * or false if it wasn't possible to queue it.
862 */
863static bool vgic_queue_irq(struct kvm_vcpu *vcpu, u8 sgi_source_id, int irq)
864{
865 struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
866 int lr;
867
868 /* Sanitize the input... */
869 BUG_ON(sgi_source_id & ~7);
870 BUG_ON(sgi_source_id && irq >= VGIC_NR_SGIS);
871 BUG_ON(irq >= VGIC_NR_IRQS);
872
873 kvm_debug("Queue IRQ%d\n", irq);
874
875 lr = vgic_cpu->vgic_irq_lr_map[irq];
876
877 /* Do we have an active interrupt for the same CPUID? */
878 if (lr != LR_EMPTY &&
879 (LR_CPUID(vgic_cpu->vgic_lr[lr]) == sgi_source_id)) {
880 kvm_debug("LR%d piggyback for IRQ%d %x\n",
881 lr, irq, vgic_cpu->vgic_lr[lr]);
882 BUG_ON(!test_bit(lr, vgic_cpu->lr_used));
883 vgic_cpu->vgic_lr[lr] |= GICH_LR_PENDING_BIT;
Marc Zyngier75da01e2013-01-31 11:25:52 +0000884 return true;
Marc Zyngier9d949dc2013-01-21 19:36:14 -0500885 }
886
887 /* Try to use another LR for this interrupt */
888 lr = find_first_zero_bit((unsigned long *)vgic_cpu->lr_used,
889 vgic_cpu->nr_lr);
890 if (lr >= vgic_cpu->nr_lr)
891 return false;
892
893 kvm_debug("LR%d allocated for IRQ%d %x\n", lr, irq, sgi_source_id);
894 vgic_cpu->vgic_lr[lr] = MK_LR_PEND(sgi_source_id, irq);
895 vgic_cpu->vgic_irq_lr_map[irq] = lr;
896 set_bit(lr, vgic_cpu->lr_used);
897
Marc Zyngier9d949dc2013-01-21 19:36:14 -0500898 if (!vgic_irq_is_edge(vcpu, irq))
899 vgic_cpu->vgic_lr[lr] |= GICH_LR_EOI;
900
901 return true;
902}
903
904static bool vgic_queue_sgi(struct kvm_vcpu *vcpu, int irq)
905{
906 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
907 unsigned long sources;
908 int vcpu_id = vcpu->vcpu_id;
909 int c;
910
911 sources = dist->irq_sgi_sources[vcpu_id][irq];
912
913 for_each_set_bit(c, &sources, VGIC_MAX_CPUS) {
914 if (vgic_queue_irq(vcpu, c, irq))
915 clear_bit(c, &sources);
916 }
917
918 dist->irq_sgi_sources[vcpu_id][irq] = sources;
919
920 /*
921 * If the sources bitmap has been cleared it means that we
922 * could queue all the SGIs onto link registers (see the
923 * clear_bit above), and therefore we are done with them in
924 * our emulated gic and can get rid of them.
925 */
926 if (!sources) {
927 vgic_dist_irq_clear(vcpu, irq);
928 vgic_cpu_irq_clear(vcpu, irq);
929 return true;
930 }
931
932 return false;
933}
934
935static bool vgic_queue_hwirq(struct kvm_vcpu *vcpu, int irq)
936{
937 if (vgic_irq_is_active(vcpu, irq))
938 return true; /* level interrupt, already queued */
939
940 if (vgic_queue_irq(vcpu, 0, irq)) {
941 if (vgic_irq_is_edge(vcpu, irq)) {
942 vgic_dist_irq_clear(vcpu, irq);
943 vgic_cpu_irq_clear(vcpu, irq);
944 } else {
945 vgic_irq_set_active(vcpu, irq);
946 }
947
948 return true;
949 }
950
951 return false;
952}
953
954/*
955 * Fill the list registers with pending interrupts before running the
956 * guest.
957 */
958static void __kvm_vgic_flush_hwstate(struct kvm_vcpu *vcpu)
959{
960 struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
961 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
962 int i, vcpu_id;
963 int overflow = 0;
964
965 vcpu_id = vcpu->vcpu_id;
966
967 /*
968 * We may not have any pending interrupt, or the interrupts
969 * may have been serviced from another vcpu. In all cases,
970 * move along.
971 */
972 if (!kvm_vgic_vcpu_pending_irq(vcpu)) {
973 pr_debug("CPU%d has no pending interrupt\n", vcpu_id);
974 goto epilog;
975 }
976
977 /* SGIs */
978 for_each_set_bit(i, vgic_cpu->pending_percpu, VGIC_NR_SGIS) {
979 if (!vgic_queue_sgi(vcpu, i))
980 overflow = 1;
981 }
982
983 /* PPIs */
984 for_each_set_bit_from(i, vgic_cpu->pending_percpu, VGIC_NR_PRIVATE_IRQS) {
985 if (!vgic_queue_hwirq(vcpu, i))
986 overflow = 1;
987 }
988
989 /* SPIs */
990 for_each_set_bit(i, vgic_cpu->pending_shared, VGIC_NR_SHARED_IRQS) {
991 if (!vgic_queue_hwirq(vcpu, i + VGIC_NR_PRIVATE_IRQS))
992 overflow = 1;
993 }
994
995epilog:
996 if (overflow) {
997 vgic_cpu->vgic_hcr |= GICH_HCR_UIE;
998 } else {
999 vgic_cpu->vgic_hcr &= ~GICH_HCR_UIE;
1000 /*
1001 * We're about to run this VCPU, and we've consumed
1002 * everything the distributor had in store for
1003 * us. Claim we don't have anything pending. We'll
1004 * adjust that if needed while exiting.
1005 */
1006 clear_bit(vcpu_id, &dist->irq_pending_on_cpu);
1007 }
1008}
1009
1010static bool vgic_process_maintenance(struct kvm_vcpu *vcpu)
1011{
1012 struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
1013 bool level_pending = false;
1014
1015 kvm_debug("MISR = %08x\n", vgic_cpu->vgic_misr);
1016
Marc Zyngier9d949dc2013-01-21 19:36:14 -05001017 if (vgic_cpu->vgic_misr & GICH_MISR_EOI) {
1018 /*
1019 * Some level interrupts have been EOIed. Clear their
1020 * active bit.
1021 */
1022 int lr, irq;
1023
1024 for_each_set_bit(lr, (unsigned long *)vgic_cpu->vgic_eisr,
1025 vgic_cpu->nr_lr) {
1026 irq = vgic_cpu->vgic_lr[lr] & GICH_LR_VIRTUALID;
1027
1028 vgic_irq_clear_active(vcpu, irq);
1029 vgic_cpu->vgic_lr[lr] &= ~GICH_LR_EOI;
1030
1031 /* Any additional pending interrupt? */
1032 if (vgic_dist_irq_is_pending(vcpu, irq)) {
1033 vgic_cpu_irq_set(vcpu, irq);
1034 level_pending = true;
1035 } else {
1036 vgic_cpu_irq_clear(vcpu, irq);
1037 }
Marc Zyngier75da01e2013-01-31 11:25:52 +00001038
1039 /*
1040 * Despite being EOIed, the LR may not have
1041 * been marked as empty.
1042 */
1043 set_bit(lr, (unsigned long *)vgic_cpu->vgic_elrsr);
1044 vgic_cpu->vgic_lr[lr] &= ~GICH_LR_ACTIVE_BIT;
Marc Zyngier9d949dc2013-01-21 19:36:14 -05001045 }
1046 }
1047
1048 if (vgic_cpu->vgic_misr & GICH_MISR_U)
1049 vgic_cpu->vgic_hcr &= ~GICH_HCR_UIE;
1050
1051 return level_pending;
1052}
1053
1054/*
Marc Zyngier33c83cb2013-02-01 18:28:30 +00001055 * Sync back the VGIC state after a guest run. The distributor lock is
1056 * needed so we don't get preempted in the middle of the state processing.
Marc Zyngier9d949dc2013-01-21 19:36:14 -05001057 */
1058static void __kvm_vgic_sync_hwstate(struct kvm_vcpu *vcpu)
1059{
1060 struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
1061 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
1062 int lr, pending;
1063 bool level_pending;
1064
1065 level_pending = vgic_process_maintenance(vcpu);
1066
1067 /* Clear mappings for empty LRs */
1068 for_each_set_bit(lr, (unsigned long *)vgic_cpu->vgic_elrsr,
1069 vgic_cpu->nr_lr) {
1070 int irq;
1071
1072 if (!test_and_clear_bit(lr, vgic_cpu->lr_used))
1073 continue;
1074
1075 irq = vgic_cpu->vgic_lr[lr] & GICH_LR_VIRTUALID;
1076
1077 BUG_ON(irq >= VGIC_NR_IRQS);
1078 vgic_cpu->vgic_irq_lr_map[irq] = LR_EMPTY;
1079 }
1080
1081 /* Check if we still have something up our sleeve... */
1082 pending = find_first_zero_bit((unsigned long *)vgic_cpu->vgic_elrsr,
1083 vgic_cpu->nr_lr);
1084 if (level_pending || pending < vgic_cpu->nr_lr)
1085 set_bit(vcpu->vcpu_id, &dist->irq_pending_on_cpu);
1086}
1087
1088void kvm_vgic_flush_hwstate(struct kvm_vcpu *vcpu)
1089{
1090 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
1091
1092 if (!irqchip_in_kernel(vcpu->kvm))
1093 return;
1094
1095 spin_lock(&dist->lock);
1096 __kvm_vgic_flush_hwstate(vcpu);
1097 spin_unlock(&dist->lock);
1098}
1099
1100void kvm_vgic_sync_hwstate(struct kvm_vcpu *vcpu)
1101{
Marc Zyngier33c83cb2013-02-01 18:28:30 +00001102 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
1103
Marc Zyngier9d949dc2013-01-21 19:36:14 -05001104 if (!irqchip_in_kernel(vcpu->kvm))
1105 return;
1106
Marc Zyngier33c83cb2013-02-01 18:28:30 +00001107 spin_lock(&dist->lock);
Marc Zyngier9d949dc2013-01-21 19:36:14 -05001108 __kvm_vgic_sync_hwstate(vcpu);
Marc Zyngier33c83cb2013-02-01 18:28:30 +00001109 spin_unlock(&dist->lock);
Marc Zyngier9d949dc2013-01-21 19:36:14 -05001110}
1111
1112int kvm_vgic_vcpu_pending_irq(struct kvm_vcpu *vcpu)
1113{
1114 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
1115
1116 if (!irqchip_in_kernel(vcpu->kvm))
1117 return 0;
1118
1119 return test_bit(vcpu->vcpu_id, &dist->irq_pending_on_cpu);
1120}
1121
Marc Zyngier5863c2c2013-01-21 19:36:15 -05001122static void vgic_kick_vcpus(struct kvm *kvm)
1123{
1124 struct kvm_vcpu *vcpu;
1125 int c;
1126
1127 /*
1128 * We've injected an interrupt, time to find out who deserves
1129 * a good kick...
1130 */
1131 kvm_for_each_vcpu(c, vcpu, kvm) {
1132 if (kvm_vgic_vcpu_pending_irq(vcpu))
1133 kvm_vcpu_kick(vcpu);
1134 }
1135}
1136
1137static int vgic_validate_injection(struct kvm_vcpu *vcpu, int irq, int level)
1138{
1139 int is_edge = vgic_irq_is_edge(vcpu, irq);
1140 int state = vgic_dist_irq_is_pending(vcpu, irq);
1141
1142 /*
1143 * Only inject an interrupt if:
1144 * - edge triggered and we have a rising edge
1145 * - level triggered and we change level
1146 */
1147 if (is_edge)
1148 return level > state;
1149 else
1150 return level != state;
1151}
1152
1153static bool vgic_update_irq_state(struct kvm *kvm, int cpuid,
1154 unsigned int irq_num, bool level)
1155{
1156 struct vgic_dist *dist = &kvm->arch.vgic;
1157 struct kvm_vcpu *vcpu;
1158 int is_edge, is_level;
1159 int enabled;
1160 bool ret = true;
1161
1162 spin_lock(&dist->lock);
1163
1164 vcpu = kvm_get_vcpu(kvm, cpuid);
1165 is_edge = vgic_irq_is_edge(vcpu, irq_num);
1166 is_level = !is_edge;
1167
1168 if (!vgic_validate_injection(vcpu, irq_num, level)) {
1169 ret = false;
1170 goto out;
1171 }
1172
1173 if (irq_num >= VGIC_NR_PRIVATE_IRQS) {
1174 cpuid = dist->irq_spi_cpu[irq_num - VGIC_NR_PRIVATE_IRQS];
1175 vcpu = kvm_get_vcpu(kvm, cpuid);
1176 }
1177
1178 kvm_debug("Inject IRQ%d level %d CPU%d\n", irq_num, level, cpuid);
1179
1180 if (level)
1181 vgic_dist_irq_set(vcpu, irq_num);
1182 else
1183 vgic_dist_irq_clear(vcpu, irq_num);
1184
1185 enabled = vgic_irq_is_enabled(vcpu, irq_num);
1186
1187 if (!enabled) {
1188 ret = false;
1189 goto out;
1190 }
1191
1192 if (is_level && vgic_irq_is_active(vcpu, irq_num)) {
1193 /*
1194 * Level interrupt in progress, will be picked up
1195 * when EOId.
1196 */
1197 ret = false;
1198 goto out;
1199 }
1200
1201 if (level) {
1202 vgic_cpu_irq_set(vcpu, irq_num);
1203 set_bit(cpuid, &dist->irq_pending_on_cpu);
1204 }
1205
1206out:
1207 spin_unlock(&dist->lock);
1208
1209 return ret;
1210}
1211
1212/**
1213 * kvm_vgic_inject_irq - Inject an IRQ from a device to the vgic
1214 * @kvm: The VM structure pointer
1215 * @cpuid: The CPU for PPIs
1216 * @irq_num: The IRQ number that is assigned to the device
1217 * @level: Edge-triggered: true: to trigger the interrupt
1218 * false: to ignore the call
1219 * Level-sensitive true: activates an interrupt
1220 * false: deactivates an interrupt
1221 *
1222 * The GIC is not concerned with devices being active-LOW or active-HIGH for
1223 * level-sensitive interrupts. You can think of the level parameter as 1
1224 * being HIGH and 0 being LOW and all devices being active-HIGH.
1225 */
1226int kvm_vgic_inject_irq(struct kvm *kvm, int cpuid, unsigned int irq_num,
1227 bool level)
1228{
1229 if (vgic_update_irq_state(kvm, cpuid, irq_num, level))
1230 vgic_kick_vcpus(kvm);
1231
1232 return 0;
1233}
1234
Marc Zyngier01ac5e32013-01-21 19:36:16 -05001235static irqreturn_t vgic_maintenance_handler(int irq, void *data)
1236{
1237 /*
1238 * We cannot rely on the vgic maintenance interrupt to be
1239 * delivered synchronously. This means we can only use it to
1240 * exit the VM, and we perform the handling of EOIed
1241 * interrupts on the exit path (see vgic_process_maintenance).
1242 */
1243 return IRQ_HANDLED;
1244}
1245
1246int kvm_vgic_vcpu_init(struct kvm_vcpu *vcpu)
1247{
1248 struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
1249 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
1250 int i;
1251
1252 if (!irqchip_in_kernel(vcpu->kvm))
1253 return 0;
1254
1255 if (vcpu->vcpu_id >= VGIC_MAX_CPUS)
1256 return -EBUSY;
1257
1258 for (i = 0; i < VGIC_NR_IRQS; i++) {
1259 if (i < VGIC_NR_PPIS)
1260 vgic_bitmap_set_irq_val(&dist->irq_enabled,
1261 vcpu->vcpu_id, i, 1);
1262 if (i < VGIC_NR_PRIVATE_IRQS)
1263 vgic_bitmap_set_irq_val(&dist->irq_cfg,
1264 vcpu->vcpu_id, i, VGIC_CFG_EDGE);
1265
1266 vgic_cpu->vgic_irq_lr_map[i] = LR_EMPTY;
1267 }
1268
1269 /*
1270 * By forcing VMCR to zero, the GIC will restore the binary
1271 * points to their reset values. Anything else resets to zero
1272 * anyway.
1273 */
1274 vgic_cpu->vgic_vmcr = 0;
1275
1276 vgic_cpu->nr_lr = vgic_nr_lr;
1277 vgic_cpu->vgic_hcr = GICH_HCR_EN; /* Get the show on the road... */
1278
1279 return 0;
1280}
1281
1282static void vgic_init_maintenance_interrupt(void *info)
1283{
1284 enable_percpu_irq(vgic_maint_irq, 0);
1285}
1286
1287static int vgic_cpu_notify(struct notifier_block *self,
1288 unsigned long action, void *cpu)
1289{
1290 switch (action) {
1291 case CPU_STARTING:
1292 case CPU_STARTING_FROZEN:
1293 vgic_init_maintenance_interrupt(NULL);
1294 break;
1295 case CPU_DYING:
1296 case CPU_DYING_FROZEN:
1297 disable_percpu_irq(vgic_maint_irq);
1298 break;
1299 }
1300
1301 return NOTIFY_OK;
1302}
1303
1304static struct notifier_block vgic_cpu_nb = {
1305 .notifier_call = vgic_cpu_notify,
1306};
1307
1308int kvm_vgic_hyp_init(void)
1309{
1310 int ret;
1311 struct resource vctrl_res;
1312 struct resource vcpu_res;
1313
1314 vgic_node = of_find_compatible_node(NULL, NULL, "arm,cortex-a15-gic");
1315 if (!vgic_node) {
1316 kvm_err("error: no compatible vgic node in DT\n");
1317 return -ENODEV;
1318 }
1319
1320 vgic_maint_irq = irq_of_parse_and_map(vgic_node, 0);
1321 if (!vgic_maint_irq) {
1322 kvm_err("error getting vgic maintenance irq from DT\n");
1323 ret = -ENXIO;
1324 goto out;
1325 }
1326
1327 ret = request_percpu_irq(vgic_maint_irq, vgic_maintenance_handler,
1328 "vgic", kvm_get_running_vcpus());
1329 if (ret) {
1330 kvm_err("Cannot register interrupt %d\n", vgic_maint_irq);
1331 goto out;
1332 }
1333
1334 ret = register_cpu_notifier(&vgic_cpu_nb);
1335 if (ret) {
1336 kvm_err("Cannot register vgic CPU notifier\n");
1337 goto out_free_irq;
1338 }
1339
1340 ret = of_address_to_resource(vgic_node, 2, &vctrl_res);
1341 if (ret) {
1342 kvm_err("Cannot obtain VCTRL resource\n");
1343 goto out_free_irq;
1344 }
1345
1346 vgic_vctrl_base = of_iomap(vgic_node, 2);
1347 if (!vgic_vctrl_base) {
1348 kvm_err("Cannot ioremap VCTRL\n");
1349 ret = -ENOMEM;
1350 goto out_free_irq;
1351 }
1352
1353 vgic_nr_lr = readl_relaxed(vgic_vctrl_base + GICH_VTR);
1354 vgic_nr_lr = (vgic_nr_lr & 0x3f) + 1;
1355
1356 ret = create_hyp_io_mappings(vgic_vctrl_base,
1357 vgic_vctrl_base + resource_size(&vctrl_res),
1358 vctrl_res.start);
1359 if (ret) {
1360 kvm_err("Cannot map VCTRL into hyp\n");
1361 goto out_unmap;
1362 }
1363
1364 kvm_info("%s@%llx IRQ%d\n", vgic_node->name,
1365 vctrl_res.start, vgic_maint_irq);
1366 on_each_cpu(vgic_init_maintenance_interrupt, NULL, 1);
1367
1368 if (of_address_to_resource(vgic_node, 3, &vcpu_res)) {
1369 kvm_err("Cannot obtain VCPU resource\n");
1370 ret = -ENXIO;
1371 goto out_unmap;
1372 }
1373 vgic_vcpu_base = vcpu_res.start;
1374
1375 goto out;
1376
1377out_unmap:
1378 iounmap(vgic_vctrl_base);
1379out_free_irq:
1380 free_percpu_irq(vgic_maint_irq, kvm_get_running_vcpus());
1381out:
1382 of_node_put(vgic_node);
1383 return ret;
1384}
1385
1386int kvm_vgic_init(struct kvm *kvm)
1387{
1388 int ret = 0, i;
1389
1390 mutex_lock(&kvm->lock);
1391
1392 if (vgic_initialized(kvm))
1393 goto out;
1394
1395 if (IS_VGIC_ADDR_UNDEF(kvm->arch.vgic.vgic_dist_base) ||
1396 IS_VGIC_ADDR_UNDEF(kvm->arch.vgic.vgic_cpu_base)) {
1397 kvm_err("Need to set vgic cpu and dist addresses first\n");
1398 ret = -ENXIO;
1399 goto out;
1400 }
1401
1402 ret = kvm_phys_addr_ioremap(kvm, kvm->arch.vgic.vgic_cpu_base,
1403 vgic_vcpu_base, KVM_VGIC_V2_CPU_SIZE);
1404 if (ret) {
1405 kvm_err("Unable to remap VGIC CPU to VCPU\n");
1406 goto out;
1407 }
1408
1409 for (i = VGIC_NR_PRIVATE_IRQS; i < VGIC_NR_IRQS; i += 4)
1410 vgic_set_target_reg(kvm, 0, i);
1411
Marc Zyngier967f8422013-01-23 13:21:59 -05001412 kvm_timer_init(kvm);
Marc Zyngier01ac5e32013-01-21 19:36:16 -05001413 kvm->arch.vgic.ready = true;
1414out:
1415 mutex_unlock(&kvm->lock);
1416 return ret;
1417}
1418
1419int kvm_vgic_create(struct kvm *kvm)
1420{
1421 int ret = 0;
1422
1423 mutex_lock(&kvm->lock);
1424
1425 if (atomic_read(&kvm->online_vcpus) || kvm->arch.vgic.vctrl_base) {
1426 ret = -EEXIST;
1427 goto out;
1428 }
1429
1430 spin_lock_init(&kvm->arch.vgic.lock);
1431 kvm->arch.vgic.vctrl_base = vgic_vctrl_base;
1432 kvm->arch.vgic.vgic_dist_base = VGIC_ADDR_UNDEF;
1433 kvm->arch.vgic.vgic_cpu_base = VGIC_ADDR_UNDEF;
1434
1435out:
1436 mutex_unlock(&kvm->lock);
1437 return ret;
1438}
1439
Christoffer Dall330690c2013-01-21 19:36:13 -05001440static bool vgic_ioaddr_overlap(struct kvm *kvm)
1441{
1442 phys_addr_t dist = kvm->arch.vgic.vgic_dist_base;
1443 phys_addr_t cpu = kvm->arch.vgic.vgic_cpu_base;
1444
1445 if (IS_VGIC_ADDR_UNDEF(dist) || IS_VGIC_ADDR_UNDEF(cpu))
1446 return 0;
1447 if ((dist <= cpu && dist + KVM_VGIC_V2_DIST_SIZE > cpu) ||
1448 (cpu <= dist && cpu + KVM_VGIC_V2_CPU_SIZE > dist))
1449 return -EBUSY;
1450 return 0;
1451}
1452
1453static int vgic_ioaddr_assign(struct kvm *kvm, phys_addr_t *ioaddr,
1454 phys_addr_t addr, phys_addr_t size)
1455{
1456 int ret;
1457
1458 if (!IS_VGIC_ADDR_UNDEF(*ioaddr))
1459 return -EEXIST;
1460 if (addr + size < addr)
1461 return -EINVAL;
1462
1463 ret = vgic_ioaddr_overlap(kvm);
1464 if (ret)
1465 return ret;
1466 *ioaddr = addr;
1467 return ret;
1468}
1469
1470int kvm_vgic_set_addr(struct kvm *kvm, unsigned long type, u64 addr)
1471{
1472 int r = 0;
1473 struct vgic_dist *vgic = &kvm->arch.vgic;
1474
1475 if (addr & ~KVM_PHYS_MASK)
1476 return -E2BIG;
1477
Marc Zyngierd7bb0772012-12-12 14:37:10 +00001478 if (addr & (SZ_4K - 1))
Christoffer Dall330690c2013-01-21 19:36:13 -05001479 return -EINVAL;
1480
1481 mutex_lock(&kvm->lock);
1482 switch (type) {
1483 case KVM_VGIC_V2_ADDR_TYPE_DIST:
1484 r = vgic_ioaddr_assign(kvm, &vgic->vgic_dist_base,
1485 addr, KVM_VGIC_V2_DIST_SIZE);
1486 break;
1487 case KVM_VGIC_V2_ADDR_TYPE_CPU:
1488 r = vgic_ioaddr_assign(kvm, &vgic->vgic_cpu_base,
1489 addr, KVM_VGIC_V2_CPU_SIZE);
1490 break;
1491 default:
1492 r = -ENODEV;
1493 }
1494
1495 mutex_unlock(&kvm->lock);
1496 return r;
1497}