blob: 65461f821a75a7ffd3d0bdab471052e7061c8824 [file] [log] [blame]
Marc Zyngier1a89dd92013-01-21 19:36:12 -05001/*
2 * Copyright (C) 2012 ARM Ltd.
3 * Author: Marc Zyngier <marc.zyngier@arm.com>
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License version 2 as
7 * published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the Free Software
16 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17 */
18
Marc Zyngier01ac5e32013-01-21 19:36:16 -050019#include <linux/cpu.h>
Marc Zyngier1a89dd92013-01-21 19:36:12 -050020#include <linux/kvm.h>
21#include <linux/kvm_host.h>
22#include <linux/interrupt.h>
23#include <linux/io.h>
Marc Zyngier01ac5e32013-01-21 19:36:16 -050024#include <linux/of.h>
25#include <linux/of_address.h>
26#include <linux/of_irq.h>
Marc Zyngier6c3d63c2014-06-23 17:37:18 +010027#include <linux/rculist.h>
Christoffer Dall2a2f3e262014-02-02 13:41:02 -080028#include <linux/uaccess.h>
Marc Zyngier01ac5e32013-01-21 19:36:16 -050029
Marc Zyngier1a89dd92013-01-21 19:36:12 -050030#include <asm/kvm_emulate.h>
Marc Zyngier01ac5e32013-01-21 19:36:16 -050031#include <asm/kvm_arm.h>
32#include <asm/kvm_mmu.h>
Eric Auger174178f2015-03-04 11:14:36 +010033#include <trace/events/kvm.h>
Andre Przywara6777f772015-03-26 14:39:34 +000034#include <asm/kvm.h>
35#include <kvm/iodev.h>
Marc Zyngier1a89dd92013-01-21 19:36:12 -050036
Christoffer Dalle21f0912015-08-30 13:57:20 +020037#define CREATE_TRACE_POINTS
38#include "trace.h"
39
Marc Zyngierb47ef922013-01-21 19:36:14 -050040/*
41 * How the whole thing works (courtesy of Christoffer Dall):
42 *
43 * - At any time, the dist->irq_pending_on_cpu is the oracle that knows if
Christoffer Dall7e362912014-06-14 22:34:04 +020044 * something is pending on the CPU interface.
45 * - Interrupts that are pending on the distributor are stored on the
46 * vgic.irq_pending vgic bitmap (this bitmap is updated by both user land
47 * ioctls and guest mmio ops, and other in-kernel peripherals such as the
48 * arch. timers).
Marc Zyngierb47ef922013-01-21 19:36:14 -050049 * - Every time the bitmap changes, the irq_pending_on_cpu oracle is
50 * recalculated
51 * - To calculate the oracle, we need info for each cpu from
52 * compute_pending_for_cpu, which considers:
Christoffer Dall227844f2014-06-09 12:27:18 +020053 * - PPI: dist->irq_pending & dist->irq_enable
54 * - SPI: dist->irq_pending & dist->irq_enable & dist->irq_spi_target
Christoffer Dall7e362912014-06-14 22:34:04 +020055 * - irq_spi_target is a 'formatted' version of the GICD_ITARGETSRn
Marc Zyngierb47ef922013-01-21 19:36:14 -050056 * registers, stored on each vcpu. We only keep one bit of
57 * information per interrupt, making sure that only one vcpu can
58 * accept the interrupt.
Christoffer Dall7e362912014-06-14 22:34:04 +020059 * - If any of the above state changes, we must recalculate the oracle.
Marc Zyngierb47ef922013-01-21 19:36:14 -050060 * - The same is true when injecting an interrupt, except that we only
61 * consider a single interrupt at a time. The irq_spi_cpu array
62 * contains the target CPU for each SPI.
63 *
64 * The handling of level interrupts adds some extra complexity. We
65 * need to track when the interrupt has been EOIed, so we can sample
66 * the 'line' again. This is achieved as such:
67 *
68 * - When a level interrupt is moved onto a vcpu, the corresponding
Christoffer Dalldbf20f92014-06-09 12:55:13 +020069 * bit in irq_queued is set. As long as this bit is set, the line
Marc Zyngierb47ef922013-01-21 19:36:14 -050070 * will be ignored for further interrupts. The interrupt is injected
71 * into the vcpu with the GICH_LR_EOI bit set (generate a
72 * maintenance interrupt on EOI).
73 * - When the interrupt is EOIed, the maintenance interrupt fires,
Christoffer Dalldbf20f92014-06-09 12:55:13 +020074 * and clears the corresponding bit in irq_queued. This allows the
Marc Zyngierb47ef922013-01-21 19:36:14 -050075 * interrupt line to be sampled again.
Christoffer Dallfaa1b462014-06-14 21:54:51 +020076 * - Note that level-triggered interrupts can also be set to pending from
77 * writes to GICD_ISPENDRn and lowering the external input line does not
78 * cause the interrupt to become inactive in such a situation.
79 * Conversely, writes to GICD_ICPENDRn do not cause the interrupt to become
80 * inactive as long as the external input line is held high.
Marc Zyngier6c3d63c2014-06-23 17:37:18 +010081 *
82 *
83 * Initialization rules: there are multiple stages to the vgic
84 * initialization, both for the distributor and the CPU interfaces.
85 *
86 * Distributor:
87 *
88 * - kvm_vgic_early_init(): initialization of static data that doesn't
89 * depend on any sizing information or emulation type. No allocation
90 * is allowed there.
91 *
92 * - vgic_init(): allocation and initialization of the generic data
93 * structures that depend on sizing information (number of CPUs,
94 * number of interrupts). Also initializes the vcpu specific data
95 * structures. Can be executed lazily for GICv2.
96 * [to be renamed to kvm_vgic_init??]
97 *
98 * CPU Interface:
99 *
100 * - kvm_vgic_cpu_early_init(): initialization of static data that
101 * doesn't depend on any sizing information or emulation type. No
102 * allocation is allowed there.
Marc Zyngierb47ef922013-01-21 19:36:14 -0500103 */
104
Andre Przywara83215812014-06-07 00:53:08 +0200105#include "vgic.h"
Christoffer Dall330690c2013-01-21 19:36:13 -0500106
Marc Zyngiera1fcb442013-01-21 19:36:15 -0500107static void vgic_retire_disabled_irqs(struct kvm_vcpu *vcpu);
Pavel Fedin212c7652015-10-27 11:37:30 +0300108static void vgic_retire_lr(int lr_nr, struct kvm_vcpu *vcpu);
Marc Zyngier8d5c6b02013-06-03 15:55:02 +0100109static struct vgic_lr vgic_get_lr(const struct kvm_vcpu *vcpu, int lr);
110static void vgic_set_lr(struct kvm_vcpu *vcpu, int lr, struct vgic_lr lr_desc);
Pavel Fedinc4cd4c12015-10-27 11:37:29 +0300111static u64 vgic_get_elrsr(struct kvm_vcpu *vcpu);
Marc Zyngier6c3d63c2014-06-23 17:37:18 +0100112static struct irq_phys_map *vgic_irq_map_search(struct kvm_vcpu *vcpu,
113 int virt_irq);
Christoffer Dall91036172015-08-25 22:50:57 +0200114static int compute_pending_for_cpu(struct kvm_vcpu *vcpu);
Marc Zyngier01ac5e32013-01-21 19:36:16 -0500115
Marc Zyngier8f186d52014-02-04 18:13:03 +0000116static const struct vgic_ops *vgic_ops;
117static const struct vgic_params *vgic;
Marc Zyngierb47ef922013-01-21 19:36:14 -0500118
Andre Przywarab26e5fd2014-06-02 16:19:12 +0200119static void add_sgi_source(struct kvm_vcpu *vcpu, int irq, int source)
120{
121 vcpu->kvm->arch.vgic.vm_ops.add_sgi_source(vcpu, irq, source);
122}
123
124static bool queue_sgi(struct kvm_vcpu *vcpu, int irq)
125{
126 return vcpu->kvm->arch.vgic.vm_ops.queue_sgi(vcpu, irq);
127}
128
129int kvm_vgic_map_resources(struct kvm *kvm)
130{
131 return kvm->arch.vgic.vm_ops.map_resources(kvm, vgic);
132}
133
Victor Kamensky9662fb42014-06-12 09:30:10 -0700134/*
Marc Zyngierc1bfb572014-07-08 12:09:01 +0100135 * struct vgic_bitmap contains a bitmap made of unsigned longs, but
136 * extracts u32s out of them.
Victor Kamensky9662fb42014-06-12 09:30:10 -0700137 *
138 * This does not work on 64-bit BE systems, because the bitmap access
139 * will store two consecutive 32-bit words with the higher-addressed
140 * register's bits at the lower index and the lower-addressed register's
141 * bits at the higher index.
142 *
143 * Therefore, swizzle the register index when accessing the 32-bit word
144 * registers to access the right register's value.
145 */
146#if defined(CONFIG_CPU_BIG_ENDIAN) && BITS_PER_LONG == 64
147#define REG_OFFSET_SWIZZLE 1
148#else
149#define REG_OFFSET_SWIZZLE 0
150#endif
Marc Zyngierb47ef922013-01-21 19:36:14 -0500151
Marc Zyngierc1bfb572014-07-08 12:09:01 +0100152static int vgic_init_bitmap(struct vgic_bitmap *b, int nr_cpus, int nr_irqs)
153{
154 int nr_longs;
155
156 nr_longs = nr_cpus + BITS_TO_LONGS(nr_irqs - VGIC_NR_PRIVATE_IRQS);
157
158 b->private = kzalloc(sizeof(unsigned long) * nr_longs, GFP_KERNEL);
159 if (!b->private)
160 return -ENOMEM;
161
162 b->shared = b->private + nr_cpus;
163
164 return 0;
165}
166
167static void vgic_free_bitmap(struct vgic_bitmap *b)
168{
169 kfree(b->private);
170 b->private = NULL;
171 b->shared = NULL;
172}
173
Christoffer Dall2df36a52014-09-28 16:04:26 +0200174/*
175 * Call this function to convert a u64 value to an unsigned long * bitmask
176 * in a way that works on both 32-bit and 64-bit LE and BE platforms.
177 *
178 * Warning: Calling this function may modify *val.
179 */
180static unsigned long *u64_to_bitmask(u64 *val)
181{
182#if defined(CONFIG_CPU_BIG_ENDIAN) && BITS_PER_LONG == 32
183 *val = (*val >> 32) | (*val << 32);
184#endif
185 return (unsigned long *)val;
186}
187
Andre Przywara83215812014-06-07 00:53:08 +0200188u32 *vgic_bitmap_get_reg(struct vgic_bitmap *x, int cpuid, u32 offset)
Marc Zyngierb47ef922013-01-21 19:36:14 -0500189{
190 offset >>= 2;
191 if (!offset)
Marc Zyngierc1bfb572014-07-08 12:09:01 +0100192 return (u32 *)(x->private + cpuid) + REG_OFFSET_SWIZZLE;
Marc Zyngierb47ef922013-01-21 19:36:14 -0500193 else
Marc Zyngierc1bfb572014-07-08 12:09:01 +0100194 return (u32 *)(x->shared) + ((offset - 1) ^ REG_OFFSET_SWIZZLE);
Marc Zyngierb47ef922013-01-21 19:36:14 -0500195}
196
197static int vgic_bitmap_get_irq_val(struct vgic_bitmap *x,
198 int cpuid, int irq)
199{
200 if (irq < VGIC_NR_PRIVATE_IRQS)
Marc Zyngierc1bfb572014-07-08 12:09:01 +0100201 return test_bit(irq, x->private + cpuid);
Marc Zyngierb47ef922013-01-21 19:36:14 -0500202
Marc Zyngierc1bfb572014-07-08 12:09:01 +0100203 return test_bit(irq - VGIC_NR_PRIVATE_IRQS, x->shared);
Marc Zyngierb47ef922013-01-21 19:36:14 -0500204}
205
Andre Przywara83215812014-06-07 00:53:08 +0200206void vgic_bitmap_set_irq_val(struct vgic_bitmap *x, int cpuid,
207 int irq, int val)
Marc Zyngierb47ef922013-01-21 19:36:14 -0500208{
209 unsigned long *reg;
210
211 if (irq < VGIC_NR_PRIVATE_IRQS) {
Marc Zyngierc1bfb572014-07-08 12:09:01 +0100212 reg = x->private + cpuid;
Marc Zyngierb47ef922013-01-21 19:36:14 -0500213 } else {
Marc Zyngierc1bfb572014-07-08 12:09:01 +0100214 reg = x->shared;
Marc Zyngierb47ef922013-01-21 19:36:14 -0500215 irq -= VGIC_NR_PRIVATE_IRQS;
216 }
217
218 if (val)
219 set_bit(irq, reg);
220 else
221 clear_bit(irq, reg);
222}
223
224static unsigned long *vgic_bitmap_get_cpu_map(struct vgic_bitmap *x, int cpuid)
225{
Marc Zyngierc1bfb572014-07-08 12:09:01 +0100226 return x->private + cpuid;
Marc Zyngierb47ef922013-01-21 19:36:14 -0500227}
228
Andre Przywara83215812014-06-07 00:53:08 +0200229unsigned long *vgic_bitmap_get_shared_map(struct vgic_bitmap *x)
Marc Zyngierb47ef922013-01-21 19:36:14 -0500230{
Marc Zyngierc1bfb572014-07-08 12:09:01 +0100231 return x->shared;
232}
233
234static int vgic_init_bytemap(struct vgic_bytemap *x, int nr_cpus, int nr_irqs)
235{
236 int size;
237
238 size = nr_cpus * VGIC_NR_PRIVATE_IRQS;
239 size += nr_irqs - VGIC_NR_PRIVATE_IRQS;
240
241 x->private = kzalloc(size, GFP_KERNEL);
242 if (!x->private)
243 return -ENOMEM;
244
245 x->shared = x->private + nr_cpus * VGIC_NR_PRIVATE_IRQS / sizeof(u32);
246 return 0;
247}
248
249static void vgic_free_bytemap(struct vgic_bytemap *b)
250{
251 kfree(b->private);
252 b->private = NULL;
253 b->shared = NULL;
Marc Zyngierb47ef922013-01-21 19:36:14 -0500254}
255
Andre Przywara83215812014-06-07 00:53:08 +0200256u32 *vgic_bytemap_get_reg(struct vgic_bytemap *x, int cpuid, u32 offset)
Marc Zyngierb47ef922013-01-21 19:36:14 -0500257{
Marc Zyngierc1bfb572014-07-08 12:09:01 +0100258 u32 *reg;
259
260 if (offset < VGIC_NR_PRIVATE_IRQS) {
261 reg = x->private;
262 offset += cpuid * VGIC_NR_PRIVATE_IRQS;
263 } else {
264 reg = x->shared;
265 offset -= VGIC_NR_PRIVATE_IRQS;
266 }
267
268 return reg + (offset / sizeof(u32));
Marc Zyngierb47ef922013-01-21 19:36:14 -0500269}
270
271#define VGIC_CFG_LEVEL 0
272#define VGIC_CFG_EDGE 1
273
274static bool vgic_irq_is_edge(struct kvm_vcpu *vcpu, int irq)
275{
276 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
277 int irq_val;
278
279 irq_val = vgic_bitmap_get_irq_val(&dist->irq_cfg, vcpu->vcpu_id, irq);
280 return irq_val == VGIC_CFG_EDGE;
281}
282
283static int vgic_irq_is_enabled(struct kvm_vcpu *vcpu, int irq)
284{
285 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
286
287 return vgic_bitmap_get_irq_val(&dist->irq_enabled, vcpu->vcpu_id, irq);
288}
289
Christoffer Dalldbf20f92014-06-09 12:55:13 +0200290static int vgic_irq_is_queued(struct kvm_vcpu *vcpu, int irq)
Marc Zyngier9d949dc2013-01-21 19:36:14 -0500291{
292 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
293
Christoffer Dalldbf20f92014-06-09 12:55:13 +0200294 return vgic_bitmap_get_irq_val(&dist->irq_queued, vcpu->vcpu_id, irq);
Marc Zyngier9d949dc2013-01-21 19:36:14 -0500295}
296
Christoffer Dall47a98b12015-03-13 17:02:54 +0000297static int vgic_irq_is_active(struct kvm_vcpu *vcpu, int irq)
298{
299 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
300
301 return vgic_bitmap_get_irq_val(&dist->irq_active, vcpu->vcpu_id, irq);
302}
303
Christoffer Dalldbf20f92014-06-09 12:55:13 +0200304static void vgic_irq_set_queued(struct kvm_vcpu *vcpu, int irq)
Marc Zyngier9d949dc2013-01-21 19:36:14 -0500305{
306 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
307
Christoffer Dalldbf20f92014-06-09 12:55:13 +0200308 vgic_bitmap_set_irq_val(&dist->irq_queued, vcpu->vcpu_id, irq, 1);
Marc Zyngier9d949dc2013-01-21 19:36:14 -0500309}
310
Christoffer Dalldbf20f92014-06-09 12:55:13 +0200311static void vgic_irq_clear_queued(struct kvm_vcpu *vcpu, int irq)
Marc Zyngier9d949dc2013-01-21 19:36:14 -0500312{
313 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
314
Christoffer Dalldbf20f92014-06-09 12:55:13 +0200315 vgic_bitmap_set_irq_val(&dist->irq_queued, vcpu->vcpu_id, irq, 0);
Marc Zyngier9d949dc2013-01-21 19:36:14 -0500316}
317
Christoffer Dall47a98b12015-03-13 17:02:54 +0000318static void vgic_irq_set_active(struct kvm_vcpu *vcpu, int irq)
319{
320 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
321
322 vgic_bitmap_set_irq_val(&dist->irq_active, vcpu->vcpu_id, irq, 1);
323}
324
325static void vgic_irq_clear_active(struct kvm_vcpu *vcpu, int irq)
326{
327 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
328
329 vgic_bitmap_set_irq_val(&dist->irq_active, vcpu->vcpu_id, irq, 0);
330}
331
Christoffer Dallfaa1b462014-06-14 21:54:51 +0200332static int vgic_dist_irq_get_level(struct kvm_vcpu *vcpu, int irq)
333{
334 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
335
336 return vgic_bitmap_get_irq_val(&dist->irq_level, vcpu->vcpu_id, irq);
337}
338
339static void vgic_dist_irq_set_level(struct kvm_vcpu *vcpu, int irq)
340{
341 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
342
343 vgic_bitmap_set_irq_val(&dist->irq_level, vcpu->vcpu_id, irq, 1);
344}
345
346static void vgic_dist_irq_clear_level(struct kvm_vcpu *vcpu, int irq)
347{
348 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
349
350 vgic_bitmap_set_irq_val(&dist->irq_level, vcpu->vcpu_id, irq, 0);
351}
352
353static int vgic_dist_irq_soft_pend(struct kvm_vcpu *vcpu, int irq)
354{
355 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
356
357 return vgic_bitmap_get_irq_val(&dist->irq_soft_pend, vcpu->vcpu_id, irq);
358}
359
360static void vgic_dist_irq_clear_soft_pend(struct kvm_vcpu *vcpu, int irq)
361{
362 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
363
364 vgic_bitmap_set_irq_val(&dist->irq_soft_pend, vcpu->vcpu_id, irq, 0);
Christoffer Dall91036172015-08-25 22:50:57 +0200365 if (!vgic_dist_irq_get_level(vcpu, irq)) {
366 vgic_dist_irq_clear_pending(vcpu, irq);
367 if (!compute_pending_for_cpu(vcpu))
368 clear_bit(vcpu->vcpu_id, dist->irq_pending_on_cpu);
369 }
Christoffer Dallfaa1b462014-06-14 21:54:51 +0200370}
371
Marc Zyngier9d949dc2013-01-21 19:36:14 -0500372static int vgic_dist_irq_is_pending(struct kvm_vcpu *vcpu, int irq)
373{
374 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
375
Christoffer Dall227844f2014-06-09 12:27:18 +0200376 return vgic_bitmap_get_irq_val(&dist->irq_pending, vcpu->vcpu_id, irq);
Marc Zyngier9d949dc2013-01-21 19:36:14 -0500377}
378
Andre Przywara83215812014-06-07 00:53:08 +0200379void vgic_dist_irq_set_pending(struct kvm_vcpu *vcpu, int irq)
Marc Zyngierb47ef922013-01-21 19:36:14 -0500380{
381 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
382
Christoffer Dall227844f2014-06-09 12:27:18 +0200383 vgic_bitmap_set_irq_val(&dist->irq_pending, vcpu->vcpu_id, irq, 1);
Marc Zyngierb47ef922013-01-21 19:36:14 -0500384}
385
Andre Przywara83215812014-06-07 00:53:08 +0200386void vgic_dist_irq_clear_pending(struct kvm_vcpu *vcpu, int irq)
Marc Zyngierb47ef922013-01-21 19:36:14 -0500387{
388 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
389
Christoffer Dall227844f2014-06-09 12:27:18 +0200390 vgic_bitmap_set_irq_val(&dist->irq_pending, vcpu->vcpu_id, irq, 0);
Marc Zyngierb47ef922013-01-21 19:36:14 -0500391}
392
393static void vgic_cpu_irq_set(struct kvm_vcpu *vcpu, int irq)
394{
395 if (irq < VGIC_NR_PRIVATE_IRQS)
396 set_bit(irq, vcpu->arch.vgic_cpu.pending_percpu);
397 else
398 set_bit(irq - VGIC_NR_PRIVATE_IRQS,
399 vcpu->arch.vgic_cpu.pending_shared);
400}
401
Andre Przywara83215812014-06-07 00:53:08 +0200402void vgic_cpu_irq_clear(struct kvm_vcpu *vcpu, int irq)
Marc Zyngierb47ef922013-01-21 19:36:14 -0500403{
404 if (irq < VGIC_NR_PRIVATE_IRQS)
405 clear_bit(irq, vcpu->arch.vgic_cpu.pending_percpu);
406 else
407 clear_bit(irq - VGIC_NR_PRIVATE_IRQS,
408 vcpu->arch.vgic_cpu.pending_shared);
409}
410
Christoffer Dalldbf20f92014-06-09 12:55:13 +0200411static bool vgic_can_sample_irq(struct kvm_vcpu *vcpu, int irq)
412{
Marc Zyngier7a67b4b2015-06-05 16:45:29 +0100413 return !vgic_irq_is_queued(vcpu, irq);
Christoffer Dalldbf20f92014-06-09 12:55:13 +0200414}
415
Marc Zyngier1a89dd92013-01-21 19:36:12 -0500416/**
417 * vgic_reg_access - access vgic register
418 * @mmio: pointer to the data describing the mmio access
419 * @reg: pointer to the virtual backing of vgic distributor data
420 * @offset: least significant 2 bits used for word offset
421 * @mode: ACCESS_ mode (see defines above)
422 *
423 * Helper to make vgic register access easier using one of the access
424 * modes defined for vgic register access
425 * (read,raz,write-ignored,setbit,clearbit,write)
426 */
Andre Przywara83215812014-06-07 00:53:08 +0200427void vgic_reg_access(struct kvm_exit_mmio *mmio, u32 *reg,
428 phys_addr_t offset, int mode)
Marc Zyngier1a89dd92013-01-21 19:36:12 -0500429{
430 int word_offset = (offset & 3) * 8;
431 u32 mask = (1UL << (mmio->len * 8)) - 1;
432 u32 regval;
433
434 /*
435 * Any alignment fault should have been delivered to the guest
436 * directly (ARM ARM B3.12.7 "Prioritization of aborts").
437 */
438
439 if (reg) {
440 regval = *reg;
441 } else {
442 BUG_ON(mode != (ACCESS_READ_RAZ | ACCESS_WRITE_IGNORED));
443 regval = 0;
444 }
445
446 if (mmio->is_write) {
447 u32 data = mmio_data_read(mmio, mask) << word_offset;
448 switch (ACCESS_WRITE_MASK(mode)) {
449 case ACCESS_WRITE_IGNORED:
450 return;
451
452 case ACCESS_WRITE_SETBIT:
453 regval |= data;
454 break;
455
456 case ACCESS_WRITE_CLEARBIT:
457 regval &= ~data;
458 break;
459
460 case ACCESS_WRITE_VALUE:
461 regval = (regval & ~(mask << word_offset)) | data;
462 break;
463 }
464 *reg = regval;
465 } else {
466 switch (ACCESS_READ_MASK(mode)) {
467 case ACCESS_READ_RAZ:
468 regval = 0;
469 /* fall through */
470
471 case ACCESS_READ_VALUE:
472 mmio_data_write(mmio, mask, regval >> word_offset);
473 }
474 }
475}
476
Andre Przywara83215812014-06-07 00:53:08 +0200477bool handle_mmio_raz_wi(struct kvm_vcpu *vcpu, struct kvm_exit_mmio *mmio,
478 phys_addr_t offset)
Marc Zyngierb47ef922013-01-21 19:36:14 -0500479{
480 vgic_reg_access(mmio, NULL, offset,
481 ACCESS_READ_RAZ | ACCESS_WRITE_IGNORED);
482 return false;
483}
484
Andre Przywara83215812014-06-07 00:53:08 +0200485bool vgic_handle_enable_reg(struct kvm *kvm, struct kvm_exit_mmio *mmio,
486 phys_addr_t offset, int vcpu_id, int access)
Marc Zyngierb47ef922013-01-21 19:36:14 -0500487{
Andre Przywarad97f6832014-06-11 14:11:49 +0200488 u32 *reg;
489 int mode = ACCESS_READ_VALUE | access;
490 struct kvm_vcpu *target_vcpu = kvm_get_vcpu(kvm, vcpu_id);
491
492 reg = vgic_bitmap_get_reg(&kvm->arch.vgic.irq_enabled, vcpu_id, offset);
493 vgic_reg_access(mmio, reg, offset, mode);
Marc Zyngierb47ef922013-01-21 19:36:14 -0500494 if (mmio->is_write) {
Andre Przywarad97f6832014-06-11 14:11:49 +0200495 if (access & ACCESS_WRITE_CLEARBIT) {
496 if (offset < 4) /* Force SGI enabled */
497 *reg |= 0xffff;
498 vgic_retire_disabled_irqs(target_vcpu);
499 }
500 vgic_update_state(kvm);
Marc Zyngierb47ef922013-01-21 19:36:14 -0500501 return true;
502 }
503
504 return false;
505}
506
Andre Przywara83215812014-06-07 00:53:08 +0200507bool vgic_handle_set_pending_reg(struct kvm *kvm,
508 struct kvm_exit_mmio *mmio,
509 phys_addr_t offset, int vcpu_id)
Marc Zyngierb47ef922013-01-21 19:36:14 -0500510{
Christoffer Dall9da48b52014-06-14 22:30:45 +0200511 u32 *reg, orig;
Christoffer Dallfaa1b462014-06-14 21:54:51 +0200512 u32 level_mask;
Andre Przywarad97f6832014-06-11 14:11:49 +0200513 int mode = ACCESS_READ_VALUE | ACCESS_WRITE_SETBIT;
514 struct vgic_dist *dist = &kvm->arch.vgic;
Christoffer Dallfaa1b462014-06-14 21:54:51 +0200515
Andre Przywarad97f6832014-06-11 14:11:49 +0200516 reg = vgic_bitmap_get_reg(&dist->irq_cfg, vcpu_id, offset);
Christoffer Dallfaa1b462014-06-14 21:54:51 +0200517 level_mask = (~(*reg));
518
519 /* Mark both level and edge triggered irqs as pending */
Andre Przywarad97f6832014-06-11 14:11:49 +0200520 reg = vgic_bitmap_get_reg(&dist->irq_pending, vcpu_id, offset);
Christoffer Dall9da48b52014-06-14 22:30:45 +0200521 orig = *reg;
Andre Przywarad97f6832014-06-11 14:11:49 +0200522 vgic_reg_access(mmio, reg, offset, mode);
Christoffer Dallfaa1b462014-06-14 21:54:51 +0200523
Marc Zyngierb47ef922013-01-21 19:36:14 -0500524 if (mmio->is_write) {
Christoffer Dallfaa1b462014-06-14 21:54:51 +0200525 /* Set the soft-pending flag only for level-triggered irqs */
526 reg = vgic_bitmap_get_reg(&dist->irq_soft_pend,
Andre Przywarad97f6832014-06-11 14:11:49 +0200527 vcpu_id, offset);
528 vgic_reg_access(mmio, reg, offset, mode);
Christoffer Dallfaa1b462014-06-14 21:54:51 +0200529 *reg &= level_mask;
530
Christoffer Dall9da48b52014-06-14 22:30:45 +0200531 /* Ignore writes to SGIs */
532 if (offset < 2) {
533 *reg &= ~0xffff;
534 *reg |= orig & 0xffff;
535 }
536
Andre Przywarad97f6832014-06-11 14:11:49 +0200537 vgic_update_state(kvm);
Marc Zyngierb47ef922013-01-21 19:36:14 -0500538 return true;
539 }
540
541 return false;
542}
543
Andre Przywara83215812014-06-07 00:53:08 +0200544bool vgic_handle_clear_pending_reg(struct kvm *kvm,
545 struct kvm_exit_mmio *mmio,
546 phys_addr_t offset, int vcpu_id)
Marc Zyngierb47ef922013-01-21 19:36:14 -0500547{
Christoffer Dallfaa1b462014-06-14 21:54:51 +0200548 u32 *level_active;
Christoffer Dall9da48b52014-06-14 22:30:45 +0200549 u32 *reg, orig;
Andre Przywarad97f6832014-06-11 14:11:49 +0200550 int mode = ACCESS_READ_VALUE | ACCESS_WRITE_CLEARBIT;
551 struct vgic_dist *dist = &kvm->arch.vgic;
Christoffer Dallfaa1b462014-06-14 21:54:51 +0200552
Andre Przywarad97f6832014-06-11 14:11:49 +0200553 reg = vgic_bitmap_get_reg(&dist->irq_pending, vcpu_id, offset);
Christoffer Dall9da48b52014-06-14 22:30:45 +0200554 orig = *reg;
Andre Przywarad97f6832014-06-11 14:11:49 +0200555 vgic_reg_access(mmio, reg, offset, mode);
Marc Zyngierb47ef922013-01-21 19:36:14 -0500556 if (mmio->is_write) {
Christoffer Dallfaa1b462014-06-14 21:54:51 +0200557 /* Re-set level triggered level-active interrupts */
558 level_active = vgic_bitmap_get_reg(&dist->irq_level,
Andre Przywarad97f6832014-06-11 14:11:49 +0200559 vcpu_id, offset);
560 reg = vgic_bitmap_get_reg(&dist->irq_pending, vcpu_id, offset);
Christoffer Dallfaa1b462014-06-14 21:54:51 +0200561 *reg |= *level_active;
562
Christoffer Dall9da48b52014-06-14 22:30:45 +0200563 /* Ignore writes to SGIs */
564 if (offset < 2) {
565 *reg &= ~0xffff;
566 *reg |= orig & 0xffff;
567 }
568
Christoffer Dallfaa1b462014-06-14 21:54:51 +0200569 /* Clear soft-pending flags */
570 reg = vgic_bitmap_get_reg(&dist->irq_soft_pend,
Andre Przywarad97f6832014-06-11 14:11:49 +0200571 vcpu_id, offset);
572 vgic_reg_access(mmio, reg, offset, mode);
Christoffer Dallfaa1b462014-06-14 21:54:51 +0200573
Andre Przywarad97f6832014-06-11 14:11:49 +0200574 vgic_update_state(kvm);
Marc Zyngierb47ef922013-01-21 19:36:14 -0500575 return true;
576 }
Marc Zyngierb47ef922013-01-21 19:36:14 -0500577 return false;
578}
579
Christoffer Dall47a98b12015-03-13 17:02:54 +0000580bool vgic_handle_set_active_reg(struct kvm *kvm,
581 struct kvm_exit_mmio *mmio,
582 phys_addr_t offset, int vcpu_id)
583{
584 u32 *reg;
585 struct vgic_dist *dist = &kvm->arch.vgic;
586
587 reg = vgic_bitmap_get_reg(&dist->irq_active, vcpu_id, offset);
588 vgic_reg_access(mmio, reg, offset,
589 ACCESS_READ_VALUE | ACCESS_WRITE_SETBIT);
590
591 if (mmio->is_write) {
592 vgic_update_state(kvm);
593 return true;
594 }
595
596 return false;
597}
598
599bool vgic_handle_clear_active_reg(struct kvm *kvm,
600 struct kvm_exit_mmio *mmio,
601 phys_addr_t offset, int vcpu_id)
602{
603 u32 *reg;
604 struct vgic_dist *dist = &kvm->arch.vgic;
605
606 reg = vgic_bitmap_get_reg(&dist->irq_active, vcpu_id, offset);
607 vgic_reg_access(mmio, reg, offset,
608 ACCESS_READ_VALUE | ACCESS_WRITE_CLEARBIT);
609
610 if (mmio->is_write) {
611 vgic_update_state(kvm);
612 return true;
613 }
614
615 return false;
616}
617
Marc Zyngierb47ef922013-01-21 19:36:14 -0500618static u32 vgic_cfg_expand(u16 val)
619{
620 u32 res = 0;
621 int i;
622
623 /*
624 * Turn a 16bit value like abcd...mnop into a 32bit word
625 * a0b0c0d0...m0n0o0p0, which is what the HW cfg register is.
626 */
627 for (i = 0; i < 16; i++)
628 res |= ((val >> i) & VGIC_CFG_EDGE) << (2 * i + 1);
629
630 return res;
631}
632
633static u16 vgic_cfg_compress(u32 val)
634{
635 u16 res = 0;
636 int i;
637
638 /*
639 * Turn a 32bit word a0b0c0d0...m0n0o0p0 into 16bit value like
640 * abcd...mnop which is what we really care about.
641 */
642 for (i = 0; i < 16; i++)
643 res |= ((val >> (i * 2 + 1)) & VGIC_CFG_EDGE) << i;
644
645 return res;
646}
647
648/*
649 * The distributor uses 2 bits per IRQ for the CFG register, but the
650 * LSB is always 0. As such, we only keep the upper bit, and use the
651 * two above functions to compress/expand the bits
652 */
Andre Przywara83215812014-06-07 00:53:08 +0200653bool vgic_handle_cfg_reg(u32 *reg, struct kvm_exit_mmio *mmio,
654 phys_addr_t offset)
Marc Zyngierb47ef922013-01-21 19:36:14 -0500655{
656 u32 val;
Marc Zyngier6545eae2013-08-29 11:08:23 +0100657
Andre Przywaraf2ae85b2014-04-11 00:07:18 +0200658 if (offset & 4)
Marc Zyngierb47ef922013-01-21 19:36:14 -0500659 val = *reg >> 16;
660 else
661 val = *reg & 0xffff;
662
663 val = vgic_cfg_expand(val);
664 vgic_reg_access(mmio, &val, offset,
665 ACCESS_READ_VALUE | ACCESS_WRITE_VALUE);
666 if (mmio->is_write) {
Christoffer Dall8bf9a702015-08-30 14:42:16 +0200667 /* Ignore writes to read-only SGI and PPI bits */
668 if (offset < 8)
Marc Zyngierb47ef922013-01-21 19:36:14 -0500669 return false;
Marc Zyngierb47ef922013-01-21 19:36:14 -0500670
671 val = vgic_cfg_compress(val);
Andre Przywaraf2ae85b2014-04-11 00:07:18 +0200672 if (offset & 4) {
Marc Zyngierb47ef922013-01-21 19:36:14 -0500673 *reg &= 0xffff;
674 *reg |= val << 16;
675 } else {
676 *reg &= 0xffff << 16;
677 *reg |= val;
678 }
679 }
680
681 return false;
682}
683
Christoffer Dallcbd333a2013-11-15 20:51:31 -0800684/**
Christoffer Dall47a98b12015-03-13 17:02:54 +0000685 * vgic_unqueue_irqs - move pending/active IRQs from LRs to the distributor
Christoffer Dallcbd333a2013-11-15 20:51:31 -0800686 * @vgic_cpu: Pointer to the vgic_cpu struct holding the LRs
687 *
Christoffer Dall47a98b12015-03-13 17:02:54 +0000688 * Move any IRQs that have already been assigned to LRs back to the
Christoffer Dallcbd333a2013-11-15 20:51:31 -0800689 * emulated distributor state so that the complete emulated state can be read
690 * from the main emulation structures without investigating the LRs.
Christoffer Dallcbd333a2013-11-15 20:51:31 -0800691 */
Andre Przywara83215812014-06-07 00:53:08 +0200692void vgic_unqueue_irqs(struct kvm_vcpu *vcpu)
Christoffer Dallcbd333a2013-11-15 20:51:31 -0800693{
Christoffer Dallcbd333a2013-11-15 20:51:31 -0800694 struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
Pavel Fedinc4cd4c12015-10-27 11:37:29 +0300695 u64 elrsr = vgic_get_elrsr(vcpu);
696 unsigned long *elrsr_ptr = u64_to_bitmask(&elrsr);
Marc Zyngier8d5c6b02013-06-03 15:55:02 +0100697 int i;
Christoffer Dallcbd333a2013-11-15 20:51:31 -0800698
Pavel Fedinc4cd4c12015-10-27 11:37:29 +0300699 for_each_clear_bit(i, elrsr_ptr, vgic_cpu->nr_lr) {
Marc Zyngier8d5c6b02013-06-03 15:55:02 +0100700 struct vgic_lr lr = vgic_get_lr(vcpu, i);
Christoffer Dallcbd333a2013-11-15 20:51:31 -0800701
702 /*
703 * There are three options for the state bits:
704 *
705 * 01: pending
706 * 10: active
707 * 11: pending and active
Christoffer Dallcbd333a2013-11-15 20:51:31 -0800708 */
Christoffer Dall47a98b12015-03-13 17:02:54 +0000709 BUG_ON(!(lr.state & LR_STATE_MASK));
710
711 /* Reestablish SGI source for pending and active IRQs */
712 if (lr.irq < VGIC_NR_SGIS)
713 add_sgi_source(vcpu, lr.irq, lr.source);
714
715 /*
716 * If the LR holds an active (10) or a pending and active (11)
717 * interrupt then move the active state to the
718 * distributor tracking bit.
719 */
Pavel Fedin212c7652015-10-27 11:37:30 +0300720 if (lr.state & LR_STATE_ACTIVE)
Christoffer Dall47a98b12015-03-13 17:02:54 +0000721 vgic_irq_set_active(vcpu, lr.irq);
Christoffer Dallcbd333a2013-11-15 20:51:31 -0800722
723 /*
724 * Reestablish the pending state on the distributor and the
Pavel Fedin212c7652015-10-27 11:37:30 +0300725 * CPU interface and mark the LR as free for other use.
Christoffer Dallcbd333a2013-11-15 20:51:31 -0800726 */
Pavel Fedin212c7652015-10-27 11:37:30 +0300727 vgic_retire_lr(i, vcpu);
Christoffer Dallcbd333a2013-11-15 20:51:31 -0800728
729 /* Finally update the VGIC state. */
730 vgic_update_state(vcpu->kvm);
731 }
732}
733
Andre Przywara83215812014-06-07 00:53:08 +0200734const
Andre Przywaracf50a1e2015-03-26 14:39:32 +0000735struct vgic_io_range *vgic_find_range(const struct vgic_io_range *ranges,
Andre Przywara9f199d02015-03-26 14:39:33 +0000736 int len, gpa_t offset)
Marc Zyngier1a89dd92013-01-21 19:36:12 -0500737{
Andre Przywara9f199d02015-03-26 14:39:33 +0000738 while (ranges->len) {
739 if (offset >= ranges->base &&
740 (offset + len) <= (ranges->base + ranges->len))
741 return ranges;
742 ranges++;
Marc Zyngier1a89dd92013-01-21 19:36:12 -0500743 }
744
745 return NULL;
746}
747
Marc Zyngierc3c91832014-07-08 12:09:04 +0100748static bool vgic_validate_access(const struct vgic_dist *dist,
Andre Przywaracf50a1e2015-03-26 14:39:32 +0000749 const struct vgic_io_range *range,
Marc Zyngierc3c91832014-07-08 12:09:04 +0100750 unsigned long offset)
751{
752 int irq;
753
754 if (!range->bits_per_irq)
755 return true; /* Not an irq-based access */
756
757 irq = offset * 8 / range->bits_per_irq;
758 if (irq >= dist->nr_irqs)
759 return false;
760
761 return true;
762}
763
Andre Przywara05bc8aa2014-06-05 16:07:50 +0200764/*
765 * Call the respective handler function for the given range.
766 * We split up any 64 bit accesses into two consecutive 32 bit
767 * handler calls and merge the result afterwards.
768 * We do this in a little endian fashion regardless of the host's
769 * or guest's endianness, because the GIC is always LE and the rest of
770 * the code (vgic_reg_access) also puts it in a LE fashion already.
771 * At this point we have already identified the handle function, so
772 * range points to that one entry and offset is relative to this.
773 */
774static bool call_range_handler(struct kvm_vcpu *vcpu,
775 struct kvm_exit_mmio *mmio,
776 unsigned long offset,
Andre Przywaracf50a1e2015-03-26 14:39:32 +0000777 const struct vgic_io_range *range)
Andre Przywara05bc8aa2014-06-05 16:07:50 +0200778{
Andre Przywara05bc8aa2014-06-05 16:07:50 +0200779 struct kvm_exit_mmio mmio32;
780 bool ret;
781
782 if (likely(mmio->len <= 4))
783 return range->handle_mmio(vcpu, mmio, offset);
784
785 /*
786 * Any access bigger than 4 bytes (that we currently handle in KVM)
787 * is actually 8 bytes long, caused by a 64-bit access
788 */
789
790 mmio32.len = 4;
791 mmio32.is_write = mmio->is_write;
Andre Przywara9fedf142014-11-13 16:21:35 +0000792 mmio32.private = mmio->private;
Andre Przywara05bc8aa2014-06-05 16:07:50 +0200793
794 mmio32.phys_addr = mmio->phys_addr + 4;
Andre Przywara950324a2015-03-28 01:13:13 +0000795 mmio32.data = &((u32 *)mmio->data)[1];
Andre Przywara05bc8aa2014-06-05 16:07:50 +0200796 ret = range->handle_mmio(vcpu, &mmio32, offset + 4);
Andre Przywara05bc8aa2014-06-05 16:07:50 +0200797
798 mmio32.phys_addr = mmio->phys_addr;
Andre Przywara950324a2015-03-28 01:13:13 +0000799 mmio32.data = &((u32 *)mmio->data)[0];
Andre Przywara05bc8aa2014-06-05 16:07:50 +0200800 ret |= range->handle_mmio(vcpu, &mmio32, offset);
Andre Przywara05bc8aa2014-06-05 16:07:50 +0200801
802 return ret;
803}
804
Marc Zyngier1a89dd92013-01-21 19:36:12 -0500805/**
Andre Przywara6777f772015-03-26 14:39:34 +0000806 * vgic_handle_mmio_access - handle an in-kernel MMIO access
807 * This is called by the read/write KVM IO device wrappers below.
Marc Zyngier1a89dd92013-01-21 19:36:12 -0500808 * @vcpu: pointer to the vcpu performing the access
Andre Przywara6777f772015-03-26 14:39:34 +0000809 * @this: pointer to the KVM IO device in charge
810 * @addr: guest physical address of the access
811 * @len: size of the access
812 * @val: pointer to the data region
813 * @is_write: read or write access
Marc Zyngier1a89dd92013-01-21 19:36:12 -0500814 *
Andre Przywara96415252014-06-02 22:44:37 +0200815 * returns true if the MMIO access could be performed
Marc Zyngier1a89dd92013-01-21 19:36:12 -0500816 */
Andre Przywara6777f772015-03-26 14:39:34 +0000817static int vgic_handle_mmio_access(struct kvm_vcpu *vcpu,
818 struct kvm_io_device *this, gpa_t addr,
819 int len, void *val, bool is_write)
Marc Zyngier1a89dd92013-01-21 19:36:12 -0500820{
Marc Zyngierb47ef922013-01-21 19:36:14 -0500821 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
Andre Przywara6777f772015-03-26 14:39:34 +0000822 struct vgic_io_device *iodev = container_of(this,
823 struct vgic_io_device, dev);
824 struct kvm_run *run = vcpu->run;
825 const struct vgic_io_range *range;
826 struct kvm_exit_mmio mmio;
Marc Zyngierb47ef922013-01-21 19:36:14 -0500827 bool updated_state;
Andre Przywara6777f772015-03-26 14:39:34 +0000828 gpa_t offset;
Marc Zyngierb47ef922013-01-21 19:36:14 -0500829
Andre Przywara6777f772015-03-26 14:39:34 +0000830 offset = addr - iodev->addr;
831 range = vgic_find_range(iodev->reg_ranges, len, offset);
Marc Zyngierb47ef922013-01-21 19:36:14 -0500832 if (unlikely(!range || !range->handle_mmio)) {
Andre Przywara6777f772015-03-26 14:39:34 +0000833 pr_warn("Unhandled access %d %08llx %d\n", is_write, addr, len);
834 return -ENXIO;
Marc Zyngierb47ef922013-01-21 19:36:14 -0500835 }
836
Andre Przywara6777f772015-03-26 14:39:34 +0000837 mmio.phys_addr = addr;
838 mmio.len = len;
839 mmio.is_write = is_write;
Andre Przywara950324a2015-03-28 01:13:13 +0000840 mmio.data = val;
Andre Przywara6777f772015-03-26 14:39:34 +0000841 mmio.private = iodev->redist_vcpu;
842
843 spin_lock(&dist->lock);
Andre Przywara96415252014-06-02 22:44:37 +0200844 offset -= range->base;
Marc Zyngierc3c91832014-07-08 12:09:04 +0100845 if (vgic_validate_access(dist, range, offset)) {
Andre Przywara6777f772015-03-26 14:39:34 +0000846 updated_state = call_range_handler(vcpu, &mmio, offset, range);
Marc Zyngierc3c91832014-07-08 12:09:04 +0100847 } else {
Andre Przywara6777f772015-03-26 14:39:34 +0000848 if (!is_write)
849 memset(val, 0, len);
Marc Zyngierc3c91832014-07-08 12:09:04 +0100850 updated_state = false;
851 }
Andre Przywara6777f772015-03-26 14:39:34 +0000852 spin_unlock(&dist->lock);
Andre Przywara950324a2015-03-28 01:13:13 +0000853 run->mmio.is_write = is_write;
854 run->mmio.len = len;
855 run->mmio.phys_addr = addr;
856 memcpy(run->mmio.data, val, len);
857
Marc Zyngierb47ef922013-01-21 19:36:14 -0500858 kvm_handle_mmio_return(vcpu, run);
859
Marc Zyngier5863c2c2013-01-21 19:36:15 -0500860 if (updated_state)
861 vgic_kick_vcpus(vcpu->kvm);
862
Andre Przywara6777f772015-03-26 14:39:34 +0000863 return 0;
Marc Zyngierb47ef922013-01-21 19:36:14 -0500864}
865
Andre Przywara6777f772015-03-26 14:39:34 +0000866static int vgic_handle_mmio_read(struct kvm_vcpu *vcpu,
867 struct kvm_io_device *this,
868 gpa_t addr, int len, void *val)
Andre Przywara96415252014-06-02 22:44:37 +0200869{
Andre Przywara6777f772015-03-26 14:39:34 +0000870 return vgic_handle_mmio_access(vcpu, this, addr, len, val, false);
871}
Andre Przywara96415252014-06-02 22:44:37 +0200872
Andre Przywara6777f772015-03-26 14:39:34 +0000873static int vgic_handle_mmio_write(struct kvm_vcpu *vcpu,
874 struct kvm_io_device *this,
875 gpa_t addr, int len, const void *val)
876{
877 return vgic_handle_mmio_access(vcpu, this, addr, len, (void *)val,
878 true);
879}
880
881struct kvm_io_device_ops vgic_io_ops = {
882 .read = vgic_handle_mmio_read,
883 .write = vgic_handle_mmio_write,
884};
885
886/**
887 * vgic_register_kvm_io_dev - register VGIC register frame on the KVM I/O bus
888 * @kvm: The VM structure pointer
889 * @base: The (guest) base address for the register frame
890 * @len: Length of the register frame window
891 * @ranges: Describing the handler functions for each register
892 * @redist_vcpu_id: The VCPU ID to pass on to the handlers on call
893 * @iodev: Points to memory to be passed on to the handler
894 *
895 * @iodev stores the parameters of this function to be usable by the handler
896 * respectively the dispatcher function (since the KVM I/O bus framework lacks
897 * an opaque parameter). Initialization is done in this function, but the
898 * reference should be valid and unique for the whole VGIC lifetime.
899 * If the register frame is not mapped for a specific VCPU, pass -1 to
900 * @redist_vcpu_id.
901 */
902int vgic_register_kvm_io_dev(struct kvm *kvm, gpa_t base, int len,
903 const struct vgic_io_range *ranges,
904 int redist_vcpu_id,
905 struct vgic_io_device *iodev)
906{
907 struct kvm_vcpu *vcpu = NULL;
908 int ret;
909
910 if (redist_vcpu_id >= 0)
911 vcpu = kvm_get_vcpu(kvm, redist_vcpu_id);
912
913 iodev->addr = base;
914 iodev->len = len;
915 iodev->reg_ranges = ranges;
916 iodev->redist_vcpu = vcpu;
917
918 kvm_iodevice_init(&iodev->dev, &vgic_io_ops);
919
920 mutex_lock(&kvm->slots_lock);
921
922 ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS, base, len,
923 &iodev->dev);
924 mutex_unlock(&kvm->slots_lock);
925
926 /* Mark the iodev as invalid if registration fails. */
927 if (ret)
928 iodev->dev.ops = NULL;
929
930 return ret;
Andre Przywara96415252014-06-02 22:44:37 +0200931}
932
Marc Zyngierfb65ab62014-07-08 12:09:02 +0100933static int vgic_nr_shared_irqs(struct vgic_dist *dist)
934{
935 return dist->nr_irqs - VGIC_NR_PRIVATE_IRQS;
936}
937
Christoffer Dall47a98b12015-03-13 17:02:54 +0000938static int compute_active_for_cpu(struct kvm_vcpu *vcpu)
939{
940 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
941 unsigned long *active, *enabled, *act_percpu, *act_shared;
942 unsigned long active_private, active_shared;
943 int nr_shared = vgic_nr_shared_irqs(dist);
944 int vcpu_id;
945
946 vcpu_id = vcpu->vcpu_id;
947 act_percpu = vcpu->arch.vgic_cpu.active_percpu;
948 act_shared = vcpu->arch.vgic_cpu.active_shared;
949
950 active = vgic_bitmap_get_cpu_map(&dist->irq_active, vcpu_id);
951 enabled = vgic_bitmap_get_cpu_map(&dist->irq_enabled, vcpu_id);
952 bitmap_and(act_percpu, active, enabled, VGIC_NR_PRIVATE_IRQS);
953
954 active = vgic_bitmap_get_shared_map(&dist->irq_active);
955 enabled = vgic_bitmap_get_shared_map(&dist->irq_enabled);
956 bitmap_and(act_shared, active, enabled, nr_shared);
957 bitmap_and(act_shared, act_shared,
958 vgic_bitmap_get_shared_map(&dist->irq_spi_target[vcpu_id]),
959 nr_shared);
960
961 active_private = find_first_bit(act_percpu, VGIC_NR_PRIVATE_IRQS);
962 active_shared = find_first_bit(act_shared, nr_shared);
963
964 return (active_private < VGIC_NR_PRIVATE_IRQS ||
965 active_shared < nr_shared);
966}
967
Marc Zyngierb47ef922013-01-21 19:36:14 -0500968static int compute_pending_for_cpu(struct kvm_vcpu *vcpu)
969{
Marc Zyngier9d949dc2013-01-21 19:36:14 -0500970 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
971 unsigned long *pending, *enabled, *pend_percpu, *pend_shared;
972 unsigned long pending_private, pending_shared;
Marc Zyngierfb65ab62014-07-08 12:09:02 +0100973 int nr_shared = vgic_nr_shared_irqs(dist);
Marc Zyngier9d949dc2013-01-21 19:36:14 -0500974 int vcpu_id;
975
976 vcpu_id = vcpu->vcpu_id;
977 pend_percpu = vcpu->arch.vgic_cpu.pending_percpu;
978 pend_shared = vcpu->arch.vgic_cpu.pending_shared;
979
Christoffer Dall0d997492015-10-17 19:05:27 +0200980 if (!dist->enabled) {
981 bitmap_zero(pend_percpu, VGIC_NR_PRIVATE_IRQS);
982 bitmap_zero(pend_shared, nr_shared);
983 return 0;
984 }
985
Christoffer Dall227844f2014-06-09 12:27:18 +0200986 pending = vgic_bitmap_get_cpu_map(&dist->irq_pending, vcpu_id);
Marc Zyngier9d949dc2013-01-21 19:36:14 -0500987 enabled = vgic_bitmap_get_cpu_map(&dist->irq_enabled, vcpu_id);
988 bitmap_and(pend_percpu, pending, enabled, VGIC_NR_PRIVATE_IRQS);
989
Christoffer Dall227844f2014-06-09 12:27:18 +0200990 pending = vgic_bitmap_get_shared_map(&dist->irq_pending);
Marc Zyngier9d949dc2013-01-21 19:36:14 -0500991 enabled = vgic_bitmap_get_shared_map(&dist->irq_enabled);
Marc Zyngierfb65ab62014-07-08 12:09:02 +0100992 bitmap_and(pend_shared, pending, enabled, nr_shared);
Marc Zyngier9d949dc2013-01-21 19:36:14 -0500993 bitmap_and(pend_shared, pend_shared,
994 vgic_bitmap_get_shared_map(&dist->irq_spi_target[vcpu_id]),
Marc Zyngierfb65ab62014-07-08 12:09:02 +0100995 nr_shared);
Marc Zyngier9d949dc2013-01-21 19:36:14 -0500996
997 pending_private = find_first_bit(pend_percpu, VGIC_NR_PRIVATE_IRQS);
Marc Zyngierfb65ab62014-07-08 12:09:02 +0100998 pending_shared = find_first_bit(pend_shared, nr_shared);
Marc Zyngier9d949dc2013-01-21 19:36:14 -0500999 return (pending_private < VGIC_NR_PRIVATE_IRQS ||
Marc Zyngierfb65ab62014-07-08 12:09:02 +01001000 pending_shared < vgic_nr_shared_irqs(dist));
Marc Zyngierb47ef922013-01-21 19:36:14 -05001001}
1002
1003/*
1004 * Update the interrupt state and determine which CPUs have pending
Christoffer Dall47a98b12015-03-13 17:02:54 +00001005 * or active interrupts. Must be called with distributor lock held.
Marc Zyngierb47ef922013-01-21 19:36:14 -05001006 */
Andre Przywara83215812014-06-07 00:53:08 +02001007void vgic_update_state(struct kvm *kvm)
Marc Zyngierb47ef922013-01-21 19:36:14 -05001008{
1009 struct vgic_dist *dist = &kvm->arch.vgic;
1010 struct kvm_vcpu *vcpu;
1011 int c;
1012
Marc Zyngierb47ef922013-01-21 19:36:14 -05001013 kvm_for_each_vcpu(c, vcpu, kvm) {
Christoffer Dall47a98b12015-03-13 17:02:54 +00001014 if (compute_pending_for_cpu(vcpu))
Marc Zyngierc1bfb572014-07-08 12:09:01 +01001015 set_bit(c, dist->irq_pending_on_cpu);
Christoffer Dall47a98b12015-03-13 17:02:54 +00001016
1017 if (compute_active_for_cpu(vcpu))
1018 set_bit(c, dist->irq_active_on_cpu);
1019 else
1020 clear_bit(c, dist->irq_active_on_cpu);
Marc Zyngierb47ef922013-01-21 19:36:14 -05001021 }
Marc Zyngier1a89dd92013-01-21 19:36:12 -05001022}
Christoffer Dall330690c2013-01-21 19:36:13 -05001023
Marc Zyngier8d5c6b02013-06-03 15:55:02 +01001024static struct vgic_lr vgic_get_lr(const struct kvm_vcpu *vcpu, int lr)
1025{
Marc Zyngier8f186d52014-02-04 18:13:03 +00001026 return vgic_ops->get_lr(vcpu, lr);
Marc Zyngier8d5c6b02013-06-03 15:55:02 +01001027}
1028
1029static void vgic_set_lr(struct kvm_vcpu *vcpu, int lr,
1030 struct vgic_lr vlr)
1031{
Marc Zyngier8f186d52014-02-04 18:13:03 +00001032 vgic_ops->set_lr(vcpu, lr, vlr);
Marc Zyngier8d5c6b02013-06-03 15:55:02 +01001033}
1034
Marc Zyngier69bb2c92013-06-04 10:29:39 +01001035static inline u64 vgic_get_elrsr(struct kvm_vcpu *vcpu)
1036{
Marc Zyngier8f186d52014-02-04 18:13:03 +00001037 return vgic_ops->get_elrsr(vcpu);
Marc Zyngier69bb2c92013-06-04 10:29:39 +01001038}
1039
Marc Zyngier8d6a0312013-06-04 10:33:43 +01001040static inline u64 vgic_get_eisr(struct kvm_vcpu *vcpu)
1041{
Marc Zyngier8f186d52014-02-04 18:13:03 +00001042 return vgic_ops->get_eisr(vcpu);
Marc Zyngier8d6a0312013-06-04 10:33:43 +01001043}
1044
Christoffer Dallae705932015-03-13 17:02:56 +00001045static inline void vgic_clear_eisr(struct kvm_vcpu *vcpu)
1046{
1047 vgic_ops->clear_eisr(vcpu);
1048}
1049
Marc Zyngier495dd852013-06-04 11:02:10 +01001050static inline u32 vgic_get_interrupt_status(struct kvm_vcpu *vcpu)
1051{
Marc Zyngier8f186d52014-02-04 18:13:03 +00001052 return vgic_ops->get_interrupt_status(vcpu);
Marc Zyngier495dd852013-06-04 11:02:10 +01001053}
1054
Marc Zyngier909d9b52013-06-04 11:24:17 +01001055static inline void vgic_enable_underflow(struct kvm_vcpu *vcpu)
1056{
Marc Zyngier8f186d52014-02-04 18:13:03 +00001057 vgic_ops->enable_underflow(vcpu);
Marc Zyngier909d9b52013-06-04 11:24:17 +01001058}
1059
1060static inline void vgic_disable_underflow(struct kvm_vcpu *vcpu)
1061{
Marc Zyngier8f186d52014-02-04 18:13:03 +00001062 vgic_ops->disable_underflow(vcpu);
Marc Zyngier909d9b52013-06-04 11:24:17 +01001063}
1064
Andre Przywara83215812014-06-07 00:53:08 +02001065void vgic_get_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr)
Marc Zyngierbeee38b2014-02-04 17:48:10 +00001066{
Marc Zyngier8f186d52014-02-04 18:13:03 +00001067 vgic_ops->get_vmcr(vcpu, vmcr);
Marc Zyngierbeee38b2014-02-04 17:48:10 +00001068}
1069
Andre Przywara83215812014-06-07 00:53:08 +02001070void vgic_set_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr)
Marc Zyngierbeee38b2014-02-04 17:48:10 +00001071{
Marc Zyngier8f186d52014-02-04 18:13:03 +00001072 vgic_ops->set_vmcr(vcpu, vmcr);
Marc Zyngierbeee38b2014-02-04 17:48:10 +00001073}
1074
Marc Zyngierda8dafd12013-06-04 11:36:38 +01001075static inline void vgic_enable(struct kvm_vcpu *vcpu)
1076{
Marc Zyngier8f186d52014-02-04 18:13:03 +00001077 vgic_ops->enable(vcpu);
Marc Zyngierda8dafd12013-06-04 11:36:38 +01001078}
1079
Pavel Fedin212c7652015-10-27 11:37:30 +03001080static void vgic_retire_lr(int lr_nr, struct kvm_vcpu *vcpu)
Marc Zyngier8d5c6b02013-06-03 15:55:02 +01001081{
Marc Zyngier8d5c6b02013-06-03 15:55:02 +01001082 struct vgic_lr vlr = vgic_get_lr(vcpu, lr_nr);
1083
Pavel Fedin212c7652015-10-27 11:37:30 +03001084 vgic_irq_clear_queued(vcpu, vlr.irq);
1085
Christoffer Dallcff92112015-10-16 12:41:21 +02001086 /*
1087 * We must transfer the pending state back to the distributor before
1088 * retiring the LR, otherwise we may loose edge-triggered interrupts.
1089 */
1090 if (vlr.state & LR_STATE_PENDING) {
Pavel Fedin212c7652015-10-27 11:37:30 +03001091 vgic_dist_irq_set_pending(vcpu, vlr.irq);
Christoffer Dallcff92112015-10-16 12:41:21 +02001092 vlr.hwirq = 0;
1093 }
1094
Marc Zyngier8d5c6b02013-06-03 15:55:02 +01001095 vlr.state = 0;
1096 vgic_set_lr(vcpu, lr_nr, vlr);
Marc Zyngier8d5c6b02013-06-03 15:55:02 +01001097}
Marc Zyngiera1fcb442013-01-21 19:36:15 -05001098
Christoffer Dall0e3dfda2015-11-24 16:23:05 +01001099static bool dist_active_irq(struct kvm_vcpu *vcpu)
1100{
1101 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
1102
1103 return test_bit(vcpu->vcpu_id, dist->irq_active_on_cpu);
1104}
1105
1106bool kvm_vgic_map_is_active(struct kvm_vcpu *vcpu, struct irq_phys_map *map)
1107{
1108 int i;
1109
1110 for (i = 0; i < vcpu->arch.vgic_cpu.nr_lr; i++) {
1111 struct vgic_lr vlr = vgic_get_lr(vcpu, i);
1112
1113 if (vlr.irq == map->virt_irq && vlr.state & LR_STATE_ACTIVE)
1114 return true;
1115 }
1116
1117 return dist_active_irq(vcpu);
1118}
1119
Marc Zyngiera1fcb442013-01-21 19:36:15 -05001120/*
1121 * An interrupt may have been disabled after being made pending on the
1122 * CPU interface (the classic case is a timer running while we're
1123 * rebooting the guest - the interrupt would kick as soon as the CPU
1124 * interface gets enabled, with deadly consequences).
1125 *
1126 * The solution is to examine already active LRs, and check the
1127 * interrupt is still enabled. If not, just retire it.
1128 */
1129static void vgic_retire_disabled_irqs(struct kvm_vcpu *vcpu)
1130{
Pavel Fedinc4cd4c12015-10-27 11:37:29 +03001131 u64 elrsr = vgic_get_elrsr(vcpu);
1132 unsigned long *elrsr_ptr = u64_to_bitmask(&elrsr);
Marc Zyngiera1fcb442013-01-21 19:36:15 -05001133 int lr;
1134
Pavel Fedinc4cd4c12015-10-27 11:37:29 +03001135 for_each_clear_bit(lr, elrsr_ptr, vgic->nr_lr) {
Marc Zyngier8d5c6b02013-06-03 15:55:02 +01001136 struct vgic_lr vlr = vgic_get_lr(vcpu, lr);
Marc Zyngiera1fcb442013-01-21 19:36:15 -05001137
Pavel Fedin212c7652015-10-27 11:37:30 +03001138 if (!vgic_irq_is_enabled(vcpu, vlr.irq))
1139 vgic_retire_lr(lr, vcpu);
Marc Zyngiera1fcb442013-01-21 19:36:15 -05001140 }
1141}
1142
Alex Bennée71760952015-03-13 17:02:53 +00001143static void vgic_queue_irq_to_lr(struct kvm_vcpu *vcpu, int irq,
1144 int lr_nr, struct vgic_lr vlr)
1145{
Christoffer Dall47a98b12015-03-13 17:02:54 +00001146 if (vgic_irq_is_active(vcpu, irq)) {
1147 vlr.state |= LR_STATE_ACTIVE;
1148 kvm_debug("Set active, clear distributor: 0x%x\n", vlr.state);
1149 vgic_irq_clear_active(vcpu, irq);
1150 vgic_update_state(vcpu->kvm);
Pavel Fedin437f9962015-09-25 17:00:29 +03001151 } else {
1152 WARN_ON(!vgic_dist_irq_is_pending(vcpu, irq));
Alex Bennée71760952015-03-13 17:02:53 +00001153 vlr.state |= LR_STATE_PENDING;
1154 kvm_debug("Set pending: 0x%x\n", vlr.state);
1155 }
1156
1157 if (!vgic_irq_is_edge(vcpu, irq))
1158 vlr.state |= LR_EOI_INT;
1159
Marc Zyngier08fd6462015-06-08 16:06:13 +01001160 if (vlr.irq >= VGIC_NR_SGIS) {
1161 struct irq_phys_map *map;
1162 map = vgic_irq_map_search(vcpu, irq);
1163
Marc Zyngier08fd6462015-06-08 16:06:13 +01001164 if (map) {
Marc Zyngier08fd6462015-06-08 16:06:13 +01001165 vlr.hwirq = map->phys_irq;
1166 vlr.state |= LR_HW;
1167 vlr.state &= ~LR_EOI_INT;
1168
Marc Zyngier08fd6462015-06-08 16:06:13 +01001169 /*
1170 * Make sure we're not going to sample this
1171 * again, as a HW-backed interrupt cannot be
1172 * in the PENDING_ACTIVE stage.
1173 */
1174 vgic_irq_set_queued(vcpu, irq);
1175 }
1176 }
1177
Alex Bennée71760952015-03-13 17:02:53 +00001178 vgic_set_lr(vcpu, lr_nr, vlr);
1179}
1180
Marc Zyngier9d949dc2013-01-21 19:36:14 -05001181/*
1182 * Queue an interrupt to a CPU virtual interface. Return true on success,
1183 * or false if it wasn't possible to queue it.
Andre Przywara1d916222014-06-07 00:53:08 +02001184 * sgi_source must be zero for any non-SGI interrupts.
Marc Zyngier9d949dc2013-01-21 19:36:14 -05001185 */
Andre Przywara83215812014-06-07 00:53:08 +02001186bool vgic_queue_irq(struct kvm_vcpu *vcpu, u8 sgi_source_id, int irq)
Marc Zyngier9d949dc2013-01-21 19:36:14 -05001187{
Marc Zyngier5fb66da2014-07-08 12:09:05 +01001188 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
Pavel Fedinc4cd4c12015-10-27 11:37:29 +03001189 u64 elrsr = vgic_get_elrsr(vcpu);
1190 unsigned long *elrsr_ptr = u64_to_bitmask(&elrsr);
Marc Zyngier8d5c6b02013-06-03 15:55:02 +01001191 struct vgic_lr vlr;
Marc Zyngier9d949dc2013-01-21 19:36:14 -05001192 int lr;
1193
1194 /* Sanitize the input... */
1195 BUG_ON(sgi_source_id & ~7);
1196 BUG_ON(sgi_source_id && irq >= VGIC_NR_SGIS);
Marc Zyngier5fb66da2014-07-08 12:09:05 +01001197 BUG_ON(irq >= dist->nr_irqs);
Marc Zyngier9d949dc2013-01-21 19:36:14 -05001198
1199 kvm_debug("Queue IRQ%d\n", irq);
1200
Marc Zyngier9d949dc2013-01-21 19:36:14 -05001201 /* Do we have an active interrupt for the same CPUID? */
Pavel Fedinc4cd4c12015-10-27 11:37:29 +03001202 for_each_clear_bit(lr, elrsr_ptr, vgic->nr_lr) {
Marc Zyngier8d5c6b02013-06-03 15:55:02 +01001203 vlr = vgic_get_lr(vcpu, lr);
Pavel Fedinc4cd4c12015-10-27 11:37:29 +03001204 if (vlr.irq == irq && vlr.source == sgi_source_id) {
Marc Zyngier8d5c6b02013-06-03 15:55:02 +01001205 kvm_debug("LR%d piggyback for IRQ%d\n", lr, vlr.irq);
Alex Bennée71760952015-03-13 17:02:53 +00001206 vgic_queue_irq_to_lr(vcpu, irq, lr, vlr);
Marc Zyngier8d5c6b02013-06-03 15:55:02 +01001207 return true;
1208 }
Marc Zyngier9d949dc2013-01-21 19:36:14 -05001209 }
1210
1211 /* Try to use another LR for this interrupt */
Pavel Fedinc4cd4c12015-10-27 11:37:29 +03001212 lr = find_first_bit(elrsr_ptr, vgic->nr_lr);
Marc Zyngier8f186d52014-02-04 18:13:03 +00001213 if (lr >= vgic->nr_lr)
Marc Zyngier9d949dc2013-01-21 19:36:14 -05001214 return false;
1215
1216 kvm_debug("LR%d allocated for IRQ%d %x\n", lr, irq, sgi_source_id);
Marc Zyngier9d949dc2013-01-21 19:36:14 -05001217
Marc Zyngier8d5c6b02013-06-03 15:55:02 +01001218 vlr.irq = irq;
1219 vlr.source = sgi_source_id;
Alex Bennée71760952015-03-13 17:02:53 +00001220 vlr.state = 0;
1221 vgic_queue_irq_to_lr(vcpu, irq, lr, vlr);
Marc Zyngier9d949dc2013-01-21 19:36:14 -05001222
1223 return true;
1224}
1225
Marc Zyngier9d949dc2013-01-21 19:36:14 -05001226static bool vgic_queue_hwirq(struct kvm_vcpu *vcpu, int irq)
1227{
Christoffer Dalldbf20f92014-06-09 12:55:13 +02001228 if (!vgic_can_sample_irq(vcpu, irq))
Marc Zyngier9d949dc2013-01-21 19:36:14 -05001229 return true; /* level interrupt, already queued */
1230
1231 if (vgic_queue_irq(vcpu, 0, irq)) {
1232 if (vgic_irq_is_edge(vcpu, irq)) {
Christoffer Dall227844f2014-06-09 12:27:18 +02001233 vgic_dist_irq_clear_pending(vcpu, irq);
Marc Zyngier9d949dc2013-01-21 19:36:14 -05001234 vgic_cpu_irq_clear(vcpu, irq);
1235 } else {
Christoffer Dalldbf20f92014-06-09 12:55:13 +02001236 vgic_irq_set_queued(vcpu, irq);
Marc Zyngier9d949dc2013-01-21 19:36:14 -05001237 }
1238
1239 return true;
1240 }
1241
1242 return false;
1243}
1244
1245/*
1246 * Fill the list registers with pending interrupts before running the
1247 * guest.
1248 */
1249static void __kvm_vgic_flush_hwstate(struct kvm_vcpu *vcpu)
1250{
1251 struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
1252 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
Christoffer Dall47a98b12015-03-13 17:02:54 +00001253 unsigned long *pa_percpu, *pa_shared;
Christoffer Dallcff92112015-10-16 12:41:21 +02001254 int i, vcpu_id;
Marc Zyngier9d949dc2013-01-21 19:36:14 -05001255 int overflow = 0;
Christoffer Dall47a98b12015-03-13 17:02:54 +00001256 int nr_shared = vgic_nr_shared_irqs(dist);
Marc Zyngier9d949dc2013-01-21 19:36:14 -05001257
1258 vcpu_id = vcpu->vcpu_id;
1259
Christoffer Dall47a98b12015-03-13 17:02:54 +00001260 pa_percpu = vcpu->arch.vgic_cpu.pend_act_percpu;
1261 pa_shared = vcpu->arch.vgic_cpu.pend_act_shared;
1262
1263 bitmap_or(pa_percpu, vgic_cpu->pending_percpu, vgic_cpu->active_percpu,
1264 VGIC_NR_PRIVATE_IRQS);
1265 bitmap_or(pa_shared, vgic_cpu->pending_shared, vgic_cpu->active_shared,
1266 nr_shared);
Marc Zyngier9d949dc2013-01-21 19:36:14 -05001267 /*
1268 * We may not have any pending interrupt, or the interrupts
1269 * may have been serviced from another vcpu. In all cases,
1270 * move along.
1271 */
Christoffer Dall0e3dfda2015-11-24 16:23:05 +01001272 if (!kvm_vgic_vcpu_pending_irq(vcpu) && !dist_active_irq(vcpu))
Marc Zyngier9d949dc2013-01-21 19:36:14 -05001273 goto epilog;
Marc Zyngier9d949dc2013-01-21 19:36:14 -05001274
1275 /* SGIs */
Christoffer Dall47a98b12015-03-13 17:02:54 +00001276 for_each_set_bit(i, pa_percpu, VGIC_NR_SGIS) {
Andre Przywarab26e5fd2014-06-02 16:19:12 +02001277 if (!queue_sgi(vcpu, i))
Marc Zyngier9d949dc2013-01-21 19:36:14 -05001278 overflow = 1;
1279 }
1280
1281 /* PPIs */
Christoffer Dall47a98b12015-03-13 17:02:54 +00001282 for_each_set_bit_from(i, pa_percpu, VGIC_NR_PRIVATE_IRQS) {
Marc Zyngier9d949dc2013-01-21 19:36:14 -05001283 if (!vgic_queue_hwirq(vcpu, i))
1284 overflow = 1;
1285 }
1286
1287 /* SPIs */
Christoffer Dall47a98b12015-03-13 17:02:54 +00001288 for_each_set_bit(i, pa_shared, nr_shared) {
Marc Zyngier9d949dc2013-01-21 19:36:14 -05001289 if (!vgic_queue_hwirq(vcpu, i + VGIC_NR_PRIVATE_IRQS))
1290 overflow = 1;
1291 }
1292
Christoffer Dall47a98b12015-03-13 17:02:54 +00001293
1294
1295
Marc Zyngier9d949dc2013-01-21 19:36:14 -05001296epilog:
1297 if (overflow) {
Marc Zyngier909d9b52013-06-04 11:24:17 +01001298 vgic_enable_underflow(vcpu);
Marc Zyngier9d949dc2013-01-21 19:36:14 -05001299 } else {
Marc Zyngier909d9b52013-06-04 11:24:17 +01001300 vgic_disable_underflow(vcpu);
Marc Zyngier9d949dc2013-01-21 19:36:14 -05001301 /*
1302 * We're about to run this VCPU, and we've consumed
1303 * everything the distributor had in store for
1304 * us. Claim we don't have anything pending. We'll
1305 * adjust that if needed while exiting.
1306 */
Marc Zyngierc1bfb572014-07-08 12:09:01 +01001307 clear_bit(vcpu_id, dist->irq_pending_on_cpu);
Marc Zyngier9d949dc2013-01-21 19:36:14 -05001308 }
1309}
1310
Christoffer Dall8fe2f192015-09-04 21:25:12 +02001311static int process_queued_irq(struct kvm_vcpu *vcpu,
1312 int lr, struct vgic_lr vlr)
Christoffer Dall91036172015-08-25 22:50:57 +02001313{
Christoffer Dall8fe2f192015-09-04 21:25:12 +02001314 int pending = 0;
Christoffer Dall91036172015-08-25 22:50:57 +02001315
1316 /*
1317 * If the IRQ was EOIed (called from vgic_process_maintenance) or it
1318 * went from active to non-active (called from vgic_sync_hwirq) it was
1319 * also ACKed and we we therefore assume we can clear the soft pending
1320 * state (should it had been set) for this interrupt.
1321 *
1322 * Note: if the IRQ soft pending state was set after the IRQ was
1323 * acked, it actually shouldn't be cleared, but we have no way of
1324 * knowing that unless we start trapping ACKs when the soft-pending
1325 * state is set.
1326 */
1327 vgic_dist_irq_clear_soft_pend(vcpu, vlr.irq);
1328
1329 /*
Christoffer Dall8fe2f192015-09-04 21:25:12 +02001330 * Tell the gic to start sampling this interrupt again.
Christoffer Dall91036172015-08-25 22:50:57 +02001331 */
1332 vgic_irq_clear_queued(vcpu, vlr.irq);
1333
1334 /* Any additional pending interrupt? */
Christoffer Dall8fe2f192015-09-04 21:25:12 +02001335 if (vgic_irq_is_edge(vcpu, vlr.irq)) {
1336 BUG_ON(!(vlr.state & LR_HW));
1337 pending = vgic_dist_irq_is_pending(vcpu, vlr.irq);
Christoffer Dall91036172015-08-25 22:50:57 +02001338 } else {
Christoffer Dall8fe2f192015-09-04 21:25:12 +02001339 if (vgic_dist_irq_get_level(vcpu, vlr.irq)) {
1340 vgic_cpu_irq_set(vcpu, vlr.irq);
1341 pending = 1;
1342 } else {
1343 vgic_dist_irq_clear_pending(vcpu, vlr.irq);
1344 vgic_cpu_irq_clear(vcpu, vlr.irq);
1345 }
Christoffer Dall91036172015-08-25 22:50:57 +02001346 }
1347
1348 /*
1349 * Despite being EOIed, the LR may not have
1350 * been marked as empty.
1351 */
Christoffer Dall8fe2f192015-09-04 21:25:12 +02001352 vlr.state = 0;
1353 vlr.hwirq = 0;
1354 vgic_set_lr(vcpu, lr, vlr);
1355
Christoffer Dall8fe2f192015-09-04 21:25:12 +02001356 return pending;
Christoffer Dall91036172015-08-25 22:50:57 +02001357}
1358
Marc Zyngier9d949dc2013-01-21 19:36:14 -05001359static bool vgic_process_maintenance(struct kvm_vcpu *vcpu)
1360{
Marc Zyngier495dd852013-06-04 11:02:10 +01001361 u32 status = vgic_get_interrupt_status(vcpu);
Eric Auger649cf732015-03-04 11:14:35 +01001362 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
Eric Auger174178f2015-03-04 11:14:36 +01001363 struct kvm *kvm = vcpu->kvm;
Christoffer Dall91036172015-08-25 22:50:57 +02001364 int level_pending = 0;
Marc Zyngier9d949dc2013-01-21 19:36:14 -05001365
Marc Zyngier495dd852013-06-04 11:02:10 +01001366 kvm_debug("STATUS = %08x\n", status);
Marc Zyngier9d949dc2013-01-21 19:36:14 -05001367
Marc Zyngier495dd852013-06-04 11:02:10 +01001368 if (status & INT_STATUS_EOI) {
Marc Zyngier9d949dc2013-01-21 19:36:14 -05001369 /*
1370 * Some level interrupts have been EOIed. Clear their
1371 * active bit.
1372 */
Marc Zyngier8d6a0312013-06-04 10:33:43 +01001373 u64 eisr = vgic_get_eisr(vcpu);
Christoffer Dall2df36a52014-09-28 16:04:26 +02001374 unsigned long *eisr_ptr = u64_to_bitmask(&eisr);
Marc Zyngier8d5c6b02013-06-03 15:55:02 +01001375 int lr;
Marc Zyngier9d949dc2013-01-21 19:36:14 -05001376
Marc Zyngier8f186d52014-02-04 18:13:03 +00001377 for_each_set_bit(lr, eisr_ptr, vgic->nr_lr) {
Marc Zyngier8d5c6b02013-06-03 15:55:02 +01001378 struct vgic_lr vlr = vgic_get_lr(vcpu, lr);
Christoffer Dall91036172015-08-25 22:50:57 +02001379
Christoffer Dallfaa1b462014-06-14 21:54:51 +02001380 WARN_ON(vgic_irq_is_edge(vcpu, vlr.irq));
Marc Zyngier8d5c6b02013-06-03 15:55:02 +01001381 WARN_ON(vlr.state & LR_STATE_MASK);
Marc Zyngier9d949dc2013-01-21 19:36:14 -05001382
Christoffer Dallfaa1b462014-06-14 21:54:51 +02001383
Eric Auger174178f2015-03-04 11:14:36 +01001384 /*
1385 * kvm_notify_acked_irq calls kvm_set_irq()
Christoffer Dall91036172015-08-25 22:50:57 +02001386 * to reset the IRQ level, which grabs the dist->lock
1387 * so we call this before taking the dist->lock.
Eric Auger174178f2015-03-04 11:14:36 +01001388 */
Eric Auger174178f2015-03-04 11:14:36 +01001389 kvm_notify_acked_irq(kvm, 0,
1390 vlr.irq - VGIC_NR_PRIVATE_IRQS);
Christoffer Dall91036172015-08-25 22:50:57 +02001391
Eric Auger174178f2015-03-04 11:14:36 +01001392 spin_lock(&dist->lock);
Christoffer Dall8fe2f192015-09-04 21:25:12 +02001393 level_pending |= process_queued_irq(vcpu, lr, vlr);
Eric Auger649cf732015-03-04 11:14:35 +01001394 spin_unlock(&dist->lock);
Marc Zyngier9d949dc2013-01-21 19:36:14 -05001395 }
1396 }
1397
Marc Zyngier495dd852013-06-04 11:02:10 +01001398 if (status & INT_STATUS_UNDERFLOW)
Marc Zyngier909d9b52013-06-04 11:24:17 +01001399 vgic_disable_underflow(vcpu);
Marc Zyngier9d949dc2013-01-21 19:36:14 -05001400
Christoffer Dallae705932015-03-13 17:02:56 +00001401 /*
1402 * In the next iterations of the vcpu loop, if we sync the vgic state
1403 * after flushing it, but before entering the guest (this happens for
1404 * pending signals and vmid rollovers), then make sure we don't pick
1405 * up any old maintenance interrupts here.
1406 */
1407 vgic_clear_eisr(vcpu);
1408
Marc Zyngier9d949dc2013-01-21 19:36:14 -05001409 return level_pending;
1410}
1411
Marc Zyngier08fd6462015-06-08 16:06:13 +01001412/*
1413 * Save the physical active state, and reset it to inactive.
1414 *
Christoffer Dall8fe2f192015-09-04 21:25:12 +02001415 * Return true if there's a pending forwarded interrupt to queue.
Marc Zyngier08fd6462015-06-08 16:06:13 +01001416 */
Christoffer Dall4b4b4512015-08-30 15:01:27 +02001417static bool vgic_sync_hwirq(struct kvm_vcpu *vcpu, int lr, struct vgic_lr vlr)
Marc Zyngier08fd6462015-06-08 16:06:13 +01001418{
Christoffer Dall4b4b4512015-08-30 15:01:27 +02001419 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
Christoffer Dall4b4b4512015-08-30 15:01:27 +02001420 bool level_pending;
Marc Zyngier08fd6462015-06-08 16:06:13 +01001421
1422 if (!(vlr.state & LR_HW))
Christoffer Dall4b4b4512015-08-30 15:01:27 +02001423 return false;
Marc Zyngier08fd6462015-06-08 16:06:13 +01001424
Christoffer Dall9f958c12015-11-24 16:34:31 +01001425 if (vlr.state & LR_STATE_ACTIVE)
1426 return false;
Marc Zyngier08fd6462015-06-08 16:06:13 +01001427
Christoffer Dall4b4b4512015-08-30 15:01:27 +02001428 spin_lock(&dist->lock);
Christoffer Dall8fe2f192015-09-04 21:25:12 +02001429 level_pending = process_queued_irq(vcpu, lr, vlr);
Christoffer Dall4b4b4512015-08-30 15:01:27 +02001430 spin_unlock(&dist->lock);
1431 return level_pending;
Marc Zyngier08fd6462015-06-08 16:06:13 +01001432}
1433
Eric Auger649cf732015-03-04 11:14:35 +01001434/* Sync back the VGIC state after a guest run */
Marc Zyngier9d949dc2013-01-21 19:36:14 -05001435static void __kvm_vgic_sync_hwstate(struct kvm_vcpu *vcpu)
1436{
Marc Zyngier9d949dc2013-01-21 19:36:14 -05001437 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
Marc Zyngier69bb2c92013-06-04 10:29:39 +01001438 u64 elrsr;
1439 unsigned long *elrsr_ptr;
Marc Zyngier9d949dc2013-01-21 19:36:14 -05001440 int lr, pending;
1441 bool level_pending;
1442
1443 level_pending = vgic_process_maintenance(vcpu);
1444
Marc Zyngier08fd6462015-06-08 16:06:13 +01001445 /* Deal with HW interrupts, and clear mappings for empty LRs */
1446 for (lr = 0; lr < vgic->nr_lr; lr++) {
Pavel Fedinc4cd4c12015-10-27 11:37:29 +03001447 struct vgic_lr vlr = vgic_get_lr(vcpu, lr);
Marc Zyngier9d949dc2013-01-21 19:36:14 -05001448
Pavel Fedinc4cd4c12015-10-27 11:37:29 +03001449 level_pending |= vgic_sync_hwirq(vcpu, lr, vlr);
Marc Zyngier5fb66da2014-07-08 12:09:05 +01001450 BUG_ON(vlr.irq >= dist->nr_irqs);
Marc Zyngier9d949dc2013-01-21 19:36:14 -05001451 }
1452
1453 /* Check if we still have something up our sleeve... */
Pavel Fedin26caea72015-10-27 11:37:31 +03001454 elrsr = vgic_get_elrsr(vcpu);
1455 elrsr_ptr = u64_to_bitmask(&elrsr);
Marc Zyngier8f186d52014-02-04 18:13:03 +00001456 pending = find_first_zero_bit(elrsr_ptr, vgic->nr_lr);
1457 if (level_pending || pending < vgic->nr_lr)
Marc Zyngierc1bfb572014-07-08 12:09:01 +01001458 set_bit(vcpu->vcpu_id, dist->irq_pending_on_cpu);
Marc Zyngier9d949dc2013-01-21 19:36:14 -05001459}
1460
1461void kvm_vgic_flush_hwstate(struct kvm_vcpu *vcpu)
1462{
1463 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
1464
1465 if (!irqchip_in_kernel(vcpu->kvm))
1466 return;
1467
1468 spin_lock(&dist->lock);
1469 __kvm_vgic_flush_hwstate(vcpu);
1470 spin_unlock(&dist->lock);
1471}
1472
1473void kvm_vgic_sync_hwstate(struct kvm_vcpu *vcpu)
1474{
1475 if (!irqchip_in_kernel(vcpu->kvm))
1476 return;
1477
1478 __kvm_vgic_sync_hwstate(vcpu);
1479}
1480
1481int kvm_vgic_vcpu_pending_irq(struct kvm_vcpu *vcpu)
1482{
1483 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
1484
1485 if (!irqchip_in_kernel(vcpu->kvm))
1486 return 0;
1487
Marc Zyngierc1bfb572014-07-08 12:09:01 +01001488 return test_bit(vcpu->vcpu_id, dist->irq_pending_on_cpu);
Marc Zyngier9d949dc2013-01-21 19:36:14 -05001489}
1490
Andre Przywara83215812014-06-07 00:53:08 +02001491void vgic_kick_vcpus(struct kvm *kvm)
Marc Zyngier5863c2c2013-01-21 19:36:15 -05001492{
1493 struct kvm_vcpu *vcpu;
1494 int c;
1495
1496 /*
1497 * We've injected an interrupt, time to find out who deserves
1498 * a good kick...
1499 */
1500 kvm_for_each_vcpu(c, vcpu, kvm) {
1501 if (kvm_vgic_vcpu_pending_irq(vcpu))
1502 kvm_vcpu_kick(vcpu);
1503 }
1504}
1505
1506static int vgic_validate_injection(struct kvm_vcpu *vcpu, int irq, int level)
1507{
Christoffer Dall227844f2014-06-09 12:27:18 +02001508 int edge_triggered = vgic_irq_is_edge(vcpu, irq);
Marc Zyngier5863c2c2013-01-21 19:36:15 -05001509
1510 /*
1511 * Only inject an interrupt if:
1512 * - edge triggered and we have a rising edge
1513 * - level triggered and we change level
1514 */
Christoffer Dallfaa1b462014-06-14 21:54:51 +02001515 if (edge_triggered) {
1516 int state = vgic_dist_irq_is_pending(vcpu, irq);
Marc Zyngier5863c2c2013-01-21 19:36:15 -05001517 return level > state;
Christoffer Dallfaa1b462014-06-14 21:54:51 +02001518 } else {
1519 int state = vgic_dist_irq_get_level(vcpu, irq);
Marc Zyngier5863c2c2013-01-21 19:36:15 -05001520 return level != state;
Christoffer Dallfaa1b462014-06-14 21:54:51 +02001521 }
Marc Zyngier5863c2c2013-01-21 19:36:15 -05001522}
1523
Shannon Zhao016ed392014-11-19 10:11:25 +00001524static int vgic_update_irq_pending(struct kvm *kvm, int cpuid,
Marc Zyngier773299a2015-07-24 11:30:43 +01001525 struct irq_phys_map *map,
1526 unsigned int irq_num, bool level)
Marc Zyngier5863c2c2013-01-21 19:36:15 -05001527{
1528 struct vgic_dist *dist = &kvm->arch.vgic;
1529 struct kvm_vcpu *vcpu;
Christoffer Dall227844f2014-06-09 12:27:18 +02001530 int edge_triggered, level_triggered;
Marc Zyngier5863c2c2013-01-21 19:36:15 -05001531 int enabled;
Andre Przywaraa0675c22014-06-07 00:54:51 +02001532 bool ret = true, can_inject = true;
Marc Zyngier5863c2c2013-01-21 19:36:15 -05001533
Christoffer Dalle21f0912015-08-30 13:57:20 +02001534 trace_vgic_update_irq_pending(cpuid, irq_num, level);
1535
Marc Zyngier773299a2015-07-24 11:30:43 +01001536 if (irq_num >= min(kvm->arch.vgic.nr_irqs, 1020))
1537 return -EINVAL;
1538
Marc Zyngier5863c2c2013-01-21 19:36:15 -05001539 spin_lock(&dist->lock);
1540
1541 vcpu = kvm_get_vcpu(kvm, cpuid);
Christoffer Dall227844f2014-06-09 12:27:18 +02001542 edge_triggered = vgic_irq_is_edge(vcpu, irq_num);
1543 level_triggered = !edge_triggered;
Marc Zyngier5863c2c2013-01-21 19:36:15 -05001544
1545 if (!vgic_validate_injection(vcpu, irq_num, level)) {
1546 ret = false;
1547 goto out;
1548 }
1549
1550 if (irq_num >= VGIC_NR_PRIVATE_IRQS) {
1551 cpuid = dist->irq_spi_cpu[irq_num - VGIC_NR_PRIVATE_IRQS];
Andre Przywaraa0675c22014-06-07 00:54:51 +02001552 if (cpuid == VCPU_NOT_ALLOCATED) {
1553 /* Pretend we use CPU0, and prevent injection */
1554 cpuid = 0;
1555 can_inject = false;
1556 }
Marc Zyngier5863c2c2013-01-21 19:36:15 -05001557 vcpu = kvm_get_vcpu(kvm, cpuid);
1558 }
1559
1560 kvm_debug("Inject IRQ%d level %d CPU%d\n", irq_num, level, cpuid);
1561
Christoffer Dallfaa1b462014-06-14 21:54:51 +02001562 if (level) {
1563 if (level_triggered)
1564 vgic_dist_irq_set_level(vcpu, irq_num);
Christoffer Dall227844f2014-06-09 12:27:18 +02001565 vgic_dist_irq_set_pending(vcpu, irq_num);
Christoffer Dallfaa1b462014-06-14 21:54:51 +02001566 } else {
1567 if (level_triggered) {
1568 vgic_dist_irq_clear_level(vcpu, irq_num);
Pavel Fedin437f9962015-09-25 17:00:29 +03001569 if (!vgic_dist_irq_soft_pend(vcpu, irq_num)) {
Christoffer Dallfaa1b462014-06-14 21:54:51 +02001570 vgic_dist_irq_clear_pending(vcpu, irq_num);
Pavel Fedin437f9962015-09-25 17:00:29 +03001571 vgic_cpu_irq_clear(vcpu, irq_num);
1572 if (!compute_pending_for_cpu(vcpu))
1573 clear_bit(cpuid, dist->irq_pending_on_cpu);
1574 }
Christoffer Dallfaa1b462014-06-14 21:54:51 +02001575 }
wanghaibin7d39f9e32014-11-17 09:27:37 +00001576
1577 ret = false;
1578 goto out;
Christoffer Dallfaa1b462014-06-14 21:54:51 +02001579 }
Marc Zyngier5863c2c2013-01-21 19:36:15 -05001580
1581 enabled = vgic_irq_is_enabled(vcpu, irq_num);
1582
Andre Przywaraa0675c22014-06-07 00:54:51 +02001583 if (!enabled || !can_inject) {
Marc Zyngier5863c2c2013-01-21 19:36:15 -05001584 ret = false;
1585 goto out;
1586 }
1587
Christoffer Dalldbf20f92014-06-09 12:55:13 +02001588 if (!vgic_can_sample_irq(vcpu, irq_num)) {
Marc Zyngier5863c2c2013-01-21 19:36:15 -05001589 /*
1590 * Level interrupt in progress, will be picked up
1591 * when EOId.
1592 */
1593 ret = false;
1594 goto out;
1595 }
1596
1597 if (level) {
1598 vgic_cpu_irq_set(vcpu, irq_num);
Marc Zyngierc1bfb572014-07-08 12:09:01 +01001599 set_bit(cpuid, dist->irq_pending_on_cpu);
Marc Zyngier5863c2c2013-01-21 19:36:15 -05001600 }
1601
1602out:
1603 spin_unlock(&dist->lock);
1604
Marc Zyngier773299a2015-07-24 11:30:43 +01001605 if (ret) {
1606 /* kick the specified vcpu */
1607 kvm_vcpu_kick(kvm_get_vcpu(kvm, cpuid));
1608 }
1609
1610 return 0;
Marc Zyngier5863c2c2013-01-21 19:36:15 -05001611}
1612
Marc Zyngier773299a2015-07-24 11:30:43 +01001613static int vgic_lazy_init(struct kvm *kvm)
Marc Zyngier5863c2c2013-01-21 19:36:15 -05001614{
Christoffer Dallca7d9c82014-12-09 14:35:33 +01001615 int ret = 0;
Marc Zyngier5863c2c2013-01-21 19:36:15 -05001616
Christoffer Dallca7d9c82014-12-09 14:35:33 +01001617 if (unlikely(!vgic_initialized(kvm))) {
Andre Przywara598921362014-06-03 09:33:10 +02001618 /*
1619 * We only provide the automatic initialization of the VGIC
1620 * for the legacy case of a GICv2. Any other type must
1621 * be explicitly initialized once setup with the respective
1622 * KVM device call.
1623 */
Marc Zyngier773299a2015-07-24 11:30:43 +01001624 if (kvm->arch.vgic.vgic_model != KVM_DEV_TYPE_ARM_VGIC_V2)
1625 return -EBUSY;
1626
Christoffer Dallca7d9c82014-12-09 14:35:33 +01001627 mutex_lock(&kvm->lock);
1628 ret = vgic_init(kvm);
1629 mutex_unlock(&kvm->lock);
Shannon Zhao016ed392014-11-19 10:11:25 +00001630 }
Marc Zyngier5863c2c2013-01-21 19:36:15 -05001631
Marc Zyngier773299a2015-07-24 11:30:43 +01001632 return ret;
1633}
1634
1635/**
1636 * kvm_vgic_inject_irq - Inject an IRQ from a device to the vgic
1637 * @kvm: The VM structure pointer
1638 * @cpuid: The CPU for PPIs
1639 * @irq_num: The IRQ number that is assigned to the device. This IRQ
1640 * must not be mapped to a HW interrupt.
1641 * @level: Edge-triggered: true: to trigger the interrupt
1642 * false: to ignore the call
1643 * Level-sensitive true: raise the input signal
1644 * false: lower the input signal
1645 *
1646 * The GIC is not concerned with devices being active-LOW or active-HIGH for
1647 * level-sensitive interrupts. You can think of the level parameter as 1
1648 * being HIGH and 0 being LOW and all devices being active-HIGH.
1649 */
1650int kvm_vgic_inject_irq(struct kvm *kvm, int cpuid, unsigned int irq_num,
1651 bool level)
1652{
1653 struct irq_phys_map *map;
1654 int ret;
1655
1656 ret = vgic_lazy_init(kvm);
1657 if (ret)
1658 return ret;
1659
1660 map = vgic_irq_map_search(kvm_get_vcpu(kvm, cpuid), irq_num);
1661 if (map)
Andre Przywarafd1d0dd2015-04-10 16:17:59 +01001662 return -EINVAL;
1663
Marc Zyngier773299a2015-07-24 11:30:43 +01001664 return vgic_update_irq_pending(kvm, cpuid, NULL, irq_num, level);
1665}
Christoffer Dallca7d9c82014-12-09 14:35:33 +01001666
Marc Zyngier773299a2015-07-24 11:30:43 +01001667/**
1668 * kvm_vgic_inject_mapped_irq - Inject a physically mapped IRQ to the vgic
1669 * @kvm: The VM structure pointer
1670 * @cpuid: The CPU for PPIs
1671 * @map: Pointer to a irq_phys_map structure describing the mapping
1672 * @level: Edge-triggered: true: to trigger the interrupt
1673 * false: to ignore the call
1674 * Level-sensitive true: raise the input signal
1675 * false: lower the input signal
1676 *
1677 * The GIC is not concerned with devices being active-LOW or active-HIGH for
1678 * level-sensitive interrupts. You can think of the level parameter as 1
1679 * being HIGH and 0 being LOW and all devices being active-HIGH.
1680 */
1681int kvm_vgic_inject_mapped_irq(struct kvm *kvm, int cpuid,
1682 struct irq_phys_map *map, bool level)
1683{
1684 int ret;
1685
1686 ret = vgic_lazy_init(kvm);
1687 if (ret)
1688 return ret;
1689
1690 return vgic_update_irq_pending(kvm, cpuid, map, map->virt_irq, level);
Marc Zyngier5863c2c2013-01-21 19:36:15 -05001691}
1692
Marc Zyngier01ac5e32013-01-21 19:36:16 -05001693static irqreturn_t vgic_maintenance_handler(int irq, void *data)
1694{
1695 /*
1696 * We cannot rely on the vgic maintenance interrupt to be
1697 * delivered synchronously. This means we can only use it to
1698 * exit the VM, and we perform the handling of EOIed
1699 * interrupts on the exit path (see vgic_process_maintenance).
1700 */
1701 return IRQ_HANDLED;
1702}
1703
Marc Zyngier6c3d63c2014-06-23 17:37:18 +01001704static struct list_head *vgic_get_irq_phys_map_list(struct kvm_vcpu *vcpu,
1705 int virt_irq)
1706{
1707 if (virt_irq < VGIC_NR_PRIVATE_IRQS)
1708 return &vcpu->arch.vgic_cpu.irq_phys_map_list;
1709 else
1710 return &vcpu->kvm->arch.vgic.irq_phys_map_list;
1711}
1712
1713/**
1714 * kvm_vgic_map_phys_irq - map a virtual IRQ to a physical IRQ
1715 * @vcpu: The VCPU pointer
1716 * @virt_irq: The virtual irq number
1717 * @irq: The Linux IRQ number
1718 *
1719 * Establish a mapping between a guest visible irq (@virt_irq) and a
1720 * Linux irq (@irq). On injection, @virt_irq will be associated with
1721 * the physical interrupt represented by @irq. This mapping can be
1722 * established multiple times as long as the parameters are the same.
1723 *
1724 * Returns a valid pointer on success, and an error pointer otherwise
1725 */
1726struct irq_phys_map *kvm_vgic_map_phys_irq(struct kvm_vcpu *vcpu,
1727 int virt_irq, int irq)
1728{
1729 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
1730 struct list_head *root = vgic_get_irq_phys_map_list(vcpu, virt_irq);
1731 struct irq_phys_map *map;
1732 struct irq_phys_map_entry *entry;
1733 struct irq_desc *desc;
1734 struct irq_data *data;
1735 int phys_irq;
1736
1737 desc = irq_to_desc(irq);
1738 if (!desc) {
1739 kvm_err("%s: no interrupt descriptor\n", __func__);
1740 return ERR_PTR(-EINVAL);
1741 }
1742
1743 data = irq_desc_get_irq_data(desc);
1744 while (data->parent_data)
1745 data = data->parent_data;
1746
1747 phys_irq = data->hwirq;
1748
1749 /* Create a new mapping */
1750 entry = kzalloc(sizeof(*entry), GFP_KERNEL);
1751 if (!entry)
1752 return ERR_PTR(-ENOMEM);
1753
1754 spin_lock(&dist->irq_phys_map_lock);
1755
1756 /* Try to match an existing mapping */
1757 map = vgic_irq_map_search(vcpu, virt_irq);
1758 if (map) {
1759 /* Make sure this mapping matches */
1760 if (map->phys_irq != phys_irq ||
1761 map->irq != irq)
1762 map = ERR_PTR(-EINVAL);
1763
1764 /* Found an existing, valid mapping */
1765 goto out;
1766 }
1767
1768 map = &entry->map;
1769 map->virt_irq = virt_irq;
1770 map->phys_irq = phys_irq;
1771 map->irq = irq;
1772
1773 list_add_tail_rcu(&entry->entry, root);
1774
1775out:
1776 spin_unlock(&dist->irq_phys_map_lock);
1777 /* If we've found a hit in the existing list, free the useless
1778 * entry */
1779 if (IS_ERR(map) || map != &entry->map)
1780 kfree(entry);
1781 return map;
1782}
1783
1784static struct irq_phys_map *vgic_irq_map_search(struct kvm_vcpu *vcpu,
1785 int virt_irq)
1786{
1787 struct list_head *root = vgic_get_irq_phys_map_list(vcpu, virt_irq);
1788 struct irq_phys_map_entry *entry;
1789 struct irq_phys_map *map;
1790
1791 rcu_read_lock();
1792
1793 list_for_each_entry_rcu(entry, root, entry) {
1794 map = &entry->map;
1795 if (map->virt_irq == virt_irq) {
1796 rcu_read_unlock();
1797 return map;
1798 }
1799 }
1800
1801 rcu_read_unlock();
1802
1803 return NULL;
1804}
1805
1806static void vgic_free_phys_irq_map_rcu(struct rcu_head *rcu)
1807{
1808 struct irq_phys_map_entry *entry;
1809
1810 entry = container_of(rcu, struct irq_phys_map_entry, rcu);
1811 kfree(entry);
1812}
1813
1814/**
1815 * kvm_vgic_unmap_phys_irq - Remove a virtual to physical IRQ mapping
1816 * @vcpu: The VCPU pointer
1817 * @map: The pointer to a mapping obtained through kvm_vgic_map_phys_irq
1818 *
1819 * Remove an existing mapping between virtual and physical interrupts.
1820 */
1821int kvm_vgic_unmap_phys_irq(struct kvm_vcpu *vcpu, struct irq_phys_map *map)
1822{
1823 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
1824 struct irq_phys_map_entry *entry;
1825 struct list_head *root;
1826
1827 if (!map)
1828 return -EINVAL;
1829
1830 root = vgic_get_irq_phys_map_list(vcpu, map->virt_irq);
1831
1832 spin_lock(&dist->irq_phys_map_lock);
1833
1834 list_for_each_entry(entry, root, entry) {
1835 if (&entry->map == map) {
1836 list_del_rcu(&entry->entry);
1837 call_rcu(&entry->rcu, vgic_free_phys_irq_map_rcu);
1838 break;
1839 }
1840 }
1841
1842 spin_unlock(&dist->irq_phys_map_lock);
1843
1844 return 0;
1845}
1846
1847static void vgic_destroy_irq_phys_map(struct kvm *kvm, struct list_head *root)
1848{
1849 struct vgic_dist *dist = &kvm->arch.vgic;
1850 struct irq_phys_map_entry *entry;
1851
1852 spin_lock(&dist->irq_phys_map_lock);
1853
1854 list_for_each_entry(entry, root, entry) {
1855 list_del_rcu(&entry->entry);
1856 call_rcu(&entry->rcu, vgic_free_phys_irq_map_rcu);
1857 }
1858
1859 spin_unlock(&dist->irq_phys_map_lock);
1860}
1861
Marc Zyngierc1bfb572014-07-08 12:09:01 +01001862void kvm_vgic_vcpu_destroy(struct kvm_vcpu *vcpu)
1863{
1864 struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
1865
1866 kfree(vgic_cpu->pending_shared);
Christoffer Dall47a98b12015-03-13 17:02:54 +00001867 kfree(vgic_cpu->active_shared);
1868 kfree(vgic_cpu->pend_act_shared);
Marc Zyngier6c3d63c2014-06-23 17:37:18 +01001869 vgic_destroy_irq_phys_map(vcpu->kvm, &vgic_cpu->irq_phys_map_list);
Marc Zyngierc1bfb572014-07-08 12:09:01 +01001870 vgic_cpu->pending_shared = NULL;
Christoffer Dall47a98b12015-03-13 17:02:54 +00001871 vgic_cpu->active_shared = NULL;
1872 vgic_cpu->pend_act_shared = NULL;
Marc Zyngierc1bfb572014-07-08 12:09:01 +01001873}
1874
1875static int vgic_vcpu_init_maps(struct kvm_vcpu *vcpu, int nr_irqs)
1876{
1877 struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
1878
1879 int sz = (nr_irqs - VGIC_NR_PRIVATE_IRQS) / 8;
1880 vgic_cpu->pending_shared = kzalloc(sz, GFP_KERNEL);
Christoffer Dall47a98b12015-03-13 17:02:54 +00001881 vgic_cpu->active_shared = kzalloc(sz, GFP_KERNEL);
1882 vgic_cpu->pend_act_shared = kzalloc(sz, GFP_KERNEL);
Marc Zyngierc1bfb572014-07-08 12:09:01 +01001883
Christoffer Dall47a98b12015-03-13 17:02:54 +00001884 if (!vgic_cpu->pending_shared
1885 || !vgic_cpu->active_shared
Pavel Fedinc4cd4c12015-10-27 11:37:29 +03001886 || !vgic_cpu->pend_act_shared) {
Marc Zyngierc1bfb572014-07-08 12:09:01 +01001887 kvm_vgic_vcpu_destroy(vcpu);
1888 return -ENOMEM;
1889 }
1890
Marc Zyngier01ac5e32013-01-21 19:36:16 -05001891 /*
Marc Zyngierca85f622013-06-18 19:17:28 +01001892 * Store the number of LRs per vcpu, so we don't have to go
1893 * all the way to the distributor structure to find out. Only
1894 * assembly code should use this one.
Marc Zyngier01ac5e32013-01-21 19:36:16 -05001895 */
Marc Zyngier8f186d52014-02-04 18:13:03 +00001896 vgic_cpu->nr_lr = vgic->nr_lr;
Marc Zyngier01ac5e32013-01-21 19:36:16 -05001897
Peter Maydell6d3cfbe2014-12-04 15:02:24 +00001898 return 0;
Marc Zyngier01ac5e32013-01-21 19:36:16 -05001899}
1900
Andre Przywara3caa2d82014-06-02 16:26:01 +02001901/**
Marc Zyngier6c3d63c2014-06-23 17:37:18 +01001902 * kvm_vgic_vcpu_early_init - Earliest possible per-vcpu vgic init stage
1903 *
1904 * No memory allocation should be performed here, only static init.
1905 */
1906void kvm_vgic_vcpu_early_init(struct kvm_vcpu *vcpu)
1907{
1908 struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
1909 INIT_LIST_HEAD(&vgic_cpu->irq_phys_map_list);
1910}
1911
1912/**
Andre Przywara3caa2d82014-06-02 16:26:01 +02001913 * kvm_vgic_get_max_vcpus - Get the maximum number of VCPUs allowed by HW
1914 *
1915 * The host's GIC naturally limits the maximum amount of VCPUs a guest
1916 * can use.
1917 */
1918int kvm_vgic_get_max_vcpus(void)
1919{
1920 return vgic->max_gic_vcpus;
1921}
1922
Marc Zyngierc1bfb572014-07-08 12:09:01 +01001923void kvm_vgic_destroy(struct kvm *kvm)
1924{
1925 struct vgic_dist *dist = &kvm->arch.vgic;
1926 struct kvm_vcpu *vcpu;
1927 int i;
1928
1929 kvm_for_each_vcpu(i, vcpu, kvm)
1930 kvm_vgic_vcpu_destroy(vcpu);
1931
1932 vgic_free_bitmap(&dist->irq_enabled);
1933 vgic_free_bitmap(&dist->irq_level);
1934 vgic_free_bitmap(&dist->irq_pending);
1935 vgic_free_bitmap(&dist->irq_soft_pend);
1936 vgic_free_bitmap(&dist->irq_queued);
1937 vgic_free_bitmap(&dist->irq_cfg);
1938 vgic_free_bytemap(&dist->irq_priority);
1939 if (dist->irq_spi_target) {
1940 for (i = 0; i < dist->nr_cpus; i++)
1941 vgic_free_bitmap(&dist->irq_spi_target[i]);
1942 }
1943 kfree(dist->irq_sgi_sources);
1944 kfree(dist->irq_spi_cpu);
Andre Przywaraa0675c22014-06-07 00:54:51 +02001945 kfree(dist->irq_spi_mpidr);
Marc Zyngierc1bfb572014-07-08 12:09:01 +01001946 kfree(dist->irq_spi_target);
1947 kfree(dist->irq_pending_on_cpu);
Christoffer Dall47a98b12015-03-13 17:02:54 +00001948 kfree(dist->irq_active_on_cpu);
Marc Zyngier6c3d63c2014-06-23 17:37:18 +01001949 vgic_destroy_irq_phys_map(kvm, &dist->irq_phys_map_list);
Marc Zyngierc1bfb572014-07-08 12:09:01 +01001950 dist->irq_sgi_sources = NULL;
1951 dist->irq_spi_cpu = NULL;
1952 dist->irq_spi_target = NULL;
1953 dist->irq_pending_on_cpu = NULL;
Christoffer Dall47a98b12015-03-13 17:02:54 +00001954 dist->irq_active_on_cpu = NULL;
Christoffer Dall1f57be22014-12-09 14:30:36 +01001955 dist->nr_cpus = 0;
Marc Zyngierc1bfb572014-07-08 12:09:01 +01001956}
1957
1958/*
1959 * Allocate and initialize the various data structures. Must be called
1960 * with kvm->lock held!
1961 */
Andre Przywara83215812014-06-07 00:53:08 +02001962int vgic_init(struct kvm *kvm)
Marc Zyngierc1bfb572014-07-08 12:09:01 +01001963{
1964 struct vgic_dist *dist = &kvm->arch.vgic;
1965 struct kvm_vcpu *vcpu;
1966 int nr_cpus, nr_irqs;
Peter Maydell6d3cfbe2014-12-04 15:02:24 +00001967 int ret, i, vcpu_id;
Marc Zyngierc1bfb572014-07-08 12:09:01 +01001968
Christoffer Dall1f57be22014-12-09 14:30:36 +01001969 if (vgic_initialized(kvm))
Marc Zyngier4956f2b2014-07-08 12:09:06 +01001970 return 0;
Marc Zyngier5fb66da2014-07-08 12:09:05 +01001971
Marc Zyngier4956f2b2014-07-08 12:09:06 +01001972 nr_cpus = dist->nr_cpus = atomic_read(&kvm->online_vcpus);
1973 if (!nr_cpus) /* No vcpus? Can't be good... */
Eric Auger66b030e2014-12-15 18:43:32 +01001974 return -ENODEV;
Marc Zyngier4956f2b2014-07-08 12:09:06 +01001975
1976 /*
1977 * If nobody configured the number of interrupts, use the
1978 * legacy one.
1979 */
Marc Zyngier5fb66da2014-07-08 12:09:05 +01001980 if (!dist->nr_irqs)
1981 dist->nr_irqs = VGIC_NR_IRQS_LEGACY;
1982
1983 nr_irqs = dist->nr_irqs;
Marc Zyngierc1bfb572014-07-08 12:09:01 +01001984
1985 ret = vgic_init_bitmap(&dist->irq_enabled, nr_cpus, nr_irqs);
1986 ret |= vgic_init_bitmap(&dist->irq_level, nr_cpus, nr_irqs);
1987 ret |= vgic_init_bitmap(&dist->irq_pending, nr_cpus, nr_irqs);
1988 ret |= vgic_init_bitmap(&dist->irq_soft_pend, nr_cpus, nr_irqs);
1989 ret |= vgic_init_bitmap(&dist->irq_queued, nr_cpus, nr_irqs);
Christoffer Dall47a98b12015-03-13 17:02:54 +00001990 ret |= vgic_init_bitmap(&dist->irq_active, nr_cpus, nr_irqs);
Marc Zyngierc1bfb572014-07-08 12:09:01 +01001991 ret |= vgic_init_bitmap(&dist->irq_cfg, nr_cpus, nr_irqs);
1992 ret |= vgic_init_bytemap(&dist->irq_priority, nr_cpus, nr_irqs);
1993
1994 if (ret)
1995 goto out;
1996
1997 dist->irq_sgi_sources = kzalloc(nr_cpus * VGIC_NR_SGIS, GFP_KERNEL);
1998 dist->irq_spi_cpu = kzalloc(nr_irqs - VGIC_NR_PRIVATE_IRQS, GFP_KERNEL);
1999 dist->irq_spi_target = kzalloc(sizeof(*dist->irq_spi_target) * nr_cpus,
2000 GFP_KERNEL);
2001 dist->irq_pending_on_cpu = kzalloc(BITS_TO_LONGS(nr_cpus) * sizeof(long),
2002 GFP_KERNEL);
Christoffer Dall47a98b12015-03-13 17:02:54 +00002003 dist->irq_active_on_cpu = kzalloc(BITS_TO_LONGS(nr_cpus) * sizeof(long),
2004 GFP_KERNEL);
Marc Zyngierc1bfb572014-07-08 12:09:01 +01002005 if (!dist->irq_sgi_sources ||
2006 !dist->irq_spi_cpu ||
2007 !dist->irq_spi_target ||
Christoffer Dall47a98b12015-03-13 17:02:54 +00002008 !dist->irq_pending_on_cpu ||
2009 !dist->irq_active_on_cpu) {
Marc Zyngierc1bfb572014-07-08 12:09:01 +01002010 ret = -ENOMEM;
2011 goto out;
2012 }
2013
2014 for (i = 0; i < nr_cpus; i++)
2015 ret |= vgic_init_bitmap(&dist->irq_spi_target[i],
2016 nr_cpus, nr_irqs);
2017
2018 if (ret)
2019 goto out;
2020
Andre Przywarab26e5fd2014-06-02 16:19:12 +02002021 ret = kvm->arch.vgic.vm_ops.init_model(kvm);
2022 if (ret)
2023 goto out;
Peter Maydell6d3cfbe2014-12-04 15:02:24 +00002024
2025 kvm_for_each_vcpu(vcpu_id, vcpu, kvm) {
Marc Zyngierc1bfb572014-07-08 12:09:01 +01002026 ret = vgic_vcpu_init_maps(vcpu, nr_irqs);
2027 if (ret) {
2028 kvm_err("VGIC: Failed to allocate vcpu memory\n");
2029 break;
2030 }
Marc Zyngierc1bfb572014-07-08 12:09:01 +01002031
Christoffer Dall54723bb2015-08-30 14:45:20 +02002032 /*
Christoffer Dall4b4b4512015-08-30 15:01:27 +02002033 * Enable and configure all SGIs to be edge-triggere and
2034 * configure all PPIs as level-triggered.
Christoffer Dall54723bb2015-08-30 14:45:20 +02002035 */
2036 for (i = 0; i < VGIC_NR_PRIVATE_IRQS; i++) {
Christoffer Dall4b4b4512015-08-30 15:01:27 +02002037 if (i < VGIC_NR_SGIS) {
2038 /* SGIs */
Peter Maydell6d3cfbe2014-12-04 15:02:24 +00002039 vgic_bitmap_set_irq_val(&dist->irq_enabled,
2040 vcpu->vcpu_id, i, 1);
Peter Maydell6d3cfbe2014-12-04 15:02:24 +00002041 vgic_bitmap_set_irq_val(&dist->irq_cfg,
2042 vcpu->vcpu_id, i,
2043 VGIC_CFG_EDGE);
Christoffer Dall4b4b4512015-08-30 15:01:27 +02002044 } else if (i < VGIC_NR_PRIVATE_IRQS) {
2045 /* PPIs */
2046 vgic_bitmap_set_irq_val(&dist->irq_cfg,
2047 vcpu->vcpu_id, i,
2048 VGIC_CFG_LEVEL);
2049 }
Peter Maydell6d3cfbe2014-12-04 15:02:24 +00002050 }
2051
2052 vgic_enable(vcpu);
2053 }
Marc Zyngier4956f2b2014-07-08 12:09:06 +01002054
Marc Zyngierc1bfb572014-07-08 12:09:01 +01002055out:
2056 if (ret)
2057 kvm_vgic_destroy(kvm);
2058
2059 return ret;
2060}
2061
Andre Przywarab26e5fd2014-06-02 16:19:12 +02002062static int init_vgic_model(struct kvm *kvm, int type)
2063{
2064 switch (type) {
2065 case KVM_DEV_TYPE_ARM_VGIC_V2:
2066 vgic_v2_init_emulation(kvm);
2067 break;
Jean-Philippe Brucker4f64cb62015-10-01 13:47:19 +01002068#ifdef CONFIG_KVM_ARM_VGIC_V3
Andre Przywarab5d84ff2014-06-03 10:26:03 +02002069 case KVM_DEV_TYPE_ARM_VGIC_V3:
2070 vgic_v3_init_emulation(kvm);
2071 break;
2072#endif
Andre Przywarab26e5fd2014-06-02 16:19:12 +02002073 default:
2074 return -ENODEV;
2075 }
2076
Andre Przywara3caa2d82014-06-02 16:26:01 +02002077 if (atomic_read(&kvm->online_vcpus) > kvm->arch.max_vcpus)
2078 return -E2BIG;
2079
Andre Przywarab26e5fd2014-06-02 16:19:12 +02002080 return 0;
2081}
2082
Marc Zyngier6c3d63c2014-06-23 17:37:18 +01002083/**
2084 * kvm_vgic_early_init - Earliest possible vgic initialization stage
2085 *
2086 * No memory allocation should be performed here, only static init.
2087 */
2088void kvm_vgic_early_init(struct kvm *kvm)
2089{
2090 spin_lock_init(&kvm->arch.vgic.lock);
2091 spin_lock_init(&kvm->arch.vgic.irq_phys_map_lock);
2092 INIT_LIST_HEAD(&kvm->arch.vgic.irq_phys_map_list);
2093}
2094
Andre Przywara598921362014-06-03 09:33:10 +02002095int kvm_vgic_create(struct kvm *kvm, u32 type)
Marc Zyngier01ac5e32013-01-21 19:36:16 -05002096{
Christoffer Dall6b50f542014-11-06 11:47:39 +00002097 int i, vcpu_lock_idx = -1, ret;
Christoffer Dall73306722013-10-25 17:29:18 +01002098 struct kvm_vcpu *vcpu;
Marc Zyngier01ac5e32013-01-21 19:36:16 -05002099
2100 mutex_lock(&kvm->lock);
2101
Andre Przywara4ce7ebd2014-10-26 23:18:14 +00002102 if (irqchip_in_kernel(kvm)) {
Marc Zyngier01ac5e32013-01-21 19:36:16 -05002103 ret = -EEXIST;
2104 goto out;
2105 }
2106
Christoffer Dall73306722013-10-25 17:29:18 +01002107 /*
Andre Przywarab5d84ff2014-06-03 10:26:03 +02002108 * This function is also called by the KVM_CREATE_IRQCHIP handler,
2109 * which had no chance yet to check the availability of the GICv2
2110 * emulation. So check this here again. KVM_CREATE_DEVICE does
2111 * the proper checks already.
2112 */
Wei Yongjunb52104e2015-02-27 19:41:45 +08002113 if (type == KVM_DEV_TYPE_ARM_VGIC_V2 && !vgic->can_emulate_gicv2) {
2114 ret = -ENODEV;
2115 goto out;
2116 }
Andre Przywarab5d84ff2014-06-03 10:26:03 +02002117
2118 /*
Christoffer Dall73306722013-10-25 17:29:18 +01002119 * Any time a vcpu is run, vcpu_load is called which tries to grab the
2120 * vcpu->mutex. By grabbing the vcpu->mutex of all VCPUs we ensure
2121 * that no other VCPUs are run while we create the vgic.
2122 */
Christoffer Dall6b50f542014-11-06 11:47:39 +00002123 ret = -EBUSY;
Christoffer Dall73306722013-10-25 17:29:18 +01002124 kvm_for_each_vcpu(i, vcpu, kvm) {
2125 if (!mutex_trylock(&vcpu->mutex))
2126 goto out_unlock;
2127 vcpu_lock_idx = i;
2128 }
2129
2130 kvm_for_each_vcpu(i, vcpu, kvm) {
Christoffer Dall6b50f542014-11-06 11:47:39 +00002131 if (vcpu->arch.has_run_once)
Christoffer Dall73306722013-10-25 17:29:18 +01002132 goto out_unlock;
Christoffer Dall73306722013-10-25 17:29:18 +01002133 }
Christoffer Dall6b50f542014-11-06 11:47:39 +00002134 ret = 0;
Christoffer Dall73306722013-10-25 17:29:18 +01002135
Andre Przywarab26e5fd2014-06-02 16:19:12 +02002136 ret = init_vgic_model(kvm, type);
2137 if (ret)
2138 goto out_unlock;
2139
Marc Zyngierf982cf42014-05-15 10:03:25 +01002140 kvm->arch.vgic.in_kernel = true;
Andre Przywara598921362014-06-03 09:33:10 +02002141 kvm->arch.vgic.vgic_model = type;
Marc Zyngier8f186d52014-02-04 18:13:03 +00002142 kvm->arch.vgic.vctrl_base = vgic->vctrl_base;
Marc Zyngier01ac5e32013-01-21 19:36:16 -05002143 kvm->arch.vgic.vgic_dist_base = VGIC_ADDR_UNDEF;
2144 kvm->arch.vgic.vgic_cpu_base = VGIC_ADDR_UNDEF;
Andre Przywaraa0675c22014-06-07 00:54:51 +02002145 kvm->arch.vgic.vgic_redist_base = VGIC_ADDR_UNDEF;
Marc Zyngier01ac5e32013-01-21 19:36:16 -05002146
Christoffer Dall73306722013-10-25 17:29:18 +01002147out_unlock:
2148 for (; vcpu_lock_idx >= 0; vcpu_lock_idx--) {
2149 vcpu = kvm_get_vcpu(kvm, vcpu_lock_idx);
2150 mutex_unlock(&vcpu->mutex);
2151 }
2152
Marc Zyngier01ac5e32013-01-21 19:36:16 -05002153out:
2154 mutex_unlock(&kvm->lock);
2155 return ret;
2156}
2157
Will Deacon1fa451b2014-08-26 15:13:24 +01002158static int vgic_ioaddr_overlap(struct kvm *kvm)
Christoffer Dall330690c2013-01-21 19:36:13 -05002159{
2160 phys_addr_t dist = kvm->arch.vgic.vgic_dist_base;
2161 phys_addr_t cpu = kvm->arch.vgic.vgic_cpu_base;
2162
2163 if (IS_VGIC_ADDR_UNDEF(dist) || IS_VGIC_ADDR_UNDEF(cpu))
2164 return 0;
2165 if ((dist <= cpu && dist + KVM_VGIC_V2_DIST_SIZE > cpu) ||
2166 (cpu <= dist && cpu + KVM_VGIC_V2_CPU_SIZE > dist))
2167 return -EBUSY;
2168 return 0;
2169}
2170
2171static int vgic_ioaddr_assign(struct kvm *kvm, phys_addr_t *ioaddr,
2172 phys_addr_t addr, phys_addr_t size)
2173{
2174 int ret;
2175
Christoffer Dallce01e4e2013-09-23 14:55:56 -07002176 if (addr & ~KVM_PHYS_MASK)
2177 return -E2BIG;
2178
2179 if (addr & (SZ_4K - 1))
2180 return -EINVAL;
2181
Christoffer Dall330690c2013-01-21 19:36:13 -05002182 if (!IS_VGIC_ADDR_UNDEF(*ioaddr))
2183 return -EEXIST;
2184 if (addr + size < addr)
2185 return -EINVAL;
2186
Haibin Wang30c21172014-04-29 14:49:17 +08002187 *ioaddr = addr;
Christoffer Dall330690c2013-01-21 19:36:13 -05002188 ret = vgic_ioaddr_overlap(kvm);
2189 if (ret)
Haibin Wang30c21172014-04-29 14:49:17 +08002190 *ioaddr = VGIC_ADDR_UNDEF;
2191
Christoffer Dall330690c2013-01-21 19:36:13 -05002192 return ret;
2193}
2194
Christoffer Dallce01e4e2013-09-23 14:55:56 -07002195/**
2196 * kvm_vgic_addr - set or get vgic VM base addresses
2197 * @kvm: pointer to the vm struct
Andre Przywaraac3d3732014-06-03 10:26:30 +02002198 * @type: the VGIC addr type, one of KVM_VGIC_V[23]_ADDR_TYPE_XXX
Christoffer Dallce01e4e2013-09-23 14:55:56 -07002199 * @addr: pointer to address value
2200 * @write: if true set the address in the VM address space, if false read the
2201 * address
2202 *
2203 * Set or get the vgic base addresses for the distributor and the virtual CPU
2204 * interface in the VM physical address space. These addresses are properties
2205 * of the emulated core/SoC and therefore user space initially knows this
2206 * information.
2207 */
2208int kvm_vgic_addr(struct kvm *kvm, unsigned long type, u64 *addr, bool write)
Christoffer Dall330690c2013-01-21 19:36:13 -05002209{
2210 int r = 0;
2211 struct vgic_dist *vgic = &kvm->arch.vgic;
Andre Przywaraac3d3732014-06-03 10:26:30 +02002212 int type_needed;
2213 phys_addr_t *addr_ptr, block_size;
Andre Przywara4fa96afd2015-01-13 12:02:13 +00002214 phys_addr_t alignment;
Christoffer Dall330690c2013-01-21 19:36:13 -05002215
Christoffer Dall330690c2013-01-21 19:36:13 -05002216 mutex_lock(&kvm->lock);
2217 switch (type) {
2218 case KVM_VGIC_V2_ADDR_TYPE_DIST:
Andre Przywaraac3d3732014-06-03 10:26:30 +02002219 type_needed = KVM_DEV_TYPE_ARM_VGIC_V2;
2220 addr_ptr = &vgic->vgic_dist_base;
2221 block_size = KVM_VGIC_V2_DIST_SIZE;
Andre Przywara4fa96afd2015-01-13 12:02:13 +00002222 alignment = SZ_4K;
Christoffer Dall330690c2013-01-21 19:36:13 -05002223 break;
2224 case KVM_VGIC_V2_ADDR_TYPE_CPU:
Andre Przywaraac3d3732014-06-03 10:26:30 +02002225 type_needed = KVM_DEV_TYPE_ARM_VGIC_V2;
2226 addr_ptr = &vgic->vgic_cpu_base;
2227 block_size = KVM_VGIC_V2_CPU_SIZE;
Andre Przywara4fa96afd2015-01-13 12:02:13 +00002228 alignment = SZ_4K;
Christoffer Dall330690c2013-01-21 19:36:13 -05002229 break;
Jean-Philippe Brucker4f64cb62015-10-01 13:47:19 +01002230#ifdef CONFIG_KVM_ARM_VGIC_V3
Andre Przywaraac3d3732014-06-03 10:26:30 +02002231 case KVM_VGIC_V3_ADDR_TYPE_DIST:
2232 type_needed = KVM_DEV_TYPE_ARM_VGIC_V3;
2233 addr_ptr = &vgic->vgic_dist_base;
2234 block_size = KVM_VGIC_V3_DIST_SIZE;
Andre Przywara4fa96afd2015-01-13 12:02:13 +00002235 alignment = SZ_64K;
Andre Przywaraac3d3732014-06-03 10:26:30 +02002236 break;
2237 case KVM_VGIC_V3_ADDR_TYPE_REDIST:
2238 type_needed = KVM_DEV_TYPE_ARM_VGIC_V3;
2239 addr_ptr = &vgic->vgic_redist_base;
2240 block_size = KVM_VGIC_V3_REDIST_SIZE;
Andre Przywara4fa96afd2015-01-13 12:02:13 +00002241 alignment = SZ_64K;
Andre Przywaraac3d3732014-06-03 10:26:30 +02002242 break;
2243#endif
Christoffer Dall330690c2013-01-21 19:36:13 -05002244 default:
2245 r = -ENODEV;
Andre Przywaraac3d3732014-06-03 10:26:30 +02002246 goto out;
Christoffer Dall330690c2013-01-21 19:36:13 -05002247 }
2248
Andre Przywaraac3d3732014-06-03 10:26:30 +02002249 if (vgic->vgic_model != type_needed) {
2250 r = -ENODEV;
2251 goto out;
2252 }
2253
Andre Przywara4fa96afd2015-01-13 12:02:13 +00002254 if (write) {
2255 if (!IS_ALIGNED(*addr, alignment))
2256 r = -EINVAL;
2257 else
2258 r = vgic_ioaddr_assign(kvm, addr_ptr, *addr,
2259 block_size);
2260 } else {
Andre Przywaraac3d3732014-06-03 10:26:30 +02002261 *addr = *addr_ptr;
Andre Przywara4fa96afd2015-01-13 12:02:13 +00002262 }
Andre Przywaraac3d3732014-06-03 10:26:30 +02002263
2264out:
Christoffer Dall330690c2013-01-21 19:36:13 -05002265 mutex_unlock(&kvm->lock);
2266 return r;
2267}
Christoffer Dall73306722013-10-25 17:29:18 +01002268
Andre Przywara83215812014-06-07 00:53:08 +02002269int vgic_set_common_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
Christoffer Dall73306722013-10-25 17:29:18 +01002270{
Christoffer Dallce01e4e2013-09-23 14:55:56 -07002271 int r;
2272
2273 switch (attr->group) {
2274 case KVM_DEV_ARM_VGIC_GRP_ADDR: {
2275 u64 __user *uaddr = (u64 __user *)(long)attr->addr;
2276 u64 addr;
2277 unsigned long type = (unsigned long)attr->attr;
2278
2279 if (copy_from_user(&addr, uaddr, sizeof(addr)))
2280 return -EFAULT;
2281
2282 r = kvm_vgic_addr(dev->kvm, type, &addr, true);
2283 return (r == -ENODEV) ? -ENXIO : r;
2284 }
Marc Zyngiera98f26f2014-07-08 12:09:07 +01002285 case KVM_DEV_ARM_VGIC_GRP_NR_IRQS: {
2286 u32 __user *uaddr = (u32 __user *)(long)attr->addr;
2287 u32 val;
2288 int ret = 0;
2289
2290 if (get_user(val, uaddr))
2291 return -EFAULT;
2292
2293 /*
2294 * We require:
2295 * - at least 32 SPIs on top of the 16 SGIs and 16 PPIs
2296 * - at most 1024 interrupts
2297 * - a multiple of 32 interrupts
2298 */
2299 if (val < (VGIC_NR_PRIVATE_IRQS + 32) ||
2300 val > VGIC_MAX_IRQS ||
2301 (val & 31))
2302 return -EINVAL;
2303
2304 mutex_lock(&dev->kvm->lock);
2305
Christoffer Dallc52edf52014-12-09 14:28:09 +01002306 if (vgic_ready(dev->kvm) || dev->kvm->arch.vgic.nr_irqs)
Marc Zyngiera98f26f2014-07-08 12:09:07 +01002307 ret = -EBUSY;
2308 else
2309 dev->kvm->arch.vgic.nr_irqs = val;
2310
2311 mutex_unlock(&dev->kvm->lock);
2312
2313 return ret;
2314 }
Eric Auger065c0032014-12-15 18:43:33 +01002315 case KVM_DEV_ARM_VGIC_GRP_CTRL: {
2316 switch (attr->attr) {
2317 case KVM_DEV_ARM_VGIC_CTRL_INIT:
2318 r = vgic_init(dev->kvm);
2319 return r;
2320 }
2321 break;
2322 }
Christoffer Dallce01e4e2013-09-23 14:55:56 -07002323 }
2324
Christoffer Dall73306722013-10-25 17:29:18 +01002325 return -ENXIO;
2326}
2327
Andre Przywara83215812014-06-07 00:53:08 +02002328int vgic_get_common_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
Christoffer Dall73306722013-10-25 17:29:18 +01002329{
Christoffer Dallce01e4e2013-09-23 14:55:56 -07002330 int r = -ENXIO;
2331
2332 switch (attr->group) {
2333 case KVM_DEV_ARM_VGIC_GRP_ADDR: {
2334 u64 __user *uaddr = (u64 __user *)(long)attr->addr;
2335 u64 addr;
2336 unsigned long type = (unsigned long)attr->attr;
2337
2338 r = kvm_vgic_addr(dev->kvm, type, &addr, false);
2339 if (r)
2340 return (r == -ENODEV) ? -ENXIO : r;
2341
2342 if (copy_to_user(uaddr, &addr, sizeof(addr)))
2343 return -EFAULT;
Christoffer Dallc07a0192013-10-25 21:17:31 +01002344 break;
Christoffer Dallce01e4e2013-09-23 14:55:56 -07002345 }
Marc Zyngiera98f26f2014-07-08 12:09:07 +01002346 case KVM_DEV_ARM_VGIC_GRP_NR_IRQS: {
2347 u32 __user *uaddr = (u32 __user *)(long)attr->addr;
Andre Przywarab60da142014-08-21 11:08:27 +01002348
Marc Zyngiera98f26f2014-07-08 12:09:07 +01002349 r = put_user(dev->kvm->arch.vgic.nr_irqs, uaddr);
2350 break;
2351 }
Christoffer Dallc07a0192013-10-25 21:17:31 +01002352
Christoffer Dallce01e4e2013-09-23 14:55:56 -07002353 }
2354
2355 return r;
Christoffer Dall73306722013-10-25 17:29:18 +01002356}
2357
Andre Przywaracf50a1e2015-03-26 14:39:32 +00002358int vgic_has_attr_regs(const struct vgic_io_range *ranges, phys_addr_t offset)
Christoffer Dallc07a0192013-10-25 21:17:31 +01002359{
Andre Przywara9f199d02015-03-26 14:39:33 +00002360 if (vgic_find_range(ranges, 4, offset))
Christoffer Dallc07a0192013-10-25 21:17:31 +01002361 return 0;
2362 else
2363 return -ENXIO;
2364}
2365
Will Deaconc06a8412014-09-02 10:27:34 +01002366static void vgic_init_maintenance_interrupt(void *info)
2367{
2368 enable_percpu_irq(vgic->maint_irq, 0);
2369}
2370
2371static int vgic_cpu_notify(struct notifier_block *self,
2372 unsigned long action, void *cpu)
2373{
2374 switch (action) {
2375 case CPU_STARTING:
2376 case CPU_STARTING_FROZEN:
2377 vgic_init_maintenance_interrupt(NULL);
2378 break;
2379 case CPU_DYING:
2380 case CPU_DYING_FROZEN:
2381 disable_percpu_irq(vgic->maint_irq);
2382 break;
2383 }
2384
2385 return NOTIFY_OK;
2386}
2387
2388static struct notifier_block vgic_cpu_nb = {
2389 .notifier_call = vgic_cpu_notify,
2390};
2391
2392static const struct of_device_id vgic_ids[] = {
Mark Rutland0f3724752015-03-05 14:47:44 +00002393 { .compatible = "arm,cortex-a15-gic", .data = vgic_v2_probe, },
2394 { .compatible = "arm,cortex-a7-gic", .data = vgic_v2_probe, },
2395 { .compatible = "arm,gic-400", .data = vgic_v2_probe, },
2396 { .compatible = "arm,gic-v3", .data = vgic_v3_probe, },
Will Deaconc06a8412014-09-02 10:27:34 +01002397 {},
2398};
2399
2400int kvm_vgic_hyp_init(void)
2401{
2402 const struct of_device_id *matched_id;
Christoffer Dalla875daf2014-09-18 18:15:32 -07002403 const int (*vgic_probe)(struct device_node *,const struct vgic_ops **,
2404 const struct vgic_params **);
Will Deaconc06a8412014-09-02 10:27:34 +01002405 struct device_node *vgic_node;
2406 int ret;
2407
2408 vgic_node = of_find_matching_node_and_match(NULL,
2409 vgic_ids, &matched_id);
2410 if (!vgic_node) {
2411 kvm_err("error: no compatible GIC node found\n");
2412 return -ENODEV;
2413 }
2414
2415 vgic_probe = matched_id->data;
2416 ret = vgic_probe(vgic_node, &vgic_ops, &vgic);
2417 if (ret)
2418 return ret;
2419
2420 ret = request_percpu_irq(vgic->maint_irq, vgic_maintenance_handler,
2421 "vgic", kvm_get_running_vcpus());
2422 if (ret) {
2423 kvm_err("Cannot register interrupt %d\n", vgic->maint_irq);
2424 return ret;
2425 }
2426
2427 ret = __register_cpu_notifier(&vgic_cpu_nb);
2428 if (ret) {
2429 kvm_err("Cannot register vgic CPU notifier\n");
2430 goto out_free_irq;
2431 }
2432
Will Deaconc06a8412014-09-02 10:27:34 +01002433 on_each_cpu(vgic_init_maintenance_interrupt, NULL, 1);
2434
Andre Przywaraea2f83a2014-10-26 23:17:00 +00002435 return 0;
Will Deaconc06a8412014-09-02 10:27:34 +01002436
2437out_free_irq:
2438 free_percpu_irq(vgic->maint_irq, kvm_get_running_vcpus());
2439 return ret;
2440}
Eric Auger174178f2015-03-04 11:14:36 +01002441
2442int kvm_irq_map_gsi(struct kvm *kvm,
2443 struct kvm_kernel_irq_routing_entry *entries,
2444 int gsi)
2445{
Eric Auger0b3289e2015-04-13 15:01:59 +02002446 return 0;
Eric Auger174178f2015-03-04 11:14:36 +01002447}
2448
2449int kvm_irq_map_chip_pin(struct kvm *kvm, unsigned irqchip, unsigned pin)
2450{
2451 return pin;
2452}
2453
2454int kvm_set_irq(struct kvm *kvm, int irq_source_id,
2455 u32 irq, int level, bool line_status)
2456{
2457 unsigned int spi = irq + VGIC_NR_PRIVATE_IRQS;
2458
2459 trace_kvm_set_irq(irq, level, irq_source_id);
2460
2461 BUG_ON(!vgic_initialized(kvm));
2462
Eric Auger174178f2015-03-04 11:14:36 +01002463 return kvm_vgic_inject_irq(kvm, 0, spi, level);
Eric Auger174178f2015-03-04 11:14:36 +01002464}
2465
2466/* MSI not implemented yet */
2467int kvm_set_msi(struct kvm_kernel_irq_routing_entry *e,
2468 struct kvm *kvm, int irq_source_id,
2469 int level, bool line_status)
2470{
2471 return 0;
2472}