blob: 4a37292855bc7f8bed299b99407d68882b98c84e [file] [log] [blame]
Marc Zyngier74fe55d2017-10-27 15:28:38 +01001/*
2 * Copyright (C) 2017 ARM Ltd.
3 * Author: Marc Zyngier <marc.zyngier@arm.com>
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License version 2 as
7 * published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program. If not, see <http://www.gnu.org/licenses/>.
16 */
17
18#include <linux/interrupt.h>
Marc Zyngierbdb2d2c2017-10-27 15:28:48 +010019#include <linux/irq.h>
Marc Zyngier74fe55d2017-10-27 15:28:38 +010020#include <linux/irqdomain.h>
21#include <linux/kvm_host.h>
Marc Zyngier196b1362017-10-27 15:28:39 +010022#include <linux/irqchip/arm-gic-v3.h>
Marc Zyngier74fe55d2017-10-27 15:28:38 +010023
24#include "vgic.h"
25
Marc Zyngiered8703a2017-10-27 15:28:55 +010026/*
27 * How KVM uses GICv4 (insert rude comments here):
28 *
29 * The vgic-v4 layer acts as a bridge between several entities:
30 * - The GICv4 ITS representation offered by the ITS driver
31 * - VFIO, which is in charge of the PCI endpoint
32 * - The virtual ITS, which is the only thing the guest sees
33 *
34 * The configuration of VLPIs is triggered by a callback from VFIO,
35 * instructing KVM that a PCI device has been configured to deliver
36 * MSIs to a vITS.
37 *
38 * kvm_vgic_v4_set_forwarding() is thus called with the routing entry,
39 * and this is used to find the corresponding vITS data structures
40 * (ITS instance, device, event and irq) using a process that is
41 * extremely similar to the injection of an MSI.
42 *
43 * At this stage, we can link the guest's view of an LPI (uniquely
44 * identified by the routing entry) and the host irq, using the GICv4
45 * driver mapping operation. Should the mapping succeed, we've then
46 * successfully upgraded the guest's LPI to a VLPI. We can then start
47 * with updating GICv4's view of the property table and generating an
48 * INValidation in order to kickstart the delivery of this VLPI to the
49 * guest directly, without software intervention. Well, almost.
50 *
51 * When the PCI endpoint is deconfigured, this operation is reversed
52 * with VFIO calling kvm_vgic_v4_unset_forwarding().
53 *
54 * Once the VLPI has been mapped, it needs to follow any change the
55 * guest performs on its LPI through the vITS. For that, a number of
56 * command handlers have hooks to communicate these changes to the HW:
57 * - Any invalidation triggers a call to its_prop_update_vlpi()
58 * - The INT command results in a irq_set_irqchip_state(), which
59 * generates an INT on the corresponding VLPI.
60 * - The CLEAR command results in a irq_set_irqchip_state(), which
61 * generates an CLEAR on the corresponding VLPI.
62 * - DISCARD translates into an unmap, similar to a call to
63 * kvm_vgic_v4_unset_forwarding().
64 * - MOVI is translated by an update of the existing mapping, changing
65 * the target vcpu, resulting in a VMOVI being generated.
66 * - MOVALL is translated by a string of mapping updates (similar to
67 * the handling of MOVI). MOVALL is horrible.
68 *
69 * Note that a DISCARD/MAPTI sequence emitted from the guest without
70 * reprogramming the PCI endpoint after MAPTI does not result in a
71 * VLPI being mapped, as there is no callback from VFIO (the guest
72 * will get the interrupt via the normal SW injection). Fixing this is
73 * not trivial, and requires some horrible messing with the VFIO
74 * internals. Not fun. Don't do that.
75 *
76 * Then there is the scheduling. Each time a vcpu is about to run on a
77 * physical CPU, KVM must tell the corresponding redistributor about
78 * it. And if we've migrated our vcpu from one CPU to another, we must
79 * tell the ITS (so that the messages reach the right redistributor).
80 * This is done in two steps: first issue a irq_set_affinity() on the
81 * irq corresponding to the vcpu, then call its_schedule_vpe(). You
82 * must be in a non-preemptible context. On exit, another call to
83 * its_schedule_vpe() tells the redistributor that we're done with the
84 * vcpu.
85 *
86 * Finally, the doorbell handling: Each vcpu is allocated an interrupt
87 * which will fire each time a VLPI is made pending whilst the vcpu is
88 * not running. Each time the vcpu gets blocked, the doorbell
89 * interrupt gets enabled. When the vcpu is unblocked (for whatever
90 * reason), the doorbell interrupt is disabled.
91 */
92
Marc Zyngierd3d83f72017-10-27 15:28:53 +010093#define DB_IRQ_FLAGS (IRQ_NOAUTOEN | IRQ_DISABLE_UNLAZY | IRQ_NO_BALANCING)
94
Marc Zyngierbdb2d2c2017-10-27 15:28:48 +010095static irqreturn_t vgic_v4_doorbell_handler(int irq, void *info)
96{
97 struct kvm_vcpu *vcpu = info;
98
99 vcpu->arch.vgic_cpu.vgic_v3.its_vpe.pending_last = true;
100 kvm_make_request(KVM_REQ_IRQ_PENDING, vcpu);
101 kvm_vcpu_kick(vcpu);
102
103 return IRQ_HANDLED;
104}
105
Marc Zyngier74fe55d2017-10-27 15:28:38 +0100106/**
107 * vgic_v4_init - Initialize the GICv4 data structures
108 * @kvm: Pointer to the VM being initialized
109 *
110 * We may be called each time a vITS is created, or when the
111 * vgic is initialized. This relies on kvm->lock to be
112 * held. In both cases, the number of vcpus should now be
113 * fixed.
114 */
115int vgic_v4_init(struct kvm *kvm)
116{
117 struct vgic_dist *dist = &kvm->arch.vgic;
118 struct kvm_vcpu *vcpu;
119 int i, nr_vcpus, ret;
120
Christoffer Dall3d1ad642017-11-10 09:16:23 +0100121 if (!vgic_supports_direct_msis(kvm))
122 return 0; /* Nothing to see here... move along. */
123
Marc Zyngier74fe55d2017-10-27 15:28:38 +0100124 if (dist->its_vm.vpes)
125 return 0;
126
127 nr_vcpus = atomic_read(&kvm->online_vcpus);
128
129 dist->its_vm.vpes = kzalloc(sizeof(*dist->its_vm.vpes) * nr_vcpus,
130 GFP_KERNEL);
131 if (!dist->its_vm.vpes)
132 return -ENOMEM;
133
134 dist->its_vm.nr_vpes = nr_vcpus;
135
136 kvm_for_each_vcpu(i, vcpu, kvm)
137 dist->its_vm.vpes[i] = &vcpu->arch.vgic_cpu.vgic_v3.its_vpe;
138
139 ret = its_alloc_vcpu_irqs(&dist->its_vm);
140 if (ret < 0) {
141 kvm_err("VPE IRQ allocation failure\n");
142 kfree(dist->its_vm.vpes);
143 dist->its_vm.nr_vpes = 0;
144 dist->its_vm.vpes = NULL;
145 return ret;
146 }
147
Marc Zyngierbdb2d2c2017-10-27 15:28:48 +0100148 kvm_for_each_vcpu(i, vcpu, kvm) {
149 int irq = dist->its_vm.vpes[i]->irq;
150
151 /*
152 * Don't automatically enable the doorbell, as we're
153 * flipping it back and forth when the vcpu gets
154 * blocked. Also disable the lazy disabling, as the
155 * doorbell could kick us out of the guest too
156 * early...
157 */
Marc Zyngierd3d83f72017-10-27 15:28:53 +0100158 irq_set_status_flags(irq, DB_IRQ_FLAGS);
Marc Zyngierbdb2d2c2017-10-27 15:28:48 +0100159 ret = request_irq(irq, vgic_v4_doorbell_handler,
160 0, "vcpu", vcpu);
161 if (ret) {
162 kvm_err("failed to allocate vcpu IRQ%d\n", irq);
163 /*
164 * Trick: adjust the number of vpes so we know
165 * how many to nuke on teardown...
166 */
167 dist->its_vm.nr_vpes = i;
168 break;
169 }
170 }
171
172 if (ret)
173 vgic_v4_teardown(kvm);
174
Marc Zyngier74fe55d2017-10-27 15:28:38 +0100175 return ret;
176}
177
178/**
179 * vgic_v4_teardown - Free the GICv4 data structures
180 * @kvm: Pointer to the VM being destroyed
181 *
182 * Relies on kvm->lock to be held.
183 */
184void vgic_v4_teardown(struct kvm *kvm)
185{
186 struct its_vm *its_vm = &kvm->arch.vgic.its_vm;
Marc Zyngierbdb2d2c2017-10-27 15:28:48 +0100187 int i;
Marc Zyngier74fe55d2017-10-27 15:28:38 +0100188
189 if (!its_vm->vpes)
190 return;
191
Marc Zyngierbdb2d2c2017-10-27 15:28:48 +0100192 for (i = 0; i < its_vm->nr_vpes; i++) {
193 struct kvm_vcpu *vcpu = kvm_get_vcpu(kvm, i);
194 int irq = its_vm->vpes[i]->irq;
195
Marc Zyngierd3d83f72017-10-27 15:28:53 +0100196 irq_clear_status_flags(irq, DB_IRQ_FLAGS);
Marc Zyngierbdb2d2c2017-10-27 15:28:48 +0100197 free_irq(irq, vcpu);
198 }
199
Marc Zyngier74fe55d2017-10-27 15:28:38 +0100200 its_free_vcpu_irqs(its_vm);
201 kfree(its_vm->vpes);
202 its_vm->nr_vpes = 0;
203 its_vm->vpes = NULL;
204}
Marc Zyngier196b1362017-10-27 15:28:39 +0100205
Marc Zyngier62775792017-10-27 15:28:50 +0100206int vgic_v4_sync_hwstate(struct kvm_vcpu *vcpu)
207{
208 if (!vgic_supports_direct_msis(vcpu->kvm))
209 return 0;
210
211 return its_schedule_vpe(&vcpu->arch.vgic_cpu.vgic_v3.its_vpe, false);
212}
213
214int vgic_v4_flush_hwstate(struct kvm_vcpu *vcpu)
215{
216 int irq = vcpu->arch.vgic_cpu.vgic_v3.its_vpe.irq;
217 int err;
218
219 if (!vgic_supports_direct_msis(vcpu->kvm))
220 return 0;
221
222 /*
223 * Before making the VPE resident, make sure the redistributor
224 * corresponding to our current CPU expects us here. See the
225 * doc in drivers/irqchip/irq-gic-v4.c to understand how this
226 * turns into a VMOVP command at the ITS level.
227 */
228 err = irq_set_affinity(irq, cpumask_of(smp_processor_id()));
229 if (err)
230 return err;
231
232 err = its_schedule_vpe(&vcpu->arch.vgic_cpu.vgic_v3.its_vpe, true);
233 if (err)
234 return err;
235
236 /*
237 * Now that the VPE is resident, let's get rid of a potential
238 * doorbell interrupt that would still be pending.
239 */
240 err = irq_set_irqchip_state(irq, IRQCHIP_STATE_PENDING, false);
241
242 return err;
243}
244
Marc Zyngier196b1362017-10-27 15:28:39 +0100245static struct vgic_its *vgic_get_its(struct kvm *kvm,
246 struct kvm_kernel_irq_routing_entry *irq_entry)
247{
248 struct kvm_msi msi = (struct kvm_msi) {
249 .address_lo = irq_entry->msi.address_lo,
250 .address_hi = irq_entry->msi.address_hi,
251 .data = irq_entry->msi.data,
252 .flags = irq_entry->msi.flags,
253 .devid = irq_entry->msi.devid,
254 };
255
256 return vgic_msi_to_its(kvm, &msi);
257}
258
259int kvm_vgic_v4_set_forwarding(struct kvm *kvm, int virq,
260 struct kvm_kernel_irq_routing_entry *irq_entry)
261{
262 struct vgic_its *its;
263 struct vgic_irq *irq;
264 struct its_vlpi_map map;
265 int ret;
266
267 if (!vgic_supports_direct_msis(kvm))
268 return 0;
269
270 /*
271 * Get the ITS, and escape early on error (not a valid
272 * doorbell for any of our vITSs).
273 */
274 its = vgic_get_its(kvm, irq_entry);
275 if (IS_ERR(its))
276 return 0;
277
278 mutex_lock(&its->its_lock);
279
280 /* Perform then actual DevID/EventID -> LPI translation. */
281 ret = vgic_its_resolve_lpi(kvm, its, irq_entry->msi.devid,
282 irq_entry->msi.data, &irq);
283 if (ret)
284 goto out;
285
286 /*
287 * Emit the mapping request. If it fails, the ITS probably
288 * isn't v4 compatible, so let's silently bail out. Holding
289 * the ITS lock should ensure that nothing can modify the
290 * target vcpu.
291 */
292 map = (struct its_vlpi_map) {
293 .vm = &kvm->arch.vgic.its_vm,
294 .vpe = &irq->target_vcpu->arch.vgic_cpu.vgic_v3.its_vpe,
295 .vintid = irq->intid,
296 .properties = ((irq->priority & 0xfc) |
297 (irq->enabled ? LPI_PROP_ENABLED : 0) |
298 LPI_PROP_GROUP1),
299 .db_enabled = true,
300 };
301
302 ret = its_map_vlpi(virq, &map);
303 if (ret)
304 goto out;
305
306 irq->hw = true;
307 irq->host_irq = virq;
308
309out:
310 mutex_unlock(&its->its_lock);
311 return ret;
312}
313
314int kvm_vgic_v4_unset_forwarding(struct kvm *kvm, int virq,
315 struct kvm_kernel_irq_routing_entry *irq_entry)
316{
317 struct vgic_its *its;
318 struct vgic_irq *irq;
319 int ret;
320
321 if (!vgic_supports_direct_msis(kvm))
322 return 0;
323
324 /*
325 * Get the ITS, and escape early on error (not a valid
326 * doorbell for any of our vITSs).
327 */
328 its = vgic_get_its(kvm, irq_entry);
329 if (IS_ERR(its))
330 return 0;
331
332 mutex_lock(&its->its_lock);
333
334 ret = vgic_its_resolve_lpi(kvm, its, irq_entry->msi.devid,
335 irq_entry->msi.data, &irq);
336 if (ret)
337 goto out;
338
339 WARN_ON(!(irq->hw && irq->host_irq == virq));
Marc Zyngiera05d1c02017-11-16 17:58:19 +0000340 if (irq->hw) {
341 irq->hw = false;
342 ret = its_unmap_vlpi(virq);
343 }
Marc Zyngier196b1362017-10-27 15:28:39 +0100344
345out:
346 mutex_unlock(&its->its_lock);
347 return ret;
348}
Marc Zyngierdf9ba952017-10-27 15:28:49 +0100349
350void kvm_vgic_v4_enable_doorbell(struct kvm_vcpu *vcpu)
351{
352 if (vgic_supports_direct_msis(vcpu->kvm)) {
353 int irq = vcpu->arch.vgic_cpu.vgic_v3.its_vpe.irq;
354 if (irq)
355 enable_irq(irq);
356 }
357}
358
359void kvm_vgic_v4_disable_doorbell(struct kvm_vcpu *vcpu)
360{
361 if (vgic_supports_direct_msis(vcpu->kvm)) {
362 int irq = vcpu->arch.vgic_cpu.vgic_v3.its_vpe.irq;
363 if (irq)
364 disable_irq(irq);
365 }
366}