blob: 5e893ebb9de7b2829f1f88c9894c129f58837c6a [file] [log] [blame]
Christoffer Dall749cf76c2013-01-20 18:28:06 -05001/*
2 * Copyright (C) 2012 - Virtual Open Systems and Columbia University
3 * Author: Christoffer Dall <c.dall@virtualopensystems.com>
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License, version 2, as
7 * published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the Free Software
16 * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
17 */
18
Marc Zyngierd157f4a2013-04-12 19:12:07 +010019#include <linux/cpu.h>
Lorenzo Pieralisi1fcf7ce2013-08-05 15:04:46 +010020#include <linux/cpu_pm.h>
Christoffer Dall749cf76c2013-01-20 18:28:06 -050021#include <linux/errno.h>
22#include <linux/err.h>
23#include <linux/kvm_host.h>
24#include <linux/module.h>
25#include <linux/vmalloc.h>
26#include <linux/fs.h>
27#include <linux/mman.h>
28#include <linux/sched.h>
Christoffer Dall86ce85352013-01-20 18:28:08 -050029#include <linux/kvm.h>
Christoffer Dall749cf76c2013-01-20 18:28:06 -050030#include <trace/events/kvm.h>
31
32#define CREATE_TRACE_POINTS
33#include "trace.h"
34
Christoffer Dall749cf76c2013-01-20 18:28:06 -050035#include <asm/uaccess.h>
36#include <asm/ptrace.h>
37#include <asm/mman.h>
Christoffer Dall342cd0a2013-01-20 18:28:06 -050038#include <asm/tlbflush.h>
Christoffer Dall5b3e5e52013-01-20 18:28:09 -050039#include <asm/cacheflush.h>
Christoffer Dall342cd0a2013-01-20 18:28:06 -050040#include <asm/virt.h>
41#include <asm/kvm_arm.h>
42#include <asm/kvm_asm.h>
43#include <asm/kvm_mmu.h>
Christoffer Dallf7ed45b2013-01-20 18:47:42 -050044#include <asm/kvm_emulate.h>
Christoffer Dall5b3e5e52013-01-20 18:28:09 -050045#include <asm/kvm_coproc.h>
Marc Zyngieraa024c2f2013-01-20 18:28:13 -050046#include <asm/kvm_psci.h>
Christoffer Dall749cf76c2013-01-20 18:28:06 -050047
48#ifdef REQUIRES_VIRT
49__asm__(".arch_extension virt");
50#endif
51
Christoffer Dall342cd0a2013-01-20 18:28:06 -050052static DEFINE_PER_CPU(unsigned long, kvm_arm_hyp_stack_page);
Marc Zyngier3de50da2013-04-08 16:47:19 +010053static kvm_cpu_context_t __percpu *kvm_host_cpu_state;
Christoffer Dall342cd0a2013-01-20 18:28:06 -050054static unsigned long hyp_default_vectors;
55
Marc Zyngier1638a122013-01-21 19:36:11 -050056/* Per-CPU variable containing the currently running vcpu. */
57static DEFINE_PER_CPU(struct kvm_vcpu *, kvm_arm_running_vcpu);
58
Christoffer Dallf7ed45b2013-01-20 18:47:42 -050059/* The VMID used in the VTTBR */
60static atomic64_t kvm_vmid_gen = ATOMIC64_INIT(1);
61static u8 kvm_next_vmid;
62static DEFINE_SPINLOCK(kvm_vmid_lock);
Christoffer Dall342cd0a2013-01-20 18:28:06 -050063
Marc Zyngier1638a122013-01-21 19:36:11 -050064static void kvm_arm_set_running_vcpu(struct kvm_vcpu *vcpu)
65{
66 BUG_ON(preemptible());
Christoph Lameter1436c1a2013-10-21 13:17:08 +010067 __this_cpu_write(kvm_arm_running_vcpu, vcpu);
Marc Zyngier1638a122013-01-21 19:36:11 -050068}
69
70/**
71 * kvm_arm_get_running_vcpu - get the vcpu running on the current CPU.
72 * Must be called from non-preemptible context
73 */
74struct kvm_vcpu *kvm_arm_get_running_vcpu(void)
75{
76 BUG_ON(preemptible());
Christoph Lameter1436c1a2013-10-21 13:17:08 +010077 return __this_cpu_read(kvm_arm_running_vcpu);
Marc Zyngier1638a122013-01-21 19:36:11 -050078}
79
80/**
81 * kvm_arm_get_running_vcpus - get the per-CPU array of currently running vcpus.
82 */
Will Deacon4000be42014-08-26 15:13:21 +010083struct kvm_vcpu * __percpu *kvm_get_running_vcpus(void)
Marc Zyngier1638a122013-01-21 19:36:11 -050084{
85 return &kvm_arm_running_vcpu;
86}
87
Radim Krčmář13a34e02014-08-28 15:13:03 +020088int kvm_arch_hardware_enable(void)
Christoffer Dall749cf76c2013-01-20 18:28:06 -050089{
90 return 0;
91}
92
93int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
94{
95 return kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE;
96}
97
Christoffer Dall749cf76c2013-01-20 18:28:06 -050098int kvm_arch_hardware_setup(void)
99{
100 return 0;
101}
102
Christoffer Dall749cf76c2013-01-20 18:28:06 -0500103void kvm_arch_check_processor_compat(void *rtn)
104{
105 *(int *)rtn = 0;
106}
107
Christoffer Dall749cf76c2013-01-20 18:28:06 -0500108
Christoffer Dalld5d81842013-01-20 18:28:07 -0500109/**
110 * kvm_arch_init_vm - initializes a VM data structure
111 * @kvm: pointer to the KVM struct
112 */
Christoffer Dall749cf76c2013-01-20 18:28:06 -0500113int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
114{
Christoffer Dalld5d81842013-01-20 18:28:07 -0500115 int ret = 0;
116
Christoffer Dall749cf76c2013-01-20 18:28:06 -0500117 if (type)
118 return -EINVAL;
119
Christoffer Dalld5d81842013-01-20 18:28:07 -0500120 ret = kvm_alloc_stage2_pgd(kvm);
121 if (ret)
122 goto out_fail_alloc;
123
124 ret = create_hyp_mappings(kvm, kvm + 1);
125 if (ret)
126 goto out_free_stage2_pgd;
127
Christoffer Dalla1a64382013-11-16 10:51:25 -0800128 kvm_timer_init(kvm);
129
Christoffer Dalld5d81842013-01-20 18:28:07 -0500130 /* Mark the initial VMID generation invalid */
131 kvm->arch.vmid_gen = 0;
132
Andre Przywara3caa2d82014-06-02 16:26:01 +0200133 /* The maximum number of VCPUs is limited by the host's GIC model */
134 kvm->arch.max_vcpus = kvm_vgic_get_max_vcpus();
135
Christoffer Dalld5d81842013-01-20 18:28:07 -0500136 return ret;
137out_free_stage2_pgd:
138 kvm_free_stage2_pgd(kvm);
139out_fail_alloc:
140 return ret;
Christoffer Dall749cf76c2013-01-20 18:28:06 -0500141}
142
143int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
144{
145 return VM_FAULT_SIGBUS;
146}
147
Christoffer Dall749cf76c2013-01-20 18:28:06 -0500148
Christoffer Dalld5d81842013-01-20 18:28:07 -0500149/**
150 * kvm_arch_destroy_vm - destroy the VM data structure
151 * @kvm: pointer to the KVM struct
152 */
Christoffer Dall749cf76c2013-01-20 18:28:06 -0500153void kvm_arch_destroy_vm(struct kvm *kvm)
154{
155 int i;
156
Christoffer Dalld5d81842013-01-20 18:28:07 -0500157 kvm_free_stage2_pgd(kvm);
158
Christoffer Dall749cf76c2013-01-20 18:28:06 -0500159 for (i = 0; i < KVM_MAX_VCPUS; ++i) {
160 if (kvm->vcpus[i]) {
161 kvm_arch_vcpu_free(kvm->vcpus[i]);
162 kvm->vcpus[i] = NULL;
163 }
164 }
Marc Zyngierc1bfb572014-07-08 12:09:01 +0100165
166 kvm_vgic_destroy(kvm);
Christoffer Dall749cf76c2013-01-20 18:28:06 -0500167}
168
Alexander Graf784aa3d2014-07-14 18:27:35 +0200169int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
Christoffer Dall749cf76c2013-01-20 18:28:06 -0500170{
171 int r;
172 switch (ext) {
Marc Zyngier1a89dd92013-01-21 19:36:12 -0500173 case KVM_CAP_IRQCHIP:
Christoffer Dall73306722013-10-25 17:29:18 +0100174 case KVM_CAP_DEVICE_CTRL:
Christoffer Dall749cf76c2013-01-20 18:28:06 -0500175 case KVM_CAP_USER_MEMORY:
176 case KVM_CAP_SYNC_MMU:
177 case KVM_CAP_DESTROY_MEMORY_REGION_WORKS:
178 case KVM_CAP_ONE_REG:
Marc Zyngieraa024c2f2013-01-20 18:28:13 -0500179 case KVM_CAP_ARM_PSCI:
Anup Patel4447a202014-04-29 11:24:25 +0530180 case KVM_CAP_ARM_PSCI_0_2:
Christoffer Dall98047882014-08-19 12:18:04 +0200181 case KVM_CAP_READONLY_MEM:
Christoffer Dall749cf76c2013-01-20 18:28:06 -0500182 r = 1;
183 break;
184 case KVM_CAP_COALESCED_MMIO:
185 r = KVM_COALESCED_MMIO_PAGE_OFFSET;
186 break;
Christoffer Dall3401d5462013-01-23 13:18:04 -0500187 case KVM_CAP_ARM_SET_DEVICE_ADDR:
188 r = 1;
Marc Zyngierca46e102013-04-03 10:43:13 +0100189 break;
Christoffer Dall749cf76c2013-01-20 18:28:06 -0500190 case KVM_CAP_NR_VCPUS:
191 r = num_online_cpus();
192 break;
193 case KVM_CAP_MAX_VCPUS:
194 r = KVM_MAX_VCPUS;
195 break;
196 default:
Marc Zyngier17b1e312013-04-08 16:47:18 +0100197 r = kvm_arch_dev_ioctl_check_extension(ext);
Christoffer Dall749cf76c2013-01-20 18:28:06 -0500198 break;
199 }
200 return r;
201}
202
203long kvm_arch_dev_ioctl(struct file *filp,
204 unsigned int ioctl, unsigned long arg)
205{
206 return -EINVAL;
207}
208
Christoffer Dall749cf76c2013-01-20 18:28:06 -0500209
210struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm, unsigned int id)
211{
212 int err;
213 struct kvm_vcpu *vcpu;
214
Christoffer Dall716139d2014-12-09 14:33:45 +0100215 if (irqchip_in_kernel(kvm) && vgic_initialized(kvm)) {
216 err = -EBUSY;
217 goto out;
218 }
219
Andre Przywara3caa2d82014-06-02 16:26:01 +0200220 if (id >= kvm->arch.max_vcpus) {
221 err = -EINVAL;
222 goto out;
223 }
224
Christoffer Dall749cf76c2013-01-20 18:28:06 -0500225 vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
226 if (!vcpu) {
227 err = -ENOMEM;
228 goto out;
229 }
230
231 err = kvm_vcpu_init(vcpu, kvm, id);
232 if (err)
233 goto free_vcpu;
234
Christoffer Dalld5d81842013-01-20 18:28:07 -0500235 err = create_hyp_mappings(vcpu, vcpu + 1);
236 if (err)
237 goto vcpu_uninit;
238
Christoffer Dall749cf76c2013-01-20 18:28:06 -0500239 return vcpu;
Christoffer Dalld5d81842013-01-20 18:28:07 -0500240vcpu_uninit:
241 kvm_vcpu_uninit(vcpu);
Christoffer Dall749cf76c2013-01-20 18:28:06 -0500242free_vcpu:
243 kmem_cache_free(kvm_vcpu_cache, vcpu);
244out:
245 return ERR_PTR(err);
246}
247
Dominik Dingel31928aa2014-12-04 15:47:07 +0100248void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
Christoffer Dall749cf76c2013-01-20 18:28:06 -0500249{
Christoffer Dall749cf76c2013-01-20 18:28:06 -0500250}
251
252void kvm_arch_vcpu_free(struct kvm_vcpu *vcpu)
253{
Christoffer Dalld5d81842013-01-20 18:28:07 -0500254 kvm_mmu_free_memory_caches(vcpu);
Marc Zyngier967f8422013-01-23 13:21:59 -0500255 kvm_timer_vcpu_terminate(vcpu);
Marc Zyngierc1bfb572014-07-08 12:09:01 +0100256 kvm_vgic_vcpu_destroy(vcpu);
Christoffer Dalld5d81842013-01-20 18:28:07 -0500257 kmem_cache_free(kvm_vcpu_cache, vcpu);
Christoffer Dall749cf76c2013-01-20 18:28:06 -0500258}
259
260void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
261{
262 kvm_arch_vcpu_free(vcpu);
263}
264
265int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu)
266{
267 return 0;
268}
269
Christoffer Dall749cf76c2013-01-20 18:28:06 -0500270int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
271{
Christoffer Dallf7ed45b2013-01-20 18:47:42 -0500272 /* Force users to call KVM_ARM_VCPU_INIT */
273 vcpu->arch.target = -1;
Christoffer Dallf7fa034d2014-10-16 16:40:53 +0200274 bitmap_zero(vcpu->arch.features, KVM_VCPU_MAX_FEATURES);
Marc Zyngier1a89dd92013-01-21 19:36:12 -0500275
Marc Zyngier967f8422013-01-23 13:21:59 -0500276 /* Set up the timer */
277 kvm_timer_vcpu_init(vcpu);
278
Christoffer Dall749cf76c2013-01-20 18:28:06 -0500279 return 0;
280}
281
Christoffer Dall749cf76c2013-01-20 18:28:06 -0500282void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
283{
Christoffer Dall86ce85352013-01-20 18:28:08 -0500284 vcpu->cpu = cpu;
Marc Zyngier3de50da2013-04-08 16:47:19 +0100285 vcpu->arch.host_cpu_context = this_cpu_ptr(kvm_host_cpu_state);
Christoffer Dall5b3e5e52013-01-20 18:28:09 -0500286
Marc Zyngier1638a122013-01-21 19:36:11 -0500287 kvm_arm_set_running_vcpu(vcpu);
Christoffer Dall749cf76c2013-01-20 18:28:06 -0500288}
289
290void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
291{
Christoffer Dalle9b152c2013-12-11 20:29:11 -0800292 /*
293 * The arch-generic KVM code expects the cpu field of a vcpu to be -1
294 * if the vcpu is no longer assigned to a cpu. This is used for the
295 * optimized make_all_cpus_request path.
296 */
297 vcpu->cpu = -1;
298
Marc Zyngier1638a122013-01-21 19:36:11 -0500299 kvm_arm_set_running_vcpu(NULL);
Christoffer Dall749cf76c2013-01-20 18:28:06 -0500300}
301
302int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
303 struct kvm_guest_debug *dbg)
304{
305 return -EINVAL;
306}
307
308
309int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
310 struct kvm_mp_state *mp_state)
311{
312 return -EINVAL;
313}
314
315int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
316 struct kvm_mp_state *mp_state)
317{
318 return -EINVAL;
319}
320
Christoffer Dall5b3e5e52013-01-20 18:28:09 -0500321/**
322 * kvm_arch_vcpu_runnable - determine if the vcpu can be scheduled
323 * @v: The VCPU pointer
324 *
325 * If the guest CPU is not waiting for interrupts or an interrupt line is
326 * asserted, the CPU is by definition runnable.
327 */
Christoffer Dall749cf76c2013-01-20 18:28:06 -0500328int kvm_arch_vcpu_runnable(struct kvm_vcpu *v)
329{
Marc Zyngier1a89dd92013-01-21 19:36:12 -0500330 return !!v->arch.irq_lines || kvm_vgic_vcpu_pending_irq(v);
Christoffer Dall749cf76c2013-01-20 18:28:06 -0500331}
332
Christoffer Dallf7ed45b2013-01-20 18:47:42 -0500333/* Just ensure a guest exit from a particular CPU */
334static void exit_vm_noop(void *info)
335{
336}
337
338void force_vm_exit(const cpumask_t *mask)
339{
340 smp_call_function_many(mask, exit_vm_noop, NULL, true);
341}
342
343/**
344 * need_new_vmid_gen - check that the VMID is still valid
345 * @kvm: The VM's VMID to checkt
346 *
347 * return true if there is a new generation of VMIDs being used
348 *
349 * The hardware supports only 256 values with the value zero reserved for the
350 * host, so we check if an assigned value belongs to a previous generation,
351 * which which requires us to assign a new value. If we're the first to use a
352 * VMID for the new generation, we must flush necessary caches and TLBs on all
353 * CPUs.
354 */
355static bool need_new_vmid_gen(struct kvm *kvm)
356{
357 return unlikely(kvm->arch.vmid_gen != atomic64_read(&kvm_vmid_gen));
358}
359
360/**
361 * update_vttbr - Update the VTTBR with a valid VMID before the guest runs
362 * @kvm The guest that we are about to run
363 *
364 * Called from kvm_arch_vcpu_ioctl_run before entering the guest to ensure the
365 * VM has a valid VMID, otherwise assigns a new one and flushes corresponding
366 * caches and TLBs.
367 */
368static void update_vttbr(struct kvm *kvm)
369{
370 phys_addr_t pgd_phys;
371 u64 vmid;
372
373 if (!need_new_vmid_gen(kvm))
374 return;
375
376 spin_lock(&kvm_vmid_lock);
377
378 /*
379 * We need to re-check the vmid_gen here to ensure that if another vcpu
380 * already allocated a valid vmid for this vm, then this vcpu should
381 * use the same vmid.
382 */
383 if (!need_new_vmid_gen(kvm)) {
384 spin_unlock(&kvm_vmid_lock);
385 return;
386 }
387
388 /* First user of a new VMID generation? */
389 if (unlikely(kvm_next_vmid == 0)) {
390 atomic64_inc(&kvm_vmid_gen);
391 kvm_next_vmid = 1;
392
393 /*
394 * On SMP we know no other CPUs can use this CPU's or each
395 * other's VMID after force_vm_exit returns since the
396 * kvm_vmid_lock blocks them from reentry to the guest.
397 */
398 force_vm_exit(cpu_all_mask);
399 /*
400 * Now broadcast TLB + ICACHE invalidation over the inner
401 * shareable domain to make sure all data structures are
402 * clean.
403 */
404 kvm_call_hyp(__kvm_flush_vm_context);
405 }
406
407 kvm->arch.vmid_gen = atomic64_read(&kvm_vmid_gen);
408 kvm->arch.vmid = kvm_next_vmid;
409 kvm_next_vmid++;
410
411 /* update vttbr to be used with the new vmid */
Christoffer Dall38f791a2014-10-10 12:14:28 +0200412 pgd_phys = virt_to_phys(kvm_get_hwpgd(kvm));
Joel Schoppdbff1242014-07-09 11:17:04 -0500413 BUG_ON(pgd_phys & ~VTTBR_BADDR_MASK);
Christoffer Dallf7ed45b2013-01-20 18:47:42 -0500414 vmid = ((u64)(kvm->arch.vmid) << VTTBR_VMID_SHIFT) & VTTBR_VMID_MASK;
Joel Schoppdbff1242014-07-09 11:17:04 -0500415 kvm->arch.vttbr = pgd_phys | vmid;
Christoffer Dallf7ed45b2013-01-20 18:47:42 -0500416
417 spin_unlock(&kvm_vmid_lock);
418}
419
Christoffer Dallf7ed45b2013-01-20 18:47:42 -0500420static int kvm_vcpu_first_run_init(struct kvm_vcpu *vcpu)
421{
Christoffer Dall05971122014-12-12 21:19:23 +0100422 struct kvm *kvm = vcpu->kvm;
Christoffer Dalle1ba0202013-09-23 14:55:55 -0700423 int ret;
424
Christoffer Dallf7ed45b2013-01-20 18:47:42 -0500425 if (likely(vcpu->arch.has_run_once))
426 return 0;
427
428 vcpu->arch.has_run_once = true;
Marc Zyngieraa024c2f2013-01-20 18:28:13 -0500429
430 /*
Peter Maydell6d3cfbe2014-12-04 15:02:24 +0000431 * Map the VGIC hardware resources before running a vcpu the first
432 * time on this VM.
Marc Zyngier01ac5e32013-01-21 19:36:16 -0500433 */
Christoffer Dall05971122014-12-12 21:19:23 +0100434 if (unlikely(!vgic_ready(kvm))) {
435 ret = kvm_vgic_map_resources(kvm);
Marc Zyngier01ac5e32013-01-21 19:36:16 -0500436 if (ret)
437 return ret;
438 }
439
Christoffer Dall05971122014-12-12 21:19:23 +0100440 /*
441 * Enable the arch timers only if we have an in-kernel VGIC
442 * and it has been properly initialized, since we cannot handle
443 * interrupts from the virtual timer with a userspace gic.
444 */
445 if (irqchip_in_kernel(kvm) && vgic_initialized(kvm))
446 kvm_timer_enable(kvm);
447
Christoffer Dallf7ed45b2013-01-20 18:47:42 -0500448 return 0;
449}
450
Eric Augerc1426e42015-03-04 11:14:34 +0100451bool kvm_arch_intc_initialized(struct kvm *kvm)
452{
453 return vgic_initialized(kvm);
454}
455
Marc Zyngieraa024c2f2013-01-20 18:28:13 -0500456static void vcpu_pause(struct kvm_vcpu *vcpu)
457{
458 wait_queue_head_t *wq = kvm_arch_vcpu_wq(vcpu);
459
460 wait_event_interruptible(*wq, !vcpu->arch.pause);
461}
462
Andre Przywarae8180dc2013-05-09 00:28:06 +0200463static int kvm_vcpu_initialized(struct kvm_vcpu *vcpu)
464{
465 return vcpu->arch.target >= 0;
466}
467
Christoffer Dallf7ed45b2013-01-20 18:47:42 -0500468/**
469 * kvm_arch_vcpu_ioctl_run - the main VCPU run function to execute guest code
470 * @vcpu: The VCPU pointer
471 * @run: The kvm_run structure pointer used for userspace state exchange
472 *
473 * This function is called through the VCPU_RUN ioctl called from user space. It
474 * will execute VM code in a loop until the time slice for the process is used
475 * or some emulation is needed from user space in which case the function will
476 * return with return value 0 and with the kvm_run structure filled in with the
477 * required data for the requested emulation.
478 */
Christoffer Dall749cf76c2013-01-20 18:28:06 -0500479int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *run)
480{
Christoffer Dallf7ed45b2013-01-20 18:47:42 -0500481 int ret;
482 sigset_t sigsaved;
483
Andre Przywarae8180dc2013-05-09 00:28:06 +0200484 if (unlikely(!kvm_vcpu_initialized(vcpu)))
Christoffer Dallf7ed45b2013-01-20 18:47:42 -0500485 return -ENOEXEC;
486
487 ret = kvm_vcpu_first_run_init(vcpu);
488 if (ret)
489 return ret;
490
Christoffer Dall45e96ea2013-01-20 18:43:58 -0500491 if (run->exit_reason == KVM_EXIT_MMIO) {
492 ret = kvm_handle_mmio_return(vcpu, vcpu->run);
493 if (ret)
494 return ret;
495 }
496
Christoffer Dallf7ed45b2013-01-20 18:47:42 -0500497 if (vcpu->sigset_active)
498 sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
499
500 ret = 1;
501 run->exit_reason = KVM_EXIT_UNKNOWN;
502 while (ret > 0) {
503 /*
504 * Check conditions before entering the guest
505 */
506 cond_resched();
507
508 update_vttbr(vcpu->kvm);
509
Marc Zyngieraa024c2f2013-01-20 18:28:13 -0500510 if (vcpu->arch.pause)
511 vcpu_pause(vcpu);
512
Marc Zyngier1a89dd92013-01-21 19:36:12 -0500513 kvm_vgic_flush_hwstate(vcpu);
Marc Zyngierc7e3ba62013-01-23 13:21:59 -0500514 kvm_timer_flush_hwstate(vcpu);
Marc Zyngier1a89dd92013-01-21 19:36:12 -0500515
Christoffer Dallf7ed45b2013-01-20 18:47:42 -0500516 local_irq_disable();
517
518 /*
519 * Re-check atomic conditions
520 */
521 if (signal_pending(current)) {
522 ret = -EINTR;
523 run->exit_reason = KVM_EXIT_INTR;
524 }
525
526 if (ret <= 0 || need_new_vmid_gen(vcpu->kvm)) {
527 local_irq_enable();
Marc Zyngierc7e3ba62013-01-23 13:21:59 -0500528 kvm_timer_sync_hwstate(vcpu);
Marc Zyngier1a89dd92013-01-21 19:36:12 -0500529 kvm_vgic_sync_hwstate(vcpu);
Christoffer Dallf7ed45b2013-01-20 18:47:42 -0500530 continue;
531 }
532
533 /**************************************************************
534 * Enter the guest
535 */
536 trace_kvm_entry(*vcpu_pc(vcpu));
537 kvm_guest_enter();
538 vcpu->mode = IN_GUEST_MODE;
539
540 ret = kvm_call_hyp(__kvm_vcpu_run, vcpu);
541
542 vcpu->mode = OUTSIDE_GUEST_MODE;
543 kvm_guest_exit();
Wei Huang91314cb2015-01-30 13:09:26 -0500544 trace_kvm_exit(kvm_vcpu_trap_get_class(vcpu), *vcpu_pc(vcpu));
Christoffer Dallf7ed45b2013-01-20 18:47:42 -0500545 /*
546 * We may have taken a host interrupt in HYP mode (ie
547 * while executing the guest). This interrupt is still
548 * pending, as we haven't serviced it yet!
549 *
550 * We're now back in SVC mode, with interrupts
551 * disabled. Enabling the interrupts now will have
552 * the effect of taking the interrupt again, in SVC
553 * mode this time.
554 */
555 local_irq_enable();
556
557 /*
558 * Back from guest
559 *************************************************************/
560
Marc Zyngierc7e3ba62013-01-23 13:21:59 -0500561 kvm_timer_sync_hwstate(vcpu);
Marc Zyngier1a89dd92013-01-21 19:36:12 -0500562 kvm_vgic_sync_hwstate(vcpu);
563
Christoffer Dallf7ed45b2013-01-20 18:47:42 -0500564 ret = handle_exit(vcpu, run, ret);
565 }
566
567 if (vcpu->sigset_active)
568 sigprocmask(SIG_SETMASK, &sigsaved, NULL);
569 return ret;
Christoffer Dall749cf76c2013-01-20 18:28:06 -0500570}
571
Christoffer Dall86ce85352013-01-20 18:28:08 -0500572static int vcpu_interrupt_line(struct kvm_vcpu *vcpu, int number, bool level)
573{
574 int bit_index;
575 bool set;
576 unsigned long *ptr;
577
578 if (number == KVM_ARM_IRQ_CPU_IRQ)
579 bit_index = __ffs(HCR_VI);
580 else /* KVM_ARM_IRQ_CPU_FIQ */
581 bit_index = __ffs(HCR_VF);
582
583 ptr = (unsigned long *)&vcpu->arch.irq_lines;
584 if (level)
585 set = test_and_set_bit(bit_index, ptr);
586 else
587 set = test_and_clear_bit(bit_index, ptr);
588
589 /*
590 * If we didn't change anything, no need to wake up or kick other CPUs
591 */
592 if (set == level)
593 return 0;
594
595 /*
596 * The vcpu irq_lines field was updated, wake up sleeping VCPUs and
597 * trigger a world-switch round on the running physical CPU to set the
598 * virtual IRQ/FIQ fields in the HCR appropriately.
599 */
600 kvm_vcpu_kick(vcpu);
601
602 return 0;
603}
604
Alexander Graf79558f12013-04-16 19:21:41 +0200605int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_level,
606 bool line_status)
Christoffer Dall86ce85352013-01-20 18:28:08 -0500607{
608 u32 irq = irq_level->irq;
609 unsigned int irq_type, vcpu_idx, irq_num;
610 int nrcpus = atomic_read(&kvm->online_vcpus);
611 struct kvm_vcpu *vcpu = NULL;
612 bool level = irq_level->level;
613
614 irq_type = (irq >> KVM_ARM_IRQ_TYPE_SHIFT) & KVM_ARM_IRQ_TYPE_MASK;
615 vcpu_idx = (irq >> KVM_ARM_IRQ_VCPU_SHIFT) & KVM_ARM_IRQ_VCPU_MASK;
616 irq_num = (irq >> KVM_ARM_IRQ_NUM_SHIFT) & KVM_ARM_IRQ_NUM_MASK;
617
618 trace_kvm_irq_line(irq_type, vcpu_idx, irq_num, irq_level->level);
619
Marc Zyngier5863c2c2013-01-21 19:36:15 -0500620 switch (irq_type) {
621 case KVM_ARM_IRQ_TYPE_CPU:
622 if (irqchip_in_kernel(kvm))
623 return -ENXIO;
Christoffer Dall86ce85352013-01-20 18:28:08 -0500624
Marc Zyngier5863c2c2013-01-21 19:36:15 -0500625 if (vcpu_idx >= nrcpus)
626 return -EINVAL;
Christoffer Dall86ce85352013-01-20 18:28:08 -0500627
Marc Zyngier5863c2c2013-01-21 19:36:15 -0500628 vcpu = kvm_get_vcpu(kvm, vcpu_idx);
629 if (!vcpu)
630 return -EINVAL;
Christoffer Dall86ce85352013-01-20 18:28:08 -0500631
Marc Zyngier5863c2c2013-01-21 19:36:15 -0500632 if (irq_num > KVM_ARM_IRQ_CPU_FIQ)
633 return -EINVAL;
Christoffer Dall86ce85352013-01-20 18:28:08 -0500634
Marc Zyngier5863c2c2013-01-21 19:36:15 -0500635 return vcpu_interrupt_line(vcpu, irq_num, level);
636 case KVM_ARM_IRQ_TYPE_PPI:
637 if (!irqchip_in_kernel(kvm))
638 return -ENXIO;
639
640 if (vcpu_idx >= nrcpus)
641 return -EINVAL;
642
643 vcpu = kvm_get_vcpu(kvm, vcpu_idx);
644 if (!vcpu)
645 return -EINVAL;
646
647 if (irq_num < VGIC_NR_SGIS || irq_num >= VGIC_NR_PRIVATE_IRQS)
648 return -EINVAL;
649
650 return kvm_vgic_inject_irq(kvm, vcpu->vcpu_id, irq_num, level);
651 case KVM_ARM_IRQ_TYPE_SPI:
652 if (!irqchip_in_kernel(kvm))
653 return -ENXIO;
654
655 if (irq_num < VGIC_NR_PRIVATE_IRQS ||
656 irq_num > KVM_ARM_IRQ_GIC_MAX)
657 return -EINVAL;
658
659 return kvm_vgic_inject_irq(kvm, 0, irq_num, level);
660 }
661
662 return -EINVAL;
Christoffer Dall86ce85352013-01-20 18:28:08 -0500663}
664
Christoffer Dallf7fa034d2014-10-16 16:40:53 +0200665static int kvm_vcpu_set_target(struct kvm_vcpu *vcpu,
666 const struct kvm_vcpu_init *init)
667{
668 unsigned int i;
669 int phys_target = kvm_target_cpu();
670
671 if (init->target != phys_target)
672 return -EINVAL;
673
674 /*
675 * Secondary and subsequent calls to KVM_ARM_VCPU_INIT must
676 * use the same target.
677 */
678 if (vcpu->arch.target != -1 && vcpu->arch.target != init->target)
679 return -EINVAL;
680
681 /* -ENOENT for unknown features, -EINVAL for invalid combinations. */
682 for (i = 0; i < sizeof(init->features) * 8; i++) {
683 bool set = (init->features[i / 32] & (1 << (i % 32)));
684
685 if (set && i >= KVM_VCPU_MAX_FEATURES)
686 return -ENOENT;
687
688 /*
689 * Secondary and subsequent calls to KVM_ARM_VCPU_INIT must
690 * use the same feature set.
691 */
692 if (vcpu->arch.target != -1 && i < KVM_VCPU_MAX_FEATURES &&
693 test_bit(i, vcpu->arch.features) != set)
694 return -EINVAL;
695
696 if (set)
697 set_bit(i, vcpu->arch.features);
698 }
699
700 vcpu->arch.target = phys_target;
701
702 /* Now we know what it is, we can reset it. */
703 return kvm_reset_vcpu(vcpu);
704}
705
706
Christoffer Dall478a8232013-11-19 17:43:19 -0800707static int kvm_arch_vcpu_ioctl_vcpu_init(struct kvm_vcpu *vcpu,
708 struct kvm_vcpu_init *init)
709{
710 int ret;
711
712 ret = kvm_vcpu_set_target(vcpu, init);
713 if (ret)
714 return ret;
715
Christoffer Dall957db102014-11-27 10:35:03 +0100716 /*
717 * Ensure a rebooted VM will fault in RAM pages and detect if the
718 * guest MMU is turned off and flush the caches as needed.
719 */
720 if (vcpu->arch.has_run_once)
721 stage2_unmap_vm(vcpu->kvm);
722
Christoffer Dallb856a592014-10-16 17:21:16 +0200723 vcpu_reset_hcr(vcpu);
724
Christoffer Dall478a8232013-11-19 17:43:19 -0800725 /*
726 * Handle the "start in power-off" case by marking the VCPU as paused.
727 */
Christoffer Dall03f1d4c2014-12-02 15:27:51 +0100728 if (test_bit(KVM_ARM_VCPU_POWER_OFF, vcpu->arch.features))
Christoffer Dall478a8232013-11-19 17:43:19 -0800729 vcpu->arch.pause = true;
Christoffer Dall3ad8b3d2014-10-16 16:14:43 +0200730 else
731 vcpu->arch.pause = false;
Christoffer Dall478a8232013-11-19 17:43:19 -0800732
733 return 0;
734}
735
Christoffer Dall749cf76c2013-01-20 18:28:06 -0500736long kvm_arch_vcpu_ioctl(struct file *filp,
737 unsigned int ioctl, unsigned long arg)
738{
739 struct kvm_vcpu *vcpu = filp->private_data;
740 void __user *argp = (void __user *)arg;
741
742 switch (ioctl) {
743 case KVM_ARM_VCPU_INIT: {
744 struct kvm_vcpu_init init;
745
746 if (copy_from_user(&init, argp, sizeof(init)))
747 return -EFAULT;
748
Christoffer Dall478a8232013-11-19 17:43:19 -0800749 return kvm_arch_vcpu_ioctl_vcpu_init(vcpu, &init);
Christoffer Dall749cf76c2013-01-20 18:28:06 -0500750 }
751 case KVM_SET_ONE_REG:
752 case KVM_GET_ONE_REG: {
753 struct kvm_one_reg reg;
Andre Przywarae8180dc2013-05-09 00:28:06 +0200754
755 if (unlikely(!kvm_vcpu_initialized(vcpu)))
756 return -ENOEXEC;
757
Christoffer Dall749cf76c2013-01-20 18:28:06 -0500758 if (copy_from_user(&reg, argp, sizeof(reg)))
759 return -EFAULT;
760 if (ioctl == KVM_SET_ONE_REG)
761 return kvm_arm_set_reg(vcpu, &reg);
762 else
763 return kvm_arm_get_reg(vcpu, &reg);
764 }
765 case KVM_GET_REG_LIST: {
766 struct kvm_reg_list __user *user_list = argp;
767 struct kvm_reg_list reg_list;
768 unsigned n;
769
Andre Przywarae8180dc2013-05-09 00:28:06 +0200770 if (unlikely(!kvm_vcpu_initialized(vcpu)))
771 return -ENOEXEC;
772
Christoffer Dall749cf76c2013-01-20 18:28:06 -0500773 if (copy_from_user(&reg_list, user_list, sizeof(reg_list)))
774 return -EFAULT;
775 n = reg_list.n;
776 reg_list.n = kvm_arm_num_regs(vcpu);
777 if (copy_to_user(user_list, &reg_list, sizeof(reg_list)))
778 return -EFAULT;
779 if (n < reg_list.n)
780 return -E2BIG;
781 return kvm_arm_copy_reg_indices(vcpu, user_list->reg);
782 }
783 default:
784 return -EINVAL;
785 }
786}
787
Mario Smarduch53c810c2015-01-15 15:58:57 -0800788/**
789 * kvm_vm_ioctl_get_dirty_log - get and clear the log of dirty pages in a slot
790 * @kvm: kvm instance
791 * @log: slot id and address to which we copy the log
792 *
793 * Steps 1-4 below provide general overview of dirty page logging. See
794 * kvm_get_dirty_log_protect() function description for additional details.
795 *
796 * We call kvm_get_dirty_log_protect() to handle steps 1-3, upon return we
797 * always flush the TLB (step 4) even if previous step failed and the dirty
798 * bitmap may be corrupt. Regardless of previous outcome the KVM logging API
799 * does not preclude user space subsequent dirty log read. Flushing TLB ensures
800 * writes will be marked dirty for next log read.
801 *
802 * 1. Take a snapshot of the bit and clear it if needed.
803 * 2. Write protect the corresponding page.
804 * 3. Copy the snapshot to the userspace.
805 * 4. Flush TLB's if needed.
806 */
Christoffer Dall749cf76c2013-01-20 18:28:06 -0500807int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, struct kvm_dirty_log *log)
808{
Mario Smarduch53c810c2015-01-15 15:58:57 -0800809 bool is_dirty = false;
810 int r;
811
812 mutex_lock(&kvm->slots_lock);
813
814 r = kvm_get_dirty_log_protect(kvm, log, &is_dirty);
815
816 if (is_dirty)
817 kvm_flush_remote_tlbs(kvm);
818
819 mutex_unlock(&kvm->slots_lock);
820 return r;
Christoffer Dall749cf76c2013-01-20 18:28:06 -0500821}
822
Christoffer Dall3401d5462013-01-23 13:18:04 -0500823static int kvm_vm_ioctl_set_device_addr(struct kvm *kvm,
824 struct kvm_arm_device_addr *dev_addr)
825{
Christoffer Dall330690c2013-01-21 19:36:13 -0500826 unsigned long dev_id, type;
827
828 dev_id = (dev_addr->id & KVM_ARM_DEVICE_ID_MASK) >>
829 KVM_ARM_DEVICE_ID_SHIFT;
830 type = (dev_addr->id & KVM_ARM_DEVICE_TYPE_MASK) >>
831 KVM_ARM_DEVICE_TYPE_SHIFT;
832
833 switch (dev_id) {
834 case KVM_ARM_DEVICE_VGIC_V2:
Christoffer Dallce01e4e2013-09-23 14:55:56 -0700835 return kvm_vgic_addr(kvm, type, &dev_addr->addr, true);
Christoffer Dall330690c2013-01-21 19:36:13 -0500836 default:
837 return -ENODEV;
838 }
Christoffer Dall3401d5462013-01-23 13:18:04 -0500839}
840
Christoffer Dall749cf76c2013-01-20 18:28:06 -0500841long kvm_arch_vm_ioctl(struct file *filp,
842 unsigned int ioctl, unsigned long arg)
843{
Christoffer Dall3401d5462013-01-23 13:18:04 -0500844 struct kvm *kvm = filp->private_data;
845 void __user *argp = (void __user *)arg;
846
847 switch (ioctl) {
Marc Zyngier5863c2c2013-01-21 19:36:15 -0500848 case KVM_CREATE_IRQCHIP: {
Paolo Bonzini69ff5c62015-03-05 12:26:06 +0100849 return kvm_vgic_create(kvm, KVM_DEV_TYPE_ARM_VGIC_V2);
Marc Zyngier5863c2c2013-01-21 19:36:15 -0500850 }
Christoffer Dall3401d5462013-01-23 13:18:04 -0500851 case KVM_ARM_SET_DEVICE_ADDR: {
852 struct kvm_arm_device_addr dev_addr;
853
854 if (copy_from_user(&dev_addr, argp, sizeof(dev_addr)))
855 return -EFAULT;
856 return kvm_vm_ioctl_set_device_addr(kvm, &dev_addr);
857 }
Anup Patel42c4e0c2013-09-30 14:20:07 +0530858 case KVM_ARM_PREFERRED_TARGET: {
859 int err;
860 struct kvm_vcpu_init init;
861
862 err = kvm_vcpu_preferred_target(&init);
863 if (err)
864 return err;
865
866 if (copy_to_user(argp, &init, sizeof(init)))
867 return -EFAULT;
868
869 return 0;
870 }
Christoffer Dall3401d5462013-01-23 13:18:04 -0500871 default:
872 return -EINVAL;
873 }
Christoffer Dall749cf76c2013-01-20 18:28:06 -0500874}
875
Marc Zyngierd157f4a2013-04-12 19:12:07 +0100876static void cpu_init_hyp_mode(void *dummy)
Christoffer Dall342cd0a2013-01-20 18:28:06 -0500877{
Marc Zyngierdac288f2013-05-14 12:11:37 +0100878 phys_addr_t boot_pgd_ptr;
879 phys_addr_t pgd_ptr;
Christoffer Dall342cd0a2013-01-20 18:28:06 -0500880 unsigned long hyp_stack_ptr;
881 unsigned long stack_page;
882 unsigned long vector_ptr;
883
884 /* Switch from the HYP stub to our own HYP init vector */
Marc Zyngier5a677ce2013-04-12 19:12:06 +0100885 __hyp_set_vectors(kvm_get_idmap_vector());
Christoffer Dall342cd0a2013-01-20 18:28:06 -0500886
Marc Zyngierdac288f2013-05-14 12:11:37 +0100887 boot_pgd_ptr = kvm_mmu_get_boot_httbr();
888 pgd_ptr = kvm_mmu_get_httbr();
Christoph Lameter1436c1a2013-10-21 13:17:08 +0100889 stack_page = __this_cpu_read(kvm_arm_hyp_stack_page);
Christoffer Dall342cd0a2013-01-20 18:28:06 -0500890 hyp_stack_ptr = stack_page + PAGE_SIZE;
891 vector_ptr = (unsigned long)__kvm_hyp_vector;
892
Marc Zyngier5a677ce2013-04-12 19:12:06 +0100893 __cpu_init_hyp_mode(boot_pgd_ptr, pgd_ptr, hyp_stack_ptr, vector_ptr);
Christoffer Dall342cd0a2013-01-20 18:28:06 -0500894}
895
Marc Zyngierd157f4a2013-04-12 19:12:07 +0100896static int hyp_init_cpu_notify(struct notifier_block *self,
897 unsigned long action, void *cpu)
898{
899 switch (action) {
900 case CPU_STARTING:
901 case CPU_STARTING_FROZEN:
Vladimir Murzin37a34ac2014-09-22 15:52:48 +0100902 if (__hyp_get_vectors() == hyp_default_vectors)
903 cpu_init_hyp_mode(NULL);
Marc Zyngierd157f4a2013-04-12 19:12:07 +0100904 break;
905 }
906
907 return NOTIFY_OK;
908}
909
910static struct notifier_block hyp_init_cpu_nb = {
911 .notifier_call = hyp_init_cpu_notify,
912};
913
Lorenzo Pieralisi1fcf7ce2013-08-05 15:04:46 +0100914#ifdef CONFIG_CPU_PM
915static int hyp_init_cpu_pm_notifier(struct notifier_block *self,
916 unsigned long cmd,
917 void *v)
918{
Marc Zyngierb20c9f22014-02-26 18:47:36 +0000919 if (cmd == CPU_PM_EXIT &&
920 __hyp_get_vectors() == hyp_default_vectors) {
Lorenzo Pieralisi1fcf7ce2013-08-05 15:04:46 +0100921 cpu_init_hyp_mode(NULL);
922 return NOTIFY_OK;
923 }
924
925 return NOTIFY_DONE;
926}
927
928static struct notifier_block hyp_init_cpu_pm_nb = {
929 .notifier_call = hyp_init_cpu_pm_notifier,
930};
931
932static void __init hyp_cpu_pm_init(void)
933{
934 cpu_pm_register_notifier(&hyp_init_cpu_pm_nb);
935}
936#else
937static inline void hyp_cpu_pm_init(void)
938{
939}
940#endif
941
Christoffer Dall342cd0a2013-01-20 18:28:06 -0500942/**
943 * Inits Hyp-mode on all online CPUs
944 */
945static int init_hyp_mode(void)
946{
Christoffer Dall342cd0a2013-01-20 18:28:06 -0500947 int cpu;
948 int err = 0;
949
950 /*
951 * Allocate Hyp PGD and setup Hyp identity mapping
952 */
953 err = kvm_mmu_init();
954 if (err)
955 goto out_err;
956
957 /*
958 * It is probably enough to obtain the default on one
959 * CPU. It's unlikely to be different on the others.
960 */
961 hyp_default_vectors = __hyp_get_vectors();
962
963 /*
964 * Allocate stack pages for Hypervisor-mode
965 */
966 for_each_possible_cpu(cpu) {
967 unsigned long stack_page;
968
969 stack_page = __get_free_page(GFP_KERNEL);
970 if (!stack_page) {
971 err = -ENOMEM;
972 goto out_free_stack_pages;
973 }
974
975 per_cpu(kvm_arm_hyp_stack_page, cpu) = stack_page;
976 }
977
978 /*
Christoffer Dall342cd0a2013-01-20 18:28:06 -0500979 * Map the Hyp-code called directly from the host
980 */
981 err = create_hyp_mappings(__kvm_hyp_code_start, __kvm_hyp_code_end);
982 if (err) {
983 kvm_err("Cannot map world-switch code\n");
984 goto out_free_mappings;
985 }
986
987 /*
988 * Map the Hyp stack pages
989 */
990 for_each_possible_cpu(cpu) {
991 char *stack_page = (char *)per_cpu(kvm_arm_hyp_stack_page, cpu);
992 err = create_hyp_mappings(stack_page, stack_page + PAGE_SIZE);
993
994 if (err) {
995 kvm_err("Cannot map hyp stack\n");
996 goto out_free_mappings;
997 }
998 }
999
1000 /*
Marc Zyngier3de50da2013-04-08 16:47:19 +01001001 * Map the host CPU structures
Christoffer Dall342cd0a2013-01-20 18:28:06 -05001002 */
Marc Zyngier3de50da2013-04-08 16:47:19 +01001003 kvm_host_cpu_state = alloc_percpu(kvm_cpu_context_t);
1004 if (!kvm_host_cpu_state) {
Christoffer Dall342cd0a2013-01-20 18:28:06 -05001005 err = -ENOMEM;
Marc Zyngier3de50da2013-04-08 16:47:19 +01001006 kvm_err("Cannot allocate host CPU state\n");
Christoffer Dall342cd0a2013-01-20 18:28:06 -05001007 goto out_free_mappings;
1008 }
1009
1010 for_each_possible_cpu(cpu) {
Marc Zyngier3de50da2013-04-08 16:47:19 +01001011 kvm_cpu_context_t *cpu_ctxt;
Christoffer Dall342cd0a2013-01-20 18:28:06 -05001012
Marc Zyngier3de50da2013-04-08 16:47:19 +01001013 cpu_ctxt = per_cpu_ptr(kvm_host_cpu_state, cpu);
1014 err = create_hyp_mappings(cpu_ctxt, cpu_ctxt + 1);
Christoffer Dall342cd0a2013-01-20 18:28:06 -05001015
1016 if (err) {
Marc Zyngier3de50da2013-04-08 16:47:19 +01001017 kvm_err("Cannot map host CPU state: %d\n", err);
1018 goto out_free_context;
Christoffer Dall342cd0a2013-01-20 18:28:06 -05001019 }
1020 }
1021
Marc Zyngier1a89dd92013-01-21 19:36:12 -05001022 /*
Marc Zyngierd157f4a2013-04-12 19:12:07 +01001023 * Execute the init code on each CPU.
1024 */
1025 on_each_cpu(cpu_init_hyp_mode, NULL, 1);
1026
1027 /*
Marc Zyngier1a89dd92013-01-21 19:36:12 -05001028 * Init HYP view of VGIC
1029 */
1030 err = kvm_vgic_hyp_init();
1031 if (err)
Marc Zyngier3de50da2013-04-08 16:47:19 +01001032 goto out_free_context;
Marc Zyngier1a89dd92013-01-21 19:36:12 -05001033
Marc Zyngier967f8422013-01-23 13:21:59 -05001034 /*
1035 * Init HYP architected timer support
1036 */
1037 err = kvm_timer_hyp_init();
1038 if (err)
1039 goto out_free_mappings;
1040
Marc Zyngierd157f4a2013-04-12 19:12:07 +01001041#ifndef CONFIG_HOTPLUG_CPU
1042 free_boot_hyp_pgd();
1043#endif
1044
Marc Zyngier210552c2013-03-05 03:18:00 +00001045 kvm_perf_init();
1046
Christoffer Dall342cd0a2013-01-20 18:28:06 -05001047 kvm_info("Hyp mode initialized successfully\n");
Marc Zyngier210552c2013-03-05 03:18:00 +00001048
Christoffer Dall342cd0a2013-01-20 18:28:06 -05001049 return 0;
Marc Zyngier3de50da2013-04-08 16:47:19 +01001050out_free_context:
1051 free_percpu(kvm_host_cpu_state);
Christoffer Dall342cd0a2013-01-20 18:28:06 -05001052out_free_mappings:
Marc Zyngier4f728272013-04-12 19:12:05 +01001053 free_hyp_pgds();
Christoffer Dall342cd0a2013-01-20 18:28:06 -05001054out_free_stack_pages:
1055 for_each_possible_cpu(cpu)
1056 free_page(per_cpu(kvm_arm_hyp_stack_page, cpu));
1057out_err:
1058 kvm_err("error initializing Hyp mode: %d\n", err);
1059 return err;
1060}
1061
Andre Przywarad4e071c2013-04-17 12:52:01 +02001062static void check_kvm_target_cpu(void *ret)
1063{
1064 *(int *)ret = kvm_target_cpu();
1065}
1066
Andre Przywara4429fc62014-06-02 15:37:13 +02001067struct kvm_vcpu *kvm_mpidr_to_vcpu(struct kvm *kvm, unsigned long mpidr)
1068{
1069 struct kvm_vcpu *vcpu;
1070 int i;
1071
1072 mpidr &= MPIDR_HWID_BITMASK;
1073 kvm_for_each_vcpu(i, vcpu, kvm) {
1074 if (mpidr == kvm_vcpu_get_mpidr_aff(vcpu))
1075 return vcpu;
1076 }
1077 return NULL;
1078}
1079
Christoffer Dall342cd0a2013-01-20 18:28:06 -05001080/**
1081 * Initialize Hyp-mode and memory mappings on all CPUs.
1082 */
Christoffer Dall749cf76c2013-01-20 18:28:06 -05001083int kvm_arch_init(void *opaque)
1084{
Christoffer Dall342cd0a2013-01-20 18:28:06 -05001085 int err;
Andre Przywarad4e071c2013-04-17 12:52:01 +02001086 int ret, cpu;
Christoffer Dall342cd0a2013-01-20 18:28:06 -05001087
1088 if (!is_hyp_mode_available()) {
1089 kvm_err("HYP mode not available\n");
1090 return -ENODEV;
1091 }
1092
Andre Przywarad4e071c2013-04-17 12:52:01 +02001093 for_each_online_cpu(cpu) {
1094 smp_call_function_single(cpu, check_kvm_target_cpu, &ret, 1);
1095 if (ret < 0) {
1096 kvm_err("Error, CPU %d not supported!\n", cpu);
1097 return -ENODEV;
1098 }
Christoffer Dall342cd0a2013-01-20 18:28:06 -05001099 }
1100
Srivatsa S. Bhat81468752014-03-18 15:53:05 +05301101 cpu_notifier_register_begin();
1102
Christoffer Dall342cd0a2013-01-20 18:28:06 -05001103 err = init_hyp_mode();
1104 if (err)
1105 goto out_err;
1106
Srivatsa S. Bhat81468752014-03-18 15:53:05 +05301107 err = __register_cpu_notifier(&hyp_init_cpu_nb);
Marc Zyngierd157f4a2013-04-12 19:12:07 +01001108 if (err) {
1109 kvm_err("Cannot register HYP init CPU notifier (%d)\n", err);
1110 goto out_err;
1111 }
1112
Srivatsa S. Bhat81468752014-03-18 15:53:05 +05301113 cpu_notifier_register_done();
1114
Lorenzo Pieralisi1fcf7ce2013-08-05 15:04:46 +01001115 hyp_cpu_pm_init();
1116
Christoffer Dall5b3e5e52013-01-20 18:28:09 -05001117 kvm_coproc_table_init();
Christoffer Dall749cf76c2013-01-20 18:28:06 -05001118 return 0;
Christoffer Dall342cd0a2013-01-20 18:28:06 -05001119out_err:
Srivatsa S. Bhat81468752014-03-18 15:53:05 +05301120 cpu_notifier_register_done();
Christoffer Dall342cd0a2013-01-20 18:28:06 -05001121 return err;
Christoffer Dall749cf76c2013-01-20 18:28:06 -05001122}
1123
1124/* NOP: Compiling as a module not supported */
1125void kvm_arch_exit(void)
1126{
Marc Zyngier210552c2013-03-05 03:18:00 +00001127 kvm_perf_teardown();
Christoffer Dall749cf76c2013-01-20 18:28:06 -05001128}
1129
1130static int arm_init(void)
1131{
1132 int rc = kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
1133 return rc;
1134}
1135
1136module_init(arm_init);