Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 1 | /* |
| 2 | * This file is subject to the terms and conditions of the GNU General Public |
| 3 | * License. See the file "COPYING" in the main directory of this archive |
| 4 | * for more details. |
| 5 | * |
| 6 | * KVM/MIPS: MIPS specific KVM APIs |
| 7 | * |
| 8 | * Copyright (C) 2012 MIPS Technologies, Inc. All rights reserved. |
| 9 | * Authors: Sanjay Lal <sanjayl@kymasys.com> |
Deng-Cheng Zhu | d116e81 | 2014-06-26 12:11:34 -0700 | [diff] [blame] | 10 | */ |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 11 | |
James Hogan | 0510870 | 2016-06-15 19:29:56 +0100 | [diff] [blame] | 12 | #include <linux/bitops.h> |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 13 | #include <linux/errno.h> |
| 14 | #include <linux/err.h> |
James Hogan | 98e91b8 | 2014-11-18 14:09:12 +0000 | [diff] [blame] | 15 | #include <linux/kdebug.h> |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 16 | #include <linux/module.h> |
| 17 | #include <linux/vmalloc.h> |
| 18 | #include <linux/fs.h> |
| 19 | #include <linux/bootmem.h> |
James Hogan | f798217 | 2015-02-04 17:06:37 +0000 | [diff] [blame] | 20 | #include <asm/fpu.h> |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 21 | #include <asm/page.h> |
| 22 | #include <asm/cacheflush.h> |
| 23 | #include <asm/mmu_context.h> |
James Hogan | c4c6f2c | 2015-02-04 10:52:03 +0000 | [diff] [blame] | 24 | #include <asm/pgtable.h> |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 25 | |
| 26 | #include <linux/kvm_host.h> |
| 27 | |
Deng-Cheng Zhu | d7d5b05 | 2014-06-26 12:11:38 -0700 | [diff] [blame] | 28 | #include "interrupt.h" |
| 29 | #include "commpage.h" |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 30 | |
| 31 | #define CREATE_TRACE_POINTS |
| 32 | #include "trace.h" |
| 33 | |
| 34 | #ifndef VECTORSPACING |
| 35 | #define VECTORSPACING 0x100 /* for EI/VI mode */ |
| 36 | #endif |
| 37 | |
Deng-Cheng Zhu | d116e81 | 2014-06-26 12:11:34 -0700 | [diff] [blame] | 38 | #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x) |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 39 | struct kvm_stats_debugfs_item debugfs_entries[] = { |
Deng-Cheng Zhu | d116e81 | 2014-06-26 12:11:34 -0700 | [diff] [blame] | 40 | { "wait", VCPU_STAT(wait_exits), KVM_STAT_VCPU }, |
| 41 | { "cache", VCPU_STAT(cache_exits), KVM_STAT_VCPU }, |
| 42 | { "signal", VCPU_STAT(signal_exits), KVM_STAT_VCPU }, |
| 43 | { "interrupt", VCPU_STAT(int_exits), KVM_STAT_VCPU }, |
| 44 | { "cop_unsuable", VCPU_STAT(cop_unusable_exits), KVM_STAT_VCPU }, |
| 45 | { "tlbmod", VCPU_STAT(tlbmod_exits), KVM_STAT_VCPU }, |
| 46 | { "tlbmiss_ld", VCPU_STAT(tlbmiss_ld_exits), KVM_STAT_VCPU }, |
| 47 | { "tlbmiss_st", VCPU_STAT(tlbmiss_st_exits), KVM_STAT_VCPU }, |
| 48 | { "addrerr_st", VCPU_STAT(addrerr_st_exits), KVM_STAT_VCPU }, |
| 49 | { "addrerr_ld", VCPU_STAT(addrerr_ld_exits), KVM_STAT_VCPU }, |
| 50 | { "syscall", VCPU_STAT(syscall_exits), KVM_STAT_VCPU }, |
| 51 | { "resvd_inst", VCPU_STAT(resvd_inst_exits), KVM_STAT_VCPU }, |
| 52 | { "break_inst", VCPU_STAT(break_inst_exits), KVM_STAT_VCPU }, |
James Hogan | 0a56042 | 2015-02-06 16:03:57 +0000 | [diff] [blame] | 53 | { "trap_inst", VCPU_STAT(trap_inst_exits), KVM_STAT_VCPU }, |
James Hogan | c2537ed | 2015-02-06 10:56:27 +0000 | [diff] [blame] | 54 | { "msa_fpe", VCPU_STAT(msa_fpe_exits), KVM_STAT_VCPU }, |
James Hogan | 1c0cd66 | 2015-02-06 10:56:27 +0000 | [diff] [blame] | 55 | { "fpe", VCPU_STAT(fpe_exits), KVM_STAT_VCPU }, |
James Hogan | c2537ed | 2015-02-06 10:56:27 +0000 | [diff] [blame] | 56 | { "msa_disabled", VCPU_STAT(msa_disabled_exits), KVM_STAT_VCPU }, |
Deng-Cheng Zhu | d116e81 | 2014-06-26 12:11:34 -0700 | [diff] [blame] | 57 | { "flush_dcache", VCPU_STAT(flush_dcache_exits), KVM_STAT_VCPU }, |
Paolo Bonzini | f781951 | 2015-02-04 18:20:58 +0100 | [diff] [blame] | 58 | { "halt_successful_poll", VCPU_STAT(halt_successful_poll), KVM_STAT_VCPU }, |
Paolo Bonzini | 62bea5b | 2015-09-15 18:27:57 +0200 | [diff] [blame] | 59 | { "halt_attempted_poll", VCPU_STAT(halt_attempted_poll), KVM_STAT_VCPU }, |
Christian Borntraeger | 3491caf | 2016-05-13 12:16:35 +0200 | [diff] [blame] | 60 | { "halt_poll_invalid", VCPU_STAT(halt_poll_invalid), KVM_STAT_VCPU }, |
Deng-Cheng Zhu | d116e81 | 2014-06-26 12:11:34 -0700 | [diff] [blame] | 61 | { "halt_wakeup", VCPU_STAT(halt_wakeup), KVM_STAT_VCPU }, |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 62 | {NULL} |
| 63 | }; |
| 64 | |
| 65 | static int kvm_mips_reset_vcpu(struct kvm_vcpu *vcpu) |
| 66 | { |
| 67 | int i; |
Deng-Cheng Zhu | d116e81 | 2014-06-26 12:11:34 -0700 | [diff] [blame] | 68 | |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 69 | for_each_possible_cpu(i) { |
| 70 | vcpu->arch.guest_kernel_asid[i] = 0; |
| 71 | vcpu->arch.guest_user_asid[i] = 0; |
| 72 | } |
Deng-Cheng Zhu | d116e81 | 2014-06-26 12:11:34 -0700 | [diff] [blame] | 73 | |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 74 | return 0; |
| 75 | } |
| 76 | |
Deng-Cheng Zhu | d116e81 | 2014-06-26 12:11:34 -0700 | [diff] [blame] | 77 | /* |
| 78 | * XXXKYMA: We are simulatoring a processor that has the WII bit set in |
| 79 | * Config7, so we are "runnable" if interrupts are pending |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 80 | */ |
| 81 | int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu) |
| 82 | { |
| 83 | return !!(vcpu->arch.pending_exceptions); |
| 84 | } |
| 85 | |
| 86 | int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu) |
| 87 | { |
| 88 | return 1; |
| 89 | } |
| 90 | |
Radim Krčmář | 13a34e0 | 2014-08-28 15:13:03 +0200 | [diff] [blame] | 91 | int kvm_arch_hardware_enable(void) |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 92 | { |
| 93 | return 0; |
| 94 | } |
| 95 | |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 96 | int kvm_arch_hardware_setup(void) |
| 97 | { |
| 98 | return 0; |
| 99 | } |
| 100 | |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 101 | void kvm_arch_check_processor_compat(void *rtn) |
| 102 | { |
Deng-Cheng Zhu | d98403a | 2014-06-26 12:11:36 -0700 | [diff] [blame] | 103 | *(int *)rtn = 0; |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 104 | } |
| 105 | |
| 106 | static void kvm_mips_init_tlbs(struct kvm *kvm) |
| 107 | { |
| 108 | unsigned long wired; |
| 109 | |
Deng-Cheng Zhu | d116e81 | 2014-06-26 12:11:34 -0700 | [diff] [blame] | 110 | /* |
| 111 | * Add a wired entry to the TLB, it is used to map the commpage to |
| 112 | * the Guest kernel |
| 113 | */ |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 114 | wired = read_c0_wired(); |
| 115 | write_c0_wired(wired + 1); |
| 116 | mtc0_tlbw_hazard(); |
| 117 | kvm->arch.commpage_tlb = wired; |
| 118 | |
| 119 | kvm_debug("[%d] commpage TLB: %d\n", smp_processor_id(), |
| 120 | kvm->arch.commpage_tlb); |
| 121 | } |
| 122 | |
| 123 | static void kvm_mips_init_vm_percpu(void *arg) |
| 124 | { |
| 125 | struct kvm *kvm = (struct kvm *)arg; |
| 126 | |
| 127 | kvm_mips_init_tlbs(kvm); |
| 128 | kvm_mips_callbacks->vm_init(kvm); |
| 129 | |
| 130 | } |
| 131 | |
| 132 | int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) |
| 133 | { |
| 134 | if (atomic_inc_return(&kvm_mips_instance) == 1) { |
James Hogan | 6e95bfd | 2014-05-29 10:16:43 +0100 | [diff] [blame] | 135 | kvm_debug("%s: 1st KVM instance, setup host TLB parameters\n", |
| 136 | __func__); |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 137 | on_each_cpu(kvm_mips_init_vm_percpu, kvm, 1); |
| 138 | } |
| 139 | |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 140 | return 0; |
| 141 | } |
| 142 | |
Luiz Capitulino | 235539b | 2016-09-07 14:47:23 -0400 | [diff] [blame] | 143 | bool kvm_arch_has_vcpu_debugfs(void) |
| 144 | { |
| 145 | return false; |
| 146 | } |
| 147 | |
| 148 | int kvm_arch_create_vcpu_debugfs(struct kvm_vcpu *vcpu) |
| 149 | { |
| 150 | return 0; |
| 151 | } |
| 152 | |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 153 | void kvm_mips_free_vcpus(struct kvm *kvm) |
| 154 | { |
| 155 | unsigned int i; |
| 156 | struct kvm_vcpu *vcpu; |
| 157 | |
| 158 | /* Put the pages we reserved for the guest pmap */ |
| 159 | for (i = 0; i < kvm->arch.guest_pmap_npages; i++) { |
| 160 | if (kvm->arch.guest_pmap[i] != KVM_INVALID_PAGE) |
James Hogan | 9befad2 | 2016-06-09 14:19:11 +0100 | [diff] [blame] | 161 | kvm_release_pfn_clean(kvm->arch.guest_pmap[i]); |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 162 | } |
James Hogan | c6c0a66 | 2014-05-29 10:16:44 +0100 | [diff] [blame] | 163 | kfree(kvm->arch.guest_pmap); |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 164 | |
| 165 | kvm_for_each_vcpu(i, vcpu, kvm) { |
| 166 | kvm_arch_vcpu_free(vcpu); |
| 167 | } |
| 168 | |
| 169 | mutex_lock(&kvm->lock); |
| 170 | |
| 171 | for (i = 0; i < atomic_read(&kvm->online_vcpus); i++) |
| 172 | kvm->vcpus[i] = NULL; |
| 173 | |
| 174 | atomic_set(&kvm->online_vcpus, 0); |
| 175 | |
| 176 | mutex_unlock(&kvm->lock); |
| 177 | } |
| 178 | |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 179 | static void kvm_mips_uninit_tlbs(void *arg) |
| 180 | { |
| 181 | /* Restore wired count */ |
| 182 | write_c0_wired(0); |
| 183 | mtc0_tlbw_hazard(); |
| 184 | /* Clear out all the TLBs */ |
| 185 | kvm_local_flush_tlb_all(); |
| 186 | } |
| 187 | |
| 188 | void kvm_arch_destroy_vm(struct kvm *kvm) |
| 189 | { |
| 190 | kvm_mips_free_vcpus(kvm); |
| 191 | |
| 192 | /* If this is the last instance, restore wired count */ |
| 193 | if (atomic_dec_return(&kvm_mips_instance) == 0) { |
James Hogan | 6e95bfd | 2014-05-29 10:16:43 +0100 | [diff] [blame] | 194 | kvm_debug("%s: last KVM instance, restoring TLB parameters\n", |
| 195 | __func__); |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 196 | on_each_cpu(kvm_mips_uninit_tlbs, NULL, 1); |
| 197 | } |
| 198 | } |
| 199 | |
Deng-Cheng Zhu | d116e81 | 2014-06-26 12:11:34 -0700 | [diff] [blame] | 200 | long kvm_arch_dev_ioctl(struct file *filp, unsigned int ioctl, |
| 201 | unsigned long arg) |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 202 | { |
David Daney | ed82985 | 2013-05-23 09:49:10 -0700 | [diff] [blame] | 203 | return -ENOIOCTLCMD; |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 204 | } |
| 205 | |
Aneesh Kumar K.V | 5587027 | 2013-10-07 22:18:00 +0530 | [diff] [blame] | 206 | int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot, |
| 207 | unsigned long npages) |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 208 | { |
| 209 | return 0; |
| 210 | } |
| 211 | |
| 212 | int kvm_arch_prepare_memory_region(struct kvm *kvm, |
Deng-Cheng Zhu | d116e81 | 2014-06-26 12:11:34 -0700 | [diff] [blame] | 213 | struct kvm_memory_slot *memslot, |
Paolo Bonzini | 09170a4 | 2015-05-18 13:59:39 +0200 | [diff] [blame] | 214 | const struct kvm_userspace_memory_region *mem, |
Deng-Cheng Zhu | d116e81 | 2014-06-26 12:11:34 -0700 | [diff] [blame] | 215 | enum kvm_mr_change change) |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 216 | { |
| 217 | return 0; |
| 218 | } |
| 219 | |
| 220 | void kvm_arch_commit_memory_region(struct kvm *kvm, |
Paolo Bonzini | 09170a4 | 2015-05-18 13:59:39 +0200 | [diff] [blame] | 221 | const struct kvm_userspace_memory_region *mem, |
Deng-Cheng Zhu | d116e81 | 2014-06-26 12:11:34 -0700 | [diff] [blame] | 222 | const struct kvm_memory_slot *old, |
Paolo Bonzini | f36f3f2 | 2015-05-18 13:20:23 +0200 | [diff] [blame] | 223 | const struct kvm_memory_slot *new, |
Deng-Cheng Zhu | d116e81 | 2014-06-26 12:11:34 -0700 | [diff] [blame] | 224 | enum kvm_mr_change change) |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 225 | { |
| 226 | unsigned long npages = 0; |
Deng-Cheng Zhu | d98403a | 2014-06-26 12:11:36 -0700 | [diff] [blame] | 227 | int i; |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 228 | |
| 229 | kvm_debug("%s: kvm: %p slot: %d, GPA: %llx, size: %llx, QVA: %llx\n", |
| 230 | __func__, kvm, mem->slot, mem->guest_phys_addr, |
| 231 | mem->memory_size, mem->userspace_addr); |
| 232 | |
| 233 | /* Setup Guest PMAP table */ |
| 234 | if (!kvm->arch.guest_pmap) { |
| 235 | if (mem->slot == 0) |
| 236 | npages = mem->memory_size >> PAGE_SHIFT; |
| 237 | |
| 238 | if (npages) { |
| 239 | kvm->arch.guest_pmap_npages = npages; |
| 240 | kvm->arch.guest_pmap = |
| 241 | kzalloc(npages * sizeof(unsigned long), GFP_KERNEL); |
| 242 | |
| 243 | if (!kvm->arch.guest_pmap) { |
James Hogan | f7fdcb6 | 2015-12-16 23:49:39 +0000 | [diff] [blame] | 244 | kvm_err("Failed to allocate guest PMAP\n"); |
Deng-Cheng Zhu | d98403a | 2014-06-26 12:11:36 -0700 | [diff] [blame] | 245 | return; |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 246 | } |
| 247 | |
James Hogan | 6e95bfd | 2014-05-29 10:16:43 +0100 | [diff] [blame] | 248 | kvm_debug("Allocated space for Guest PMAP Table (%ld pages) @ %p\n", |
| 249 | npages, kvm->arch.guest_pmap); |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 250 | |
| 251 | /* Now setup the page table */ |
Deng-Cheng Zhu | d116e81 | 2014-06-26 12:11:34 -0700 | [diff] [blame] | 252 | for (i = 0; i < npages; i++) |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 253 | kvm->arch.guest_pmap[i] = KVM_INVALID_PAGE; |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 254 | } |
| 255 | } |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 256 | } |
| 257 | |
James Hogan | d7b8f89 | 2016-06-23 17:34:40 +0100 | [diff] [blame] | 258 | static inline void dump_handler(const char *symbol, void *start, void *end) |
| 259 | { |
| 260 | u32 *p; |
| 261 | |
| 262 | pr_debug("LEAF(%s)\n", symbol); |
| 263 | |
| 264 | pr_debug("\t.set push\n"); |
| 265 | pr_debug("\t.set noreorder\n"); |
| 266 | |
| 267 | for (p = start; p < (u32 *)end; ++p) |
| 268 | pr_debug("\t.word\t0x%08x\t\t# %p\n", *p, p); |
| 269 | |
| 270 | pr_debug("\t.set\tpop\n"); |
| 271 | |
| 272 | pr_debug("\tEND(%s)\n", symbol); |
| 273 | } |
| 274 | |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 275 | struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm, unsigned int id) |
| 276 | { |
James Hogan | 90e9311 | 2016-06-23 17:34:39 +0100 | [diff] [blame] | 277 | int err, size; |
James Hogan | 1f9ca62 | 2016-06-23 17:34:46 +0100 | [diff] [blame] | 278 | void *gebase, *p, *handler; |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 279 | int i; |
| 280 | |
| 281 | struct kvm_vcpu *vcpu = kzalloc(sizeof(struct kvm_vcpu), GFP_KERNEL); |
| 282 | |
| 283 | if (!vcpu) { |
| 284 | err = -ENOMEM; |
| 285 | goto out; |
| 286 | } |
| 287 | |
| 288 | err = kvm_vcpu_init(vcpu, kvm, id); |
| 289 | |
| 290 | if (err) |
| 291 | goto out_free_cpu; |
| 292 | |
James Hogan | 6e95bfd | 2014-05-29 10:16:43 +0100 | [diff] [blame] | 293 | kvm_debug("kvm @ %p: create cpu %d at %p\n", kvm, id, vcpu); |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 294 | |
Deng-Cheng Zhu | d116e81 | 2014-06-26 12:11:34 -0700 | [diff] [blame] | 295 | /* |
| 296 | * Allocate space for host mode exception handlers that handle |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 297 | * guest mode exits |
| 298 | */ |
Deng-Cheng Zhu | d116e81 | 2014-06-26 12:11:34 -0700 | [diff] [blame] | 299 | if (cpu_has_veic || cpu_has_vint) |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 300 | size = 0x200 + VECTORSPACING * 64; |
Deng-Cheng Zhu | d116e81 | 2014-06-26 12:11:34 -0700 | [diff] [blame] | 301 | else |
James Hogan | 7006e2d | 2014-05-29 10:16:23 +0100 | [diff] [blame] | 302 | size = 0x4000; |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 303 | |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 304 | gebase = kzalloc(ALIGN(size, PAGE_SIZE), GFP_KERNEL); |
| 305 | |
| 306 | if (!gebase) { |
| 307 | err = -ENOMEM; |
James Hogan | 585bb8f | 2015-11-11 14:21:20 +0000 | [diff] [blame] | 308 | goto out_uninit_cpu; |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 309 | } |
James Hogan | 6e95bfd | 2014-05-29 10:16:43 +0100 | [diff] [blame] | 310 | kvm_debug("Allocated %d bytes for KVM Exception Handlers @ %p\n", |
| 311 | ALIGN(size, PAGE_SIZE), gebase); |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 312 | |
James Hogan | 2a06dab | 2016-07-08 11:53:26 +0100 | [diff] [blame] | 313 | /* |
| 314 | * Check new ebase actually fits in CP0_EBase. The lack of a write gate |
| 315 | * limits us to the low 512MB of physical address space. If the memory |
| 316 | * we allocate is out of range, just give up now. |
| 317 | */ |
| 318 | if (!cpu_has_ebase_wg && virt_to_phys(gebase) >= 0x20000000) { |
| 319 | kvm_err("CP0_EBase.WG required for guest exception base %pK\n", |
| 320 | gebase); |
| 321 | err = -ENOMEM; |
| 322 | goto out_free_gebase; |
| 323 | } |
| 324 | |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 325 | /* Save new ebase */ |
| 326 | vcpu->arch.guest_ebase = gebase; |
| 327 | |
James Hogan | 90e9311 | 2016-06-23 17:34:39 +0100 | [diff] [blame] | 328 | /* Build guest exception vectors dynamically in unmapped memory */ |
James Hogan | 1f9ca62 | 2016-06-23 17:34:46 +0100 | [diff] [blame] | 329 | handler = gebase + 0x2000; |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 330 | |
| 331 | /* TLB Refill, EXL = 0 */ |
James Hogan | 1f9ca62 | 2016-06-23 17:34:46 +0100 | [diff] [blame] | 332 | kvm_mips_build_exception(gebase, handler); |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 333 | |
| 334 | /* General Exception Entry point */ |
James Hogan | 1f9ca62 | 2016-06-23 17:34:46 +0100 | [diff] [blame] | 335 | kvm_mips_build_exception(gebase + 0x180, handler); |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 336 | |
| 337 | /* For vectored interrupts poke the exception code @ all offsets 0-7 */ |
| 338 | for (i = 0; i < 8; i++) { |
| 339 | kvm_debug("L1 Vectored handler @ %p\n", |
| 340 | gebase + 0x200 + (i * VECTORSPACING)); |
James Hogan | 1f9ca62 | 2016-06-23 17:34:46 +0100 | [diff] [blame] | 341 | kvm_mips_build_exception(gebase + 0x200 + i * VECTORSPACING, |
| 342 | handler); |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 343 | } |
| 344 | |
James Hogan | 90e9311 | 2016-06-23 17:34:39 +0100 | [diff] [blame] | 345 | /* General exit handler */ |
James Hogan | 1f9ca62 | 2016-06-23 17:34:46 +0100 | [diff] [blame] | 346 | p = handler; |
James Hogan | 90e9311 | 2016-06-23 17:34:39 +0100 | [diff] [blame] | 347 | p = kvm_mips_build_exit(p); |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 348 | |
James Hogan | 90e9311 | 2016-06-23 17:34:39 +0100 | [diff] [blame] | 349 | /* Guest entry routine */ |
| 350 | vcpu->arch.vcpu_run = p; |
| 351 | p = kvm_mips_build_vcpu_run(p); |
James Hogan | 797179b | 2016-06-09 10:50:43 +0100 | [diff] [blame] | 352 | |
James Hogan | d7b8f89 | 2016-06-23 17:34:40 +0100 | [diff] [blame] | 353 | /* Dump the generated code */ |
| 354 | pr_debug("#include <asm/asm.h>\n"); |
| 355 | pr_debug("#include <asm/regdef.h>\n"); |
| 356 | pr_debug("\n"); |
| 357 | dump_handler("kvm_vcpu_run", vcpu->arch.vcpu_run, p); |
| 358 | dump_handler("kvm_gen_exc", gebase + 0x180, gebase + 0x200); |
| 359 | dump_handler("kvm_exit", gebase + 0x2000, vcpu->arch.vcpu_run); |
| 360 | |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 361 | /* Invalidate the icache for these ranges */ |
James Hogan | facaaec | 2014-05-29 10:16:25 +0100 | [diff] [blame] | 362 | local_flush_icache_range((unsigned long)gebase, |
| 363 | (unsigned long)gebase + ALIGN(size, PAGE_SIZE)); |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 364 | |
Deng-Cheng Zhu | d116e81 | 2014-06-26 12:11:34 -0700 | [diff] [blame] | 365 | /* |
| 366 | * Allocate comm page for guest kernel, a TLB will be reserved for |
| 367 | * mapping GVA @ 0xFFFF8000 to this page |
| 368 | */ |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 369 | vcpu->arch.kseg0_commpage = kzalloc(PAGE_SIZE << 1, GFP_KERNEL); |
| 370 | |
| 371 | if (!vcpu->arch.kseg0_commpage) { |
| 372 | err = -ENOMEM; |
| 373 | goto out_free_gebase; |
| 374 | } |
| 375 | |
James Hogan | 6e95bfd | 2014-05-29 10:16:43 +0100 | [diff] [blame] | 376 | kvm_debug("Allocated COMM page @ %p\n", vcpu->arch.kseg0_commpage); |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 377 | kvm_mips_commpage_init(vcpu); |
| 378 | |
| 379 | /* Init */ |
| 380 | vcpu->arch.last_sched_cpu = -1; |
| 381 | |
| 382 | /* Start off the timer */ |
James Hogan | e30492b | 2014-05-29 10:16:35 +0100 | [diff] [blame] | 383 | kvm_mips_init_count(vcpu); |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 384 | |
| 385 | return vcpu; |
| 386 | |
| 387 | out_free_gebase: |
| 388 | kfree(gebase); |
| 389 | |
James Hogan | 585bb8f | 2015-11-11 14:21:20 +0000 | [diff] [blame] | 390 | out_uninit_cpu: |
| 391 | kvm_vcpu_uninit(vcpu); |
| 392 | |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 393 | out_free_cpu: |
| 394 | kfree(vcpu); |
| 395 | |
| 396 | out: |
| 397 | return ERR_PTR(err); |
| 398 | } |
| 399 | |
| 400 | void kvm_arch_vcpu_free(struct kvm_vcpu *vcpu) |
| 401 | { |
| 402 | hrtimer_cancel(&vcpu->arch.comparecount_timer); |
| 403 | |
| 404 | kvm_vcpu_uninit(vcpu); |
| 405 | |
| 406 | kvm_mips_dump_stats(vcpu); |
| 407 | |
James Hogan | c6c0a66 | 2014-05-29 10:16:44 +0100 | [diff] [blame] | 408 | kfree(vcpu->arch.guest_ebase); |
| 409 | kfree(vcpu->arch.kseg0_commpage); |
Deng-Cheng Zhu | 8c9eb04 | 2014-06-24 10:31:08 -0700 | [diff] [blame] | 410 | kfree(vcpu); |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 411 | } |
| 412 | |
| 413 | void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) |
| 414 | { |
| 415 | kvm_arch_vcpu_free(vcpu); |
| 416 | } |
| 417 | |
Deng-Cheng Zhu | d116e81 | 2014-06-26 12:11:34 -0700 | [diff] [blame] | 418 | int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu, |
| 419 | struct kvm_guest_debug *dbg) |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 420 | { |
David Daney | ed82985 | 2013-05-23 09:49:10 -0700 | [diff] [blame] | 421 | return -ENOIOCTLCMD; |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 422 | } |
| 423 | |
James Hogan | 25b08c7 | 2016-09-16 00:06:43 +0100 | [diff] [blame] | 424 | /* Must be called with preemption disabled, just before entering guest */ |
| 425 | static void kvm_mips_check_asids(struct kvm_vcpu *vcpu) |
| 426 | { |
| 427 | struct mips_coproc *cop0 = vcpu->arch.cop0; |
| 428 | int cpu = smp_processor_id(); |
| 429 | unsigned int gasid; |
| 430 | |
| 431 | /* |
| 432 | * Lazy host ASID regeneration for guest user mode. |
| 433 | * If the guest ASID has changed since the last guest usermode |
| 434 | * execution, regenerate the host ASID so as to invalidate stale TLB |
| 435 | * entries. |
| 436 | */ |
| 437 | if (!KVM_GUEST_KERNEL_MODE(vcpu)) { |
| 438 | gasid = kvm_read_c0_guest_entryhi(cop0) & KVM_ENTRYHI_ASID; |
| 439 | if (gasid != vcpu->arch.last_user_gasid) { |
| 440 | kvm_get_new_mmu_context(&vcpu->arch.guest_user_mm, cpu, |
| 441 | vcpu); |
| 442 | vcpu->arch.guest_user_asid[cpu] = |
| 443 | vcpu->arch.guest_user_mm.context.asid[cpu]; |
| 444 | vcpu->arch.last_user_gasid = gasid; |
| 445 | } |
| 446 | } |
| 447 | } |
| 448 | |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 449 | int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *run) |
| 450 | { |
| 451 | int r = 0; |
| 452 | sigset_t sigsaved; |
| 453 | |
| 454 | if (vcpu->sigset_active) |
| 455 | sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved); |
| 456 | |
| 457 | if (vcpu->mmio_needed) { |
| 458 | if (!vcpu->mmio_is_write) |
| 459 | kvm_mips_complete_mmio_load(vcpu, run); |
| 460 | vcpu->mmio_needed = 0; |
| 461 | } |
| 462 | |
James Hogan | f798217 | 2015-02-04 17:06:37 +0000 | [diff] [blame] | 463 | lose_fpu(1); |
| 464 | |
James Hogan | 044f0f0 | 2014-05-29 10:16:32 +0100 | [diff] [blame] | 465 | local_irq_disable(); |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 466 | /* Check if we have any exceptions/interrupts pending */ |
| 467 | kvm_mips_deliver_interrupts(vcpu, |
| 468 | kvm_read_c0_guest_cause(vcpu->arch.cop0)); |
| 469 | |
Paolo Bonzini | 6edaa53 | 2016-06-15 15:18:26 +0200 | [diff] [blame] | 470 | guest_enter_irqoff(); |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 471 | |
James Hogan | c4c6f2c | 2015-02-04 10:52:03 +0000 | [diff] [blame] | 472 | /* Disable hardware page table walking while in guest */ |
| 473 | htw_stop(); |
| 474 | |
James Hogan | 9325860 | 2016-06-14 09:40:14 +0100 | [diff] [blame] | 475 | trace_kvm_enter(vcpu); |
James Hogan | 25b08c7 | 2016-09-16 00:06:43 +0100 | [diff] [blame] | 476 | |
| 477 | kvm_mips_check_asids(vcpu); |
| 478 | |
James Hogan | 797179b | 2016-06-09 10:50:43 +0100 | [diff] [blame] | 479 | r = vcpu->arch.vcpu_run(run, vcpu); |
James Hogan | 9325860 | 2016-06-14 09:40:14 +0100 | [diff] [blame] | 480 | trace_kvm_out(vcpu); |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 481 | |
James Hogan | c4c6f2c | 2015-02-04 10:52:03 +0000 | [diff] [blame] | 482 | /* Re-enable HTW before enabling interrupts */ |
| 483 | htw_start(); |
| 484 | |
Paolo Bonzini | 6edaa53 | 2016-06-15 15:18:26 +0200 | [diff] [blame] | 485 | guest_exit_irqoff(); |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 486 | local_irq_enable(); |
| 487 | |
| 488 | if (vcpu->sigset_active) |
| 489 | sigprocmask(SIG_SETMASK, &sigsaved, NULL); |
| 490 | |
| 491 | return r; |
| 492 | } |
| 493 | |
Deng-Cheng Zhu | d116e81 | 2014-06-26 12:11:34 -0700 | [diff] [blame] | 494 | int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu, |
| 495 | struct kvm_mips_interrupt *irq) |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 496 | { |
| 497 | int intr = (int)irq->irq; |
| 498 | struct kvm_vcpu *dvcpu = NULL; |
| 499 | |
| 500 | if (intr == 3 || intr == -3 || intr == 4 || intr == -4) |
| 501 | kvm_debug("%s: CPU: %d, INTR: %d\n", __func__, irq->cpu, |
| 502 | (int)intr); |
| 503 | |
| 504 | if (irq->cpu == -1) |
| 505 | dvcpu = vcpu; |
| 506 | else |
| 507 | dvcpu = vcpu->kvm->vcpus[irq->cpu]; |
| 508 | |
| 509 | if (intr == 2 || intr == 3 || intr == 4) { |
| 510 | kvm_mips_callbacks->queue_io_int(dvcpu, irq); |
| 511 | |
| 512 | } else if (intr == -2 || intr == -3 || intr == -4) { |
| 513 | kvm_mips_callbacks->dequeue_io_int(dvcpu, irq); |
| 514 | } else { |
| 515 | kvm_err("%s: invalid interrupt ioctl (%d:%d)\n", __func__, |
| 516 | irq->cpu, irq->irq); |
| 517 | return -EINVAL; |
| 518 | } |
| 519 | |
| 520 | dvcpu->arch.wait = 0; |
| 521 | |
Marcelo Tosatti | 8577370 | 2016-02-19 09:46:39 +0100 | [diff] [blame] | 522 | if (swait_active(&dvcpu->wq)) |
| 523 | swake_up(&dvcpu->wq); |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 524 | |
| 525 | return 0; |
| 526 | } |
| 527 | |
Deng-Cheng Zhu | d116e81 | 2014-06-26 12:11:34 -0700 | [diff] [blame] | 528 | int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu, |
| 529 | struct kvm_mp_state *mp_state) |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 530 | { |
David Daney | ed82985 | 2013-05-23 09:49:10 -0700 | [diff] [blame] | 531 | return -ENOIOCTLCMD; |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 532 | } |
| 533 | |
Deng-Cheng Zhu | d116e81 | 2014-06-26 12:11:34 -0700 | [diff] [blame] | 534 | int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu, |
| 535 | struct kvm_mp_state *mp_state) |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 536 | { |
David Daney | ed82985 | 2013-05-23 09:49:10 -0700 | [diff] [blame] | 537 | return -ENOIOCTLCMD; |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 538 | } |
| 539 | |
David Daney | 4c73fb2 | 2013-05-23 09:49:09 -0700 | [diff] [blame] | 540 | static u64 kvm_mips_get_one_regs[] = { |
| 541 | KVM_REG_MIPS_R0, |
| 542 | KVM_REG_MIPS_R1, |
| 543 | KVM_REG_MIPS_R2, |
| 544 | KVM_REG_MIPS_R3, |
| 545 | KVM_REG_MIPS_R4, |
| 546 | KVM_REG_MIPS_R5, |
| 547 | KVM_REG_MIPS_R6, |
| 548 | KVM_REG_MIPS_R7, |
| 549 | KVM_REG_MIPS_R8, |
| 550 | KVM_REG_MIPS_R9, |
| 551 | KVM_REG_MIPS_R10, |
| 552 | KVM_REG_MIPS_R11, |
| 553 | KVM_REG_MIPS_R12, |
| 554 | KVM_REG_MIPS_R13, |
| 555 | KVM_REG_MIPS_R14, |
| 556 | KVM_REG_MIPS_R15, |
| 557 | KVM_REG_MIPS_R16, |
| 558 | KVM_REG_MIPS_R17, |
| 559 | KVM_REG_MIPS_R18, |
| 560 | KVM_REG_MIPS_R19, |
| 561 | KVM_REG_MIPS_R20, |
| 562 | KVM_REG_MIPS_R21, |
| 563 | KVM_REG_MIPS_R22, |
| 564 | KVM_REG_MIPS_R23, |
| 565 | KVM_REG_MIPS_R24, |
| 566 | KVM_REG_MIPS_R25, |
| 567 | KVM_REG_MIPS_R26, |
| 568 | KVM_REG_MIPS_R27, |
| 569 | KVM_REG_MIPS_R28, |
| 570 | KVM_REG_MIPS_R29, |
| 571 | KVM_REG_MIPS_R30, |
| 572 | KVM_REG_MIPS_R31, |
| 573 | |
James Hogan | 70e92c7e | 2016-07-04 19:35:11 +0100 | [diff] [blame] | 574 | #ifndef CONFIG_CPU_MIPSR6 |
David Daney | 4c73fb2 | 2013-05-23 09:49:09 -0700 | [diff] [blame] | 575 | KVM_REG_MIPS_HI, |
| 576 | KVM_REG_MIPS_LO, |
James Hogan | 70e92c7e | 2016-07-04 19:35:11 +0100 | [diff] [blame] | 577 | #endif |
David Daney | 4c73fb2 | 2013-05-23 09:49:09 -0700 | [diff] [blame] | 578 | KVM_REG_MIPS_PC, |
| 579 | |
| 580 | KVM_REG_MIPS_CP0_INDEX, |
| 581 | KVM_REG_MIPS_CP0_CONTEXT, |
James Hogan | 7767b7d | 2014-05-29 10:16:30 +0100 | [diff] [blame] | 582 | KVM_REG_MIPS_CP0_USERLOCAL, |
David Daney | 4c73fb2 | 2013-05-23 09:49:09 -0700 | [diff] [blame] | 583 | KVM_REG_MIPS_CP0_PAGEMASK, |
| 584 | KVM_REG_MIPS_CP0_WIRED, |
James Hogan | 16fd5c1 | 2014-05-29 10:16:31 +0100 | [diff] [blame] | 585 | KVM_REG_MIPS_CP0_HWRENA, |
David Daney | 4c73fb2 | 2013-05-23 09:49:09 -0700 | [diff] [blame] | 586 | KVM_REG_MIPS_CP0_BADVADDR, |
James Hogan | f8be02d | 2014-05-29 10:16:29 +0100 | [diff] [blame] | 587 | KVM_REG_MIPS_CP0_COUNT, |
David Daney | 4c73fb2 | 2013-05-23 09:49:09 -0700 | [diff] [blame] | 588 | KVM_REG_MIPS_CP0_ENTRYHI, |
James Hogan | f8be02d | 2014-05-29 10:16:29 +0100 | [diff] [blame] | 589 | KVM_REG_MIPS_CP0_COMPARE, |
David Daney | 4c73fb2 | 2013-05-23 09:49:09 -0700 | [diff] [blame] | 590 | KVM_REG_MIPS_CP0_STATUS, |
| 591 | KVM_REG_MIPS_CP0_CAUSE, |
James Hogan | fb6df0c | 2014-05-29 10:16:27 +0100 | [diff] [blame] | 592 | KVM_REG_MIPS_CP0_EPC, |
James Hogan | 1068eaa | 2014-06-26 13:56:52 +0100 | [diff] [blame] | 593 | KVM_REG_MIPS_CP0_PRID, |
David Daney | 4c73fb2 | 2013-05-23 09:49:09 -0700 | [diff] [blame] | 594 | KVM_REG_MIPS_CP0_CONFIG, |
| 595 | KVM_REG_MIPS_CP0_CONFIG1, |
| 596 | KVM_REG_MIPS_CP0_CONFIG2, |
| 597 | KVM_REG_MIPS_CP0_CONFIG3, |
James Hogan | c771607 | 2014-06-26 15:11:29 +0100 | [diff] [blame] | 598 | KVM_REG_MIPS_CP0_CONFIG4, |
| 599 | KVM_REG_MIPS_CP0_CONFIG5, |
David Daney | 4c73fb2 | 2013-05-23 09:49:09 -0700 | [diff] [blame] | 600 | KVM_REG_MIPS_CP0_CONFIG7, |
James Hogan | f823934 | 2014-05-29 10:16:37 +0100 | [diff] [blame] | 601 | KVM_REG_MIPS_CP0_ERROREPC, |
| 602 | |
| 603 | KVM_REG_MIPS_COUNT_CTL, |
| 604 | KVM_REG_MIPS_COUNT_RESUME, |
James Hogan | f74a8e2 | 2014-05-29 10:16:38 +0100 | [diff] [blame] | 605 | KVM_REG_MIPS_COUNT_HZ, |
David Daney | 4c73fb2 | 2013-05-23 09:49:09 -0700 | [diff] [blame] | 606 | }; |
| 607 | |
James Hogan | e577593 | 2016-06-15 19:29:51 +0100 | [diff] [blame] | 608 | static u64 kvm_mips_get_one_regs_fpu[] = { |
| 609 | KVM_REG_MIPS_FCR_IR, |
| 610 | KVM_REG_MIPS_FCR_CSR, |
| 611 | }; |
| 612 | |
| 613 | static u64 kvm_mips_get_one_regs_msa[] = { |
| 614 | KVM_REG_MIPS_MSA_IR, |
| 615 | KVM_REG_MIPS_MSA_CSR, |
| 616 | }; |
| 617 | |
James Hogan | 0510870 | 2016-06-15 19:29:56 +0100 | [diff] [blame] | 618 | static u64 kvm_mips_get_one_regs_kscratch[] = { |
| 619 | KVM_REG_MIPS_CP0_KSCRATCH1, |
| 620 | KVM_REG_MIPS_CP0_KSCRATCH2, |
| 621 | KVM_REG_MIPS_CP0_KSCRATCH3, |
| 622 | KVM_REG_MIPS_CP0_KSCRATCH4, |
| 623 | KVM_REG_MIPS_CP0_KSCRATCH5, |
| 624 | KVM_REG_MIPS_CP0_KSCRATCH6, |
| 625 | }; |
| 626 | |
James Hogan | f5c43bd | 2016-06-15 19:29:49 +0100 | [diff] [blame] | 627 | static unsigned long kvm_mips_num_regs(struct kvm_vcpu *vcpu) |
| 628 | { |
| 629 | unsigned long ret; |
| 630 | |
| 631 | ret = ARRAY_SIZE(kvm_mips_get_one_regs); |
James Hogan | e577593 | 2016-06-15 19:29:51 +0100 | [diff] [blame] | 632 | if (kvm_mips_guest_can_have_fpu(&vcpu->arch)) { |
| 633 | ret += ARRAY_SIZE(kvm_mips_get_one_regs_fpu) + 48; |
| 634 | /* odd doubles */ |
| 635 | if (boot_cpu_data.fpu_id & MIPS_FPIR_F64) |
| 636 | ret += 16; |
| 637 | } |
| 638 | if (kvm_mips_guest_can_have_msa(&vcpu->arch)) |
| 639 | ret += ARRAY_SIZE(kvm_mips_get_one_regs_msa) + 32; |
James Hogan | 0510870 | 2016-06-15 19:29:56 +0100 | [diff] [blame] | 640 | ret += __arch_hweight8(vcpu->arch.kscratch_enabled); |
James Hogan | f5c43bd | 2016-06-15 19:29:49 +0100 | [diff] [blame] | 641 | ret += kvm_mips_callbacks->num_regs(vcpu); |
| 642 | |
| 643 | return ret; |
| 644 | } |
| 645 | |
| 646 | static int kvm_mips_copy_reg_indices(struct kvm_vcpu *vcpu, u64 __user *indices) |
| 647 | { |
James Hogan | e577593 | 2016-06-15 19:29:51 +0100 | [diff] [blame] | 648 | u64 index; |
| 649 | unsigned int i; |
| 650 | |
James Hogan | f5c43bd | 2016-06-15 19:29:49 +0100 | [diff] [blame] | 651 | if (copy_to_user(indices, kvm_mips_get_one_regs, |
| 652 | sizeof(kvm_mips_get_one_regs))) |
| 653 | return -EFAULT; |
| 654 | indices += ARRAY_SIZE(kvm_mips_get_one_regs); |
| 655 | |
James Hogan | e577593 | 2016-06-15 19:29:51 +0100 | [diff] [blame] | 656 | if (kvm_mips_guest_can_have_fpu(&vcpu->arch)) { |
| 657 | if (copy_to_user(indices, kvm_mips_get_one_regs_fpu, |
| 658 | sizeof(kvm_mips_get_one_regs_fpu))) |
| 659 | return -EFAULT; |
| 660 | indices += ARRAY_SIZE(kvm_mips_get_one_regs_fpu); |
| 661 | |
| 662 | for (i = 0; i < 32; ++i) { |
| 663 | index = KVM_REG_MIPS_FPR_32(i); |
| 664 | if (copy_to_user(indices, &index, sizeof(index))) |
| 665 | return -EFAULT; |
| 666 | ++indices; |
| 667 | |
| 668 | /* skip odd doubles if no F64 */ |
| 669 | if (i & 1 && !(boot_cpu_data.fpu_id & MIPS_FPIR_F64)) |
| 670 | continue; |
| 671 | |
| 672 | index = KVM_REG_MIPS_FPR_64(i); |
| 673 | if (copy_to_user(indices, &index, sizeof(index))) |
| 674 | return -EFAULT; |
| 675 | ++indices; |
| 676 | } |
| 677 | } |
| 678 | |
| 679 | if (kvm_mips_guest_can_have_msa(&vcpu->arch)) { |
| 680 | if (copy_to_user(indices, kvm_mips_get_one_regs_msa, |
| 681 | sizeof(kvm_mips_get_one_regs_msa))) |
| 682 | return -EFAULT; |
| 683 | indices += ARRAY_SIZE(kvm_mips_get_one_regs_msa); |
| 684 | |
| 685 | for (i = 0; i < 32; ++i) { |
| 686 | index = KVM_REG_MIPS_VEC_128(i); |
| 687 | if (copy_to_user(indices, &index, sizeof(index))) |
| 688 | return -EFAULT; |
| 689 | ++indices; |
| 690 | } |
| 691 | } |
| 692 | |
James Hogan | 0510870 | 2016-06-15 19:29:56 +0100 | [diff] [blame] | 693 | for (i = 0; i < 6; ++i) { |
| 694 | if (!(vcpu->arch.kscratch_enabled & BIT(i + 2))) |
| 695 | continue; |
| 696 | |
| 697 | if (copy_to_user(indices, &kvm_mips_get_one_regs_kscratch[i], |
| 698 | sizeof(kvm_mips_get_one_regs_kscratch[i]))) |
| 699 | return -EFAULT; |
| 700 | ++indices; |
| 701 | } |
| 702 | |
James Hogan | f5c43bd | 2016-06-15 19:29:49 +0100 | [diff] [blame] | 703 | return kvm_mips_callbacks->copy_reg_indices(vcpu, indices); |
| 704 | } |
| 705 | |
David Daney | 4c73fb2 | 2013-05-23 09:49:09 -0700 | [diff] [blame] | 706 | static int kvm_mips_get_reg(struct kvm_vcpu *vcpu, |
| 707 | const struct kvm_one_reg *reg) |
| 708 | { |
David Daney | 4c73fb2 | 2013-05-23 09:49:09 -0700 | [diff] [blame] | 709 | struct mips_coproc *cop0 = vcpu->arch.cop0; |
James Hogan | 379245c | 2014-12-02 15:48:24 +0000 | [diff] [blame] | 710 | struct mips_fpu_struct *fpu = &vcpu->arch.fpu; |
James Hogan | f8be02d | 2014-05-29 10:16:29 +0100 | [diff] [blame] | 711 | int ret; |
David Daney | 4c73fb2 | 2013-05-23 09:49:09 -0700 | [diff] [blame] | 712 | s64 v; |
James Hogan | ab86bd6 | 2014-12-02 15:48:24 +0000 | [diff] [blame] | 713 | s64 vs[2]; |
James Hogan | 379245c | 2014-12-02 15:48:24 +0000 | [diff] [blame] | 714 | unsigned int idx; |
David Daney | 4c73fb2 | 2013-05-23 09:49:09 -0700 | [diff] [blame] | 715 | |
| 716 | switch (reg->id) { |
James Hogan | 379245c | 2014-12-02 15:48:24 +0000 | [diff] [blame] | 717 | /* General purpose registers */ |
David Daney | 4c73fb2 | 2013-05-23 09:49:09 -0700 | [diff] [blame] | 718 | case KVM_REG_MIPS_R0 ... KVM_REG_MIPS_R31: |
| 719 | v = (long)vcpu->arch.gprs[reg->id - KVM_REG_MIPS_R0]; |
| 720 | break; |
James Hogan | 70e92c7e | 2016-07-04 19:35:11 +0100 | [diff] [blame] | 721 | #ifndef CONFIG_CPU_MIPSR6 |
David Daney | 4c73fb2 | 2013-05-23 09:49:09 -0700 | [diff] [blame] | 722 | case KVM_REG_MIPS_HI: |
| 723 | v = (long)vcpu->arch.hi; |
| 724 | break; |
| 725 | case KVM_REG_MIPS_LO: |
| 726 | v = (long)vcpu->arch.lo; |
| 727 | break; |
James Hogan | 70e92c7e | 2016-07-04 19:35:11 +0100 | [diff] [blame] | 728 | #endif |
David Daney | 4c73fb2 | 2013-05-23 09:49:09 -0700 | [diff] [blame] | 729 | case KVM_REG_MIPS_PC: |
| 730 | v = (long)vcpu->arch.pc; |
| 731 | break; |
| 732 | |
James Hogan | 379245c | 2014-12-02 15:48:24 +0000 | [diff] [blame] | 733 | /* Floating point registers */ |
| 734 | case KVM_REG_MIPS_FPR_32(0) ... KVM_REG_MIPS_FPR_32(31): |
| 735 | if (!kvm_mips_guest_has_fpu(&vcpu->arch)) |
| 736 | return -EINVAL; |
| 737 | idx = reg->id - KVM_REG_MIPS_FPR_32(0); |
| 738 | /* Odd singles in top of even double when FR=0 */ |
| 739 | if (kvm_read_c0_guest_status(cop0) & ST0_FR) |
| 740 | v = get_fpr32(&fpu->fpr[idx], 0); |
| 741 | else |
| 742 | v = get_fpr32(&fpu->fpr[idx & ~1], idx & 1); |
| 743 | break; |
| 744 | case KVM_REG_MIPS_FPR_64(0) ... KVM_REG_MIPS_FPR_64(31): |
| 745 | if (!kvm_mips_guest_has_fpu(&vcpu->arch)) |
| 746 | return -EINVAL; |
| 747 | idx = reg->id - KVM_REG_MIPS_FPR_64(0); |
| 748 | /* Can't access odd doubles in FR=0 mode */ |
| 749 | if (idx & 1 && !(kvm_read_c0_guest_status(cop0) & ST0_FR)) |
| 750 | return -EINVAL; |
| 751 | v = get_fpr64(&fpu->fpr[idx], 0); |
| 752 | break; |
| 753 | case KVM_REG_MIPS_FCR_IR: |
| 754 | if (!kvm_mips_guest_has_fpu(&vcpu->arch)) |
| 755 | return -EINVAL; |
| 756 | v = boot_cpu_data.fpu_id; |
| 757 | break; |
| 758 | case KVM_REG_MIPS_FCR_CSR: |
| 759 | if (!kvm_mips_guest_has_fpu(&vcpu->arch)) |
| 760 | return -EINVAL; |
| 761 | v = fpu->fcr31; |
| 762 | break; |
| 763 | |
James Hogan | ab86bd6 | 2014-12-02 15:48:24 +0000 | [diff] [blame] | 764 | /* MIPS SIMD Architecture (MSA) registers */ |
| 765 | case KVM_REG_MIPS_VEC_128(0) ... KVM_REG_MIPS_VEC_128(31): |
| 766 | if (!kvm_mips_guest_has_msa(&vcpu->arch)) |
| 767 | return -EINVAL; |
| 768 | /* Can't access MSA registers in FR=0 mode */ |
| 769 | if (!(kvm_read_c0_guest_status(cop0) & ST0_FR)) |
| 770 | return -EINVAL; |
| 771 | idx = reg->id - KVM_REG_MIPS_VEC_128(0); |
| 772 | #ifdef CONFIG_CPU_LITTLE_ENDIAN |
| 773 | /* least significant byte first */ |
| 774 | vs[0] = get_fpr64(&fpu->fpr[idx], 0); |
| 775 | vs[1] = get_fpr64(&fpu->fpr[idx], 1); |
| 776 | #else |
| 777 | /* most significant byte first */ |
| 778 | vs[0] = get_fpr64(&fpu->fpr[idx], 1); |
| 779 | vs[1] = get_fpr64(&fpu->fpr[idx], 0); |
| 780 | #endif |
| 781 | break; |
| 782 | case KVM_REG_MIPS_MSA_IR: |
| 783 | if (!kvm_mips_guest_has_msa(&vcpu->arch)) |
| 784 | return -EINVAL; |
| 785 | v = boot_cpu_data.msa_id; |
| 786 | break; |
| 787 | case KVM_REG_MIPS_MSA_CSR: |
| 788 | if (!kvm_mips_guest_has_msa(&vcpu->arch)) |
| 789 | return -EINVAL; |
| 790 | v = fpu->msacsr; |
| 791 | break; |
| 792 | |
James Hogan | 379245c | 2014-12-02 15:48:24 +0000 | [diff] [blame] | 793 | /* Co-processor 0 registers */ |
David Daney | 4c73fb2 | 2013-05-23 09:49:09 -0700 | [diff] [blame] | 794 | case KVM_REG_MIPS_CP0_INDEX: |
| 795 | v = (long)kvm_read_c0_guest_index(cop0); |
| 796 | break; |
| 797 | case KVM_REG_MIPS_CP0_CONTEXT: |
| 798 | v = (long)kvm_read_c0_guest_context(cop0); |
| 799 | break; |
James Hogan | 7767b7d | 2014-05-29 10:16:30 +0100 | [diff] [blame] | 800 | case KVM_REG_MIPS_CP0_USERLOCAL: |
| 801 | v = (long)kvm_read_c0_guest_userlocal(cop0); |
| 802 | break; |
David Daney | 4c73fb2 | 2013-05-23 09:49:09 -0700 | [diff] [blame] | 803 | case KVM_REG_MIPS_CP0_PAGEMASK: |
| 804 | v = (long)kvm_read_c0_guest_pagemask(cop0); |
| 805 | break; |
| 806 | case KVM_REG_MIPS_CP0_WIRED: |
| 807 | v = (long)kvm_read_c0_guest_wired(cop0); |
| 808 | break; |
James Hogan | 16fd5c1 | 2014-05-29 10:16:31 +0100 | [diff] [blame] | 809 | case KVM_REG_MIPS_CP0_HWRENA: |
| 810 | v = (long)kvm_read_c0_guest_hwrena(cop0); |
| 811 | break; |
David Daney | 4c73fb2 | 2013-05-23 09:49:09 -0700 | [diff] [blame] | 812 | case KVM_REG_MIPS_CP0_BADVADDR: |
| 813 | v = (long)kvm_read_c0_guest_badvaddr(cop0); |
| 814 | break; |
| 815 | case KVM_REG_MIPS_CP0_ENTRYHI: |
| 816 | v = (long)kvm_read_c0_guest_entryhi(cop0); |
| 817 | break; |
James Hogan | f8be02d | 2014-05-29 10:16:29 +0100 | [diff] [blame] | 818 | case KVM_REG_MIPS_CP0_COMPARE: |
| 819 | v = (long)kvm_read_c0_guest_compare(cop0); |
| 820 | break; |
David Daney | 4c73fb2 | 2013-05-23 09:49:09 -0700 | [diff] [blame] | 821 | case KVM_REG_MIPS_CP0_STATUS: |
| 822 | v = (long)kvm_read_c0_guest_status(cop0); |
| 823 | break; |
| 824 | case KVM_REG_MIPS_CP0_CAUSE: |
| 825 | v = (long)kvm_read_c0_guest_cause(cop0); |
| 826 | break; |
James Hogan | fb6df0c | 2014-05-29 10:16:27 +0100 | [diff] [blame] | 827 | case KVM_REG_MIPS_CP0_EPC: |
| 828 | v = (long)kvm_read_c0_guest_epc(cop0); |
| 829 | break; |
James Hogan | 1068eaa | 2014-06-26 13:56:52 +0100 | [diff] [blame] | 830 | case KVM_REG_MIPS_CP0_PRID: |
| 831 | v = (long)kvm_read_c0_guest_prid(cop0); |
| 832 | break; |
David Daney | 4c73fb2 | 2013-05-23 09:49:09 -0700 | [diff] [blame] | 833 | case KVM_REG_MIPS_CP0_CONFIG: |
| 834 | v = (long)kvm_read_c0_guest_config(cop0); |
| 835 | break; |
| 836 | case KVM_REG_MIPS_CP0_CONFIG1: |
| 837 | v = (long)kvm_read_c0_guest_config1(cop0); |
| 838 | break; |
| 839 | case KVM_REG_MIPS_CP0_CONFIG2: |
| 840 | v = (long)kvm_read_c0_guest_config2(cop0); |
| 841 | break; |
| 842 | case KVM_REG_MIPS_CP0_CONFIG3: |
| 843 | v = (long)kvm_read_c0_guest_config3(cop0); |
| 844 | break; |
James Hogan | c771607 | 2014-06-26 15:11:29 +0100 | [diff] [blame] | 845 | case KVM_REG_MIPS_CP0_CONFIG4: |
| 846 | v = (long)kvm_read_c0_guest_config4(cop0); |
| 847 | break; |
| 848 | case KVM_REG_MIPS_CP0_CONFIG5: |
| 849 | v = (long)kvm_read_c0_guest_config5(cop0); |
| 850 | break; |
David Daney | 4c73fb2 | 2013-05-23 09:49:09 -0700 | [diff] [blame] | 851 | case KVM_REG_MIPS_CP0_CONFIG7: |
| 852 | v = (long)kvm_read_c0_guest_config7(cop0); |
| 853 | break; |
James Hogan | e93d4c1 | 2014-06-26 13:47:22 +0100 | [diff] [blame] | 854 | case KVM_REG_MIPS_CP0_ERROREPC: |
| 855 | v = (long)kvm_read_c0_guest_errorepc(cop0); |
| 856 | break; |
James Hogan | 0510870 | 2016-06-15 19:29:56 +0100 | [diff] [blame] | 857 | case KVM_REG_MIPS_CP0_KSCRATCH1 ... KVM_REG_MIPS_CP0_KSCRATCH6: |
| 858 | idx = reg->id - KVM_REG_MIPS_CP0_KSCRATCH1 + 2; |
| 859 | if (!(vcpu->arch.kscratch_enabled & BIT(idx))) |
| 860 | return -EINVAL; |
| 861 | switch (idx) { |
| 862 | case 2: |
| 863 | v = (long)kvm_read_c0_guest_kscratch1(cop0); |
| 864 | break; |
| 865 | case 3: |
| 866 | v = (long)kvm_read_c0_guest_kscratch2(cop0); |
| 867 | break; |
| 868 | case 4: |
| 869 | v = (long)kvm_read_c0_guest_kscratch3(cop0); |
| 870 | break; |
| 871 | case 5: |
| 872 | v = (long)kvm_read_c0_guest_kscratch4(cop0); |
| 873 | break; |
| 874 | case 6: |
| 875 | v = (long)kvm_read_c0_guest_kscratch5(cop0); |
| 876 | break; |
| 877 | case 7: |
| 878 | v = (long)kvm_read_c0_guest_kscratch6(cop0); |
| 879 | break; |
| 880 | } |
| 881 | break; |
James Hogan | f8be02d | 2014-05-29 10:16:29 +0100 | [diff] [blame] | 882 | /* registers to be handled specially */ |
James Hogan | cc68d22 | 2016-06-15 19:29:48 +0100 | [diff] [blame] | 883 | default: |
James Hogan | f8be02d | 2014-05-29 10:16:29 +0100 | [diff] [blame] | 884 | ret = kvm_mips_callbacks->get_one_reg(vcpu, reg, &v); |
| 885 | if (ret) |
| 886 | return ret; |
| 887 | break; |
David Daney | 4c73fb2 | 2013-05-23 09:49:09 -0700 | [diff] [blame] | 888 | } |
David Daney | 681865d | 2013-06-10 12:33:48 -0700 | [diff] [blame] | 889 | if ((reg->id & KVM_REG_SIZE_MASK) == KVM_REG_SIZE_U64) { |
| 890 | u64 __user *uaddr64 = (u64 __user *)(long)reg->addr; |
Deng-Cheng Zhu | d116e81 | 2014-06-26 12:11:34 -0700 | [diff] [blame] | 891 | |
David Daney | 681865d | 2013-06-10 12:33:48 -0700 | [diff] [blame] | 892 | return put_user(v, uaddr64); |
| 893 | } else if ((reg->id & KVM_REG_SIZE_MASK) == KVM_REG_SIZE_U32) { |
| 894 | u32 __user *uaddr32 = (u32 __user *)(long)reg->addr; |
| 895 | u32 v32 = (u32)v; |
Deng-Cheng Zhu | d116e81 | 2014-06-26 12:11:34 -0700 | [diff] [blame] | 896 | |
David Daney | 681865d | 2013-06-10 12:33:48 -0700 | [diff] [blame] | 897 | return put_user(v32, uaddr32); |
James Hogan | ab86bd6 | 2014-12-02 15:48:24 +0000 | [diff] [blame] | 898 | } else if ((reg->id & KVM_REG_SIZE_MASK) == KVM_REG_SIZE_U128) { |
| 899 | void __user *uaddr = (void __user *)(long)reg->addr; |
| 900 | |
Michael S. Tsirkin | 0178fd7 | 2016-02-28 17:35:59 +0200 | [diff] [blame] | 901 | return copy_to_user(uaddr, vs, 16) ? -EFAULT : 0; |
David Daney | 681865d | 2013-06-10 12:33:48 -0700 | [diff] [blame] | 902 | } else { |
| 903 | return -EINVAL; |
| 904 | } |
David Daney | 4c73fb2 | 2013-05-23 09:49:09 -0700 | [diff] [blame] | 905 | } |
| 906 | |
| 907 | static int kvm_mips_set_reg(struct kvm_vcpu *vcpu, |
| 908 | const struct kvm_one_reg *reg) |
| 909 | { |
David Daney | 4c73fb2 | 2013-05-23 09:49:09 -0700 | [diff] [blame] | 910 | struct mips_coproc *cop0 = vcpu->arch.cop0; |
James Hogan | 379245c | 2014-12-02 15:48:24 +0000 | [diff] [blame] | 911 | struct mips_fpu_struct *fpu = &vcpu->arch.fpu; |
| 912 | s64 v; |
James Hogan | ab86bd6 | 2014-12-02 15:48:24 +0000 | [diff] [blame] | 913 | s64 vs[2]; |
James Hogan | 379245c | 2014-12-02 15:48:24 +0000 | [diff] [blame] | 914 | unsigned int idx; |
David Daney | 4c73fb2 | 2013-05-23 09:49:09 -0700 | [diff] [blame] | 915 | |
David Daney | 681865d | 2013-06-10 12:33:48 -0700 | [diff] [blame] | 916 | if ((reg->id & KVM_REG_SIZE_MASK) == KVM_REG_SIZE_U64) { |
| 917 | u64 __user *uaddr64 = (u64 __user *)(long)reg->addr; |
| 918 | |
| 919 | if (get_user(v, uaddr64) != 0) |
| 920 | return -EFAULT; |
| 921 | } else if ((reg->id & KVM_REG_SIZE_MASK) == KVM_REG_SIZE_U32) { |
| 922 | u32 __user *uaddr32 = (u32 __user *)(long)reg->addr; |
| 923 | s32 v32; |
| 924 | |
| 925 | if (get_user(v32, uaddr32) != 0) |
| 926 | return -EFAULT; |
| 927 | v = (s64)v32; |
James Hogan | ab86bd6 | 2014-12-02 15:48:24 +0000 | [diff] [blame] | 928 | } else if ((reg->id & KVM_REG_SIZE_MASK) == KVM_REG_SIZE_U128) { |
| 929 | void __user *uaddr = (void __user *)(long)reg->addr; |
| 930 | |
Michael S. Tsirkin | 0178fd7 | 2016-02-28 17:35:59 +0200 | [diff] [blame] | 931 | return copy_from_user(vs, uaddr, 16) ? -EFAULT : 0; |
David Daney | 681865d | 2013-06-10 12:33:48 -0700 | [diff] [blame] | 932 | } else { |
| 933 | return -EINVAL; |
| 934 | } |
David Daney | 4c73fb2 | 2013-05-23 09:49:09 -0700 | [diff] [blame] | 935 | |
| 936 | switch (reg->id) { |
James Hogan | 379245c | 2014-12-02 15:48:24 +0000 | [diff] [blame] | 937 | /* General purpose registers */ |
David Daney | 4c73fb2 | 2013-05-23 09:49:09 -0700 | [diff] [blame] | 938 | case KVM_REG_MIPS_R0: |
| 939 | /* Silently ignore requests to set $0 */ |
| 940 | break; |
| 941 | case KVM_REG_MIPS_R1 ... KVM_REG_MIPS_R31: |
| 942 | vcpu->arch.gprs[reg->id - KVM_REG_MIPS_R0] = v; |
| 943 | break; |
James Hogan | 70e92c7e | 2016-07-04 19:35:11 +0100 | [diff] [blame] | 944 | #ifndef CONFIG_CPU_MIPSR6 |
David Daney | 4c73fb2 | 2013-05-23 09:49:09 -0700 | [diff] [blame] | 945 | case KVM_REG_MIPS_HI: |
| 946 | vcpu->arch.hi = v; |
| 947 | break; |
| 948 | case KVM_REG_MIPS_LO: |
| 949 | vcpu->arch.lo = v; |
| 950 | break; |
James Hogan | 70e92c7e | 2016-07-04 19:35:11 +0100 | [diff] [blame] | 951 | #endif |
David Daney | 4c73fb2 | 2013-05-23 09:49:09 -0700 | [diff] [blame] | 952 | case KVM_REG_MIPS_PC: |
| 953 | vcpu->arch.pc = v; |
| 954 | break; |
| 955 | |
James Hogan | 379245c | 2014-12-02 15:48:24 +0000 | [diff] [blame] | 956 | /* Floating point registers */ |
| 957 | case KVM_REG_MIPS_FPR_32(0) ... KVM_REG_MIPS_FPR_32(31): |
| 958 | if (!kvm_mips_guest_has_fpu(&vcpu->arch)) |
| 959 | return -EINVAL; |
| 960 | idx = reg->id - KVM_REG_MIPS_FPR_32(0); |
| 961 | /* Odd singles in top of even double when FR=0 */ |
| 962 | if (kvm_read_c0_guest_status(cop0) & ST0_FR) |
| 963 | set_fpr32(&fpu->fpr[idx], 0, v); |
| 964 | else |
| 965 | set_fpr32(&fpu->fpr[idx & ~1], idx & 1, v); |
| 966 | break; |
| 967 | case KVM_REG_MIPS_FPR_64(0) ... KVM_REG_MIPS_FPR_64(31): |
| 968 | if (!kvm_mips_guest_has_fpu(&vcpu->arch)) |
| 969 | return -EINVAL; |
| 970 | idx = reg->id - KVM_REG_MIPS_FPR_64(0); |
| 971 | /* Can't access odd doubles in FR=0 mode */ |
| 972 | if (idx & 1 && !(kvm_read_c0_guest_status(cop0) & ST0_FR)) |
| 973 | return -EINVAL; |
| 974 | set_fpr64(&fpu->fpr[idx], 0, v); |
| 975 | break; |
| 976 | case KVM_REG_MIPS_FCR_IR: |
| 977 | if (!kvm_mips_guest_has_fpu(&vcpu->arch)) |
| 978 | return -EINVAL; |
| 979 | /* Read-only */ |
| 980 | break; |
| 981 | case KVM_REG_MIPS_FCR_CSR: |
| 982 | if (!kvm_mips_guest_has_fpu(&vcpu->arch)) |
| 983 | return -EINVAL; |
| 984 | fpu->fcr31 = v; |
| 985 | break; |
| 986 | |
James Hogan | ab86bd6 | 2014-12-02 15:48:24 +0000 | [diff] [blame] | 987 | /* MIPS SIMD Architecture (MSA) registers */ |
| 988 | case KVM_REG_MIPS_VEC_128(0) ... KVM_REG_MIPS_VEC_128(31): |
| 989 | if (!kvm_mips_guest_has_msa(&vcpu->arch)) |
| 990 | return -EINVAL; |
| 991 | idx = reg->id - KVM_REG_MIPS_VEC_128(0); |
| 992 | #ifdef CONFIG_CPU_LITTLE_ENDIAN |
| 993 | /* least significant byte first */ |
| 994 | set_fpr64(&fpu->fpr[idx], 0, vs[0]); |
| 995 | set_fpr64(&fpu->fpr[idx], 1, vs[1]); |
| 996 | #else |
| 997 | /* most significant byte first */ |
| 998 | set_fpr64(&fpu->fpr[idx], 1, vs[0]); |
| 999 | set_fpr64(&fpu->fpr[idx], 0, vs[1]); |
| 1000 | #endif |
| 1001 | break; |
| 1002 | case KVM_REG_MIPS_MSA_IR: |
| 1003 | if (!kvm_mips_guest_has_msa(&vcpu->arch)) |
| 1004 | return -EINVAL; |
| 1005 | /* Read-only */ |
| 1006 | break; |
| 1007 | case KVM_REG_MIPS_MSA_CSR: |
| 1008 | if (!kvm_mips_guest_has_msa(&vcpu->arch)) |
| 1009 | return -EINVAL; |
| 1010 | fpu->msacsr = v; |
| 1011 | break; |
| 1012 | |
James Hogan | 379245c | 2014-12-02 15:48:24 +0000 | [diff] [blame] | 1013 | /* Co-processor 0 registers */ |
David Daney | 4c73fb2 | 2013-05-23 09:49:09 -0700 | [diff] [blame] | 1014 | case KVM_REG_MIPS_CP0_INDEX: |
| 1015 | kvm_write_c0_guest_index(cop0, v); |
| 1016 | break; |
| 1017 | case KVM_REG_MIPS_CP0_CONTEXT: |
| 1018 | kvm_write_c0_guest_context(cop0, v); |
| 1019 | break; |
James Hogan | 7767b7d | 2014-05-29 10:16:30 +0100 | [diff] [blame] | 1020 | case KVM_REG_MIPS_CP0_USERLOCAL: |
| 1021 | kvm_write_c0_guest_userlocal(cop0, v); |
| 1022 | break; |
David Daney | 4c73fb2 | 2013-05-23 09:49:09 -0700 | [diff] [blame] | 1023 | case KVM_REG_MIPS_CP0_PAGEMASK: |
| 1024 | kvm_write_c0_guest_pagemask(cop0, v); |
| 1025 | break; |
| 1026 | case KVM_REG_MIPS_CP0_WIRED: |
| 1027 | kvm_write_c0_guest_wired(cop0, v); |
| 1028 | break; |
James Hogan | 16fd5c1 | 2014-05-29 10:16:31 +0100 | [diff] [blame] | 1029 | case KVM_REG_MIPS_CP0_HWRENA: |
| 1030 | kvm_write_c0_guest_hwrena(cop0, v); |
| 1031 | break; |
David Daney | 4c73fb2 | 2013-05-23 09:49:09 -0700 | [diff] [blame] | 1032 | case KVM_REG_MIPS_CP0_BADVADDR: |
| 1033 | kvm_write_c0_guest_badvaddr(cop0, v); |
| 1034 | break; |
| 1035 | case KVM_REG_MIPS_CP0_ENTRYHI: |
| 1036 | kvm_write_c0_guest_entryhi(cop0, v); |
| 1037 | break; |
| 1038 | case KVM_REG_MIPS_CP0_STATUS: |
| 1039 | kvm_write_c0_guest_status(cop0, v); |
| 1040 | break; |
James Hogan | fb6df0c | 2014-05-29 10:16:27 +0100 | [diff] [blame] | 1041 | case KVM_REG_MIPS_CP0_EPC: |
| 1042 | kvm_write_c0_guest_epc(cop0, v); |
| 1043 | break; |
James Hogan | 1068eaa | 2014-06-26 13:56:52 +0100 | [diff] [blame] | 1044 | case KVM_REG_MIPS_CP0_PRID: |
| 1045 | kvm_write_c0_guest_prid(cop0, v); |
| 1046 | break; |
David Daney | 4c73fb2 | 2013-05-23 09:49:09 -0700 | [diff] [blame] | 1047 | case KVM_REG_MIPS_CP0_ERROREPC: |
| 1048 | kvm_write_c0_guest_errorepc(cop0, v); |
| 1049 | break; |
James Hogan | 0510870 | 2016-06-15 19:29:56 +0100 | [diff] [blame] | 1050 | case KVM_REG_MIPS_CP0_KSCRATCH1 ... KVM_REG_MIPS_CP0_KSCRATCH6: |
| 1051 | idx = reg->id - KVM_REG_MIPS_CP0_KSCRATCH1 + 2; |
| 1052 | if (!(vcpu->arch.kscratch_enabled & BIT(idx))) |
| 1053 | return -EINVAL; |
| 1054 | switch (idx) { |
| 1055 | case 2: |
| 1056 | kvm_write_c0_guest_kscratch1(cop0, v); |
| 1057 | break; |
| 1058 | case 3: |
| 1059 | kvm_write_c0_guest_kscratch2(cop0, v); |
| 1060 | break; |
| 1061 | case 4: |
| 1062 | kvm_write_c0_guest_kscratch3(cop0, v); |
| 1063 | break; |
| 1064 | case 5: |
| 1065 | kvm_write_c0_guest_kscratch4(cop0, v); |
| 1066 | break; |
| 1067 | case 6: |
| 1068 | kvm_write_c0_guest_kscratch5(cop0, v); |
| 1069 | break; |
| 1070 | case 7: |
| 1071 | kvm_write_c0_guest_kscratch6(cop0, v); |
| 1072 | break; |
| 1073 | } |
| 1074 | break; |
James Hogan | f8be02d | 2014-05-29 10:16:29 +0100 | [diff] [blame] | 1075 | /* registers to be handled specially */ |
David Daney | 4c73fb2 | 2013-05-23 09:49:09 -0700 | [diff] [blame] | 1076 | default: |
James Hogan | cc68d22 | 2016-06-15 19:29:48 +0100 | [diff] [blame] | 1077 | return kvm_mips_callbacks->set_one_reg(vcpu, reg, v); |
David Daney | 4c73fb2 | 2013-05-23 09:49:09 -0700 | [diff] [blame] | 1078 | } |
| 1079 | return 0; |
| 1080 | } |
| 1081 | |
James Hogan | 5fafd874 | 2014-12-08 23:07:56 +0000 | [diff] [blame] | 1082 | static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu, |
| 1083 | struct kvm_enable_cap *cap) |
| 1084 | { |
| 1085 | int r = 0; |
| 1086 | |
| 1087 | if (!kvm_vm_ioctl_check_extension(vcpu->kvm, cap->cap)) |
| 1088 | return -EINVAL; |
| 1089 | if (cap->flags) |
| 1090 | return -EINVAL; |
| 1091 | if (cap->args[0]) |
| 1092 | return -EINVAL; |
| 1093 | |
| 1094 | switch (cap->cap) { |
| 1095 | case KVM_CAP_MIPS_FPU: |
| 1096 | vcpu->arch.fpu_enabled = true; |
| 1097 | break; |
James Hogan | d952bd0 | 2014-12-08 23:07:56 +0000 | [diff] [blame] | 1098 | case KVM_CAP_MIPS_MSA: |
| 1099 | vcpu->arch.msa_enabled = true; |
| 1100 | break; |
James Hogan | 5fafd874 | 2014-12-08 23:07:56 +0000 | [diff] [blame] | 1101 | default: |
| 1102 | r = -EINVAL; |
| 1103 | break; |
| 1104 | } |
| 1105 | |
| 1106 | return r; |
| 1107 | } |
| 1108 | |
Deng-Cheng Zhu | d116e81 | 2014-06-26 12:11:34 -0700 | [diff] [blame] | 1109 | long kvm_arch_vcpu_ioctl(struct file *filp, unsigned int ioctl, |
| 1110 | unsigned long arg) |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 1111 | { |
| 1112 | struct kvm_vcpu *vcpu = filp->private_data; |
| 1113 | void __user *argp = (void __user *)arg; |
| 1114 | long r; |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 1115 | |
| 1116 | switch (ioctl) { |
David Daney | 4c73fb2 | 2013-05-23 09:49:09 -0700 | [diff] [blame] | 1117 | case KVM_SET_ONE_REG: |
| 1118 | case KVM_GET_ONE_REG: { |
| 1119 | struct kvm_one_reg reg; |
Deng-Cheng Zhu | d116e81 | 2014-06-26 12:11:34 -0700 | [diff] [blame] | 1120 | |
David Daney | 4c73fb2 | 2013-05-23 09:49:09 -0700 | [diff] [blame] | 1121 | if (copy_from_user(®, argp, sizeof(reg))) |
| 1122 | return -EFAULT; |
| 1123 | if (ioctl == KVM_SET_ONE_REG) |
| 1124 | return kvm_mips_set_reg(vcpu, ®); |
| 1125 | else |
| 1126 | return kvm_mips_get_reg(vcpu, ®); |
| 1127 | } |
| 1128 | case KVM_GET_REG_LIST: { |
| 1129 | struct kvm_reg_list __user *user_list = argp; |
David Daney | 4c73fb2 | 2013-05-23 09:49:09 -0700 | [diff] [blame] | 1130 | struct kvm_reg_list reg_list; |
| 1131 | unsigned n; |
| 1132 | |
| 1133 | if (copy_from_user(®_list, user_list, sizeof(reg_list))) |
| 1134 | return -EFAULT; |
| 1135 | n = reg_list.n; |
James Hogan | f5c43bd | 2016-06-15 19:29:49 +0100 | [diff] [blame] | 1136 | reg_list.n = kvm_mips_num_regs(vcpu); |
David Daney | 4c73fb2 | 2013-05-23 09:49:09 -0700 | [diff] [blame] | 1137 | if (copy_to_user(user_list, ®_list, sizeof(reg_list))) |
| 1138 | return -EFAULT; |
| 1139 | if (n < reg_list.n) |
| 1140 | return -E2BIG; |
James Hogan | f5c43bd | 2016-06-15 19:29:49 +0100 | [diff] [blame] | 1141 | return kvm_mips_copy_reg_indices(vcpu, user_list->reg); |
David Daney | 4c73fb2 | 2013-05-23 09:49:09 -0700 | [diff] [blame] | 1142 | } |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 1143 | case KVM_NMI: |
| 1144 | /* Treat the NMI as a CPU reset */ |
| 1145 | r = kvm_mips_reset_vcpu(vcpu); |
| 1146 | break; |
| 1147 | case KVM_INTERRUPT: |
| 1148 | { |
| 1149 | struct kvm_mips_interrupt irq; |
Deng-Cheng Zhu | d116e81 | 2014-06-26 12:11:34 -0700 | [diff] [blame] | 1150 | |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 1151 | r = -EFAULT; |
| 1152 | if (copy_from_user(&irq, argp, sizeof(irq))) |
| 1153 | goto out; |
| 1154 | |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 1155 | kvm_debug("[%d] %s: irq: %d\n", vcpu->vcpu_id, __func__, |
| 1156 | irq.irq); |
| 1157 | |
| 1158 | r = kvm_vcpu_ioctl_interrupt(vcpu, &irq); |
| 1159 | break; |
| 1160 | } |
James Hogan | 5fafd874 | 2014-12-08 23:07:56 +0000 | [diff] [blame] | 1161 | case KVM_ENABLE_CAP: { |
| 1162 | struct kvm_enable_cap cap; |
| 1163 | |
| 1164 | r = -EFAULT; |
| 1165 | if (copy_from_user(&cap, argp, sizeof(cap))) |
| 1166 | goto out; |
| 1167 | r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap); |
| 1168 | break; |
| 1169 | } |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 1170 | default: |
David Daney | 4c73fb2 | 2013-05-23 09:49:09 -0700 | [diff] [blame] | 1171 | r = -ENOIOCTLCMD; |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 1172 | } |
| 1173 | |
| 1174 | out: |
| 1175 | return r; |
| 1176 | } |
| 1177 | |
Deng-Cheng Zhu | d116e81 | 2014-06-26 12:11:34 -0700 | [diff] [blame] | 1178 | /* Get (and clear) the dirty memory log for a memory slot. */ |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 1179 | int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, struct kvm_dirty_log *log) |
| 1180 | { |
Paolo Bonzini | 9f6b802 | 2015-05-17 16:20:07 +0200 | [diff] [blame] | 1181 | struct kvm_memslots *slots; |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 1182 | struct kvm_memory_slot *memslot; |
| 1183 | unsigned long ga, ga_end; |
| 1184 | int is_dirty = 0; |
| 1185 | int r; |
| 1186 | unsigned long n; |
| 1187 | |
| 1188 | mutex_lock(&kvm->slots_lock); |
| 1189 | |
| 1190 | r = kvm_get_dirty_log(kvm, log, &is_dirty); |
| 1191 | if (r) |
| 1192 | goto out; |
| 1193 | |
| 1194 | /* If nothing is dirty, don't bother messing with page tables. */ |
| 1195 | if (is_dirty) { |
Paolo Bonzini | 9f6b802 | 2015-05-17 16:20:07 +0200 | [diff] [blame] | 1196 | slots = kvm_memslots(kvm); |
| 1197 | memslot = id_to_memslot(slots, log->slot); |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 1198 | |
| 1199 | ga = memslot->base_gfn << PAGE_SHIFT; |
| 1200 | ga_end = ga + (memslot->npages << PAGE_SHIFT); |
| 1201 | |
Deng-Cheng Zhu | 6ad78a5 | 2014-06-26 12:11:35 -0700 | [diff] [blame] | 1202 | kvm_info("%s: dirty, ga: %#lx, ga_end %#lx\n", __func__, ga, |
| 1203 | ga_end); |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 1204 | |
| 1205 | n = kvm_dirty_bitmap_bytes(memslot); |
| 1206 | memset(memslot->dirty_bitmap, 0, n); |
| 1207 | } |
| 1208 | |
| 1209 | r = 0; |
| 1210 | out: |
| 1211 | mutex_unlock(&kvm->slots_lock); |
| 1212 | return r; |
| 1213 | |
| 1214 | } |
| 1215 | |
| 1216 | long kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) |
| 1217 | { |
| 1218 | long r; |
| 1219 | |
| 1220 | switch (ioctl) { |
| 1221 | default: |
David Daney | ed82985 | 2013-05-23 09:49:10 -0700 | [diff] [blame] | 1222 | r = -ENOIOCTLCMD; |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 1223 | } |
| 1224 | |
| 1225 | return r; |
| 1226 | } |
| 1227 | |
| 1228 | int kvm_arch_init(void *opaque) |
| 1229 | { |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 1230 | if (kvm_mips_callbacks) { |
| 1231 | kvm_err("kvm: module already exists\n"); |
| 1232 | return -EEXIST; |
| 1233 | } |
| 1234 | |
Deng-Cheng Zhu | d98403a | 2014-06-26 12:11:36 -0700 | [diff] [blame] | 1235 | return kvm_mips_emulation_init(&kvm_mips_callbacks); |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 1236 | } |
| 1237 | |
| 1238 | void kvm_arch_exit(void) |
| 1239 | { |
| 1240 | kvm_mips_callbacks = NULL; |
| 1241 | } |
| 1242 | |
Deng-Cheng Zhu | d116e81 | 2014-06-26 12:11:34 -0700 | [diff] [blame] | 1243 | int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu, |
| 1244 | struct kvm_sregs *sregs) |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 1245 | { |
David Daney | ed82985 | 2013-05-23 09:49:10 -0700 | [diff] [blame] | 1246 | return -ENOIOCTLCMD; |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 1247 | } |
| 1248 | |
Deng-Cheng Zhu | d116e81 | 2014-06-26 12:11:34 -0700 | [diff] [blame] | 1249 | int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu, |
| 1250 | struct kvm_sregs *sregs) |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 1251 | { |
David Daney | ed82985 | 2013-05-23 09:49:10 -0700 | [diff] [blame] | 1252 | return -ENOIOCTLCMD; |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 1253 | } |
| 1254 | |
Dominik Dingel | 31928aa | 2014-12-04 15:47:07 +0100 | [diff] [blame] | 1255 | void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu) |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 1256 | { |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 1257 | } |
| 1258 | |
| 1259 | int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) |
| 1260 | { |
David Daney | ed82985 | 2013-05-23 09:49:10 -0700 | [diff] [blame] | 1261 | return -ENOIOCTLCMD; |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 1262 | } |
| 1263 | |
| 1264 | int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) |
| 1265 | { |
David Daney | ed82985 | 2013-05-23 09:49:10 -0700 | [diff] [blame] | 1266 | return -ENOIOCTLCMD; |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 1267 | } |
| 1268 | |
| 1269 | int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf) |
| 1270 | { |
| 1271 | return VM_FAULT_SIGBUS; |
| 1272 | } |
| 1273 | |
Alexander Graf | 784aa3d | 2014-07-14 18:27:35 +0200 | [diff] [blame] | 1274 | int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 1275 | { |
| 1276 | int r; |
| 1277 | |
| 1278 | switch (ext) { |
David Daney | 4c73fb2 | 2013-05-23 09:49:09 -0700 | [diff] [blame] | 1279 | case KVM_CAP_ONE_REG: |
James Hogan | 5fafd874 | 2014-12-08 23:07:56 +0000 | [diff] [blame] | 1280 | case KVM_CAP_ENABLE_CAP: |
David Daney | 4c73fb2 | 2013-05-23 09:49:09 -0700 | [diff] [blame] | 1281 | r = 1; |
| 1282 | break; |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 1283 | case KVM_CAP_COALESCED_MMIO: |
| 1284 | r = KVM_COALESCED_MMIO_PAGE_OFFSET; |
| 1285 | break; |
James Hogan | 5fafd874 | 2014-12-08 23:07:56 +0000 | [diff] [blame] | 1286 | case KVM_CAP_MIPS_FPU: |
James Hogan | 556f2a5 | 2016-04-22 10:38:48 +0100 | [diff] [blame] | 1287 | /* We don't handle systems with inconsistent cpu_has_fpu */ |
| 1288 | r = !!raw_cpu_has_fpu; |
James Hogan | 5fafd874 | 2014-12-08 23:07:56 +0000 | [diff] [blame] | 1289 | break; |
James Hogan | d952bd0 | 2014-12-08 23:07:56 +0000 | [diff] [blame] | 1290 | case KVM_CAP_MIPS_MSA: |
| 1291 | /* |
| 1292 | * We don't support MSA vector partitioning yet: |
| 1293 | * 1) It would require explicit support which can't be tested |
| 1294 | * yet due to lack of support in current hardware. |
| 1295 | * 2) It extends the state that would need to be saved/restored |
| 1296 | * by e.g. QEMU for migration. |
| 1297 | * |
| 1298 | * When vector partitioning hardware becomes available, support |
| 1299 | * could be added by requiring a flag when enabling |
| 1300 | * KVM_CAP_MIPS_MSA capability to indicate that userland knows |
| 1301 | * to save/restore the appropriate extra state. |
| 1302 | */ |
| 1303 | r = cpu_has_msa && !(boot_cpu_data.msa_id & MSA_IR_WRPF); |
| 1304 | break; |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 1305 | default: |
| 1306 | r = 0; |
| 1307 | break; |
| 1308 | } |
| 1309 | return r; |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 1310 | } |
| 1311 | |
| 1312 | int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu) |
| 1313 | { |
| 1314 | return kvm_mips_pending_timer(vcpu); |
| 1315 | } |
| 1316 | |
| 1317 | int kvm_arch_vcpu_dump_regs(struct kvm_vcpu *vcpu) |
| 1318 | { |
| 1319 | int i; |
| 1320 | struct mips_coproc *cop0; |
| 1321 | |
| 1322 | if (!vcpu) |
| 1323 | return -1; |
| 1324 | |
Deng-Cheng Zhu | 6ad78a5 | 2014-06-26 12:11:35 -0700 | [diff] [blame] | 1325 | kvm_debug("VCPU Register Dump:\n"); |
| 1326 | kvm_debug("\tpc = 0x%08lx\n", vcpu->arch.pc); |
| 1327 | kvm_debug("\texceptions: %08lx\n", vcpu->arch.pending_exceptions); |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 1328 | |
| 1329 | for (i = 0; i < 32; i += 4) { |
Deng-Cheng Zhu | 6ad78a5 | 2014-06-26 12:11:35 -0700 | [diff] [blame] | 1330 | kvm_debug("\tgpr%02d: %08lx %08lx %08lx %08lx\n", i, |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 1331 | vcpu->arch.gprs[i], |
| 1332 | vcpu->arch.gprs[i + 1], |
| 1333 | vcpu->arch.gprs[i + 2], vcpu->arch.gprs[i + 3]); |
| 1334 | } |
Deng-Cheng Zhu | 6ad78a5 | 2014-06-26 12:11:35 -0700 | [diff] [blame] | 1335 | kvm_debug("\thi: 0x%08lx\n", vcpu->arch.hi); |
| 1336 | kvm_debug("\tlo: 0x%08lx\n", vcpu->arch.lo); |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 1337 | |
| 1338 | cop0 = vcpu->arch.cop0; |
Deng-Cheng Zhu | 6ad78a5 | 2014-06-26 12:11:35 -0700 | [diff] [blame] | 1339 | kvm_debug("\tStatus: 0x%08lx, Cause: 0x%08lx\n", |
| 1340 | kvm_read_c0_guest_status(cop0), |
| 1341 | kvm_read_c0_guest_cause(cop0)); |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 1342 | |
Deng-Cheng Zhu | 6ad78a5 | 2014-06-26 12:11:35 -0700 | [diff] [blame] | 1343 | kvm_debug("\tEPC: 0x%08lx\n", kvm_read_c0_guest_epc(cop0)); |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 1344 | |
| 1345 | return 0; |
| 1346 | } |
| 1347 | |
| 1348 | int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) |
| 1349 | { |
| 1350 | int i; |
| 1351 | |
David Daney | 8d17dd0 | 2013-05-23 09:49:08 -0700 | [diff] [blame] | 1352 | for (i = 1; i < ARRAY_SIZE(vcpu->arch.gprs); i++) |
David Daney | bf32ebf | 2013-05-23 09:49:07 -0700 | [diff] [blame] | 1353 | vcpu->arch.gprs[i] = regs->gpr[i]; |
David Daney | 8d17dd0 | 2013-05-23 09:49:08 -0700 | [diff] [blame] | 1354 | vcpu->arch.gprs[0] = 0; /* zero is special, and cannot be set. */ |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 1355 | vcpu->arch.hi = regs->hi; |
| 1356 | vcpu->arch.lo = regs->lo; |
| 1357 | vcpu->arch.pc = regs->pc; |
| 1358 | |
David Daney | 4c73fb2 | 2013-05-23 09:49:09 -0700 | [diff] [blame] | 1359 | return 0; |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 1360 | } |
| 1361 | |
| 1362 | int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) |
| 1363 | { |
| 1364 | int i; |
| 1365 | |
David Daney | 8d17dd0 | 2013-05-23 09:49:08 -0700 | [diff] [blame] | 1366 | for (i = 0; i < ARRAY_SIZE(vcpu->arch.gprs); i++) |
David Daney | bf32ebf | 2013-05-23 09:49:07 -0700 | [diff] [blame] | 1367 | regs->gpr[i] = vcpu->arch.gprs[i]; |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 1368 | |
| 1369 | regs->hi = vcpu->arch.hi; |
| 1370 | regs->lo = vcpu->arch.lo; |
| 1371 | regs->pc = vcpu->arch.pc; |
| 1372 | |
David Daney | 4c73fb2 | 2013-05-23 09:49:09 -0700 | [diff] [blame] | 1373 | return 0; |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 1374 | } |
| 1375 | |
James Hogan | 0fae34f | 2014-05-29 10:16:39 +0100 | [diff] [blame] | 1376 | static void kvm_mips_comparecount_func(unsigned long data) |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 1377 | { |
| 1378 | struct kvm_vcpu *vcpu = (struct kvm_vcpu *)data; |
| 1379 | |
| 1380 | kvm_mips_callbacks->queue_timer_int(vcpu); |
| 1381 | |
| 1382 | vcpu->arch.wait = 0; |
Marcelo Tosatti | 8577370 | 2016-02-19 09:46:39 +0100 | [diff] [blame] | 1383 | if (swait_active(&vcpu->wq)) |
| 1384 | swake_up(&vcpu->wq); |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 1385 | } |
| 1386 | |
Deng-Cheng Zhu | d116e81 | 2014-06-26 12:11:34 -0700 | [diff] [blame] | 1387 | /* low level hrtimer wake routine */ |
James Hogan | 0fae34f | 2014-05-29 10:16:39 +0100 | [diff] [blame] | 1388 | static enum hrtimer_restart kvm_mips_comparecount_wakeup(struct hrtimer *timer) |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 1389 | { |
| 1390 | struct kvm_vcpu *vcpu; |
| 1391 | |
| 1392 | vcpu = container_of(timer, struct kvm_vcpu, arch.comparecount_timer); |
| 1393 | kvm_mips_comparecount_func((unsigned long) vcpu); |
James Hogan | e30492b | 2014-05-29 10:16:35 +0100 | [diff] [blame] | 1394 | return kvm_mips_count_timeout(vcpu); |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 1395 | } |
| 1396 | |
| 1397 | int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu) |
| 1398 | { |
| 1399 | kvm_mips_callbacks->vcpu_init(vcpu); |
| 1400 | hrtimer_init(&vcpu->arch.comparecount_timer, CLOCK_MONOTONIC, |
| 1401 | HRTIMER_MODE_REL); |
| 1402 | vcpu->arch.comparecount_timer.function = kvm_mips_comparecount_wakeup; |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 1403 | return 0; |
| 1404 | } |
| 1405 | |
Deng-Cheng Zhu | d116e81 | 2014-06-26 12:11:34 -0700 | [diff] [blame] | 1406 | int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu, |
| 1407 | struct kvm_translation *tr) |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 1408 | { |
| 1409 | return 0; |
| 1410 | } |
| 1411 | |
| 1412 | /* Initial guest state */ |
| 1413 | int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu) |
| 1414 | { |
| 1415 | return kvm_mips_callbacks->vcpu_setup(vcpu); |
| 1416 | } |
| 1417 | |
Deng-Cheng Zhu | d116e81 | 2014-06-26 12:11:34 -0700 | [diff] [blame] | 1418 | static void kvm_mips_set_c0_status(void) |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 1419 | { |
James Hogan | 8cffd19 | 2016-06-09 14:19:08 +0100 | [diff] [blame] | 1420 | u32 status = read_c0_status(); |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 1421 | |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 1422 | if (cpu_has_dsp) |
| 1423 | status |= (ST0_MX); |
| 1424 | |
| 1425 | write_c0_status(status); |
| 1426 | ehb(); |
| 1427 | } |
| 1428 | |
| 1429 | /* |
| 1430 | * Return value is in the form (errcode<<2 | RESUME_FLAG_HOST | RESUME_FLAG_NV) |
| 1431 | */ |
| 1432 | int kvm_mips_handle_exit(struct kvm_run *run, struct kvm_vcpu *vcpu) |
| 1433 | { |
James Hogan | 8cffd19 | 2016-06-09 14:19:08 +0100 | [diff] [blame] | 1434 | u32 cause = vcpu->arch.host_cp0_cause; |
| 1435 | u32 exccode = (cause >> CAUSEB_EXCCODE) & 0x1f; |
| 1436 | u32 __user *opc = (u32 __user *) vcpu->arch.pc; |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 1437 | unsigned long badvaddr = vcpu->arch.host_cp0_badvaddr; |
| 1438 | enum emulation_result er = EMULATE_DONE; |
| 1439 | int ret = RESUME_GUEST; |
| 1440 | |
James Hogan | c4c6f2c | 2015-02-04 10:52:03 +0000 | [diff] [blame] | 1441 | /* re-enable HTW before enabling interrupts */ |
| 1442 | htw_start(); |
| 1443 | |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 1444 | /* Set a default exit reason */ |
| 1445 | run->exit_reason = KVM_EXIT_UNKNOWN; |
| 1446 | run->ready_for_interrupt_injection = 1; |
| 1447 | |
Deng-Cheng Zhu | d116e81 | 2014-06-26 12:11:34 -0700 | [diff] [blame] | 1448 | /* |
| 1449 | * Set the appropriate status bits based on host CPU features, |
| 1450 | * before we hit the scheduler |
| 1451 | */ |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 1452 | kvm_mips_set_c0_status(); |
| 1453 | |
| 1454 | local_irq_enable(); |
| 1455 | |
| 1456 | kvm_debug("kvm_mips_handle_exit: cause: %#x, PC: %p, kvm_run: %p, kvm_vcpu: %p\n", |
| 1457 | cause, opc, run, vcpu); |
James Hogan | 1e09e86 | 2016-06-14 09:40:12 +0100 | [diff] [blame] | 1458 | trace_kvm_exit(vcpu, exccode); |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 1459 | |
Deng-Cheng Zhu | d116e81 | 2014-06-26 12:11:34 -0700 | [diff] [blame] | 1460 | /* |
| 1461 | * Do a privilege check, if in UM most of these exit conditions end up |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 1462 | * causing an exception to be delivered to the Guest Kernel |
| 1463 | */ |
| 1464 | er = kvm_mips_check_privilege(cause, opc, run, vcpu); |
| 1465 | if (er == EMULATE_PRIV_FAIL) { |
| 1466 | goto skip_emul; |
| 1467 | } else if (er == EMULATE_FAIL) { |
| 1468 | run->exit_reason = KVM_EXIT_INTERNAL_ERROR; |
| 1469 | ret = RESUME_HOST; |
| 1470 | goto skip_emul; |
| 1471 | } |
| 1472 | |
| 1473 | switch (exccode) { |
James Hogan | 16d100db | 2015-12-16 23:49:33 +0000 | [diff] [blame] | 1474 | case EXCCODE_INT: |
| 1475 | kvm_debug("[%d]EXCCODE_INT @ %p\n", vcpu->vcpu_id, opc); |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 1476 | |
| 1477 | ++vcpu->stat.int_exits; |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 1478 | |
Deng-Cheng Zhu | d116e81 | 2014-06-26 12:11:34 -0700 | [diff] [blame] | 1479 | if (need_resched()) |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 1480 | cond_resched(); |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 1481 | |
| 1482 | ret = RESUME_GUEST; |
| 1483 | break; |
| 1484 | |
James Hogan | 16d100db | 2015-12-16 23:49:33 +0000 | [diff] [blame] | 1485 | case EXCCODE_CPU: |
| 1486 | kvm_debug("EXCCODE_CPU: @ PC: %p\n", opc); |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 1487 | |
| 1488 | ++vcpu->stat.cop_unusable_exits; |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 1489 | ret = kvm_mips_callbacks->handle_cop_unusable(vcpu); |
| 1490 | /* XXXKYMA: Might need to return to user space */ |
Deng-Cheng Zhu | d116e81 | 2014-06-26 12:11:34 -0700 | [diff] [blame] | 1491 | if (run->exit_reason == KVM_EXIT_IRQ_WINDOW_OPEN) |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 1492 | ret = RESUME_HOST; |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 1493 | break; |
| 1494 | |
James Hogan | 16d100db | 2015-12-16 23:49:33 +0000 | [diff] [blame] | 1495 | case EXCCODE_MOD: |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 1496 | ++vcpu->stat.tlbmod_exits; |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 1497 | ret = kvm_mips_callbacks->handle_tlb_mod(vcpu); |
| 1498 | break; |
| 1499 | |
James Hogan | 16d100db | 2015-12-16 23:49:33 +0000 | [diff] [blame] | 1500 | case EXCCODE_TLBS: |
Deng-Cheng Zhu | d116e81 | 2014-06-26 12:11:34 -0700 | [diff] [blame] | 1501 | kvm_debug("TLB ST fault: cause %#x, status %#lx, PC: %p, BadVaddr: %#lx\n", |
| 1502 | cause, kvm_read_c0_guest_status(vcpu->arch.cop0), opc, |
| 1503 | badvaddr); |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 1504 | |
| 1505 | ++vcpu->stat.tlbmiss_st_exits; |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 1506 | ret = kvm_mips_callbacks->handle_tlb_st_miss(vcpu); |
| 1507 | break; |
| 1508 | |
James Hogan | 16d100db | 2015-12-16 23:49:33 +0000 | [diff] [blame] | 1509 | case EXCCODE_TLBL: |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 1510 | kvm_debug("TLB LD fault: cause %#x, PC: %p, BadVaddr: %#lx\n", |
| 1511 | cause, opc, badvaddr); |
| 1512 | |
| 1513 | ++vcpu->stat.tlbmiss_ld_exits; |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 1514 | ret = kvm_mips_callbacks->handle_tlb_ld_miss(vcpu); |
| 1515 | break; |
| 1516 | |
James Hogan | 16d100db | 2015-12-16 23:49:33 +0000 | [diff] [blame] | 1517 | case EXCCODE_ADES: |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 1518 | ++vcpu->stat.addrerr_st_exits; |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 1519 | ret = kvm_mips_callbacks->handle_addr_err_st(vcpu); |
| 1520 | break; |
| 1521 | |
James Hogan | 16d100db | 2015-12-16 23:49:33 +0000 | [diff] [blame] | 1522 | case EXCCODE_ADEL: |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 1523 | ++vcpu->stat.addrerr_ld_exits; |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 1524 | ret = kvm_mips_callbacks->handle_addr_err_ld(vcpu); |
| 1525 | break; |
| 1526 | |
James Hogan | 16d100db | 2015-12-16 23:49:33 +0000 | [diff] [blame] | 1527 | case EXCCODE_SYS: |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 1528 | ++vcpu->stat.syscall_exits; |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 1529 | ret = kvm_mips_callbacks->handle_syscall(vcpu); |
| 1530 | break; |
| 1531 | |
James Hogan | 16d100db | 2015-12-16 23:49:33 +0000 | [diff] [blame] | 1532 | case EXCCODE_RI: |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 1533 | ++vcpu->stat.resvd_inst_exits; |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 1534 | ret = kvm_mips_callbacks->handle_res_inst(vcpu); |
| 1535 | break; |
| 1536 | |
James Hogan | 16d100db | 2015-12-16 23:49:33 +0000 | [diff] [blame] | 1537 | case EXCCODE_BP: |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 1538 | ++vcpu->stat.break_inst_exits; |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 1539 | ret = kvm_mips_callbacks->handle_break(vcpu); |
| 1540 | break; |
| 1541 | |
James Hogan | 16d100db | 2015-12-16 23:49:33 +0000 | [diff] [blame] | 1542 | case EXCCODE_TR: |
James Hogan | 0a56042 | 2015-02-06 16:03:57 +0000 | [diff] [blame] | 1543 | ++vcpu->stat.trap_inst_exits; |
James Hogan | 0a56042 | 2015-02-06 16:03:57 +0000 | [diff] [blame] | 1544 | ret = kvm_mips_callbacks->handle_trap(vcpu); |
| 1545 | break; |
| 1546 | |
James Hogan | 16d100db | 2015-12-16 23:49:33 +0000 | [diff] [blame] | 1547 | case EXCCODE_MSAFPE: |
James Hogan | c2537ed | 2015-02-06 10:56:27 +0000 | [diff] [blame] | 1548 | ++vcpu->stat.msa_fpe_exits; |
James Hogan | c2537ed | 2015-02-06 10:56:27 +0000 | [diff] [blame] | 1549 | ret = kvm_mips_callbacks->handle_msa_fpe(vcpu); |
| 1550 | break; |
| 1551 | |
James Hogan | 16d100db | 2015-12-16 23:49:33 +0000 | [diff] [blame] | 1552 | case EXCCODE_FPE: |
James Hogan | 1c0cd66 | 2015-02-06 10:56:27 +0000 | [diff] [blame] | 1553 | ++vcpu->stat.fpe_exits; |
James Hogan | 1c0cd66 | 2015-02-06 10:56:27 +0000 | [diff] [blame] | 1554 | ret = kvm_mips_callbacks->handle_fpe(vcpu); |
| 1555 | break; |
| 1556 | |
James Hogan | 16d100db | 2015-12-16 23:49:33 +0000 | [diff] [blame] | 1557 | case EXCCODE_MSADIS: |
James Hogan | c2537ed | 2015-02-06 10:56:27 +0000 | [diff] [blame] | 1558 | ++vcpu->stat.msa_disabled_exits; |
James Hogan | 98119ad | 2015-02-06 11:11:56 +0000 | [diff] [blame] | 1559 | ret = kvm_mips_callbacks->handle_msa_disabled(vcpu); |
| 1560 | break; |
| 1561 | |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 1562 | default: |
Deng-Cheng Zhu | d116e81 | 2014-06-26 12:11:34 -0700 | [diff] [blame] | 1563 | kvm_err("Exception Code: %d, not yet handled, @ PC: %p, inst: 0x%08x BadVaddr: %#lx Status: %#lx\n", |
| 1564 | exccode, opc, kvm_get_inst(opc, vcpu), badvaddr, |
| 1565 | kvm_read_c0_guest_status(vcpu->arch.cop0)); |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 1566 | kvm_arch_vcpu_dump_regs(vcpu); |
| 1567 | run->exit_reason = KVM_EXIT_INTERNAL_ERROR; |
| 1568 | ret = RESUME_HOST; |
| 1569 | break; |
| 1570 | |
| 1571 | } |
| 1572 | |
| 1573 | skip_emul: |
| 1574 | local_irq_disable(); |
| 1575 | |
| 1576 | if (er == EMULATE_DONE && !(ret & RESUME_HOST)) |
| 1577 | kvm_mips_deliver_interrupts(vcpu, cause); |
| 1578 | |
| 1579 | if (!(ret & RESUME_HOST)) { |
Deng-Cheng Zhu | d116e81 | 2014-06-26 12:11:34 -0700 | [diff] [blame] | 1580 | /* Only check for signals if not already exiting to userspace */ |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 1581 | if (signal_pending(current)) { |
| 1582 | run->exit_reason = KVM_EXIT_INTR; |
| 1583 | ret = (-EINTR << 2) | RESUME_HOST; |
| 1584 | ++vcpu->stat.signal_exits; |
James Hogan | 1e09e86 | 2016-06-14 09:40:12 +0100 | [diff] [blame] | 1585 | trace_kvm_exit(vcpu, KVM_TRACE_EXIT_SIGNAL); |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 1586 | } |
| 1587 | } |
| 1588 | |
James Hogan | 98e91b8 | 2014-11-18 14:09:12 +0000 | [diff] [blame] | 1589 | if (ret == RESUME_GUEST) { |
James Hogan | 9325860 | 2016-06-14 09:40:14 +0100 | [diff] [blame] | 1590 | trace_kvm_reenter(vcpu); |
| 1591 | |
James Hogan | 25b08c7 | 2016-09-16 00:06:43 +0100 | [diff] [blame] | 1592 | kvm_mips_check_asids(vcpu); |
| 1593 | |
James Hogan | 98e91b8 | 2014-11-18 14:09:12 +0000 | [diff] [blame] | 1594 | /* |
James Hogan | 539cb89fb | 2015-03-05 11:43:36 +0000 | [diff] [blame] | 1595 | * If FPU / MSA are enabled (i.e. the guest's FPU / MSA context |
| 1596 | * is live), restore FCR31 / MSACSR. |
James Hogan | 98e91b8 | 2014-11-18 14:09:12 +0000 | [diff] [blame] | 1597 | * |
| 1598 | * This should be before returning to the guest exception |
James Hogan | 539cb89fb | 2015-03-05 11:43:36 +0000 | [diff] [blame] | 1599 | * vector, as it may well cause an [MSA] FP exception if there |
| 1600 | * are pending exception bits unmasked. (see |
James Hogan | 98e91b8 | 2014-11-18 14:09:12 +0000 | [diff] [blame] | 1601 | * kvm_mips_csr_die_notifier() for how that is handled). |
| 1602 | */ |
| 1603 | if (kvm_mips_guest_has_fpu(&vcpu->arch) && |
| 1604 | read_c0_status() & ST0_CU1) |
| 1605 | __kvm_restore_fcsr(&vcpu->arch); |
James Hogan | 539cb89fb | 2015-03-05 11:43:36 +0000 | [diff] [blame] | 1606 | |
| 1607 | if (kvm_mips_guest_has_msa(&vcpu->arch) && |
| 1608 | read_c0_config5() & MIPS_CONF5_MSAEN) |
| 1609 | __kvm_restore_msacsr(&vcpu->arch); |
James Hogan | 98e91b8 | 2014-11-18 14:09:12 +0000 | [diff] [blame] | 1610 | } |
| 1611 | |
James Hogan | c4c6f2c | 2015-02-04 10:52:03 +0000 | [diff] [blame] | 1612 | /* Disable HTW before returning to guest or host */ |
| 1613 | htw_stop(); |
| 1614 | |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 1615 | return ret; |
| 1616 | } |
| 1617 | |
James Hogan | 98e91b8 | 2014-11-18 14:09:12 +0000 | [diff] [blame] | 1618 | /* Enable FPU for guest and restore context */ |
| 1619 | void kvm_own_fpu(struct kvm_vcpu *vcpu) |
| 1620 | { |
| 1621 | struct mips_coproc *cop0 = vcpu->arch.cop0; |
| 1622 | unsigned int sr, cfg5; |
| 1623 | |
| 1624 | preempt_disable(); |
| 1625 | |
James Hogan | 539cb89fb | 2015-03-05 11:43:36 +0000 | [diff] [blame] | 1626 | sr = kvm_read_c0_guest_status(cop0); |
| 1627 | |
| 1628 | /* |
| 1629 | * If MSA state is already live, it is undefined how it interacts with |
| 1630 | * FR=0 FPU state, and we don't want to hit reserved instruction |
| 1631 | * exceptions trying to save the MSA state later when CU=1 && FR=1, so |
| 1632 | * play it safe and save it first. |
| 1633 | * |
| 1634 | * In theory we shouldn't ever hit this case since kvm_lose_fpu() should |
| 1635 | * get called when guest CU1 is set, however we can't trust the guest |
| 1636 | * not to clobber the status register directly via the commpage. |
| 1637 | */ |
| 1638 | if (cpu_has_msa && sr & ST0_CU1 && !(sr & ST0_FR) && |
James Hogan | f943176 | 2016-06-14 09:40:10 +0100 | [diff] [blame] | 1639 | vcpu->arch.aux_inuse & KVM_MIPS_AUX_MSA) |
James Hogan | 539cb89fb | 2015-03-05 11:43:36 +0000 | [diff] [blame] | 1640 | kvm_lose_fpu(vcpu); |
| 1641 | |
James Hogan | 98e91b8 | 2014-11-18 14:09:12 +0000 | [diff] [blame] | 1642 | /* |
| 1643 | * Enable FPU for guest |
| 1644 | * We set FR and FRE according to guest context |
| 1645 | */ |
James Hogan | 98e91b8 | 2014-11-18 14:09:12 +0000 | [diff] [blame] | 1646 | change_c0_status(ST0_CU1 | ST0_FR, sr); |
| 1647 | if (cpu_has_fre) { |
| 1648 | cfg5 = kvm_read_c0_guest_config5(cop0); |
| 1649 | change_c0_config5(MIPS_CONF5_FRE, cfg5); |
| 1650 | } |
| 1651 | enable_fpu_hazard(); |
| 1652 | |
| 1653 | /* If guest FPU state not active, restore it now */ |
James Hogan | f943176 | 2016-06-14 09:40:10 +0100 | [diff] [blame] | 1654 | if (!(vcpu->arch.aux_inuse & KVM_MIPS_AUX_FPU)) { |
James Hogan | 98e91b8 | 2014-11-18 14:09:12 +0000 | [diff] [blame] | 1655 | __kvm_restore_fpu(&vcpu->arch); |
James Hogan | f943176 | 2016-06-14 09:40:10 +0100 | [diff] [blame] | 1656 | vcpu->arch.aux_inuse |= KVM_MIPS_AUX_FPU; |
James Hogan | 04ebebf | 2016-06-14 09:40:11 +0100 | [diff] [blame] | 1657 | trace_kvm_aux(vcpu, KVM_TRACE_AUX_RESTORE, KVM_TRACE_AUX_FPU); |
| 1658 | } else { |
| 1659 | trace_kvm_aux(vcpu, KVM_TRACE_AUX_ENABLE, KVM_TRACE_AUX_FPU); |
James Hogan | 98e91b8 | 2014-11-18 14:09:12 +0000 | [diff] [blame] | 1660 | } |
| 1661 | |
| 1662 | preempt_enable(); |
| 1663 | } |
| 1664 | |
James Hogan | 539cb89fb | 2015-03-05 11:43:36 +0000 | [diff] [blame] | 1665 | #ifdef CONFIG_CPU_HAS_MSA |
| 1666 | /* Enable MSA for guest and restore context */ |
| 1667 | void kvm_own_msa(struct kvm_vcpu *vcpu) |
| 1668 | { |
| 1669 | struct mips_coproc *cop0 = vcpu->arch.cop0; |
| 1670 | unsigned int sr, cfg5; |
| 1671 | |
| 1672 | preempt_disable(); |
| 1673 | |
| 1674 | /* |
| 1675 | * Enable FPU if enabled in guest, since we're restoring FPU context |
| 1676 | * anyway. We set FR and FRE according to guest context. |
| 1677 | */ |
| 1678 | if (kvm_mips_guest_has_fpu(&vcpu->arch)) { |
| 1679 | sr = kvm_read_c0_guest_status(cop0); |
| 1680 | |
| 1681 | /* |
| 1682 | * If FR=0 FPU state is already live, it is undefined how it |
| 1683 | * interacts with MSA state, so play it safe and save it first. |
| 1684 | */ |
| 1685 | if (!(sr & ST0_FR) && |
James Hogan | f943176 | 2016-06-14 09:40:10 +0100 | [diff] [blame] | 1686 | (vcpu->arch.aux_inuse & (KVM_MIPS_AUX_FPU | |
| 1687 | KVM_MIPS_AUX_MSA)) == KVM_MIPS_AUX_FPU) |
James Hogan | 539cb89fb | 2015-03-05 11:43:36 +0000 | [diff] [blame] | 1688 | kvm_lose_fpu(vcpu); |
| 1689 | |
| 1690 | change_c0_status(ST0_CU1 | ST0_FR, sr); |
| 1691 | if (sr & ST0_CU1 && cpu_has_fre) { |
| 1692 | cfg5 = kvm_read_c0_guest_config5(cop0); |
| 1693 | change_c0_config5(MIPS_CONF5_FRE, cfg5); |
| 1694 | } |
| 1695 | } |
| 1696 | |
| 1697 | /* Enable MSA for guest */ |
| 1698 | set_c0_config5(MIPS_CONF5_MSAEN); |
| 1699 | enable_fpu_hazard(); |
| 1700 | |
James Hogan | f943176 | 2016-06-14 09:40:10 +0100 | [diff] [blame] | 1701 | switch (vcpu->arch.aux_inuse & (KVM_MIPS_AUX_FPU | KVM_MIPS_AUX_MSA)) { |
| 1702 | case KVM_MIPS_AUX_FPU: |
James Hogan | 539cb89fb | 2015-03-05 11:43:36 +0000 | [diff] [blame] | 1703 | /* |
| 1704 | * Guest FPU state already loaded, only restore upper MSA state |
| 1705 | */ |
| 1706 | __kvm_restore_msa_upper(&vcpu->arch); |
James Hogan | f943176 | 2016-06-14 09:40:10 +0100 | [diff] [blame] | 1707 | vcpu->arch.aux_inuse |= KVM_MIPS_AUX_MSA; |
James Hogan | 04ebebf | 2016-06-14 09:40:11 +0100 | [diff] [blame] | 1708 | trace_kvm_aux(vcpu, KVM_TRACE_AUX_RESTORE, KVM_TRACE_AUX_MSA); |
James Hogan | 539cb89fb | 2015-03-05 11:43:36 +0000 | [diff] [blame] | 1709 | break; |
| 1710 | case 0: |
| 1711 | /* Neither FPU or MSA already active, restore full MSA state */ |
| 1712 | __kvm_restore_msa(&vcpu->arch); |
James Hogan | f943176 | 2016-06-14 09:40:10 +0100 | [diff] [blame] | 1713 | vcpu->arch.aux_inuse |= KVM_MIPS_AUX_MSA; |
James Hogan | 539cb89fb | 2015-03-05 11:43:36 +0000 | [diff] [blame] | 1714 | if (kvm_mips_guest_has_fpu(&vcpu->arch)) |
James Hogan | f943176 | 2016-06-14 09:40:10 +0100 | [diff] [blame] | 1715 | vcpu->arch.aux_inuse |= KVM_MIPS_AUX_FPU; |
James Hogan | 04ebebf | 2016-06-14 09:40:11 +0100 | [diff] [blame] | 1716 | trace_kvm_aux(vcpu, KVM_TRACE_AUX_RESTORE, |
| 1717 | KVM_TRACE_AUX_FPU_MSA); |
James Hogan | 539cb89fb | 2015-03-05 11:43:36 +0000 | [diff] [blame] | 1718 | break; |
| 1719 | default: |
James Hogan | 04ebebf | 2016-06-14 09:40:11 +0100 | [diff] [blame] | 1720 | trace_kvm_aux(vcpu, KVM_TRACE_AUX_ENABLE, KVM_TRACE_AUX_MSA); |
James Hogan | 539cb89fb | 2015-03-05 11:43:36 +0000 | [diff] [blame] | 1721 | break; |
| 1722 | } |
| 1723 | |
| 1724 | preempt_enable(); |
| 1725 | } |
| 1726 | #endif |
| 1727 | |
| 1728 | /* Drop FPU & MSA without saving it */ |
James Hogan | 98e91b8 | 2014-11-18 14:09:12 +0000 | [diff] [blame] | 1729 | void kvm_drop_fpu(struct kvm_vcpu *vcpu) |
| 1730 | { |
| 1731 | preempt_disable(); |
James Hogan | f943176 | 2016-06-14 09:40:10 +0100 | [diff] [blame] | 1732 | if (cpu_has_msa && vcpu->arch.aux_inuse & KVM_MIPS_AUX_MSA) { |
James Hogan | 539cb89fb | 2015-03-05 11:43:36 +0000 | [diff] [blame] | 1733 | disable_msa(); |
James Hogan | 04ebebf | 2016-06-14 09:40:11 +0100 | [diff] [blame] | 1734 | trace_kvm_aux(vcpu, KVM_TRACE_AUX_DISCARD, KVM_TRACE_AUX_MSA); |
James Hogan | f943176 | 2016-06-14 09:40:10 +0100 | [diff] [blame] | 1735 | vcpu->arch.aux_inuse &= ~KVM_MIPS_AUX_MSA; |
James Hogan | 539cb89fb | 2015-03-05 11:43:36 +0000 | [diff] [blame] | 1736 | } |
James Hogan | f943176 | 2016-06-14 09:40:10 +0100 | [diff] [blame] | 1737 | if (vcpu->arch.aux_inuse & KVM_MIPS_AUX_FPU) { |
James Hogan | 98e91b8 | 2014-11-18 14:09:12 +0000 | [diff] [blame] | 1738 | clear_c0_status(ST0_CU1 | ST0_FR); |
James Hogan | 04ebebf | 2016-06-14 09:40:11 +0100 | [diff] [blame] | 1739 | trace_kvm_aux(vcpu, KVM_TRACE_AUX_DISCARD, KVM_TRACE_AUX_FPU); |
James Hogan | f943176 | 2016-06-14 09:40:10 +0100 | [diff] [blame] | 1740 | vcpu->arch.aux_inuse &= ~KVM_MIPS_AUX_FPU; |
James Hogan | 98e91b8 | 2014-11-18 14:09:12 +0000 | [diff] [blame] | 1741 | } |
| 1742 | preempt_enable(); |
| 1743 | } |
| 1744 | |
James Hogan | 539cb89fb | 2015-03-05 11:43:36 +0000 | [diff] [blame] | 1745 | /* Save and disable FPU & MSA */ |
James Hogan | 98e91b8 | 2014-11-18 14:09:12 +0000 | [diff] [blame] | 1746 | void kvm_lose_fpu(struct kvm_vcpu *vcpu) |
| 1747 | { |
| 1748 | /* |
James Hogan | 539cb89fb | 2015-03-05 11:43:36 +0000 | [diff] [blame] | 1749 | * FPU & MSA get disabled in root context (hardware) when it is disabled |
| 1750 | * in guest context (software), but the register state in the hardware |
| 1751 | * may still be in use. This is why we explicitly re-enable the hardware |
James Hogan | 98e91b8 | 2014-11-18 14:09:12 +0000 | [diff] [blame] | 1752 | * before saving. |
| 1753 | */ |
| 1754 | |
| 1755 | preempt_disable(); |
James Hogan | f943176 | 2016-06-14 09:40:10 +0100 | [diff] [blame] | 1756 | if (cpu_has_msa && vcpu->arch.aux_inuse & KVM_MIPS_AUX_MSA) { |
James Hogan | 539cb89fb | 2015-03-05 11:43:36 +0000 | [diff] [blame] | 1757 | set_c0_config5(MIPS_CONF5_MSAEN); |
| 1758 | enable_fpu_hazard(); |
| 1759 | |
| 1760 | __kvm_save_msa(&vcpu->arch); |
James Hogan | 04ebebf | 2016-06-14 09:40:11 +0100 | [diff] [blame] | 1761 | trace_kvm_aux(vcpu, KVM_TRACE_AUX_SAVE, KVM_TRACE_AUX_FPU_MSA); |
James Hogan | 539cb89fb | 2015-03-05 11:43:36 +0000 | [diff] [blame] | 1762 | |
| 1763 | /* Disable MSA & FPU */ |
| 1764 | disable_msa(); |
James Hogan | f943176 | 2016-06-14 09:40:10 +0100 | [diff] [blame] | 1765 | if (vcpu->arch.aux_inuse & KVM_MIPS_AUX_FPU) { |
James Hogan | 539cb89fb | 2015-03-05 11:43:36 +0000 | [diff] [blame] | 1766 | clear_c0_status(ST0_CU1 | ST0_FR); |
James Hogan | 4ac3342 | 2016-04-22 10:38:49 +0100 | [diff] [blame] | 1767 | disable_fpu_hazard(); |
| 1768 | } |
James Hogan | f943176 | 2016-06-14 09:40:10 +0100 | [diff] [blame] | 1769 | vcpu->arch.aux_inuse &= ~(KVM_MIPS_AUX_FPU | KVM_MIPS_AUX_MSA); |
| 1770 | } else if (vcpu->arch.aux_inuse & KVM_MIPS_AUX_FPU) { |
James Hogan | 98e91b8 | 2014-11-18 14:09:12 +0000 | [diff] [blame] | 1771 | set_c0_status(ST0_CU1); |
| 1772 | enable_fpu_hazard(); |
| 1773 | |
| 1774 | __kvm_save_fpu(&vcpu->arch); |
James Hogan | f943176 | 2016-06-14 09:40:10 +0100 | [diff] [blame] | 1775 | vcpu->arch.aux_inuse &= ~KVM_MIPS_AUX_FPU; |
James Hogan | 04ebebf | 2016-06-14 09:40:11 +0100 | [diff] [blame] | 1776 | trace_kvm_aux(vcpu, KVM_TRACE_AUX_SAVE, KVM_TRACE_AUX_FPU); |
James Hogan | 98e91b8 | 2014-11-18 14:09:12 +0000 | [diff] [blame] | 1777 | |
| 1778 | /* Disable FPU */ |
| 1779 | clear_c0_status(ST0_CU1 | ST0_FR); |
James Hogan | 4ac3342 | 2016-04-22 10:38:49 +0100 | [diff] [blame] | 1780 | disable_fpu_hazard(); |
James Hogan | 98e91b8 | 2014-11-18 14:09:12 +0000 | [diff] [blame] | 1781 | } |
| 1782 | preempt_enable(); |
| 1783 | } |
| 1784 | |
| 1785 | /* |
James Hogan | 539cb89fb | 2015-03-05 11:43:36 +0000 | [diff] [blame] | 1786 | * Step over a specific ctc1 to FCSR and a specific ctcmsa to MSACSR which are |
| 1787 | * used to restore guest FCSR/MSACSR state and may trigger a "harmless" FP/MSAFP |
| 1788 | * exception if cause bits are set in the value being written. |
James Hogan | 98e91b8 | 2014-11-18 14:09:12 +0000 | [diff] [blame] | 1789 | */ |
| 1790 | static int kvm_mips_csr_die_notify(struct notifier_block *self, |
| 1791 | unsigned long cmd, void *ptr) |
| 1792 | { |
| 1793 | struct die_args *args = (struct die_args *)ptr; |
| 1794 | struct pt_regs *regs = args->regs; |
| 1795 | unsigned long pc; |
| 1796 | |
James Hogan | 539cb89fb | 2015-03-05 11:43:36 +0000 | [diff] [blame] | 1797 | /* Only interested in FPE and MSAFPE */ |
| 1798 | if (cmd != DIE_FP && cmd != DIE_MSAFP) |
James Hogan | 98e91b8 | 2014-11-18 14:09:12 +0000 | [diff] [blame] | 1799 | return NOTIFY_DONE; |
| 1800 | |
| 1801 | /* Return immediately if guest context isn't active */ |
| 1802 | if (!(current->flags & PF_VCPU)) |
| 1803 | return NOTIFY_DONE; |
| 1804 | |
| 1805 | /* Should never get here from user mode */ |
| 1806 | BUG_ON(user_mode(regs)); |
| 1807 | |
| 1808 | pc = instruction_pointer(regs); |
| 1809 | switch (cmd) { |
| 1810 | case DIE_FP: |
| 1811 | /* match 2nd instruction in __kvm_restore_fcsr */ |
| 1812 | if (pc != (unsigned long)&__kvm_restore_fcsr + 4) |
| 1813 | return NOTIFY_DONE; |
| 1814 | break; |
James Hogan | 539cb89fb | 2015-03-05 11:43:36 +0000 | [diff] [blame] | 1815 | case DIE_MSAFP: |
| 1816 | /* match 2nd/3rd instruction in __kvm_restore_msacsr */ |
| 1817 | if (!cpu_has_msa || |
| 1818 | pc < (unsigned long)&__kvm_restore_msacsr + 4 || |
| 1819 | pc > (unsigned long)&__kvm_restore_msacsr + 8) |
| 1820 | return NOTIFY_DONE; |
| 1821 | break; |
James Hogan | 98e91b8 | 2014-11-18 14:09:12 +0000 | [diff] [blame] | 1822 | } |
| 1823 | |
| 1824 | /* Move PC forward a little and continue executing */ |
| 1825 | instruction_pointer(regs) += 4; |
| 1826 | |
| 1827 | return NOTIFY_STOP; |
| 1828 | } |
| 1829 | |
| 1830 | static struct notifier_block kvm_mips_csr_die_notifier = { |
| 1831 | .notifier_call = kvm_mips_csr_die_notify, |
| 1832 | }; |
| 1833 | |
James Hogan | 2db9d23 | 2015-12-16 23:49:32 +0000 | [diff] [blame] | 1834 | static int __init kvm_mips_init(void) |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 1835 | { |
| 1836 | int ret; |
| 1837 | |
James Hogan | 1e5217f5 | 2016-06-23 17:34:45 +0100 | [diff] [blame] | 1838 | ret = kvm_mips_entry_setup(); |
| 1839 | if (ret) |
| 1840 | return ret; |
| 1841 | |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 1842 | ret = kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE); |
| 1843 | |
| 1844 | if (ret) |
| 1845 | return ret; |
| 1846 | |
James Hogan | 98e91b8 | 2014-11-18 14:09:12 +0000 | [diff] [blame] | 1847 | register_die_notifier(&kvm_mips_csr_die_notifier); |
| 1848 | |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 1849 | return 0; |
| 1850 | } |
| 1851 | |
James Hogan | 2db9d23 | 2015-12-16 23:49:32 +0000 | [diff] [blame] | 1852 | static void __exit kvm_mips_exit(void) |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 1853 | { |
| 1854 | kvm_exit(); |
| 1855 | |
James Hogan | 98e91b8 | 2014-11-18 14:09:12 +0000 | [diff] [blame] | 1856 | unregister_die_notifier(&kvm_mips_csr_die_notifier); |
Sanjay Lal | 669e846 | 2012-11-21 18:34:02 -0800 | [diff] [blame] | 1857 | } |
| 1858 | |
| 1859 | module_init(kvm_mips_init); |
| 1860 | module_exit(kvm_mips_exit); |
| 1861 | |
| 1862 | EXPORT_TRACEPOINT_SYMBOL(kvm_exit); |