Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 1 | /* |
| 2 | * Copyright 2011 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com> |
| 3 | * Copyright (C) 2009. SUSE Linux Products GmbH. All rights reserved. |
| 4 | * |
| 5 | * Authors: |
| 6 | * Paul Mackerras <paulus@au1.ibm.com> |
| 7 | * Alexander Graf <agraf@suse.de> |
| 8 | * Kevin Wolf <mail@kevin-wolf.de> |
| 9 | * |
| 10 | * Description: KVM functions specific to running on Book 3S |
| 11 | * processors in hypervisor mode (specifically POWER7 and later). |
| 12 | * |
| 13 | * This file is derived from arch/powerpc/kvm/book3s.c, |
| 14 | * by Alexander Graf <agraf@suse.de>. |
| 15 | * |
| 16 | * This program is free software; you can redistribute it and/or modify |
| 17 | * it under the terms of the GNU General Public License, version 2, as |
| 18 | * published by the Free Software Foundation. |
| 19 | */ |
| 20 | |
| 21 | #include <linux/kvm_host.h> |
| 22 | #include <linux/err.h> |
| 23 | #include <linux/slab.h> |
| 24 | #include <linux/preempt.h> |
| 25 | #include <linux/sched.h> |
| 26 | #include <linux/delay.h> |
Paul Gortmaker | 66b15db | 2011-05-27 10:46:24 -0400 | [diff] [blame] | 27 | #include <linux/export.h> |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 28 | #include <linux/fs.h> |
| 29 | #include <linux/anon_inodes.h> |
Gavin Shan | 07f8ab25 | 2016-05-11 11:15:55 +1000 | [diff] [blame] | 30 | #include <linux/cpu.h> |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 31 | #include <linux/cpumask.h> |
Paul Mackerras | aa04b4c | 2011-06-29 00:25:44 +0000 | [diff] [blame] | 32 | #include <linux/spinlock.h> |
| 33 | #include <linux/page-flags.h> |
Paul Mackerras | 2c9097e | 2012-09-11 13:27:01 +0000 | [diff] [blame] | 34 | #include <linux/srcu.h> |
Alexander Graf | 398a76c | 2013-12-09 13:53:42 +0100 | [diff] [blame] | 35 | #include <linux/miscdevice.h> |
Paul Mackerras | e23a808 | 2015-03-28 14:21:01 +1100 | [diff] [blame] | 36 | #include <linux/debugfs.h> |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 37 | |
| 38 | #include <asm/reg.h> |
| 39 | #include <asm/cputable.h> |
| 40 | #include <asm/cacheflush.h> |
| 41 | #include <asm/tlbflush.h> |
| 42 | #include <asm/uaccess.h> |
| 43 | #include <asm/io.h> |
| 44 | #include <asm/kvm_ppc.h> |
| 45 | #include <asm/kvm_book3s.h> |
| 46 | #include <asm/mmu_context.h> |
| 47 | #include <asm/lppaca.h> |
| 48 | #include <asm/processor.h> |
Paul Mackerras | 371fefd | 2011-06-29 00:23:08 +0000 | [diff] [blame] | 49 | #include <asm/cputhreads.h> |
Paul Mackerras | aa04b4c | 2011-06-29 00:25:44 +0000 | [diff] [blame] | 50 | #include <asm/page.h> |
Michael Neuling | de1d924 | 2011-11-09 20:39:49 +0000 | [diff] [blame] | 51 | #include <asm/hvcall.h> |
David Howells | ae3a197 | 2012-03-28 18:30:02 +0100 | [diff] [blame] | 52 | #include <asm/switch_to.h> |
Paul Mackerras | 512691d | 2012-10-15 01:15:41 +0000 | [diff] [blame] | 53 | #include <asm/smp.h> |
Paul Mackerras | 66feed6 | 2015-03-28 14:21:12 +1100 | [diff] [blame] | 54 | #include <asm/dbell.h> |
Mahesh Salgaonkar | fd7bacb | 2016-05-15 09:44:26 +0530 | [diff] [blame] | 55 | #include <asm/hmi.h> |
Suresh Warrier | c57875f | 2016-08-19 15:35:50 +1000 | [diff] [blame] | 56 | #include <asm/pnv-pci.h> |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 57 | #include <linux/gfp.h> |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 58 | #include <linux/vmalloc.h> |
| 59 | #include <linux/highmem.h> |
Paul Mackerras | c77162d | 2011-12-12 12:31:00 +0000 | [diff] [blame] | 60 | #include <linux/hugetlb.h> |
Suresh Warrier | c57875f | 2016-08-19 15:35:50 +1000 | [diff] [blame] | 61 | #include <linux/kvm_irqfd.h> |
| 62 | #include <linux/irqbypass.h> |
Aneesh Kumar K.V | 2ba9f0d | 2013-10-07 22:17:59 +0530 | [diff] [blame] | 63 | #include <linux/module.h> |
Suraj Jitindar Singh | 7b5f827 | 2016-08-02 14:03:20 +1000 | [diff] [blame] | 64 | #include <linux/compiler.h> |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 65 | |
Aneesh Kumar K.V | 3a167bea | 2013-10-07 22:17:53 +0530 | [diff] [blame] | 66 | #include "book3s.h" |
| 67 | |
Suresh E. Warrier | 3c78f78 | 2014-12-03 18:48:10 -0600 | [diff] [blame] | 68 | #define CREATE_TRACE_POINTS |
| 69 | #include "trace_hv.h" |
| 70 | |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 71 | /* #define EXIT_DEBUG */ |
| 72 | /* #define EXIT_DEBUG_SIMPLE */ |
| 73 | /* #define EXIT_DEBUG_INT */ |
| 74 | |
Paul Mackerras | 913d3ff9a | 2012-10-15 01:16:48 +0000 | [diff] [blame] | 75 | /* Used to indicate that a guest page fault needs to be handled */ |
| 76 | #define RESUME_PAGE_FAULT (RESUME_GUEST | RESUME_FLAG_ARCH1) |
Suresh Warrier | f7af520 | 2016-08-19 15:35:52 +1000 | [diff] [blame] | 77 | /* Used to indicate that a guest passthrough interrupt needs to be handled */ |
| 78 | #define RESUME_PASSTHROUGH (RESUME_GUEST | RESUME_FLAG_ARCH2) |
Paul Mackerras | 913d3ff9a | 2012-10-15 01:16:48 +0000 | [diff] [blame] | 79 | |
Paul Mackerras | c7b6767 | 2012-10-15 01:18:07 +0000 | [diff] [blame] | 80 | /* Used as a "null" value for timebase values */ |
| 81 | #define TB_NIL (~(u64)0) |
| 82 | |
Paul Mackerras | 699a0ea | 2014-06-02 11:02:59 +1000 | [diff] [blame] | 83 | static DECLARE_BITMAP(default_enabled_hcalls, MAX_HCALL_OPCODE/4 + 1); |
| 84 | |
Paul Mackerras | b4deba5 | 2015-07-02 20:38:16 +1000 | [diff] [blame] | 85 | static int dynamic_mt_modes = 6; |
| 86 | module_param(dynamic_mt_modes, int, S_IRUGO | S_IWUSR); |
| 87 | MODULE_PARM_DESC(dynamic_mt_modes, "Set of allowed dynamic micro-threading modes: 0 (= none), 2, 4, or 6 (= 2 or 4)"); |
Paul Mackerras | ec25716 | 2015-06-24 21:18:03 +1000 | [diff] [blame] | 88 | static int target_smt_mode; |
| 89 | module_param(target_smt_mode, int, S_IRUGO | S_IWUSR); |
| 90 | MODULE_PARM_DESC(target_smt_mode, "Target threads per core (0 = max)"); |
Stewart Smith | 9678cda | 2014-07-18 14:18:43 +1000 | [diff] [blame] | 91 | |
Suresh E. Warrier | 520fe9c | 2015-12-21 16:33:57 -0600 | [diff] [blame] | 92 | #ifdef CONFIG_KVM_XICS |
| 93 | static struct kernel_param_ops module_param_ops = { |
| 94 | .set = param_set_int, |
| 95 | .get = param_get_int, |
| 96 | }; |
| 97 | |
Suresh Warrier | 644abbb | 2016-08-19 15:35:54 +1000 | [diff] [blame] | 98 | module_param_cb(kvm_irq_bypass, &module_param_ops, &kvm_irq_bypass, |
| 99 | S_IRUGO | S_IWUSR); |
| 100 | MODULE_PARM_DESC(kvm_irq_bypass, "Bypass passthrough interrupt optimization"); |
| 101 | |
Suresh E. Warrier | 520fe9c | 2015-12-21 16:33:57 -0600 | [diff] [blame] | 102 | module_param_cb(h_ipi_redirect, &module_param_ops, &h_ipi_redirect, |
| 103 | S_IRUGO | S_IWUSR); |
| 104 | MODULE_PARM_DESC(h_ipi_redirect, "Redirect H_IPI wakeup to a free host core"); |
| 105 | #endif |
| 106 | |
Suraj Jitindar Singh | 0cda69d | 2016-08-02 14:03:21 +1000 | [diff] [blame] | 107 | /* Maximum halt poll interval defaults to KVM_HALT_POLL_NS_DEFAULT */ |
| 108 | static unsigned int halt_poll_max_ns = KVM_HALT_POLL_NS_DEFAULT; |
| 109 | module_param(halt_poll_max_ns, uint, S_IRUGO | S_IWUSR); |
| 110 | MODULE_PARM_DESC(halt_poll_max_ns, "Maximum halt poll time in ns"); |
| 111 | |
| 112 | /* Factor by which the vcore halt poll interval is grown, default is to double |
| 113 | */ |
| 114 | static unsigned int halt_poll_ns_grow = 2; |
| 115 | module_param(halt_poll_ns_grow, int, S_IRUGO); |
| 116 | MODULE_PARM_DESC(halt_poll_ns_grow, "Factor halt poll time is grown by"); |
| 117 | |
| 118 | /* Factor by which the vcore halt poll interval is shrunk, default is to reset |
| 119 | */ |
| 120 | static unsigned int halt_poll_ns_shrink; |
| 121 | module_param(halt_poll_ns_shrink, int, S_IRUGO); |
| 122 | MODULE_PARM_DESC(halt_poll_ns_shrink, "Factor halt poll time is shrunk by"); |
| 123 | |
Paul Mackerras | 19ccb76 | 2011-07-23 17:42:46 +1000 | [diff] [blame] | 124 | static void kvmppc_end_cede(struct kvm_vcpu *vcpu); |
Paul Mackerras | 32fad28 | 2012-05-04 02:32:53 +0000 | [diff] [blame] | 125 | static int kvmppc_hv_setup_htab_rma(struct kvm_vcpu *vcpu); |
Paul Mackerras | 19ccb76 | 2011-07-23 17:42:46 +1000 | [diff] [blame] | 126 | |
Suraj Jitindar Singh | 7b5f827 | 2016-08-02 14:03:20 +1000 | [diff] [blame] | 127 | static inline struct kvm_vcpu *next_runnable_thread(struct kvmppc_vcore *vc, |
| 128 | int *ip) |
| 129 | { |
| 130 | int i = *ip; |
| 131 | struct kvm_vcpu *vcpu; |
| 132 | |
| 133 | while (++i < MAX_SMT_THREADS) { |
| 134 | vcpu = READ_ONCE(vc->runnable_threads[i]); |
| 135 | if (vcpu) { |
| 136 | *ip = i; |
| 137 | return vcpu; |
| 138 | } |
| 139 | } |
| 140 | return NULL; |
| 141 | } |
| 142 | |
| 143 | /* Used to traverse the list of runnable threads for a given vcore */ |
| 144 | #define for_each_runnable_thread(i, vcpu, vc) \ |
| 145 | for (i = -1; (vcpu = next_runnable_thread(vc, &i)); ) |
| 146 | |
Paul Mackerras | 66feed6 | 2015-03-28 14:21:12 +1100 | [diff] [blame] | 147 | static bool kvmppc_ipi_thread(int cpu) |
| 148 | { |
| 149 | /* On POWER8 for IPIs to threads in the same core, use msgsnd */ |
| 150 | if (cpu_has_feature(CPU_FTR_ARCH_207S)) { |
| 151 | preempt_disable(); |
| 152 | if (cpu_first_thread_sibling(cpu) == |
| 153 | cpu_first_thread_sibling(smp_processor_id())) { |
| 154 | unsigned long msg = PPC_DBELL_TYPE(PPC_DBELL_SERVER); |
| 155 | msg |= cpu_thread_in_core(cpu); |
| 156 | smp_mb(); |
| 157 | __asm__ __volatile__ (PPC_MSGSND(%0) : : "r" (msg)); |
| 158 | preempt_enable(); |
| 159 | return true; |
| 160 | } |
| 161 | preempt_enable(); |
| 162 | } |
| 163 | |
| 164 | #if defined(CONFIG_PPC_ICP_NATIVE) && defined(CONFIG_SMP) |
| 165 | if (cpu >= 0 && cpu < nr_cpu_ids && paca[cpu].kvm_hstate.xics_phys) { |
| 166 | xics_wake_cpu(cpu); |
| 167 | return true; |
| 168 | } |
| 169 | #endif |
| 170 | |
| 171 | return false; |
| 172 | } |
| 173 | |
Aneesh Kumar K.V | 3a167bea | 2013-10-07 22:17:53 +0530 | [diff] [blame] | 174 | static void kvmppc_fast_vcpu_kick_hv(struct kvm_vcpu *vcpu) |
Benjamin Herrenschmidt | 54695c3 | 2013-04-17 20:30:50 +0000 | [diff] [blame] | 175 | { |
Paul Mackerras | ec25716 | 2015-06-24 21:18:03 +1000 | [diff] [blame] | 176 | int cpu; |
Marcelo Tosatti | 8577370 | 2016-02-19 09:46:39 +0100 | [diff] [blame] | 177 | struct swait_queue_head *wqp; |
Benjamin Herrenschmidt | 54695c3 | 2013-04-17 20:30:50 +0000 | [diff] [blame] | 178 | |
| 179 | wqp = kvm_arch_vcpu_wq(vcpu); |
Marcelo Tosatti | 8577370 | 2016-02-19 09:46:39 +0100 | [diff] [blame] | 180 | if (swait_active(wqp)) { |
| 181 | swake_up(wqp); |
Benjamin Herrenschmidt | 54695c3 | 2013-04-17 20:30:50 +0000 | [diff] [blame] | 182 | ++vcpu->stat.halt_wakeup; |
| 183 | } |
| 184 | |
Paul Mackerras | ec25716 | 2015-06-24 21:18:03 +1000 | [diff] [blame] | 185 | if (kvmppc_ipi_thread(vcpu->arch.thread_cpu)) |
Paul Mackerras | 66feed6 | 2015-03-28 14:21:12 +1100 | [diff] [blame] | 186 | return; |
Benjamin Herrenschmidt | 54695c3 | 2013-04-17 20:30:50 +0000 | [diff] [blame] | 187 | |
| 188 | /* CPU points to the first thread of the core */ |
Paul Mackerras | ec25716 | 2015-06-24 21:18:03 +1000 | [diff] [blame] | 189 | cpu = vcpu->cpu; |
Paul Mackerras | 66feed6 | 2015-03-28 14:21:12 +1100 | [diff] [blame] | 190 | if (cpu >= 0 && cpu < nr_cpu_ids && cpu_online(cpu)) |
| 191 | smp_send_reschedule(cpu); |
Benjamin Herrenschmidt | 54695c3 | 2013-04-17 20:30:50 +0000 | [diff] [blame] | 192 | } |
| 193 | |
Paul Mackerras | c7b6767 | 2012-10-15 01:18:07 +0000 | [diff] [blame] | 194 | /* |
| 195 | * We use the vcpu_load/put functions to measure stolen time. |
| 196 | * Stolen time is counted as time when either the vcpu is able to |
| 197 | * run as part of a virtual core, but the task running the vcore |
| 198 | * is preempted or sleeping, or when the vcpu needs something done |
| 199 | * in the kernel by the task running the vcpu, but that task is |
| 200 | * preempted or sleeping. Those two things have to be counted |
| 201 | * separately, since one of the vcpu tasks will take on the job |
| 202 | * of running the core, and the other vcpu tasks in the vcore will |
| 203 | * sleep waiting for it to do that, but that sleep shouldn't count |
| 204 | * as stolen time. |
| 205 | * |
| 206 | * Hence we accumulate stolen time when the vcpu can run as part of |
| 207 | * a vcore using vc->stolen_tb, and the stolen time when the vcpu |
| 208 | * needs its task to do other things in the kernel (for example, |
| 209 | * service a page fault) in busy_stolen. We don't accumulate |
| 210 | * stolen time for a vcore when it is inactive, or for a vcpu |
| 211 | * when it is in state RUNNING or NOTREADY. NOTREADY is a bit of |
| 212 | * a misnomer; it means that the vcpu task is not executing in |
| 213 | * the KVM_VCPU_RUN ioctl, i.e. it is in userspace or elsewhere in |
| 214 | * the kernel. We don't have any way of dividing up that time |
| 215 | * between time that the vcpu is genuinely stopped, time that |
| 216 | * the task is actively working on behalf of the vcpu, and time |
| 217 | * that the task is preempted, so we don't count any of it as |
| 218 | * stolen. |
| 219 | * |
| 220 | * Updates to busy_stolen are protected by arch.tbacct_lock; |
Paul Mackerras | 2711e24 | 2014-12-04 16:43:28 +1100 | [diff] [blame] | 221 | * updates to vc->stolen_tb are protected by the vcore->stoltb_lock |
| 222 | * lock. The stolen times are measured in units of timebase ticks. |
| 223 | * (Note that the != TB_NIL checks below are purely defensive; |
| 224 | * they should never fail.) |
Paul Mackerras | c7b6767 | 2012-10-15 01:18:07 +0000 | [diff] [blame] | 225 | */ |
| 226 | |
Paul Mackerras | ec25716 | 2015-06-24 21:18:03 +1000 | [diff] [blame] | 227 | static void kvmppc_core_start_stolen(struct kvmppc_vcore *vc) |
| 228 | { |
| 229 | unsigned long flags; |
| 230 | |
| 231 | spin_lock_irqsave(&vc->stoltb_lock, flags); |
| 232 | vc->preempt_tb = mftb(); |
| 233 | spin_unlock_irqrestore(&vc->stoltb_lock, flags); |
| 234 | } |
| 235 | |
| 236 | static void kvmppc_core_end_stolen(struct kvmppc_vcore *vc) |
| 237 | { |
| 238 | unsigned long flags; |
| 239 | |
| 240 | spin_lock_irqsave(&vc->stoltb_lock, flags); |
| 241 | if (vc->preempt_tb != TB_NIL) { |
| 242 | vc->stolen_tb += mftb() - vc->preempt_tb; |
| 243 | vc->preempt_tb = TB_NIL; |
| 244 | } |
| 245 | spin_unlock_irqrestore(&vc->stoltb_lock, flags); |
| 246 | } |
| 247 | |
Aneesh Kumar K.V | 3a167bea | 2013-10-07 22:17:53 +0530 | [diff] [blame] | 248 | static void kvmppc_core_vcpu_load_hv(struct kvm_vcpu *vcpu, int cpu) |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 249 | { |
Paul Mackerras | 0456ec4 | 2012-02-03 00:56:21 +0000 | [diff] [blame] | 250 | struct kvmppc_vcore *vc = vcpu->arch.vcore; |
Paul Mackerras | bf3d32e | 2013-11-16 17:46:04 +1100 | [diff] [blame] | 251 | unsigned long flags; |
Paul Mackerras | 0456ec4 | 2012-02-03 00:56:21 +0000 | [diff] [blame] | 252 | |
Paul Mackerras | 2711e24 | 2014-12-04 16:43:28 +1100 | [diff] [blame] | 253 | /* |
| 254 | * We can test vc->runner without taking the vcore lock, |
| 255 | * because only this task ever sets vc->runner to this |
| 256 | * vcpu, and once it is set to this vcpu, only this task |
| 257 | * ever sets it to NULL. |
| 258 | */ |
Paul Mackerras | ec25716 | 2015-06-24 21:18:03 +1000 | [diff] [blame] | 259 | if (vc->runner == vcpu && vc->vcore_state >= VCORE_SLEEPING) |
| 260 | kvmppc_core_end_stolen(vc); |
| 261 | |
Paul Mackerras | 2711e24 | 2014-12-04 16:43:28 +1100 | [diff] [blame] | 262 | spin_lock_irqsave(&vcpu->arch.tbacct_lock, flags); |
Paul Mackerras | c7b6767 | 2012-10-15 01:18:07 +0000 | [diff] [blame] | 263 | if (vcpu->arch.state == KVMPPC_VCPU_BUSY_IN_HOST && |
| 264 | vcpu->arch.busy_preempt != TB_NIL) { |
| 265 | vcpu->arch.busy_stolen += mftb() - vcpu->arch.busy_preempt; |
| 266 | vcpu->arch.busy_preempt = TB_NIL; |
| 267 | } |
Paul Mackerras | bf3d32e | 2013-11-16 17:46:04 +1100 | [diff] [blame] | 268 | spin_unlock_irqrestore(&vcpu->arch.tbacct_lock, flags); |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 269 | } |
| 270 | |
Aneesh Kumar K.V | 3a167bea | 2013-10-07 22:17:53 +0530 | [diff] [blame] | 271 | static void kvmppc_core_vcpu_put_hv(struct kvm_vcpu *vcpu) |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 272 | { |
Paul Mackerras | 0456ec4 | 2012-02-03 00:56:21 +0000 | [diff] [blame] | 273 | struct kvmppc_vcore *vc = vcpu->arch.vcore; |
Paul Mackerras | bf3d32e | 2013-11-16 17:46:04 +1100 | [diff] [blame] | 274 | unsigned long flags; |
Paul Mackerras | 0456ec4 | 2012-02-03 00:56:21 +0000 | [diff] [blame] | 275 | |
Paul Mackerras | ec25716 | 2015-06-24 21:18:03 +1000 | [diff] [blame] | 276 | if (vc->runner == vcpu && vc->vcore_state >= VCORE_SLEEPING) |
| 277 | kvmppc_core_start_stolen(vc); |
| 278 | |
Paul Mackerras | 2711e24 | 2014-12-04 16:43:28 +1100 | [diff] [blame] | 279 | spin_lock_irqsave(&vcpu->arch.tbacct_lock, flags); |
Paul Mackerras | c7b6767 | 2012-10-15 01:18:07 +0000 | [diff] [blame] | 280 | if (vcpu->arch.state == KVMPPC_VCPU_BUSY_IN_HOST) |
| 281 | vcpu->arch.busy_preempt = mftb(); |
Paul Mackerras | bf3d32e | 2013-11-16 17:46:04 +1100 | [diff] [blame] | 282 | spin_unlock_irqrestore(&vcpu->arch.tbacct_lock, flags); |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 283 | } |
| 284 | |
Aneesh Kumar K.V | 3a167bea | 2013-10-07 22:17:53 +0530 | [diff] [blame] | 285 | static void kvmppc_set_msr_hv(struct kvm_vcpu *vcpu, u64 msr) |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 286 | { |
Paul Mackerras | c20875a | 2015-11-12 16:43:02 +1100 | [diff] [blame] | 287 | /* |
| 288 | * Check for illegal transactional state bit combination |
| 289 | * and if we find it, force the TS field to a safe state. |
| 290 | */ |
| 291 | if ((msr & MSR_TS_MASK) == MSR_TS_MASK) |
| 292 | msr &= ~MSR_TS_MASK; |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 293 | vcpu->arch.shregs.msr = msr; |
Paul Mackerras | 19ccb76 | 2011-07-23 17:42:46 +1000 | [diff] [blame] | 294 | kvmppc_end_cede(vcpu); |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 295 | } |
| 296 | |
Thomas Huth | 5358a96 | 2015-05-22 09:25:02 +0200 | [diff] [blame] | 297 | static void kvmppc_set_pvr_hv(struct kvm_vcpu *vcpu, u32 pvr) |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 298 | { |
| 299 | vcpu->arch.pvr = pvr; |
| 300 | } |
| 301 | |
Thomas Huth | 5358a96 | 2015-05-22 09:25:02 +0200 | [diff] [blame] | 302 | static int kvmppc_set_arch_compat(struct kvm_vcpu *vcpu, u32 arch_compat) |
Paul Mackerras | 388cc6e | 2013-09-21 14:35:02 +1000 | [diff] [blame] | 303 | { |
| 304 | unsigned long pcr = 0; |
| 305 | struct kvmppc_vcore *vc = vcpu->arch.vcore; |
| 306 | |
| 307 | if (arch_compat) { |
Paul Mackerras | 388cc6e | 2013-09-21 14:35:02 +1000 | [diff] [blame] | 308 | switch (arch_compat) { |
| 309 | case PVR_ARCH_205: |
Paul Mackerras | 5557ae0 | 2014-01-08 21:25:24 +1100 | [diff] [blame] | 310 | /* |
| 311 | * If an arch bit is set in PCR, all the defined |
| 312 | * higher-order arch bits also have to be set. |
| 313 | */ |
| 314 | pcr = PCR_ARCH_206 | PCR_ARCH_205; |
Paul Mackerras | 388cc6e | 2013-09-21 14:35:02 +1000 | [diff] [blame] | 315 | break; |
| 316 | case PVR_ARCH_206: |
| 317 | case PVR_ARCH_206p: |
Paul Mackerras | 5557ae0 | 2014-01-08 21:25:24 +1100 | [diff] [blame] | 318 | pcr = PCR_ARCH_206; |
| 319 | break; |
| 320 | case PVR_ARCH_207: |
Paul Mackerras | 388cc6e | 2013-09-21 14:35:02 +1000 | [diff] [blame] | 321 | break; |
| 322 | default: |
| 323 | return -EINVAL; |
| 324 | } |
Paul Mackerras | 5557ae0 | 2014-01-08 21:25:24 +1100 | [diff] [blame] | 325 | |
| 326 | if (!cpu_has_feature(CPU_FTR_ARCH_207S)) { |
| 327 | /* POWER7 can't emulate POWER8 */ |
| 328 | if (!(pcr & PCR_ARCH_206)) |
| 329 | return -EINVAL; |
| 330 | pcr &= ~PCR_ARCH_206; |
| 331 | } |
Paul Mackerras | 388cc6e | 2013-09-21 14:35:02 +1000 | [diff] [blame] | 332 | } |
| 333 | |
| 334 | spin_lock(&vc->lock); |
| 335 | vc->arch_compat = arch_compat; |
| 336 | vc->pcr = pcr; |
| 337 | spin_unlock(&vc->lock); |
| 338 | |
| 339 | return 0; |
| 340 | } |
| 341 | |
Thomas Huth | 5358a96 | 2015-05-22 09:25:02 +0200 | [diff] [blame] | 342 | static void kvmppc_dump_regs(struct kvm_vcpu *vcpu) |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 343 | { |
| 344 | int r; |
| 345 | |
| 346 | pr_err("vcpu %p (%d):\n", vcpu, vcpu->vcpu_id); |
| 347 | pr_err("pc = %.16lx msr = %.16llx trap = %x\n", |
| 348 | vcpu->arch.pc, vcpu->arch.shregs.msr, vcpu->arch.trap); |
| 349 | for (r = 0; r < 16; ++r) |
| 350 | pr_err("r%2d = %.16lx r%d = %.16lx\n", |
| 351 | r, kvmppc_get_gpr(vcpu, r), |
| 352 | r+16, kvmppc_get_gpr(vcpu, r+16)); |
| 353 | pr_err("ctr = %.16lx lr = %.16lx\n", |
| 354 | vcpu->arch.ctr, vcpu->arch.lr); |
| 355 | pr_err("srr0 = %.16llx srr1 = %.16llx\n", |
| 356 | vcpu->arch.shregs.srr0, vcpu->arch.shregs.srr1); |
| 357 | pr_err("sprg0 = %.16llx sprg1 = %.16llx\n", |
| 358 | vcpu->arch.shregs.sprg0, vcpu->arch.shregs.sprg1); |
| 359 | pr_err("sprg2 = %.16llx sprg3 = %.16llx\n", |
| 360 | vcpu->arch.shregs.sprg2, vcpu->arch.shregs.sprg3); |
| 361 | pr_err("cr = %.8x xer = %.16lx dsisr = %.8x\n", |
| 362 | vcpu->arch.cr, vcpu->arch.xer, vcpu->arch.shregs.dsisr); |
| 363 | pr_err("dar = %.16llx\n", vcpu->arch.shregs.dar); |
| 364 | pr_err("fault dar = %.16lx dsisr = %.8x\n", |
| 365 | vcpu->arch.fault_dar, vcpu->arch.fault_dsisr); |
| 366 | pr_err("SLB (%d entries):\n", vcpu->arch.slb_max); |
| 367 | for (r = 0; r < vcpu->arch.slb_max; ++r) |
| 368 | pr_err(" ESID = %.16llx VSID = %.16llx\n", |
| 369 | vcpu->arch.slb[r].orige, vcpu->arch.slb[r].origv); |
| 370 | pr_err("lpcr = %.16lx sdr1 = %.16lx last_inst = %.8x\n", |
Paul Mackerras | a0144e2 | 2013-09-20 14:52:38 +1000 | [diff] [blame] | 371 | vcpu->arch.vcore->lpcr, vcpu->kvm->arch.sdr1, |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 372 | vcpu->arch.last_inst); |
| 373 | } |
| 374 | |
Thomas Huth | 5358a96 | 2015-05-22 09:25:02 +0200 | [diff] [blame] | 375 | static struct kvm_vcpu *kvmppc_find_vcpu(struct kvm *kvm, int id) |
Paul Mackerras | a8606e2 | 2011-06-29 00:22:05 +0000 | [diff] [blame] | 376 | { |
Paul Mackerras | 5590c49 | 2019-05-23 16:36:32 +1000 | [diff] [blame] | 377 | return kvm_get_vcpu_by_id(kvm, id); |
Paul Mackerras | a8606e2 | 2011-06-29 00:22:05 +0000 | [diff] [blame] | 378 | } |
| 379 | |
| 380 | static void init_vpa(struct kvm_vcpu *vcpu, struct lppaca *vpa) |
| 381 | { |
Anton Blanchard | f13c13a | 2013-08-07 02:01:26 +1000 | [diff] [blame] | 382 | vpa->__old_status |= LPPACA_OLD_SHARED_PROC; |
Alexander Graf | 0240755 | 2014-06-11 10:34:19 +0200 | [diff] [blame] | 383 | vpa->yield_count = cpu_to_be32(1); |
Paul Mackerras | a8606e2 | 2011-06-29 00:22:05 +0000 | [diff] [blame] | 384 | } |
| 385 | |
Paul Mackerras | 55b665b | 2012-09-25 20:33:06 +0000 | [diff] [blame] | 386 | static int set_vpa(struct kvm_vcpu *vcpu, struct kvmppc_vpa *v, |
| 387 | unsigned long addr, unsigned long len) |
| 388 | { |
| 389 | /* check address is cacheline aligned */ |
| 390 | if (addr & (L1_CACHE_BYTES - 1)) |
| 391 | return -EINVAL; |
| 392 | spin_lock(&vcpu->arch.vpa_update_lock); |
| 393 | if (v->next_gpa != addr || v->len != len) { |
| 394 | v->next_gpa = addr; |
| 395 | v->len = addr ? len : 0; |
| 396 | v->update_pending = 1; |
| 397 | } |
| 398 | spin_unlock(&vcpu->arch.vpa_update_lock); |
| 399 | return 0; |
| 400 | } |
| 401 | |
Paul Mackerras | 2e25aa5 | 2012-02-19 17:46:32 +0000 | [diff] [blame] | 402 | /* Length for a per-processor buffer is passed in at offset 4 in the buffer */ |
| 403 | struct reg_vpa { |
| 404 | u32 dummy; |
| 405 | union { |
Alexander Graf | 0240755 | 2014-06-11 10:34:19 +0200 | [diff] [blame] | 406 | __be16 hword; |
| 407 | __be32 word; |
Paul Mackerras | 2e25aa5 | 2012-02-19 17:46:32 +0000 | [diff] [blame] | 408 | } length; |
| 409 | }; |
| 410 | |
| 411 | static int vpa_is_registered(struct kvmppc_vpa *vpap) |
| 412 | { |
| 413 | if (vpap->update_pending) |
| 414 | return vpap->next_gpa != 0; |
| 415 | return vpap->pinned_addr != NULL; |
| 416 | } |
| 417 | |
Paul Mackerras | a8606e2 | 2011-06-29 00:22:05 +0000 | [diff] [blame] | 418 | static unsigned long do_h_register_vpa(struct kvm_vcpu *vcpu, |
| 419 | unsigned long flags, |
| 420 | unsigned long vcpuid, unsigned long vpa) |
| 421 | { |
| 422 | struct kvm *kvm = vcpu->kvm; |
Paul Mackerras | 93e6024 | 2011-12-12 12:28:55 +0000 | [diff] [blame] | 423 | unsigned long len, nb; |
Paul Mackerras | a8606e2 | 2011-06-29 00:22:05 +0000 | [diff] [blame] | 424 | void *va; |
| 425 | struct kvm_vcpu *tvcpu; |
Paul Mackerras | 2e25aa5 | 2012-02-19 17:46:32 +0000 | [diff] [blame] | 426 | int err; |
| 427 | int subfunc; |
| 428 | struct kvmppc_vpa *vpap; |
Paul Mackerras | a8606e2 | 2011-06-29 00:22:05 +0000 | [diff] [blame] | 429 | |
| 430 | tvcpu = kvmppc_find_vcpu(kvm, vcpuid); |
| 431 | if (!tvcpu) |
| 432 | return H_PARAMETER; |
| 433 | |
Paul Mackerras | 2e25aa5 | 2012-02-19 17:46:32 +0000 | [diff] [blame] | 434 | subfunc = (flags >> H_VPA_FUNC_SHIFT) & H_VPA_FUNC_MASK; |
| 435 | if (subfunc == H_VPA_REG_VPA || subfunc == H_VPA_REG_DTL || |
| 436 | subfunc == H_VPA_REG_SLB) { |
| 437 | /* Registering new area - address must be cache-line aligned */ |
| 438 | if ((vpa & (L1_CACHE_BYTES - 1)) || !vpa) |
Paul Mackerras | a8606e2 | 2011-06-29 00:22:05 +0000 | [diff] [blame] | 439 | return H_PARAMETER; |
Paul Mackerras | 2e25aa5 | 2012-02-19 17:46:32 +0000 | [diff] [blame] | 440 | |
| 441 | /* convert logical addr to kernel addr and read length */ |
Paul Mackerras | 93e6024 | 2011-12-12 12:28:55 +0000 | [diff] [blame] | 442 | va = kvmppc_pin_guest_page(kvm, vpa, &nb); |
| 443 | if (va == NULL) |
Paul Mackerras | b2b2f16 | 2011-12-12 12:28:21 +0000 | [diff] [blame] | 444 | return H_PARAMETER; |
Paul Mackerras | 2e25aa5 | 2012-02-19 17:46:32 +0000 | [diff] [blame] | 445 | if (subfunc == H_VPA_REG_VPA) |
Alexander Graf | 0240755 | 2014-06-11 10:34:19 +0200 | [diff] [blame] | 446 | len = be16_to_cpu(((struct reg_vpa *)va)->length.hword); |
Paul Mackerras | a8606e2 | 2011-06-29 00:22:05 +0000 | [diff] [blame] | 447 | else |
Alexander Graf | 0240755 | 2014-06-11 10:34:19 +0200 | [diff] [blame] | 448 | len = be32_to_cpu(((struct reg_vpa *)va)->length.word); |
Paul Mackerras | c35635e | 2013-04-18 19:51:04 +0000 | [diff] [blame] | 449 | kvmppc_unpin_guest_page(kvm, va, vpa, false); |
Paul Mackerras | 2e25aa5 | 2012-02-19 17:46:32 +0000 | [diff] [blame] | 450 | |
| 451 | /* Check length */ |
| 452 | if (len > nb || len < sizeof(struct reg_vpa)) |
| 453 | return H_PARAMETER; |
Paul Mackerras | a8606e2 | 2011-06-29 00:22:05 +0000 | [diff] [blame] | 454 | } else { |
Paul Mackerras | 2e25aa5 | 2012-02-19 17:46:32 +0000 | [diff] [blame] | 455 | vpa = 0; |
| 456 | len = 0; |
| 457 | } |
| 458 | |
| 459 | err = H_PARAMETER; |
| 460 | vpap = NULL; |
| 461 | spin_lock(&tvcpu->arch.vpa_update_lock); |
| 462 | |
| 463 | switch (subfunc) { |
| 464 | case H_VPA_REG_VPA: /* register VPA */ |
| 465 | if (len < sizeof(struct lppaca)) |
Paul Mackerras | a8606e2 | 2011-06-29 00:22:05 +0000 | [diff] [blame] | 466 | break; |
Paul Mackerras | 2e25aa5 | 2012-02-19 17:46:32 +0000 | [diff] [blame] | 467 | vpap = &tvcpu->arch.vpa; |
| 468 | err = 0; |
| 469 | break; |
| 470 | |
| 471 | case H_VPA_REG_DTL: /* register DTL */ |
| 472 | if (len < sizeof(struct dtl_entry)) |
Paul Mackerras | a8606e2 | 2011-06-29 00:22:05 +0000 | [diff] [blame] | 473 | break; |
Paul Mackerras | 2e25aa5 | 2012-02-19 17:46:32 +0000 | [diff] [blame] | 474 | len -= len % sizeof(struct dtl_entry); |
| 475 | |
| 476 | /* Check that they have previously registered a VPA */ |
| 477 | err = H_RESOURCE; |
| 478 | if (!vpa_is_registered(&tvcpu->arch.vpa)) |
Paul Mackerras | a8606e2 | 2011-06-29 00:22:05 +0000 | [diff] [blame] | 479 | break; |
Paul Mackerras | 2e25aa5 | 2012-02-19 17:46:32 +0000 | [diff] [blame] | 480 | |
| 481 | vpap = &tvcpu->arch.dtl; |
| 482 | err = 0; |
| 483 | break; |
| 484 | |
| 485 | case H_VPA_REG_SLB: /* register SLB shadow buffer */ |
| 486 | /* Check that they have previously registered a VPA */ |
| 487 | err = H_RESOURCE; |
| 488 | if (!vpa_is_registered(&tvcpu->arch.vpa)) |
| 489 | break; |
| 490 | |
| 491 | vpap = &tvcpu->arch.slb_shadow; |
| 492 | err = 0; |
| 493 | break; |
| 494 | |
| 495 | case H_VPA_DEREG_VPA: /* deregister VPA */ |
| 496 | /* Check they don't still have a DTL or SLB buf registered */ |
| 497 | err = H_RESOURCE; |
| 498 | if (vpa_is_registered(&tvcpu->arch.dtl) || |
| 499 | vpa_is_registered(&tvcpu->arch.slb_shadow)) |
| 500 | break; |
| 501 | |
| 502 | vpap = &tvcpu->arch.vpa; |
| 503 | err = 0; |
| 504 | break; |
| 505 | |
| 506 | case H_VPA_DEREG_DTL: /* deregister DTL */ |
| 507 | vpap = &tvcpu->arch.dtl; |
| 508 | err = 0; |
| 509 | break; |
| 510 | |
| 511 | case H_VPA_DEREG_SLB: /* deregister SLB shadow buffer */ |
| 512 | vpap = &tvcpu->arch.slb_shadow; |
| 513 | err = 0; |
| 514 | break; |
| 515 | } |
| 516 | |
| 517 | if (vpap) { |
| 518 | vpap->next_gpa = vpa; |
| 519 | vpap->len = len; |
| 520 | vpap->update_pending = 1; |
| 521 | } |
| 522 | |
| 523 | spin_unlock(&tvcpu->arch.vpa_update_lock); |
| 524 | |
| 525 | return err; |
| 526 | } |
| 527 | |
Paul Mackerras | 081f323 | 2012-06-01 20:20:24 +1000 | [diff] [blame] | 528 | static void kvmppc_update_vpa(struct kvm_vcpu *vcpu, struct kvmppc_vpa *vpap) |
Paul Mackerras | 2e25aa5 | 2012-02-19 17:46:32 +0000 | [diff] [blame] | 529 | { |
Paul Mackerras | 081f323 | 2012-06-01 20:20:24 +1000 | [diff] [blame] | 530 | struct kvm *kvm = vcpu->kvm; |
Paul Mackerras | 2e25aa5 | 2012-02-19 17:46:32 +0000 | [diff] [blame] | 531 | void *va; |
| 532 | unsigned long nb; |
Paul Mackerras | 081f323 | 2012-06-01 20:20:24 +1000 | [diff] [blame] | 533 | unsigned long gpa; |
| 534 | |
| 535 | /* |
| 536 | * We need to pin the page pointed to by vpap->next_gpa, |
| 537 | * but we can't call kvmppc_pin_guest_page under the lock |
| 538 | * as it does get_user_pages() and down_read(). So we |
| 539 | * have to drop the lock, pin the page, then get the lock |
| 540 | * again and check that a new area didn't get registered |
| 541 | * in the meantime. |
| 542 | */ |
| 543 | for (;;) { |
| 544 | gpa = vpap->next_gpa; |
| 545 | spin_unlock(&vcpu->arch.vpa_update_lock); |
| 546 | va = NULL; |
| 547 | nb = 0; |
| 548 | if (gpa) |
Paul Mackerras | c35635e | 2013-04-18 19:51:04 +0000 | [diff] [blame] | 549 | va = kvmppc_pin_guest_page(kvm, gpa, &nb); |
Paul Mackerras | 081f323 | 2012-06-01 20:20:24 +1000 | [diff] [blame] | 550 | spin_lock(&vcpu->arch.vpa_update_lock); |
| 551 | if (gpa == vpap->next_gpa) |
| 552 | break; |
| 553 | /* sigh... unpin that one and try again */ |
| 554 | if (va) |
Paul Mackerras | c35635e | 2013-04-18 19:51:04 +0000 | [diff] [blame] | 555 | kvmppc_unpin_guest_page(kvm, va, gpa, false); |
Paul Mackerras | 081f323 | 2012-06-01 20:20:24 +1000 | [diff] [blame] | 556 | } |
Paul Mackerras | 2e25aa5 | 2012-02-19 17:46:32 +0000 | [diff] [blame] | 557 | |
| 558 | vpap->update_pending = 0; |
Paul Mackerras | 081f323 | 2012-06-01 20:20:24 +1000 | [diff] [blame] | 559 | if (va && nb < vpap->len) { |
| 560 | /* |
| 561 | * If it's now too short, it must be that userspace |
| 562 | * has changed the mappings underlying guest memory, |
| 563 | * so unregister the region. |
| 564 | */ |
Paul Mackerras | c35635e | 2013-04-18 19:51:04 +0000 | [diff] [blame] | 565 | kvmppc_unpin_guest_page(kvm, va, gpa, false); |
Paul Mackerras | 081f323 | 2012-06-01 20:20:24 +1000 | [diff] [blame] | 566 | va = NULL; |
Paul Mackerras | a8606e2 | 2011-06-29 00:22:05 +0000 | [diff] [blame] | 567 | } |
Paul Mackerras | 2e25aa5 | 2012-02-19 17:46:32 +0000 | [diff] [blame] | 568 | if (vpap->pinned_addr) |
Paul Mackerras | c35635e | 2013-04-18 19:51:04 +0000 | [diff] [blame] | 569 | kvmppc_unpin_guest_page(kvm, vpap->pinned_addr, vpap->gpa, |
| 570 | vpap->dirty); |
| 571 | vpap->gpa = gpa; |
Paul Mackerras | 2e25aa5 | 2012-02-19 17:46:32 +0000 | [diff] [blame] | 572 | vpap->pinned_addr = va; |
Paul Mackerras | c35635e | 2013-04-18 19:51:04 +0000 | [diff] [blame] | 573 | vpap->dirty = false; |
Paul Mackerras | 2e25aa5 | 2012-02-19 17:46:32 +0000 | [diff] [blame] | 574 | if (va) |
| 575 | vpap->pinned_end = va + vpap->len; |
| 576 | } |
Paul Mackerras | 93e6024 | 2011-12-12 12:28:55 +0000 | [diff] [blame] | 577 | |
Paul Mackerras | 2e25aa5 | 2012-02-19 17:46:32 +0000 | [diff] [blame] | 578 | static void kvmppc_update_vpas(struct kvm_vcpu *vcpu) |
| 579 | { |
Paul Mackerras | 2f12f03 | 2012-10-15 01:17:17 +0000 | [diff] [blame] | 580 | if (!(vcpu->arch.vpa.update_pending || |
| 581 | vcpu->arch.slb_shadow.update_pending || |
| 582 | vcpu->arch.dtl.update_pending)) |
| 583 | return; |
| 584 | |
Paul Mackerras | 2e25aa5 | 2012-02-19 17:46:32 +0000 | [diff] [blame] | 585 | spin_lock(&vcpu->arch.vpa_update_lock); |
| 586 | if (vcpu->arch.vpa.update_pending) { |
Paul Mackerras | 081f323 | 2012-06-01 20:20:24 +1000 | [diff] [blame] | 587 | kvmppc_update_vpa(vcpu, &vcpu->arch.vpa); |
Paul Mackerras | 55b665b | 2012-09-25 20:33:06 +0000 | [diff] [blame] | 588 | if (vcpu->arch.vpa.pinned_addr) |
| 589 | init_vpa(vcpu, vcpu->arch.vpa.pinned_addr); |
Paul Mackerras | 2e25aa5 | 2012-02-19 17:46:32 +0000 | [diff] [blame] | 590 | } |
| 591 | if (vcpu->arch.dtl.update_pending) { |
Paul Mackerras | 081f323 | 2012-06-01 20:20:24 +1000 | [diff] [blame] | 592 | kvmppc_update_vpa(vcpu, &vcpu->arch.dtl); |
Paul Mackerras | 2e25aa5 | 2012-02-19 17:46:32 +0000 | [diff] [blame] | 593 | vcpu->arch.dtl_ptr = vcpu->arch.dtl.pinned_addr; |
| 594 | vcpu->arch.dtl_index = 0; |
| 595 | } |
| 596 | if (vcpu->arch.slb_shadow.update_pending) |
Paul Mackerras | 081f323 | 2012-06-01 20:20:24 +1000 | [diff] [blame] | 597 | kvmppc_update_vpa(vcpu, &vcpu->arch.slb_shadow); |
Paul Mackerras | 2e25aa5 | 2012-02-19 17:46:32 +0000 | [diff] [blame] | 598 | spin_unlock(&vcpu->arch.vpa_update_lock); |
Paul Mackerras | a8606e2 | 2011-06-29 00:22:05 +0000 | [diff] [blame] | 599 | } |
| 600 | |
Paul Mackerras | c7b6767 | 2012-10-15 01:18:07 +0000 | [diff] [blame] | 601 | /* |
| 602 | * Return the accumulated stolen time for the vcore up until `now'. |
| 603 | * The caller should hold the vcore lock. |
| 604 | */ |
| 605 | static u64 vcore_stolen_time(struct kvmppc_vcore *vc, u64 now) |
| 606 | { |
| 607 | u64 p; |
Paul Mackerras | 2711e24 | 2014-12-04 16:43:28 +1100 | [diff] [blame] | 608 | unsigned long flags; |
Paul Mackerras | c7b6767 | 2012-10-15 01:18:07 +0000 | [diff] [blame] | 609 | |
Paul Mackerras | 2711e24 | 2014-12-04 16:43:28 +1100 | [diff] [blame] | 610 | spin_lock_irqsave(&vc->stoltb_lock, flags); |
| 611 | p = vc->stolen_tb; |
Paul Mackerras | c7b6767 | 2012-10-15 01:18:07 +0000 | [diff] [blame] | 612 | if (vc->vcore_state != VCORE_INACTIVE && |
Paul Mackerras | 2711e24 | 2014-12-04 16:43:28 +1100 | [diff] [blame] | 613 | vc->preempt_tb != TB_NIL) |
| 614 | p += now - vc->preempt_tb; |
| 615 | spin_unlock_irqrestore(&vc->stoltb_lock, flags); |
Paul Mackerras | c7b6767 | 2012-10-15 01:18:07 +0000 | [diff] [blame] | 616 | return p; |
| 617 | } |
| 618 | |
Paul Mackerras | 0456ec4 | 2012-02-03 00:56:21 +0000 | [diff] [blame] | 619 | static void kvmppc_create_dtl_entry(struct kvm_vcpu *vcpu, |
| 620 | struct kvmppc_vcore *vc) |
| 621 | { |
| 622 | struct dtl_entry *dt; |
| 623 | struct lppaca *vpa; |
Paul Mackerras | c7b6767 | 2012-10-15 01:18:07 +0000 | [diff] [blame] | 624 | unsigned long stolen; |
| 625 | unsigned long core_stolen; |
| 626 | u64 now; |
Paul Mackerras | 0456ec4 | 2012-02-03 00:56:21 +0000 | [diff] [blame] | 627 | |
| 628 | dt = vcpu->arch.dtl_ptr; |
| 629 | vpa = vcpu->arch.vpa.pinned_addr; |
Paul Mackerras | c7b6767 | 2012-10-15 01:18:07 +0000 | [diff] [blame] | 630 | now = mftb(); |
| 631 | core_stolen = vcore_stolen_time(vc, now); |
| 632 | stolen = core_stolen - vcpu->arch.stolen_logged; |
| 633 | vcpu->arch.stolen_logged = core_stolen; |
Paul Mackerras | bf3d32e | 2013-11-16 17:46:04 +1100 | [diff] [blame] | 634 | spin_lock_irq(&vcpu->arch.tbacct_lock); |
Paul Mackerras | c7b6767 | 2012-10-15 01:18:07 +0000 | [diff] [blame] | 635 | stolen += vcpu->arch.busy_stolen; |
| 636 | vcpu->arch.busy_stolen = 0; |
Paul Mackerras | bf3d32e | 2013-11-16 17:46:04 +1100 | [diff] [blame] | 637 | spin_unlock_irq(&vcpu->arch.tbacct_lock); |
Paul Mackerras | 0456ec4 | 2012-02-03 00:56:21 +0000 | [diff] [blame] | 638 | if (!dt || !vpa) |
| 639 | return; |
| 640 | memset(dt, 0, sizeof(struct dtl_entry)); |
| 641 | dt->dispatch_reason = 7; |
Alexander Graf | 0240755 | 2014-06-11 10:34:19 +0200 | [diff] [blame] | 642 | dt->processor_id = cpu_to_be16(vc->pcpu + vcpu->arch.ptid); |
| 643 | dt->timebase = cpu_to_be64(now + vc->tb_offset); |
| 644 | dt->enqueue_to_dispatch_time = cpu_to_be32(stolen); |
| 645 | dt->srr0 = cpu_to_be64(kvmppc_get_pc(vcpu)); |
| 646 | dt->srr1 = cpu_to_be64(vcpu->arch.shregs.msr); |
Paul Mackerras | 0456ec4 | 2012-02-03 00:56:21 +0000 | [diff] [blame] | 647 | ++dt; |
| 648 | if (dt == vcpu->arch.dtl.pinned_end) |
| 649 | dt = vcpu->arch.dtl.pinned_addr; |
| 650 | vcpu->arch.dtl_ptr = dt; |
| 651 | /* order writing *dt vs. writing vpa->dtl_idx */ |
| 652 | smp_wmb(); |
Alexander Graf | 0240755 | 2014-06-11 10:34:19 +0200 | [diff] [blame] | 653 | vpa->dtl_idx = cpu_to_be64(++vcpu->arch.dtl_index); |
Paul Mackerras | c35635e | 2013-04-18 19:51:04 +0000 | [diff] [blame] | 654 | vcpu->arch.dtl.dirty = true; |
Paul Mackerras | a8606e2 | 2011-06-29 00:22:05 +0000 | [diff] [blame] | 655 | } |
| 656 | |
Michael Neuling | 9642382 | 2014-06-02 11:03:01 +1000 | [diff] [blame] | 657 | static bool kvmppc_power8_compatible(struct kvm_vcpu *vcpu) |
| 658 | { |
| 659 | if (vcpu->arch.vcore->arch_compat >= PVR_ARCH_207) |
| 660 | return true; |
| 661 | if ((!vcpu->arch.vcore->arch_compat) && |
| 662 | cpu_has_feature(CPU_FTR_ARCH_207S)) |
| 663 | return true; |
| 664 | return false; |
| 665 | } |
| 666 | |
| 667 | static int kvmppc_h_set_mode(struct kvm_vcpu *vcpu, unsigned long mflags, |
| 668 | unsigned long resource, unsigned long value1, |
| 669 | unsigned long value2) |
| 670 | { |
| 671 | switch (resource) { |
| 672 | case H_SET_MODE_RESOURCE_SET_CIABR: |
| 673 | if (!kvmppc_power8_compatible(vcpu)) |
| 674 | return H_P2; |
| 675 | if (value2) |
| 676 | return H_P4; |
| 677 | if (mflags) |
| 678 | return H_UNSUPPORTED_FLAG_START; |
| 679 | /* Guests can't breakpoint the hypervisor */ |
| 680 | if ((value1 & CIABR_PRIV) == CIABR_PRIV_HYPER) |
| 681 | return H_P3; |
| 682 | vcpu->arch.ciabr = value1; |
| 683 | return H_SUCCESS; |
| 684 | case H_SET_MODE_RESOURCE_SET_DAWR: |
| 685 | if (!kvmppc_power8_compatible(vcpu)) |
| 686 | return H_P2; |
| 687 | if (mflags) |
| 688 | return H_UNSUPPORTED_FLAG_START; |
| 689 | if (value2 & DABRX_HYP) |
| 690 | return H_P4; |
| 691 | vcpu->arch.dawr = value1; |
| 692 | vcpu->arch.dawrx = value2; |
| 693 | return H_SUCCESS; |
| 694 | default: |
| 695 | return H_TOO_HARD; |
| 696 | } |
| 697 | } |
| 698 | |
Sam Bobroff | 90fd09f | 2014-12-03 13:30:40 +1100 | [diff] [blame] | 699 | static int kvm_arch_vcpu_yield_to(struct kvm_vcpu *target) |
| 700 | { |
| 701 | struct kvmppc_vcore *vcore = target->arch.vcore; |
| 702 | |
| 703 | /* |
| 704 | * We expect to have been called by the real mode handler |
| 705 | * (kvmppc_rm_h_confer()) which would have directly returned |
| 706 | * H_SUCCESS if the source vcore wasn't idle (e.g. if it may |
| 707 | * have useful work to do and should not confer) so we don't |
| 708 | * recheck that here. |
| 709 | */ |
| 710 | |
| 711 | spin_lock(&vcore->lock); |
| 712 | if (target->arch.state == KVMPPC_VCPU_RUNNABLE && |
Paul Mackerras | ec25716 | 2015-06-24 21:18:03 +1000 | [diff] [blame] | 713 | vcore->vcore_state != VCORE_INACTIVE && |
| 714 | vcore->runner) |
Sam Bobroff | 90fd09f | 2014-12-03 13:30:40 +1100 | [diff] [blame] | 715 | target = vcore->runner; |
| 716 | spin_unlock(&vcore->lock); |
| 717 | |
| 718 | return kvm_vcpu_yield_to(target); |
| 719 | } |
| 720 | |
| 721 | static int kvmppc_get_yield_count(struct kvm_vcpu *vcpu) |
| 722 | { |
| 723 | int yield_count = 0; |
| 724 | struct lppaca *lppaca; |
| 725 | |
| 726 | spin_lock(&vcpu->arch.vpa_update_lock); |
| 727 | lppaca = (struct lppaca *)vcpu->arch.vpa.pinned_addr; |
| 728 | if (lppaca) |
Paul Mackerras | ecb6d61 | 2015-03-20 20:39:39 +1100 | [diff] [blame] | 729 | yield_count = be32_to_cpu(lppaca->yield_count); |
Sam Bobroff | 90fd09f | 2014-12-03 13:30:40 +1100 | [diff] [blame] | 730 | spin_unlock(&vcpu->arch.vpa_update_lock); |
| 731 | return yield_count; |
| 732 | } |
| 733 | |
Paul Mackerras | a8606e2 | 2011-06-29 00:22:05 +0000 | [diff] [blame] | 734 | int kvmppc_pseries_do_hcall(struct kvm_vcpu *vcpu) |
| 735 | { |
| 736 | unsigned long req = kvmppc_get_gpr(vcpu, 3); |
| 737 | unsigned long target, ret = H_SUCCESS; |
Sam Bobroff | 90fd09f | 2014-12-03 13:30:40 +1100 | [diff] [blame] | 738 | int yield_count; |
Paul Mackerras | a8606e2 | 2011-06-29 00:22:05 +0000 | [diff] [blame] | 739 | struct kvm_vcpu *tvcpu; |
Michael Ellerman | 8e591cb | 2013-04-17 20:30:00 +0000 | [diff] [blame] | 740 | int idx, rc; |
Paul Mackerras | a8606e2 | 2011-06-29 00:22:05 +0000 | [diff] [blame] | 741 | |
Paul Mackerras | 699a0ea | 2014-06-02 11:02:59 +1000 | [diff] [blame] | 742 | if (req <= MAX_HCALL_OPCODE && |
| 743 | !test_bit(req/4, vcpu->kvm->arch.enabled_hcalls)) |
| 744 | return RESUME_HOST; |
| 745 | |
Paul Mackerras | a8606e2 | 2011-06-29 00:22:05 +0000 | [diff] [blame] | 746 | switch (req) { |
| 747 | case H_CEDE: |
Paul Mackerras | a8606e2 | 2011-06-29 00:22:05 +0000 | [diff] [blame] | 748 | break; |
| 749 | case H_PROD: |
| 750 | target = kvmppc_get_gpr(vcpu, 4); |
| 751 | tvcpu = kvmppc_find_vcpu(vcpu->kvm, target); |
| 752 | if (!tvcpu) { |
| 753 | ret = H_PARAMETER; |
| 754 | break; |
| 755 | } |
| 756 | tvcpu->arch.prodded = 1; |
| 757 | smp_mb(); |
| 758 | if (vcpu->arch.ceded) { |
Marcelo Tosatti | 8577370 | 2016-02-19 09:46:39 +0100 | [diff] [blame] | 759 | if (swait_active(&vcpu->wq)) { |
| 760 | swake_up(&vcpu->wq); |
Paul Mackerras | a8606e2 | 2011-06-29 00:22:05 +0000 | [diff] [blame] | 761 | vcpu->stat.halt_wakeup++; |
| 762 | } |
| 763 | } |
| 764 | break; |
| 765 | case H_CONFER: |
Paul Mackerras | 42d7604 | 2013-09-06 13:23:21 +1000 | [diff] [blame] | 766 | target = kvmppc_get_gpr(vcpu, 4); |
| 767 | if (target == -1) |
| 768 | break; |
| 769 | tvcpu = kvmppc_find_vcpu(vcpu->kvm, target); |
| 770 | if (!tvcpu) { |
| 771 | ret = H_PARAMETER; |
| 772 | break; |
| 773 | } |
Sam Bobroff | 90fd09f | 2014-12-03 13:30:40 +1100 | [diff] [blame] | 774 | yield_count = kvmppc_get_gpr(vcpu, 5); |
| 775 | if (kvmppc_get_yield_count(tvcpu) != yield_count) |
| 776 | break; |
| 777 | kvm_arch_vcpu_yield_to(tvcpu); |
Paul Mackerras | a8606e2 | 2011-06-29 00:22:05 +0000 | [diff] [blame] | 778 | break; |
| 779 | case H_REGISTER_VPA: |
| 780 | ret = do_h_register_vpa(vcpu, kvmppc_get_gpr(vcpu, 4), |
| 781 | kvmppc_get_gpr(vcpu, 5), |
| 782 | kvmppc_get_gpr(vcpu, 6)); |
| 783 | break; |
Michael Ellerman | 8e591cb | 2013-04-17 20:30:00 +0000 | [diff] [blame] | 784 | case H_RTAS: |
| 785 | if (list_empty(&vcpu->kvm->arch.rtas_tokens)) |
| 786 | return RESUME_HOST; |
| 787 | |
Paul Mackerras | c943809 | 2013-11-16 17:46:05 +1100 | [diff] [blame] | 788 | idx = srcu_read_lock(&vcpu->kvm->srcu); |
Michael Ellerman | 8e591cb | 2013-04-17 20:30:00 +0000 | [diff] [blame] | 789 | rc = kvmppc_rtas_hcall(vcpu); |
Paul Mackerras | c943809 | 2013-11-16 17:46:05 +1100 | [diff] [blame] | 790 | srcu_read_unlock(&vcpu->kvm->srcu, idx); |
Michael Ellerman | 8e591cb | 2013-04-17 20:30:00 +0000 | [diff] [blame] | 791 | |
| 792 | if (rc == -ENOENT) |
| 793 | return RESUME_HOST; |
| 794 | else if (rc == 0) |
| 795 | break; |
| 796 | |
| 797 | /* Send the error out to userspace via KVM_RUN */ |
| 798 | return rc; |
David Gibson | 99342cf8 | 2015-02-05 11:53:25 +1100 | [diff] [blame] | 799 | case H_LOGICAL_CI_LOAD: |
| 800 | ret = kvmppc_h_logical_ci_load(vcpu); |
| 801 | if (ret == H_TOO_HARD) |
| 802 | return RESUME_HOST; |
| 803 | break; |
| 804 | case H_LOGICAL_CI_STORE: |
| 805 | ret = kvmppc_h_logical_ci_store(vcpu); |
| 806 | if (ret == H_TOO_HARD) |
| 807 | return RESUME_HOST; |
| 808 | break; |
Michael Neuling | 9642382 | 2014-06-02 11:03:01 +1000 | [diff] [blame] | 809 | case H_SET_MODE: |
| 810 | ret = kvmppc_h_set_mode(vcpu, kvmppc_get_gpr(vcpu, 4), |
| 811 | kvmppc_get_gpr(vcpu, 5), |
| 812 | kvmppc_get_gpr(vcpu, 6), |
| 813 | kvmppc_get_gpr(vcpu, 7)); |
| 814 | if (ret == H_TOO_HARD) |
| 815 | return RESUME_HOST; |
| 816 | break; |
Benjamin Herrenschmidt | bc5ad3f | 2013-04-17 20:30:26 +0000 | [diff] [blame] | 817 | case H_XIRR: |
| 818 | case H_CPPR: |
| 819 | case H_EOI: |
| 820 | case H_IPI: |
Paul Mackerras | 8e44ddc | 2013-05-23 15:42:21 +0000 | [diff] [blame] | 821 | case H_IPOLL: |
| 822 | case H_XIRR_X: |
Benjamin Herrenschmidt | bc5ad3f | 2013-04-17 20:30:26 +0000 | [diff] [blame] | 823 | if (kvmppc_xics_enabled(vcpu)) { |
| 824 | ret = kvmppc_xics_hcall(vcpu, req); |
| 825 | break; |
Alexey Kardashevskiy | d3695aa | 2016-02-15 12:55:09 +1100 | [diff] [blame] | 826 | } |
| 827 | return RESUME_HOST; |
| 828 | case H_PUT_TCE: |
| 829 | ret = kvmppc_h_put_tce(vcpu, kvmppc_get_gpr(vcpu, 4), |
| 830 | kvmppc_get_gpr(vcpu, 5), |
| 831 | kvmppc_get_gpr(vcpu, 6)); |
| 832 | if (ret == H_TOO_HARD) |
| 833 | return RESUME_HOST; |
| 834 | break; |
| 835 | case H_PUT_TCE_INDIRECT: |
| 836 | ret = kvmppc_h_put_tce_indirect(vcpu, kvmppc_get_gpr(vcpu, 4), |
| 837 | kvmppc_get_gpr(vcpu, 5), |
| 838 | kvmppc_get_gpr(vcpu, 6), |
| 839 | kvmppc_get_gpr(vcpu, 7)); |
| 840 | if (ret == H_TOO_HARD) |
| 841 | return RESUME_HOST; |
| 842 | break; |
| 843 | case H_STUFF_TCE: |
| 844 | ret = kvmppc_h_stuff_tce(vcpu, kvmppc_get_gpr(vcpu, 4), |
| 845 | kvmppc_get_gpr(vcpu, 5), |
| 846 | kvmppc_get_gpr(vcpu, 6), |
| 847 | kvmppc_get_gpr(vcpu, 7)); |
| 848 | if (ret == H_TOO_HARD) |
| 849 | return RESUME_HOST; |
| 850 | break; |
Paul Mackerras | a8606e2 | 2011-06-29 00:22:05 +0000 | [diff] [blame] | 851 | default: |
| 852 | return RESUME_HOST; |
| 853 | } |
| 854 | kvmppc_set_gpr(vcpu, 3, ret); |
| 855 | vcpu->arch.hcall_needed = 0; |
| 856 | return RESUME_GUEST; |
| 857 | } |
| 858 | |
Paul Mackerras | ae2113a | 2014-06-02 11:03:00 +1000 | [diff] [blame] | 859 | static int kvmppc_hcall_impl_hv(unsigned long cmd) |
| 860 | { |
| 861 | switch (cmd) { |
| 862 | case H_CEDE: |
| 863 | case H_PROD: |
| 864 | case H_CONFER: |
| 865 | case H_REGISTER_VPA: |
Michael Neuling | 9642382 | 2014-06-02 11:03:01 +1000 | [diff] [blame] | 866 | case H_SET_MODE: |
David Gibson | 99342cf8 | 2015-02-05 11:53:25 +1100 | [diff] [blame] | 867 | case H_LOGICAL_CI_LOAD: |
| 868 | case H_LOGICAL_CI_STORE: |
Paul Mackerras | ae2113a | 2014-06-02 11:03:00 +1000 | [diff] [blame] | 869 | #ifdef CONFIG_KVM_XICS |
| 870 | case H_XIRR: |
| 871 | case H_CPPR: |
| 872 | case H_EOI: |
| 873 | case H_IPI: |
| 874 | case H_IPOLL: |
| 875 | case H_XIRR_X: |
| 876 | #endif |
| 877 | return 1; |
| 878 | } |
| 879 | |
| 880 | /* See if it's in the real-mode table */ |
| 881 | return kvmppc_hcall_impl_hv_realmode(cmd); |
| 882 | } |
| 883 | |
Madhavan Srinivasan | a59c1d9 | 2014-09-09 22:37:35 +0530 | [diff] [blame] | 884 | static int kvmppc_emulate_debug_inst(struct kvm_run *run, |
| 885 | struct kvm_vcpu *vcpu) |
| 886 | { |
| 887 | u32 last_inst; |
| 888 | |
| 889 | if (kvmppc_get_last_inst(vcpu, INST_GENERIC, &last_inst) != |
| 890 | EMULATE_DONE) { |
| 891 | /* |
| 892 | * Fetch failed, so return to guest and |
| 893 | * try executing it again. |
| 894 | */ |
| 895 | return RESUME_GUEST; |
| 896 | } |
| 897 | |
| 898 | if (last_inst == KVMPPC_INST_SW_BREAKPOINT) { |
| 899 | run->exit_reason = KVM_EXIT_DEBUG; |
| 900 | run->debug.arch.address = kvmppc_get_pc(vcpu); |
| 901 | return RESUME_HOST; |
| 902 | } else { |
| 903 | kvmppc_core_queue_program(vcpu, SRR1_PROGILL); |
| 904 | return RESUME_GUEST; |
| 905 | } |
| 906 | } |
| 907 | |
Aneesh Kumar K.V | 3a167bea | 2013-10-07 22:17:53 +0530 | [diff] [blame] | 908 | static int kvmppc_handle_exit_hv(struct kvm_run *run, struct kvm_vcpu *vcpu, |
| 909 | struct task_struct *tsk) |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 910 | { |
| 911 | int r = RESUME_HOST; |
| 912 | |
| 913 | vcpu->stat.sum_exits++; |
| 914 | |
Paul Mackerras | 1c9e3d5 | 2015-11-12 16:43:48 +1100 | [diff] [blame] | 915 | /* |
| 916 | * This can happen if an interrupt occurs in the last stages |
| 917 | * of guest entry or the first stages of guest exit (i.e. after |
| 918 | * setting paca->kvm_hstate.in_guest to KVM_GUEST_MODE_GUEST_HV |
| 919 | * and before setting it to KVM_GUEST_MODE_HOST_HV). |
| 920 | * That can happen due to a bug, or due to a machine check |
| 921 | * occurring at just the wrong time. |
| 922 | */ |
| 923 | if (vcpu->arch.shregs.msr & MSR_HV) { |
| 924 | printk(KERN_EMERG "KVM trap in HV mode!\n"); |
| 925 | printk(KERN_EMERG "trap=0x%x | pc=0x%lx | msr=0x%llx\n", |
| 926 | vcpu->arch.trap, kvmppc_get_pc(vcpu), |
| 927 | vcpu->arch.shregs.msr); |
| 928 | kvmppc_dump_regs(vcpu); |
| 929 | run->exit_reason = KVM_EXIT_INTERNAL_ERROR; |
| 930 | run->hw.hardware_exit_reason = vcpu->arch.trap; |
| 931 | return RESUME_HOST; |
| 932 | } |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 933 | run->exit_reason = KVM_EXIT_UNKNOWN; |
| 934 | run->ready_for_interrupt_injection = 1; |
| 935 | switch (vcpu->arch.trap) { |
| 936 | /* We're good on these - the host merely wanted to get our attention */ |
| 937 | case BOOK3S_INTERRUPT_HV_DECREMENTER: |
| 938 | vcpu->stat.dec_exits++; |
| 939 | r = RESUME_GUEST; |
| 940 | break; |
| 941 | case BOOK3S_INTERRUPT_EXTERNAL: |
Paul Mackerras | 5d00f66 | 2014-01-08 21:25:28 +1100 | [diff] [blame] | 942 | case BOOK3S_INTERRUPT_H_DOORBELL: |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 943 | vcpu->stat.ext_intr_exits++; |
| 944 | r = RESUME_GUEST; |
| 945 | break; |
Mahesh Salgaonkar | dee6f24 | 2014-11-03 15:51:57 +1100 | [diff] [blame] | 946 | /* HMI is hypervisor interrupt and host has handled it. Resume guest.*/ |
| 947 | case BOOK3S_INTERRUPT_HMI: |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 948 | case BOOK3S_INTERRUPT_PERFMON: |
| 949 | r = RESUME_GUEST; |
| 950 | break; |
Paul Mackerras | b4072df | 2012-11-23 22:37:50 +0000 | [diff] [blame] | 951 | case BOOK3S_INTERRUPT_MACHINE_CHECK: |
| 952 | /* |
| 953 | * Deliver a machine check interrupt to the guest. |
| 954 | * We have to do this, even if the host has handled the |
| 955 | * machine check, because machine checks use SRR0/1 and |
| 956 | * the interrupt might have trashed guest state in them. |
| 957 | */ |
| 958 | kvmppc_book3s_queue_irqprio(vcpu, |
| 959 | BOOK3S_INTERRUPT_MACHINE_CHECK); |
| 960 | r = RESUME_GUEST; |
| 961 | break; |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 962 | case BOOK3S_INTERRUPT_PROGRAM: |
| 963 | { |
| 964 | ulong flags; |
| 965 | /* |
| 966 | * Normally program interrupts are delivered directly |
| 967 | * to the guest by the hardware, but we can get here |
| 968 | * as a result of a hypervisor emulation interrupt |
| 969 | * (e40) getting turned into a 700 by BML RTAS. |
| 970 | */ |
| 971 | flags = vcpu->arch.shregs.msr & 0x1f0000ull; |
| 972 | kvmppc_core_queue_program(vcpu, flags); |
| 973 | r = RESUME_GUEST; |
| 974 | break; |
| 975 | } |
| 976 | case BOOK3S_INTERRUPT_SYSCALL: |
| 977 | { |
| 978 | /* hcall - punt to userspace */ |
| 979 | int i; |
| 980 | |
Liu Ping Fan | 27025a6 | 2013-11-19 14:12:48 +0800 | [diff] [blame] | 981 | /* hypercall with MSR_PR has already been handled in rmode, |
| 982 | * and never reaches here. |
| 983 | */ |
| 984 | |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 985 | run->papr_hcall.nr = kvmppc_get_gpr(vcpu, 3); |
| 986 | for (i = 0; i < 9; ++i) |
| 987 | run->papr_hcall.args[i] = kvmppc_get_gpr(vcpu, 4 + i); |
| 988 | run->exit_reason = KVM_EXIT_PAPR_HCALL; |
| 989 | vcpu->arch.hcall_needed = 1; |
| 990 | r = RESUME_HOST; |
| 991 | break; |
| 992 | } |
| 993 | /* |
Paul Mackerras | 342d3db | 2011-12-12 12:38:05 +0000 | [diff] [blame] | 994 | * We get these next two if the guest accesses a page which it thinks |
| 995 | * it has mapped but which is not actually present, either because |
| 996 | * it is for an emulated I/O device or because the corresonding |
| 997 | * host page has been paged out. Any other HDSI/HISI interrupts |
| 998 | * have been handled already. |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 999 | */ |
| 1000 | case BOOK3S_INTERRUPT_H_DATA_STORAGE: |
Paul Mackerras | 913d3ff9a | 2012-10-15 01:16:48 +0000 | [diff] [blame] | 1001 | r = RESUME_PAGE_FAULT; |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 1002 | break; |
| 1003 | case BOOK3S_INTERRUPT_H_INST_STORAGE: |
Paul Mackerras | 913d3ff9a | 2012-10-15 01:16:48 +0000 | [diff] [blame] | 1004 | vcpu->arch.fault_dar = kvmppc_get_pc(vcpu); |
| 1005 | vcpu->arch.fault_dsisr = 0; |
| 1006 | r = RESUME_PAGE_FAULT; |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 1007 | break; |
| 1008 | /* |
| 1009 | * This occurs if the guest executes an illegal instruction. |
Madhavan Srinivasan | a59c1d9 | 2014-09-09 22:37:35 +0530 | [diff] [blame] | 1010 | * If the guest debug is disabled, generate a program interrupt |
| 1011 | * to the guest. If guest debug is enabled, we need to check |
| 1012 | * whether the instruction is a software breakpoint instruction. |
| 1013 | * Accordingly return to Guest or Host. |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 1014 | */ |
| 1015 | case BOOK3S_INTERRUPT_H_EMUL_ASSIST: |
Paul Mackerras | 4a157d6 | 2014-12-03 13:30:39 +1100 | [diff] [blame] | 1016 | if (vcpu->arch.emul_inst != KVM_INST_FETCH_FAILED) |
| 1017 | vcpu->arch.last_inst = kvmppc_need_byteswap(vcpu) ? |
| 1018 | swab32(vcpu->arch.emul_inst) : |
| 1019 | vcpu->arch.emul_inst; |
Madhavan Srinivasan | a59c1d9 | 2014-09-09 22:37:35 +0530 | [diff] [blame] | 1020 | if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP) { |
| 1021 | r = kvmppc_emulate_debug_inst(run, vcpu); |
| 1022 | } else { |
| 1023 | kvmppc_core_queue_program(vcpu, SRR1_PROGILL); |
| 1024 | r = RESUME_GUEST; |
| 1025 | } |
Michael Ellerman | bd3048b | 2014-01-08 21:25:23 +1100 | [diff] [blame] | 1026 | break; |
| 1027 | /* |
| 1028 | * This occurs if the guest (kernel or userspace), does something that |
| 1029 | * is prohibited by HFSCR. We just generate a program interrupt to |
| 1030 | * the guest. |
| 1031 | */ |
| 1032 | case BOOK3S_INTERRUPT_H_FAC_UNAVAIL: |
| 1033 | kvmppc_core_queue_program(vcpu, SRR1_PROGILL); |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 1034 | r = RESUME_GUEST; |
| 1035 | break; |
Suresh Warrier | f7af520 | 2016-08-19 15:35:52 +1000 | [diff] [blame] | 1036 | case BOOK3S_INTERRUPT_HV_RM_HARD: |
| 1037 | r = RESUME_PASSTHROUGH; |
| 1038 | break; |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 1039 | default: |
| 1040 | kvmppc_dump_regs(vcpu); |
| 1041 | printk(KERN_EMERG "trap=0x%x | pc=0x%lx | msr=0x%llx\n", |
| 1042 | vcpu->arch.trap, kvmppc_get_pc(vcpu), |
| 1043 | vcpu->arch.shregs.msr); |
Paul Mackerras | f3271d4 | 2013-09-20 14:52:41 +1000 | [diff] [blame] | 1044 | run->hw.hardware_exit_reason = vcpu->arch.trap; |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 1045 | r = RESUME_HOST; |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 1046 | break; |
| 1047 | } |
| 1048 | |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 1049 | return r; |
| 1050 | } |
| 1051 | |
Aneesh Kumar K.V | 3a167bea | 2013-10-07 22:17:53 +0530 | [diff] [blame] | 1052 | static int kvm_arch_vcpu_ioctl_get_sregs_hv(struct kvm_vcpu *vcpu, |
| 1053 | struct kvm_sregs *sregs) |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 1054 | { |
| 1055 | int i; |
| 1056 | |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 1057 | memset(sregs, 0, sizeof(struct kvm_sregs)); |
Aneesh Kumar K.V | 8791644 | 2013-08-22 17:08:39 +0530 | [diff] [blame] | 1058 | sregs->pvr = vcpu->arch.pvr; |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 1059 | for (i = 0; i < vcpu->arch.slb_max; i++) { |
| 1060 | sregs->u.s.ppc64.slb[i].slbe = vcpu->arch.slb[i].orige; |
| 1061 | sregs->u.s.ppc64.slb[i].slbv = vcpu->arch.slb[i].origv; |
| 1062 | } |
| 1063 | |
| 1064 | return 0; |
| 1065 | } |
| 1066 | |
Aneesh Kumar K.V | 3a167bea | 2013-10-07 22:17:53 +0530 | [diff] [blame] | 1067 | static int kvm_arch_vcpu_ioctl_set_sregs_hv(struct kvm_vcpu *vcpu, |
| 1068 | struct kvm_sregs *sregs) |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 1069 | { |
| 1070 | int i, j; |
| 1071 | |
Paul Mackerras | 9333e6c | 2014-09-02 16:14:43 +1000 | [diff] [blame] | 1072 | /* Only accept the same PVR as the host's, since we can't spoof it */ |
| 1073 | if (sregs->pvr != vcpu->arch.pvr) |
| 1074 | return -EINVAL; |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 1075 | |
| 1076 | j = 0; |
| 1077 | for (i = 0; i < vcpu->arch.slb_nr; i++) { |
| 1078 | if (sregs->u.s.ppc64.slb[i].slbe & SLB_ESID_V) { |
| 1079 | vcpu->arch.slb[j].orige = sregs->u.s.ppc64.slb[i].slbe; |
| 1080 | vcpu->arch.slb[j].origv = sregs->u.s.ppc64.slb[i].slbv; |
| 1081 | ++j; |
| 1082 | } |
| 1083 | } |
| 1084 | vcpu->arch.slb_max = j; |
| 1085 | |
| 1086 | return 0; |
| 1087 | } |
| 1088 | |
Alexey Kardashevskiy | a084024 | 2014-07-19 17:59:34 +1000 | [diff] [blame] | 1089 | static void kvmppc_set_lpcr(struct kvm_vcpu *vcpu, u64 new_lpcr, |
| 1090 | bool preserve_top32) |
Paul Mackerras | a0144e2 | 2013-09-20 14:52:38 +1000 | [diff] [blame] | 1091 | { |
Paul Mackerras | 8f902b0 | 2015-03-20 20:39:38 +1100 | [diff] [blame] | 1092 | struct kvm *kvm = vcpu->kvm; |
Paul Mackerras | a0144e2 | 2013-09-20 14:52:38 +1000 | [diff] [blame] | 1093 | struct kvmppc_vcore *vc = vcpu->arch.vcore; |
| 1094 | u64 mask; |
| 1095 | |
| 1096 | spin_lock(&vc->lock); |
| 1097 | /* |
Anton Blanchard | d682916 | 2014-01-08 21:25:30 +1100 | [diff] [blame] | 1098 | * If ILE (interrupt little-endian) has changed, update the |
| 1099 | * MSR_LE bit in the intr_msr for each vcpu in this vcore. |
| 1100 | */ |
| 1101 | if ((new_lpcr & LPCR_ILE) != (vc->lpcr & LPCR_ILE)) { |
Anton Blanchard | d682916 | 2014-01-08 21:25:30 +1100 | [diff] [blame] | 1102 | struct kvm_vcpu *vcpu; |
| 1103 | int i; |
| 1104 | |
Anton Blanchard | d682916 | 2014-01-08 21:25:30 +1100 | [diff] [blame] | 1105 | kvm_for_each_vcpu(i, vcpu, kvm) { |
| 1106 | if (vcpu->arch.vcore != vc) |
| 1107 | continue; |
| 1108 | if (new_lpcr & LPCR_ILE) |
| 1109 | vcpu->arch.intr_msr |= MSR_LE; |
| 1110 | else |
| 1111 | vcpu->arch.intr_msr &= ~MSR_LE; |
| 1112 | } |
Anton Blanchard | d682916 | 2014-01-08 21:25:30 +1100 | [diff] [blame] | 1113 | } |
| 1114 | |
| 1115 | /* |
Paul Mackerras | a0144e2 | 2013-09-20 14:52:38 +1000 | [diff] [blame] | 1116 | * Userspace can only modify DPFD (default prefetch depth), |
| 1117 | * ILE (interrupt little-endian) and TC (translation control). |
Paul Mackerras | e0622bd | 2014-01-08 21:25:27 +1100 | [diff] [blame] | 1118 | * On POWER8 userspace can also modify AIL (alt. interrupt loc.) |
Paul Mackerras | a0144e2 | 2013-09-20 14:52:38 +1000 | [diff] [blame] | 1119 | */ |
| 1120 | mask = LPCR_DPFD | LPCR_ILE | LPCR_TC; |
Paul Mackerras | e0622bd | 2014-01-08 21:25:27 +1100 | [diff] [blame] | 1121 | if (cpu_has_feature(CPU_FTR_ARCH_207S)) |
| 1122 | mask |= LPCR_AIL; |
Alexey Kardashevskiy | a084024 | 2014-07-19 17:59:34 +1000 | [diff] [blame] | 1123 | |
| 1124 | /* Broken 32-bit version of LPCR must not clear top bits */ |
| 1125 | if (preserve_top32) |
| 1126 | mask &= 0xFFFFFFFF; |
Paul Mackerras | a0144e2 | 2013-09-20 14:52:38 +1000 | [diff] [blame] | 1127 | vc->lpcr = (vc->lpcr & ~mask) | (new_lpcr & mask); |
| 1128 | spin_unlock(&vc->lock); |
| 1129 | } |
| 1130 | |
Aneesh Kumar K.V | 3a167bea | 2013-10-07 22:17:53 +0530 | [diff] [blame] | 1131 | static int kvmppc_get_one_reg_hv(struct kvm_vcpu *vcpu, u64 id, |
| 1132 | union kvmppc_one_reg *val) |
Paul Mackerras | 31f3438 | 2011-12-12 12:26:50 +0000 | [diff] [blame] | 1133 | { |
Paul Mackerras | a136a8b | 2012-09-25 20:31:56 +0000 | [diff] [blame] | 1134 | int r = 0; |
| 1135 | long int i; |
Paul Mackerras | 31f3438 | 2011-12-12 12:26:50 +0000 | [diff] [blame] | 1136 | |
Paul Mackerras | a136a8b | 2012-09-25 20:31:56 +0000 | [diff] [blame] | 1137 | switch (id) { |
Madhavan Srinivasan | a59c1d9 | 2014-09-09 22:37:35 +0530 | [diff] [blame] | 1138 | case KVM_REG_PPC_DEBUG_INST: |
| 1139 | *val = get_reg_val(id, KVMPPC_INST_SW_BREAKPOINT); |
| 1140 | break; |
Paul Mackerras | 31f3438 | 2011-12-12 12:26:50 +0000 | [diff] [blame] | 1141 | case KVM_REG_PPC_HIOR: |
Paul Mackerras | a136a8b | 2012-09-25 20:31:56 +0000 | [diff] [blame] | 1142 | *val = get_reg_val(id, 0); |
| 1143 | break; |
| 1144 | case KVM_REG_PPC_DABR: |
| 1145 | *val = get_reg_val(id, vcpu->arch.dabr); |
| 1146 | break; |
Paul Mackerras | 8563bf5 | 2014-01-08 21:25:29 +1100 | [diff] [blame] | 1147 | case KVM_REG_PPC_DABRX: |
| 1148 | *val = get_reg_val(id, vcpu->arch.dabrx); |
| 1149 | break; |
Paul Mackerras | a136a8b | 2012-09-25 20:31:56 +0000 | [diff] [blame] | 1150 | case KVM_REG_PPC_DSCR: |
| 1151 | *val = get_reg_val(id, vcpu->arch.dscr); |
| 1152 | break; |
| 1153 | case KVM_REG_PPC_PURR: |
| 1154 | *val = get_reg_val(id, vcpu->arch.purr); |
| 1155 | break; |
| 1156 | case KVM_REG_PPC_SPURR: |
| 1157 | *val = get_reg_val(id, vcpu->arch.spurr); |
| 1158 | break; |
| 1159 | case KVM_REG_PPC_AMR: |
| 1160 | *val = get_reg_val(id, vcpu->arch.amr); |
| 1161 | break; |
| 1162 | case KVM_REG_PPC_UAMOR: |
| 1163 | *val = get_reg_val(id, vcpu->arch.uamor); |
| 1164 | break; |
Michael Neuling | b005255e | 2014-01-08 21:25:21 +1100 | [diff] [blame] | 1165 | case KVM_REG_PPC_MMCR0 ... KVM_REG_PPC_MMCRS: |
Paul Mackerras | a136a8b | 2012-09-25 20:31:56 +0000 | [diff] [blame] | 1166 | i = id - KVM_REG_PPC_MMCR0; |
| 1167 | *val = get_reg_val(id, vcpu->arch.mmcr[i]); |
| 1168 | break; |
| 1169 | case KVM_REG_PPC_PMC1 ... KVM_REG_PPC_PMC8: |
| 1170 | i = id - KVM_REG_PPC_PMC1; |
| 1171 | *val = get_reg_val(id, vcpu->arch.pmc[i]); |
Paul Mackerras | 31f3438 | 2011-12-12 12:26:50 +0000 | [diff] [blame] | 1172 | break; |
Michael Neuling | b005255e | 2014-01-08 21:25:21 +1100 | [diff] [blame] | 1173 | case KVM_REG_PPC_SPMC1 ... KVM_REG_PPC_SPMC2: |
| 1174 | i = id - KVM_REG_PPC_SPMC1; |
| 1175 | *val = get_reg_val(id, vcpu->arch.spmc[i]); |
| 1176 | break; |
Paul Mackerras | 1494178 | 2013-09-06 13:11:18 +1000 | [diff] [blame] | 1177 | case KVM_REG_PPC_SIAR: |
| 1178 | *val = get_reg_val(id, vcpu->arch.siar); |
| 1179 | break; |
| 1180 | case KVM_REG_PPC_SDAR: |
| 1181 | *val = get_reg_val(id, vcpu->arch.sdar); |
| 1182 | break; |
Michael Neuling | b005255e | 2014-01-08 21:25:21 +1100 | [diff] [blame] | 1183 | case KVM_REG_PPC_SIER: |
| 1184 | *val = get_reg_val(id, vcpu->arch.sier); |
Paul Mackerras | a8bd19e | 2012-09-25 20:32:30 +0000 | [diff] [blame] | 1185 | break; |
Michael Neuling | b005255e | 2014-01-08 21:25:21 +1100 | [diff] [blame] | 1186 | case KVM_REG_PPC_IAMR: |
| 1187 | *val = get_reg_val(id, vcpu->arch.iamr); |
Paul Mackerras | a8bd19e | 2012-09-25 20:32:30 +0000 | [diff] [blame] | 1188 | break; |
Michael Neuling | b005255e | 2014-01-08 21:25:21 +1100 | [diff] [blame] | 1189 | case KVM_REG_PPC_PSPB: |
| 1190 | *val = get_reg_val(id, vcpu->arch.pspb); |
| 1191 | break; |
Michael Neuling | b005255e | 2014-01-08 21:25:21 +1100 | [diff] [blame] | 1192 | case KVM_REG_PPC_DPDES: |
| 1193 | *val = get_reg_val(id, vcpu->arch.vcore->dpdes); |
| 1194 | break; |
Paul Mackerras | 88b02cf9 | 2016-09-15 13:42:52 +1000 | [diff] [blame] | 1195 | case KVM_REG_PPC_VTB: |
| 1196 | *val = get_reg_val(id, vcpu->arch.vcore->vtb); |
| 1197 | break; |
Michael Neuling | b005255e | 2014-01-08 21:25:21 +1100 | [diff] [blame] | 1198 | case KVM_REG_PPC_DAWR: |
| 1199 | *val = get_reg_val(id, vcpu->arch.dawr); |
| 1200 | break; |
| 1201 | case KVM_REG_PPC_DAWRX: |
| 1202 | *val = get_reg_val(id, vcpu->arch.dawrx); |
| 1203 | break; |
| 1204 | case KVM_REG_PPC_CIABR: |
| 1205 | *val = get_reg_val(id, vcpu->arch.ciabr); |
| 1206 | break; |
Michael Neuling | b005255e | 2014-01-08 21:25:21 +1100 | [diff] [blame] | 1207 | case KVM_REG_PPC_CSIGR: |
| 1208 | *val = get_reg_val(id, vcpu->arch.csigr); |
| 1209 | break; |
| 1210 | case KVM_REG_PPC_TACR: |
| 1211 | *val = get_reg_val(id, vcpu->arch.tacr); |
| 1212 | break; |
| 1213 | case KVM_REG_PPC_TCSCR: |
| 1214 | *val = get_reg_val(id, vcpu->arch.tcscr); |
| 1215 | break; |
| 1216 | case KVM_REG_PPC_PID: |
| 1217 | *val = get_reg_val(id, vcpu->arch.pid); |
| 1218 | break; |
| 1219 | case KVM_REG_PPC_ACOP: |
| 1220 | *val = get_reg_val(id, vcpu->arch.acop); |
| 1221 | break; |
| 1222 | case KVM_REG_PPC_WORT: |
| 1223 | *val = get_reg_val(id, vcpu->arch.wort); |
| 1224 | break; |
Paul Mackerras | 55b665b | 2012-09-25 20:33:06 +0000 | [diff] [blame] | 1225 | case KVM_REG_PPC_VPA_ADDR: |
| 1226 | spin_lock(&vcpu->arch.vpa_update_lock); |
| 1227 | *val = get_reg_val(id, vcpu->arch.vpa.next_gpa); |
| 1228 | spin_unlock(&vcpu->arch.vpa_update_lock); |
| 1229 | break; |
| 1230 | case KVM_REG_PPC_VPA_SLB: |
| 1231 | spin_lock(&vcpu->arch.vpa_update_lock); |
| 1232 | val->vpaval.addr = vcpu->arch.slb_shadow.next_gpa; |
| 1233 | val->vpaval.length = vcpu->arch.slb_shadow.len; |
| 1234 | spin_unlock(&vcpu->arch.vpa_update_lock); |
| 1235 | break; |
| 1236 | case KVM_REG_PPC_VPA_DTL: |
| 1237 | spin_lock(&vcpu->arch.vpa_update_lock); |
| 1238 | val->vpaval.addr = vcpu->arch.dtl.next_gpa; |
| 1239 | val->vpaval.length = vcpu->arch.dtl.len; |
| 1240 | spin_unlock(&vcpu->arch.vpa_update_lock); |
| 1241 | break; |
Paul Mackerras | 93b0f4d | 2013-09-06 13:17:46 +1000 | [diff] [blame] | 1242 | case KVM_REG_PPC_TB_OFFSET: |
| 1243 | *val = get_reg_val(id, vcpu->arch.vcore->tb_offset); |
| 1244 | break; |
Paul Mackerras | a0144e2 | 2013-09-20 14:52:38 +1000 | [diff] [blame] | 1245 | case KVM_REG_PPC_LPCR: |
Alexey Kardashevskiy | a084024 | 2014-07-19 17:59:34 +1000 | [diff] [blame] | 1246 | case KVM_REG_PPC_LPCR_64: |
Paul Mackerras | a0144e2 | 2013-09-20 14:52:38 +1000 | [diff] [blame] | 1247 | *val = get_reg_val(id, vcpu->arch.vcore->lpcr); |
| 1248 | break; |
Paul Mackerras | 4b8473c | 2013-09-20 14:52:39 +1000 | [diff] [blame] | 1249 | case KVM_REG_PPC_PPR: |
| 1250 | *val = get_reg_val(id, vcpu->arch.ppr); |
| 1251 | break; |
Michael Neuling | a7d80d0 | 2014-03-25 10:47:03 +1100 | [diff] [blame] | 1252 | #ifdef CONFIG_PPC_TRANSACTIONAL_MEM |
| 1253 | case KVM_REG_PPC_TFHAR: |
| 1254 | *val = get_reg_val(id, vcpu->arch.tfhar); |
| 1255 | break; |
| 1256 | case KVM_REG_PPC_TFIAR: |
| 1257 | *val = get_reg_val(id, vcpu->arch.tfiar); |
| 1258 | break; |
| 1259 | case KVM_REG_PPC_TEXASR: |
| 1260 | *val = get_reg_val(id, vcpu->arch.texasr); |
| 1261 | break; |
| 1262 | case KVM_REG_PPC_TM_GPR0 ... KVM_REG_PPC_TM_GPR31: |
| 1263 | i = id - KVM_REG_PPC_TM_GPR0; |
| 1264 | *val = get_reg_val(id, vcpu->arch.gpr_tm[i]); |
| 1265 | break; |
| 1266 | case KVM_REG_PPC_TM_VSR0 ... KVM_REG_PPC_TM_VSR63: |
| 1267 | { |
| 1268 | int j; |
| 1269 | i = id - KVM_REG_PPC_TM_VSR0; |
| 1270 | if (i < 32) |
| 1271 | for (j = 0; j < TS_FPRWIDTH; j++) |
| 1272 | val->vsxval[j] = vcpu->arch.fp_tm.fpr[i][j]; |
| 1273 | else { |
| 1274 | if (cpu_has_feature(CPU_FTR_ALTIVEC)) |
| 1275 | val->vval = vcpu->arch.vr_tm.vr[i-32]; |
| 1276 | else |
| 1277 | r = -ENXIO; |
| 1278 | } |
| 1279 | break; |
| 1280 | } |
| 1281 | case KVM_REG_PPC_TM_CR: |
| 1282 | *val = get_reg_val(id, vcpu->arch.cr_tm); |
| 1283 | break; |
Paul Mackerras | 75b1053 | 2016-11-07 15:09:58 +1100 | [diff] [blame] | 1284 | case KVM_REG_PPC_TM_XER: |
| 1285 | *val = get_reg_val(id, vcpu->arch.xer_tm); |
| 1286 | break; |
Michael Neuling | a7d80d0 | 2014-03-25 10:47:03 +1100 | [diff] [blame] | 1287 | case KVM_REG_PPC_TM_LR: |
| 1288 | *val = get_reg_val(id, vcpu->arch.lr_tm); |
| 1289 | break; |
| 1290 | case KVM_REG_PPC_TM_CTR: |
| 1291 | *val = get_reg_val(id, vcpu->arch.ctr_tm); |
| 1292 | break; |
| 1293 | case KVM_REG_PPC_TM_FPSCR: |
| 1294 | *val = get_reg_val(id, vcpu->arch.fp_tm.fpscr); |
| 1295 | break; |
| 1296 | case KVM_REG_PPC_TM_AMR: |
| 1297 | *val = get_reg_val(id, vcpu->arch.amr_tm); |
| 1298 | break; |
| 1299 | case KVM_REG_PPC_TM_PPR: |
| 1300 | *val = get_reg_val(id, vcpu->arch.ppr_tm); |
| 1301 | break; |
| 1302 | case KVM_REG_PPC_TM_VRSAVE: |
| 1303 | *val = get_reg_val(id, vcpu->arch.vrsave_tm); |
| 1304 | break; |
| 1305 | case KVM_REG_PPC_TM_VSCR: |
| 1306 | if (cpu_has_feature(CPU_FTR_ALTIVEC)) |
| 1307 | *val = get_reg_val(id, vcpu->arch.vr_tm.vscr.u[3]); |
| 1308 | else |
| 1309 | r = -ENXIO; |
| 1310 | break; |
| 1311 | case KVM_REG_PPC_TM_DSCR: |
| 1312 | *val = get_reg_val(id, vcpu->arch.dscr_tm); |
| 1313 | break; |
| 1314 | case KVM_REG_PPC_TM_TAR: |
| 1315 | *val = get_reg_val(id, vcpu->arch.tar_tm); |
| 1316 | break; |
| 1317 | #endif |
Paul Mackerras | 388cc6e | 2013-09-21 14:35:02 +1000 | [diff] [blame] | 1318 | case KVM_REG_PPC_ARCH_COMPAT: |
| 1319 | *val = get_reg_val(id, vcpu->arch.vcore->arch_compat); |
| 1320 | break; |
Paul Mackerras | 31f3438 | 2011-12-12 12:26:50 +0000 | [diff] [blame] | 1321 | default: |
Paul Mackerras | a136a8b | 2012-09-25 20:31:56 +0000 | [diff] [blame] | 1322 | r = -EINVAL; |
Paul Mackerras | 31f3438 | 2011-12-12 12:26:50 +0000 | [diff] [blame] | 1323 | break; |
| 1324 | } |
| 1325 | |
| 1326 | return r; |
| 1327 | } |
| 1328 | |
Aneesh Kumar K.V | 3a167bea | 2013-10-07 22:17:53 +0530 | [diff] [blame] | 1329 | static int kvmppc_set_one_reg_hv(struct kvm_vcpu *vcpu, u64 id, |
| 1330 | union kvmppc_one_reg *val) |
Paul Mackerras | 31f3438 | 2011-12-12 12:26:50 +0000 | [diff] [blame] | 1331 | { |
Paul Mackerras | a136a8b | 2012-09-25 20:31:56 +0000 | [diff] [blame] | 1332 | int r = 0; |
| 1333 | long int i; |
Paul Mackerras | 55b665b | 2012-09-25 20:33:06 +0000 | [diff] [blame] | 1334 | unsigned long addr, len; |
Paul Mackerras | 31f3438 | 2011-12-12 12:26:50 +0000 | [diff] [blame] | 1335 | |
Paul Mackerras | a136a8b | 2012-09-25 20:31:56 +0000 | [diff] [blame] | 1336 | switch (id) { |
Paul Mackerras | 31f3438 | 2011-12-12 12:26:50 +0000 | [diff] [blame] | 1337 | case KVM_REG_PPC_HIOR: |
Paul Mackerras | 31f3438 | 2011-12-12 12:26:50 +0000 | [diff] [blame] | 1338 | /* Only allow this to be set to zero */ |
Paul Mackerras | a136a8b | 2012-09-25 20:31:56 +0000 | [diff] [blame] | 1339 | if (set_reg_val(id, *val)) |
Paul Mackerras | 31f3438 | 2011-12-12 12:26:50 +0000 | [diff] [blame] | 1340 | r = -EINVAL; |
| 1341 | break; |
Paul Mackerras | a136a8b | 2012-09-25 20:31:56 +0000 | [diff] [blame] | 1342 | case KVM_REG_PPC_DABR: |
| 1343 | vcpu->arch.dabr = set_reg_val(id, *val); |
| 1344 | break; |
Paul Mackerras | 8563bf5 | 2014-01-08 21:25:29 +1100 | [diff] [blame] | 1345 | case KVM_REG_PPC_DABRX: |
| 1346 | vcpu->arch.dabrx = set_reg_val(id, *val) & ~DABRX_HYP; |
| 1347 | break; |
Paul Mackerras | a136a8b | 2012-09-25 20:31:56 +0000 | [diff] [blame] | 1348 | case KVM_REG_PPC_DSCR: |
| 1349 | vcpu->arch.dscr = set_reg_val(id, *val); |
| 1350 | break; |
| 1351 | case KVM_REG_PPC_PURR: |
| 1352 | vcpu->arch.purr = set_reg_val(id, *val); |
| 1353 | break; |
| 1354 | case KVM_REG_PPC_SPURR: |
| 1355 | vcpu->arch.spurr = set_reg_val(id, *val); |
| 1356 | break; |
| 1357 | case KVM_REG_PPC_AMR: |
| 1358 | vcpu->arch.amr = set_reg_val(id, *val); |
| 1359 | break; |
| 1360 | case KVM_REG_PPC_UAMOR: |
| 1361 | vcpu->arch.uamor = set_reg_val(id, *val); |
| 1362 | break; |
Michael Neuling | b005255e | 2014-01-08 21:25:21 +1100 | [diff] [blame] | 1363 | case KVM_REG_PPC_MMCR0 ... KVM_REG_PPC_MMCRS: |
Paul Mackerras | a136a8b | 2012-09-25 20:31:56 +0000 | [diff] [blame] | 1364 | i = id - KVM_REG_PPC_MMCR0; |
| 1365 | vcpu->arch.mmcr[i] = set_reg_val(id, *val); |
| 1366 | break; |
| 1367 | case KVM_REG_PPC_PMC1 ... KVM_REG_PPC_PMC8: |
| 1368 | i = id - KVM_REG_PPC_PMC1; |
| 1369 | vcpu->arch.pmc[i] = set_reg_val(id, *val); |
| 1370 | break; |
Michael Neuling | b005255e | 2014-01-08 21:25:21 +1100 | [diff] [blame] | 1371 | case KVM_REG_PPC_SPMC1 ... KVM_REG_PPC_SPMC2: |
| 1372 | i = id - KVM_REG_PPC_SPMC1; |
| 1373 | vcpu->arch.spmc[i] = set_reg_val(id, *val); |
| 1374 | break; |
Paul Mackerras | 1494178 | 2013-09-06 13:11:18 +1000 | [diff] [blame] | 1375 | case KVM_REG_PPC_SIAR: |
| 1376 | vcpu->arch.siar = set_reg_val(id, *val); |
| 1377 | break; |
| 1378 | case KVM_REG_PPC_SDAR: |
| 1379 | vcpu->arch.sdar = set_reg_val(id, *val); |
| 1380 | break; |
Michael Neuling | b005255e | 2014-01-08 21:25:21 +1100 | [diff] [blame] | 1381 | case KVM_REG_PPC_SIER: |
| 1382 | vcpu->arch.sier = set_reg_val(id, *val); |
Paul Mackerras | a8bd19e | 2012-09-25 20:32:30 +0000 | [diff] [blame] | 1383 | break; |
Michael Neuling | b005255e | 2014-01-08 21:25:21 +1100 | [diff] [blame] | 1384 | case KVM_REG_PPC_IAMR: |
| 1385 | vcpu->arch.iamr = set_reg_val(id, *val); |
Paul Mackerras | a8bd19e | 2012-09-25 20:32:30 +0000 | [diff] [blame] | 1386 | break; |
Michael Neuling | b005255e | 2014-01-08 21:25:21 +1100 | [diff] [blame] | 1387 | case KVM_REG_PPC_PSPB: |
| 1388 | vcpu->arch.pspb = set_reg_val(id, *val); |
| 1389 | break; |
Michael Neuling | b005255e | 2014-01-08 21:25:21 +1100 | [diff] [blame] | 1390 | case KVM_REG_PPC_DPDES: |
| 1391 | vcpu->arch.vcore->dpdes = set_reg_val(id, *val); |
| 1392 | break; |
Paul Mackerras | 88b02cf9 | 2016-09-15 13:42:52 +1000 | [diff] [blame] | 1393 | case KVM_REG_PPC_VTB: |
| 1394 | vcpu->arch.vcore->vtb = set_reg_val(id, *val); |
| 1395 | break; |
Michael Neuling | b005255e | 2014-01-08 21:25:21 +1100 | [diff] [blame] | 1396 | case KVM_REG_PPC_DAWR: |
| 1397 | vcpu->arch.dawr = set_reg_val(id, *val); |
| 1398 | break; |
| 1399 | case KVM_REG_PPC_DAWRX: |
| 1400 | vcpu->arch.dawrx = set_reg_val(id, *val) & ~DAWRX_HYP; |
| 1401 | break; |
| 1402 | case KVM_REG_PPC_CIABR: |
| 1403 | vcpu->arch.ciabr = set_reg_val(id, *val); |
| 1404 | /* Don't allow setting breakpoints in hypervisor code */ |
| 1405 | if ((vcpu->arch.ciabr & CIABR_PRIV) == CIABR_PRIV_HYPER) |
| 1406 | vcpu->arch.ciabr &= ~CIABR_PRIV; /* disable */ |
| 1407 | break; |
Michael Neuling | b005255e | 2014-01-08 21:25:21 +1100 | [diff] [blame] | 1408 | case KVM_REG_PPC_CSIGR: |
| 1409 | vcpu->arch.csigr = set_reg_val(id, *val); |
| 1410 | break; |
| 1411 | case KVM_REG_PPC_TACR: |
| 1412 | vcpu->arch.tacr = set_reg_val(id, *val); |
| 1413 | break; |
| 1414 | case KVM_REG_PPC_TCSCR: |
| 1415 | vcpu->arch.tcscr = set_reg_val(id, *val); |
| 1416 | break; |
| 1417 | case KVM_REG_PPC_PID: |
| 1418 | vcpu->arch.pid = set_reg_val(id, *val); |
| 1419 | break; |
| 1420 | case KVM_REG_PPC_ACOP: |
| 1421 | vcpu->arch.acop = set_reg_val(id, *val); |
| 1422 | break; |
| 1423 | case KVM_REG_PPC_WORT: |
| 1424 | vcpu->arch.wort = set_reg_val(id, *val); |
| 1425 | break; |
Paul Mackerras | 55b665b | 2012-09-25 20:33:06 +0000 | [diff] [blame] | 1426 | case KVM_REG_PPC_VPA_ADDR: |
| 1427 | addr = set_reg_val(id, *val); |
| 1428 | r = -EINVAL; |
| 1429 | if (!addr && (vcpu->arch.slb_shadow.next_gpa || |
| 1430 | vcpu->arch.dtl.next_gpa)) |
| 1431 | break; |
| 1432 | r = set_vpa(vcpu, &vcpu->arch.vpa, addr, sizeof(struct lppaca)); |
| 1433 | break; |
| 1434 | case KVM_REG_PPC_VPA_SLB: |
| 1435 | addr = val->vpaval.addr; |
| 1436 | len = val->vpaval.length; |
| 1437 | r = -EINVAL; |
| 1438 | if (addr && !vcpu->arch.vpa.next_gpa) |
| 1439 | break; |
| 1440 | r = set_vpa(vcpu, &vcpu->arch.slb_shadow, addr, len); |
| 1441 | break; |
| 1442 | case KVM_REG_PPC_VPA_DTL: |
| 1443 | addr = val->vpaval.addr; |
| 1444 | len = val->vpaval.length; |
| 1445 | r = -EINVAL; |
Paul Mackerras | 9f8c8c7 | 2012-10-15 01:18:37 +0000 | [diff] [blame] | 1446 | if (addr && (len < sizeof(struct dtl_entry) || |
| 1447 | !vcpu->arch.vpa.next_gpa)) |
Paul Mackerras | 55b665b | 2012-09-25 20:33:06 +0000 | [diff] [blame] | 1448 | break; |
| 1449 | len -= len % sizeof(struct dtl_entry); |
| 1450 | r = set_vpa(vcpu, &vcpu->arch.dtl, addr, len); |
| 1451 | break; |
Paul Mackerras | 93b0f4d | 2013-09-06 13:17:46 +1000 | [diff] [blame] | 1452 | case KVM_REG_PPC_TB_OFFSET: |
| 1453 | /* round up to multiple of 2^24 */ |
| 1454 | vcpu->arch.vcore->tb_offset = |
| 1455 | ALIGN(set_reg_val(id, *val), 1UL << 24); |
| 1456 | break; |
Paul Mackerras | a0144e2 | 2013-09-20 14:52:38 +1000 | [diff] [blame] | 1457 | case KVM_REG_PPC_LPCR: |
Alexey Kardashevskiy | a084024 | 2014-07-19 17:59:34 +1000 | [diff] [blame] | 1458 | kvmppc_set_lpcr(vcpu, set_reg_val(id, *val), true); |
| 1459 | break; |
| 1460 | case KVM_REG_PPC_LPCR_64: |
| 1461 | kvmppc_set_lpcr(vcpu, set_reg_val(id, *val), false); |
Paul Mackerras | a0144e2 | 2013-09-20 14:52:38 +1000 | [diff] [blame] | 1462 | break; |
Paul Mackerras | 4b8473c | 2013-09-20 14:52:39 +1000 | [diff] [blame] | 1463 | case KVM_REG_PPC_PPR: |
| 1464 | vcpu->arch.ppr = set_reg_val(id, *val); |
| 1465 | break; |
Michael Neuling | a7d80d0 | 2014-03-25 10:47:03 +1100 | [diff] [blame] | 1466 | #ifdef CONFIG_PPC_TRANSACTIONAL_MEM |
| 1467 | case KVM_REG_PPC_TFHAR: |
| 1468 | vcpu->arch.tfhar = set_reg_val(id, *val); |
| 1469 | break; |
| 1470 | case KVM_REG_PPC_TFIAR: |
| 1471 | vcpu->arch.tfiar = set_reg_val(id, *val); |
| 1472 | break; |
| 1473 | case KVM_REG_PPC_TEXASR: |
| 1474 | vcpu->arch.texasr = set_reg_val(id, *val); |
| 1475 | break; |
| 1476 | case KVM_REG_PPC_TM_GPR0 ... KVM_REG_PPC_TM_GPR31: |
| 1477 | i = id - KVM_REG_PPC_TM_GPR0; |
| 1478 | vcpu->arch.gpr_tm[i] = set_reg_val(id, *val); |
| 1479 | break; |
| 1480 | case KVM_REG_PPC_TM_VSR0 ... KVM_REG_PPC_TM_VSR63: |
| 1481 | { |
| 1482 | int j; |
| 1483 | i = id - KVM_REG_PPC_TM_VSR0; |
| 1484 | if (i < 32) |
| 1485 | for (j = 0; j < TS_FPRWIDTH; j++) |
| 1486 | vcpu->arch.fp_tm.fpr[i][j] = val->vsxval[j]; |
| 1487 | else |
| 1488 | if (cpu_has_feature(CPU_FTR_ALTIVEC)) |
| 1489 | vcpu->arch.vr_tm.vr[i-32] = val->vval; |
| 1490 | else |
| 1491 | r = -ENXIO; |
| 1492 | break; |
| 1493 | } |
| 1494 | case KVM_REG_PPC_TM_CR: |
| 1495 | vcpu->arch.cr_tm = set_reg_val(id, *val); |
| 1496 | break; |
Paul Mackerras | 75b1053 | 2016-11-07 15:09:58 +1100 | [diff] [blame] | 1497 | case KVM_REG_PPC_TM_XER: |
| 1498 | vcpu->arch.xer_tm = set_reg_val(id, *val); |
| 1499 | break; |
Michael Neuling | a7d80d0 | 2014-03-25 10:47:03 +1100 | [diff] [blame] | 1500 | case KVM_REG_PPC_TM_LR: |
| 1501 | vcpu->arch.lr_tm = set_reg_val(id, *val); |
| 1502 | break; |
| 1503 | case KVM_REG_PPC_TM_CTR: |
| 1504 | vcpu->arch.ctr_tm = set_reg_val(id, *val); |
| 1505 | break; |
| 1506 | case KVM_REG_PPC_TM_FPSCR: |
| 1507 | vcpu->arch.fp_tm.fpscr = set_reg_val(id, *val); |
| 1508 | break; |
| 1509 | case KVM_REG_PPC_TM_AMR: |
| 1510 | vcpu->arch.amr_tm = set_reg_val(id, *val); |
| 1511 | break; |
| 1512 | case KVM_REG_PPC_TM_PPR: |
| 1513 | vcpu->arch.ppr_tm = set_reg_val(id, *val); |
| 1514 | break; |
| 1515 | case KVM_REG_PPC_TM_VRSAVE: |
| 1516 | vcpu->arch.vrsave_tm = set_reg_val(id, *val); |
| 1517 | break; |
| 1518 | case KVM_REG_PPC_TM_VSCR: |
| 1519 | if (cpu_has_feature(CPU_FTR_ALTIVEC)) |
| 1520 | vcpu->arch.vr.vscr.u[3] = set_reg_val(id, *val); |
| 1521 | else |
| 1522 | r = - ENXIO; |
| 1523 | break; |
| 1524 | case KVM_REG_PPC_TM_DSCR: |
| 1525 | vcpu->arch.dscr_tm = set_reg_val(id, *val); |
| 1526 | break; |
| 1527 | case KVM_REG_PPC_TM_TAR: |
| 1528 | vcpu->arch.tar_tm = set_reg_val(id, *val); |
| 1529 | break; |
| 1530 | #endif |
Paul Mackerras | 388cc6e | 2013-09-21 14:35:02 +1000 | [diff] [blame] | 1531 | case KVM_REG_PPC_ARCH_COMPAT: |
| 1532 | r = kvmppc_set_arch_compat(vcpu, set_reg_val(id, *val)); |
| 1533 | break; |
Paul Mackerras | 31f3438 | 2011-12-12 12:26:50 +0000 | [diff] [blame] | 1534 | default: |
Paul Mackerras | a136a8b | 2012-09-25 20:31:56 +0000 | [diff] [blame] | 1535 | r = -EINVAL; |
Paul Mackerras | 31f3438 | 2011-12-12 12:26:50 +0000 | [diff] [blame] | 1536 | break; |
| 1537 | } |
| 1538 | |
| 1539 | return r; |
| 1540 | } |
| 1541 | |
Stewart Smith | de9bdd1 | 2014-07-18 14:18:42 +1000 | [diff] [blame] | 1542 | static struct kvmppc_vcore *kvmppc_vcore_create(struct kvm *kvm, int core) |
| 1543 | { |
| 1544 | struct kvmppc_vcore *vcore; |
| 1545 | |
| 1546 | vcore = kzalloc(sizeof(struct kvmppc_vcore), GFP_KERNEL); |
| 1547 | |
| 1548 | if (vcore == NULL) |
| 1549 | return NULL; |
| 1550 | |
Stewart Smith | de9bdd1 | 2014-07-18 14:18:42 +1000 | [diff] [blame] | 1551 | spin_lock_init(&vcore->lock); |
Paul Mackerras | 2711e24 | 2014-12-04 16:43:28 +1100 | [diff] [blame] | 1552 | spin_lock_init(&vcore->stoltb_lock); |
Marcelo Tosatti | 8577370 | 2016-02-19 09:46:39 +0100 | [diff] [blame] | 1553 | init_swait_queue_head(&vcore->wq); |
Stewart Smith | de9bdd1 | 2014-07-18 14:18:42 +1000 | [diff] [blame] | 1554 | vcore->preempt_tb = TB_NIL; |
| 1555 | vcore->lpcr = kvm->arch.lpcr; |
| 1556 | vcore->first_vcpuid = core * threads_per_subcore; |
| 1557 | vcore->kvm = kvm; |
Paul Mackerras | ec25716 | 2015-06-24 21:18:03 +1000 | [diff] [blame] | 1558 | INIT_LIST_HEAD(&vcore->preempt_list); |
Stewart Smith | de9bdd1 | 2014-07-18 14:18:42 +1000 | [diff] [blame] | 1559 | |
| 1560 | return vcore; |
| 1561 | } |
| 1562 | |
Paul Mackerras | b6c295d | 2015-03-28 14:21:02 +1100 | [diff] [blame] | 1563 | #ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING |
| 1564 | static struct debugfs_timings_element { |
| 1565 | const char *name; |
| 1566 | size_t offset; |
| 1567 | } timings[] = { |
| 1568 | {"rm_entry", offsetof(struct kvm_vcpu, arch.rm_entry)}, |
| 1569 | {"rm_intr", offsetof(struct kvm_vcpu, arch.rm_intr)}, |
| 1570 | {"rm_exit", offsetof(struct kvm_vcpu, arch.rm_exit)}, |
| 1571 | {"guest", offsetof(struct kvm_vcpu, arch.guest_time)}, |
| 1572 | {"cede", offsetof(struct kvm_vcpu, arch.cede_time)}, |
| 1573 | }; |
| 1574 | |
| 1575 | #define N_TIMINGS (sizeof(timings) / sizeof(timings[0])) |
| 1576 | |
| 1577 | struct debugfs_timings_state { |
| 1578 | struct kvm_vcpu *vcpu; |
| 1579 | unsigned int buflen; |
| 1580 | char buf[N_TIMINGS * 100]; |
| 1581 | }; |
| 1582 | |
| 1583 | static int debugfs_timings_open(struct inode *inode, struct file *file) |
| 1584 | { |
| 1585 | struct kvm_vcpu *vcpu = inode->i_private; |
| 1586 | struct debugfs_timings_state *p; |
| 1587 | |
| 1588 | p = kzalloc(sizeof(*p), GFP_KERNEL); |
| 1589 | if (!p) |
| 1590 | return -ENOMEM; |
| 1591 | |
| 1592 | kvm_get_kvm(vcpu->kvm); |
| 1593 | p->vcpu = vcpu; |
| 1594 | file->private_data = p; |
| 1595 | |
| 1596 | return nonseekable_open(inode, file); |
| 1597 | } |
| 1598 | |
| 1599 | static int debugfs_timings_release(struct inode *inode, struct file *file) |
| 1600 | { |
| 1601 | struct debugfs_timings_state *p = file->private_data; |
| 1602 | |
| 1603 | kvm_put_kvm(p->vcpu->kvm); |
| 1604 | kfree(p); |
| 1605 | return 0; |
| 1606 | } |
| 1607 | |
| 1608 | static ssize_t debugfs_timings_read(struct file *file, char __user *buf, |
| 1609 | size_t len, loff_t *ppos) |
| 1610 | { |
| 1611 | struct debugfs_timings_state *p = file->private_data; |
| 1612 | struct kvm_vcpu *vcpu = p->vcpu; |
| 1613 | char *s, *buf_end; |
| 1614 | struct kvmhv_tb_accumulator tb; |
| 1615 | u64 count; |
| 1616 | loff_t pos; |
| 1617 | ssize_t n; |
| 1618 | int i, loops; |
| 1619 | bool ok; |
| 1620 | |
| 1621 | if (!p->buflen) { |
| 1622 | s = p->buf; |
| 1623 | buf_end = s + sizeof(p->buf); |
| 1624 | for (i = 0; i < N_TIMINGS; ++i) { |
| 1625 | struct kvmhv_tb_accumulator *acc; |
| 1626 | |
| 1627 | acc = (struct kvmhv_tb_accumulator *) |
| 1628 | ((unsigned long)vcpu + timings[i].offset); |
| 1629 | ok = false; |
| 1630 | for (loops = 0; loops < 1000; ++loops) { |
| 1631 | count = acc->seqcount; |
| 1632 | if (!(count & 1)) { |
| 1633 | smp_rmb(); |
| 1634 | tb = *acc; |
| 1635 | smp_rmb(); |
| 1636 | if (count == acc->seqcount) { |
| 1637 | ok = true; |
| 1638 | break; |
| 1639 | } |
| 1640 | } |
| 1641 | udelay(1); |
| 1642 | } |
| 1643 | if (!ok) |
| 1644 | snprintf(s, buf_end - s, "%s: stuck\n", |
| 1645 | timings[i].name); |
| 1646 | else |
| 1647 | snprintf(s, buf_end - s, |
| 1648 | "%s: %llu %llu %llu %llu\n", |
| 1649 | timings[i].name, count / 2, |
| 1650 | tb_to_ns(tb.tb_total), |
| 1651 | tb_to_ns(tb.tb_min), |
| 1652 | tb_to_ns(tb.tb_max)); |
| 1653 | s += strlen(s); |
| 1654 | } |
| 1655 | p->buflen = s - p->buf; |
| 1656 | } |
| 1657 | |
| 1658 | pos = *ppos; |
| 1659 | if (pos >= p->buflen) |
| 1660 | return 0; |
| 1661 | if (len > p->buflen - pos) |
| 1662 | len = p->buflen - pos; |
| 1663 | n = copy_to_user(buf, p->buf + pos, len); |
| 1664 | if (n) { |
| 1665 | if (n == len) |
| 1666 | return -EFAULT; |
| 1667 | len -= n; |
| 1668 | } |
| 1669 | *ppos = pos + len; |
| 1670 | return len; |
| 1671 | } |
| 1672 | |
| 1673 | static ssize_t debugfs_timings_write(struct file *file, const char __user *buf, |
| 1674 | size_t len, loff_t *ppos) |
| 1675 | { |
| 1676 | return -EACCES; |
| 1677 | } |
| 1678 | |
| 1679 | static const struct file_operations debugfs_timings_ops = { |
| 1680 | .owner = THIS_MODULE, |
| 1681 | .open = debugfs_timings_open, |
| 1682 | .release = debugfs_timings_release, |
| 1683 | .read = debugfs_timings_read, |
| 1684 | .write = debugfs_timings_write, |
| 1685 | .llseek = generic_file_llseek, |
| 1686 | }; |
| 1687 | |
| 1688 | /* Create a debugfs directory for the vcpu */ |
| 1689 | static void debugfs_vcpu_init(struct kvm_vcpu *vcpu, unsigned int id) |
| 1690 | { |
| 1691 | char buf[16]; |
| 1692 | struct kvm *kvm = vcpu->kvm; |
| 1693 | |
| 1694 | snprintf(buf, sizeof(buf), "vcpu%u", id); |
| 1695 | if (IS_ERR_OR_NULL(kvm->arch.debugfs_dir)) |
| 1696 | return; |
| 1697 | vcpu->arch.debugfs_dir = debugfs_create_dir(buf, kvm->arch.debugfs_dir); |
| 1698 | if (IS_ERR_OR_NULL(vcpu->arch.debugfs_dir)) |
| 1699 | return; |
| 1700 | vcpu->arch.debugfs_timings = |
| 1701 | debugfs_create_file("timings", 0444, vcpu->arch.debugfs_dir, |
| 1702 | vcpu, &debugfs_timings_ops); |
| 1703 | } |
| 1704 | |
| 1705 | #else /* CONFIG_KVM_BOOK3S_HV_EXIT_TIMING */ |
| 1706 | static void debugfs_vcpu_init(struct kvm_vcpu *vcpu, unsigned int id) |
| 1707 | { |
| 1708 | } |
| 1709 | #endif /* CONFIG_KVM_BOOK3S_HV_EXIT_TIMING */ |
| 1710 | |
Aneesh Kumar K.V | 3a167bea | 2013-10-07 22:17:53 +0530 | [diff] [blame] | 1711 | static struct kvm_vcpu *kvmppc_core_vcpu_create_hv(struct kvm *kvm, |
| 1712 | unsigned int id) |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 1713 | { |
| 1714 | struct kvm_vcpu *vcpu; |
Paul Mackerras | 371fefd | 2011-06-29 00:23:08 +0000 | [diff] [blame] | 1715 | int err = -EINVAL; |
| 1716 | int core; |
| 1717 | struct kvmppc_vcore *vcore; |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 1718 | |
Michael Ellerman | 3102f78 | 2014-05-23 18:15:29 +1000 | [diff] [blame] | 1719 | core = id / threads_per_subcore; |
Paul Mackerras | 371fefd | 2011-06-29 00:23:08 +0000 | [diff] [blame] | 1720 | if (core >= KVM_MAX_VCORES) |
| 1721 | goto out; |
| 1722 | |
| 1723 | err = -ENOMEM; |
Sasha Levin | 6b75e6b | 2011-12-07 10:24:56 +0200 | [diff] [blame] | 1724 | vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL); |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 1725 | if (!vcpu) |
| 1726 | goto out; |
| 1727 | |
| 1728 | err = kvm_vcpu_init(vcpu, kvm, id); |
| 1729 | if (err) |
| 1730 | goto free_vcpu; |
| 1731 | |
| 1732 | vcpu->arch.shared = &vcpu->arch.shregs; |
Alexander Graf | 5deb8e7 | 2014-04-24 13:46:24 +0200 | [diff] [blame] | 1733 | #ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE |
| 1734 | /* |
| 1735 | * The shared struct is never shared on HV, |
| 1736 | * so we can always use host endianness |
| 1737 | */ |
| 1738 | #ifdef __BIG_ENDIAN__ |
| 1739 | vcpu->arch.shared_big_endian = true; |
| 1740 | #else |
| 1741 | vcpu->arch.shared_big_endian = false; |
| 1742 | #endif |
| 1743 | #endif |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 1744 | vcpu->arch.mmcr[0] = MMCR0_FC; |
| 1745 | vcpu->arch.ctrl = CTRL_RUNLATCH; |
| 1746 | /* default to host PVR, since we can't spoof it */ |
Aneesh Kumar K.V | 3a167bea | 2013-10-07 22:17:53 +0530 | [diff] [blame] | 1747 | kvmppc_set_pvr_hv(vcpu, mfspr(SPRN_PVR)); |
Paul Mackerras | 2e25aa5 | 2012-02-19 17:46:32 +0000 | [diff] [blame] | 1748 | spin_lock_init(&vcpu->arch.vpa_update_lock); |
Paul Mackerras | c7b6767 | 2012-10-15 01:18:07 +0000 | [diff] [blame] | 1749 | spin_lock_init(&vcpu->arch.tbacct_lock); |
| 1750 | vcpu->arch.busy_preempt = TB_NIL; |
Anton Blanchard | d682916 | 2014-01-08 21:25:30 +1100 | [diff] [blame] | 1751 | vcpu->arch.intr_msr = MSR_SF | MSR_ME; |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 1752 | |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 1753 | kvmppc_mmu_book3s_hv_init(vcpu); |
| 1754 | |
Paul Mackerras | 8455d79 | 2012-10-15 01:17:42 +0000 | [diff] [blame] | 1755 | vcpu->arch.state = KVMPPC_VCPU_NOTREADY; |
Paul Mackerras | 371fefd | 2011-06-29 00:23:08 +0000 | [diff] [blame] | 1756 | |
| 1757 | init_waitqueue_head(&vcpu->arch.cpu_run); |
| 1758 | |
| 1759 | mutex_lock(&kvm->lock); |
| 1760 | vcore = kvm->arch.vcores[core]; |
| 1761 | if (!vcore) { |
Stewart Smith | de9bdd1 | 2014-07-18 14:18:42 +1000 | [diff] [blame] | 1762 | vcore = kvmppc_vcore_create(kvm, core); |
Paul Mackerras | 371fefd | 2011-06-29 00:23:08 +0000 | [diff] [blame] | 1763 | kvm->arch.vcores[core] = vcore; |
Paul Mackerras | 1b400ba | 2012-11-21 23:28:08 +0000 | [diff] [blame] | 1764 | kvm->arch.online_vcores++; |
Paul Mackerras | 371fefd | 2011-06-29 00:23:08 +0000 | [diff] [blame] | 1765 | } |
| 1766 | mutex_unlock(&kvm->lock); |
| 1767 | |
| 1768 | if (!vcore) |
Sean Christopherson | 582fb2d | 2019-12-18 13:54:46 -0800 | [diff] [blame] | 1769 | goto uninit_vcpu; |
Paul Mackerras | 371fefd | 2011-06-29 00:23:08 +0000 | [diff] [blame] | 1770 | |
| 1771 | spin_lock(&vcore->lock); |
| 1772 | ++vcore->num_threads; |
Paul Mackerras | 371fefd | 2011-06-29 00:23:08 +0000 | [diff] [blame] | 1773 | spin_unlock(&vcore->lock); |
| 1774 | vcpu->arch.vcore = vcore; |
Paul Mackerras | e0b7ec0 | 2014-01-08 21:25:20 +1100 | [diff] [blame] | 1775 | vcpu->arch.ptid = vcpu->vcpu_id - vcore->first_vcpuid; |
Paul Mackerras | ec25716 | 2015-06-24 21:18:03 +1000 | [diff] [blame] | 1776 | vcpu->arch.thread_cpu = -1; |
Paul Mackerras | 371fefd | 2011-06-29 00:23:08 +0000 | [diff] [blame] | 1777 | |
Alexander Graf | af8f38b | 2011-08-10 13:57:08 +0200 | [diff] [blame] | 1778 | vcpu->arch.cpu_type = KVM_CPU_3S_64; |
| 1779 | kvmppc_sanity_check(vcpu); |
| 1780 | |
Paul Mackerras | b6c295d | 2015-03-28 14:21:02 +1100 | [diff] [blame] | 1781 | debugfs_vcpu_init(vcpu, id); |
| 1782 | |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 1783 | return vcpu; |
| 1784 | |
Sean Christopherson | 582fb2d | 2019-12-18 13:54:46 -0800 | [diff] [blame] | 1785 | uninit_vcpu: |
| 1786 | kvm_vcpu_uninit(vcpu); |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 1787 | free_vcpu: |
Sasha Levin | 6b75e6b | 2011-12-07 10:24:56 +0200 | [diff] [blame] | 1788 | kmem_cache_free(kvm_vcpu_cache, vcpu); |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 1789 | out: |
| 1790 | return ERR_PTR(err); |
| 1791 | } |
| 1792 | |
Paul Mackerras | c35635e | 2013-04-18 19:51:04 +0000 | [diff] [blame] | 1793 | static void unpin_vpa(struct kvm *kvm, struct kvmppc_vpa *vpa) |
| 1794 | { |
| 1795 | if (vpa->pinned_addr) |
| 1796 | kvmppc_unpin_guest_page(kvm, vpa->pinned_addr, vpa->gpa, |
| 1797 | vpa->dirty); |
| 1798 | } |
| 1799 | |
Aneesh Kumar K.V | 3a167bea | 2013-10-07 22:17:53 +0530 | [diff] [blame] | 1800 | static void kvmppc_core_vcpu_free_hv(struct kvm_vcpu *vcpu) |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 1801 | { |
Paul Mackerras | 2e25aa5 | 2012-02-19 17:46:32 +0000 | [diff] [blame] | 1802 | spin_lock(&vcpu->arch.vpa_update_lock); |
Paul Mackerras | c35635e | 2013-04-18 19:51:04 +0000 | [diff] [blame] | 1803 | unpin_vpa(vcpu->kvm, &vcpu->arch.dtl); |
| 1804 | unpin_vpa(vcpu->kvm, &vcpu->arch.slb_shadow); |
| 1805 | unpin_vpa(vcpu->kvm, &vcpu->arch.vpa); |
Paul Mackerras | 2e25aa5 | 2012-02-19 17:46:32 +0000 | [diff] [blame] | 1806 | spin_unlock(&vcpu->arch.vpa_update_lock); |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 1807 | kvm_vcpu_uninit(vcpu); |
Sasha Levin | 6b75e6b | 2011-12-07 10:24:56 +0200 | [diff] [blame] | 1808 | kmem_cache_free(kvm_vcpu_cache, vcpu); |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 1809 | } |
| 1810 | |
Aneesh Kumar K.V | 3a167bea | 2013-10-07 22:17:53 +0530 | [diff] [blame] | 1811 | static int kvmppc_core_check_requests_hv(struct kvm_vcpu *vcpu) |
| 1812 | { |
| 1813 | /* Indicate we want to get back into the guest */ |
| 1814 | return 1; |
| 1815 | } |
| 1816 | |
Paul Mackerras | 19ccb76 | 2011-07-23 17:42:46 +1000 | [diff] [blame] | 1817 | static void kvmppc_set_timer(struct kvm_vcpu *vcpu) |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 1818 | { |
Paul Mackerras | 19ccb76 | 2011-07-23 17:42:46 +1000 | [diff] [blame] | 1819 | unsigned long dec_nsec, now; |
Paul Mackerras | 371fefd | 2011-06-29 00:23:08 +0000 | [diff] [blame] | 1820 | |
Paul Mackerras | 19ccb76 | 2011-07-23 17:42:46 +1000 | [diff] [blame] | 1821 | now = get_tb(); |
| 1822 | if (now > vcpu->arch.dec_expires) { |
| 1823 | /* decrementer has already gone negative */ |
| 1824 | kvmppc_core_queue_dec(vcpu); |
Scott Wood | 7e28e60e | 2011-11-08 18:23:20 -0600 | [diff] [blame] | 1825 | kvmppc_core_prepare_to_enter(vcpu); |
Paul Mackerras | 19ccb76 | 2011-07-23 17:42:46 +1000 | [diff] [blame] | 1826 | return; |
Paul Mackerras | 371fefd | 2011-06-29 00:23:08 +0000 | [diff] [blame] | 1827 | } |
Paul Mackerras | 19ccb76 | 2011-07-23 17:42:46 +1000 | [diff] [blame] | 1828 | dec_nsec = (vcpu->arch.dec_expires - now) * NSEC_PER_SEC |
| 1829 | / tb_ticks_per_sec; |
| 1830 | hrtimer_start(&vcpu->arch.dec_timer, ktime_set(0, dec_nsec), |
| 1831 | HRTIMER_MODE_REL); |
| 1832 | vcpu->arch.timer_running = 1; |
Paul Mackerras | 371fefd | 2011-06-29 00:23:08 +0000 | [diff] [blame] | 1833 | } |
| 1834 | |
Paul Mackerras | 19ccb76 | 2011-07-23 17:42:46 +1000 | [diff] [blame] | 1835 | static void kvmppc_end_cede(struct kvm_vcpu *vcpu) |
Paul Mackerras | 371fefd | 2011-06-29 00:23:08 +0000 | [diff] [blame] | 1836 | { |
Paul Mackerras | 19ccb76 | 2011-07-23 17:42:46 +1000 | [diff] [blame] | 1837 | vcpu->arch.ceded = 0; |
| 1838 | if (vcpu->arch.timer_running) { |
| 1839 | hrtimer_try_to_cancel(&vcpu->arch.dec_timer); |
| 1840 | vcpu->arch.timer_running = 0; |
| 1841 | } |
Paul Mackerras | 371fefd | 2011-06-29 00:23:08 +0000 | [diff] [blame] | 1842 | } |
| 1843 | |
Paul Mackerras | e0b7ec0 | 2014-01-08 21:25:20 +1100 | [diff] [blame] | 1844 | extern void __kvmppc_vcore_entry(void); |
Paul Mackerras | 371fefd | 2011-06-29 00:23:08 +0000 | [diff] [blame] | 1845 | |
| 1846 | static void kvmppc_remove_runnable(struct kvmppc_vcore *vc, |
| 1847 | struct kvm_vcpu *vcpu) |
| 1848 | { |
Paul Mackerras | c7b6767 | 2012-10-15 01:18:07 +0000 | [diff] [blame] | 1849 | u64 now; |
| 1850 | |
Paul Mackerras | 371fefd | 2011-06-29 00:23:08 +0000 | [diff] [blame] | 1851 | if (vcpu->arch.state != KVMPPC_VCPU_RUNNABLE) |
| 1852 | return; |
Paul Mackerras | bf3d32e | 2013-11-16 17:46:04 +1100 | [diff] [blame] | 1853 | spin_lock_irq(&vcpu->arch.tbacct_lock); |
Paul Mackerras | c7b6767 | 2012-10-15 01:18:07 +0000 | [diff] [blame] | 1854 | now = mftb(); |
| 1855 | vcpu->arch.busy_stolen += vcore_stolen_time(vc, now) - |
| 1856 | vcpu->arch.stolen_logged; |
| 1857 | vcpu->arch.busy_preempt = now; |
| 1858 | vcpu->arch.state = KVMPPC_VCPU_BUSY_IN_HOST; |
Paul Mackerras | bf3d32e | 2013-11-16 17:46:04 +1100 | [diff] [blame] | 1859 | spin_unlock_irq(&vcpu->arch.tbacct_lock); |
Paul Mackerras | 371fefd | 2011-06-29 00:23:08 +0000 | [diff] [blame] | 1860 | --vc->n_runnable; |
Suraj Jitindar Singh | 7b5f827 | 2016-08-02 14:03:20 +1000 | [diff] [blame] | 1861 | WRITE_ONCE(vc->runnable_threads[vcpu->arch.ptid], NULL); |
Paul Mackerras | 371fefd | 2011-06-29 00:23:08 +0000 | [diff] [blame] | 1862 | } |
| 1863 | |
Paul Mackerras | f0888f7 | 2012-02-03 00:54:17 +0000 | [diff] [blame] | 1864 | static int kvmppc_grab_hwthread(int cpu) |
| 1865 | { |
| 1866 | struct paca_struct *tpaca; |
Paul Mackerras | b754c73 | 2014-09-02 16:14:42 +1000 | [diff] [blame] | 1867 | long timeout = 10000; |
Paul Mackerras | f0888f7 | 2012-02-03 00:54:17 +0000 | [diff] [blame] | 1868 | |
| 1869 | tpaca = &paca[cpu]; |
| 1870 | |
| 1871 | /* Ensure the thread won't go into the kernel if it wakes */ |
Paul Mackerras | 7b444c6 | 2012-10-15 01:16:14 +0000 | [diff] [blame] | 1872 | tpaca->kvm_hstate.kvm_vcpu = NULL; |
Paul Mackerras | b4deba5 | 2015-07-02 20:38:16 +1000 | [diff] [blame] | 1873 | tpaca->kvm_hstate.kvm_vcore = NULL; |
Paul Mackerras | 5d5b99c | 2015-03-28 14:21:06 +1100 | [diff] [blame] | 1874 | tpaca->kvm_hstate.napping = 0; |
| 1875 | smp_wmb(); |
| 1876 | tpaca->kvm_hstate.hwthread_req = 1; |
Paul Mackerras | f0888f7 | 2012-02-03 00:54:17 +0000 | [diff] [blame] | 1877 | |
| 1878 | /* |
| 1879 | * If the thread is already executing in the kernel (e.g. handling |
| 1880 | * a stray interrupt), wait for it to get back to nap mode. |
| 1881 | * The smp_mb() is to ensure that our setting of hwthread_req |
| 1882 | * is visible before we look at hwthread_state, so if this |
| 1883 | * races with the code at system_reset_pSeries and the thread |
| 1884 | * misses our setting of hwthread_req, we are sure to see its |
| 1885 | * setting of hwthread_state, and vice versa. |
| 1886 | */ |
| 1887 | smp_mb(); |
| 1888 | while (tpaca->kvm_hstate.hwthread_state == KVM_HWTHREAD_IN_KERNEL) { |
| 1889 | if (--timeout <= 0) { |
| 1890 | pr_err("KVM: couldn't grab cpu %d\n", cpu); |
| 1891 | return -EBUSY; |
| 1892 | } |
| 1893 | udelay(1); |
| 1894 | } |
| 1895 | return 0; |
| 1896 | } |
| 1897 | |
| 1898 | static void kvmppc_release_hwthread(int cpu) |
| 1899 | { |
| 1900 | struct paca_struct *tpaca; |
| 1901 | |
| 1902 | tpaca = &paca[cpu]; |
| 1903 | tpaca->kvm_hstate.hwthread_req = 0; |
| 1904 | tpaca->kvm_hstate.kvm_vcpu = NULL; |
Paul Mackerras | b4deba5 | 2015-07-02 20:38:16 +1000 | [diff] [blame] | 1905 | tpaca->kvm_hstate.kvm_vcore = NULL; |
| 1906 | tpaca->kvm_hstate.kvm_split_mode = NULL; |
Paul Mackerras | f0888f7 | 2012-02-03 00:54:17 +0000 | [diff] [blame] | 1907 | } |
| 1908 | |
Paul Mackerras | b4deba5 | 2015-07-02 20:38:16 +1000 | [diff] [blame] | 1909 | static void kvmppc_start_thread(struct kvm_vcpu *vcpu, struct kvmppc_vcore *vc) |
Paul Mackerras | 371fefd | 2011-06-29 00:23:08 +0000 | [diff] [blame] | 1910 | { |
| 1911 | int cpu; |
| 1912 | struct paca_struct *tpaca; |
Paul Mackerras | ec25716 | 2015-06-24 21:18:03 +1000 | [diff] [blame] | 1913 | struct kvmppc_vcore *mvc = vc->master_vcore; |
Paul Mackerras | 371fefd | 2011-06-29 00:23:08 +0000 | [diff] [blame] | 1914 | |
Paul Mackerras | b4deba5 | 2015-07-02 20:38:16 +1000 | [diff] [blame] | 1915 | cpu = vc->pcpu; |
| 1916 | if (vcpu) { |
| 1917 | if (vcpu->arch.timer_running) { |
| 1918 | hrtimer_try_to_cancel(&vcpu->arch.dec_timer); |
| 1919 | vcpu->arch.timer_running = 0; |
| 1920 | } |
| 1921 | cpu += vcpu->arch.ptid; |
| 1922 | vcpu->cpu = mvc->pcpu; |
| 1923 | vcpu->arch.thread_cpu = cpu; |
Paul Mackerras | 19ccb76 | 2011-07-23 17:42:46 +1000 | [diff] [blame] | 1924 | } |
Paul Mackerras | 371fefd | 2011-06-29 00:23:08 +0000 | [diff] [blame] | 1925 | tpaca = &paca[cpu]; |
Paul Mackerras | 5d5b99c | 2015-03-28 14:21:06 +1100 | [diff] [blame] | 1926 | tpaca->kvm_hstate.kvm_vcpu = vcpu; |
Paul Mackerras | ec25716 | 2015-06-24 21:18:03 +1000 | [diff] [blame] | 1927 | tpaca->kvm_hstate.ptid = cpu - mvc->pcpu; |
Paul Mackerras | ec25716 | 2015-06-24 21:18:03 +1000 | [diff] [blame] | 1928 | /* Order stores to hstate.kvm_vcpu etc. before store to kvm_vcore */ |
Paul Mackerras | 19ccb76 | 2011-07-23 17:42:46 +1000 | [diff] [blame] | 1929 | smp_wmb(); |
Paul Mackerras | b4deba5 | 2015-07-02 20:38:16 +1000 | [diff] [blame] | 1930 | tpaca->kvm_hstate.kvm_vcore = mvc; |
Paul Mackerras | 5d5b99c | 2015-03-28 14:21:06 +1100 | [diff] [blame] | 1931 | if (cpu != smp_processor_id()) |
Paul Mackerras | 66feed6 | 2015-03-28 14:21:12 +1100 | [diff] [blame] | 1932 | kvmppc_ipi_thread(cpu); |
Paul Mackerras | 371fefd | 2011-06-29 00:23:08 +0000 | [diff] [blame] | 1933 | } |
| 1934 | |
Paul Mackerras | 5d5b99c | 2015-03-28 14:21:06 +1100 | [diff] [blame] | 1935 | static void kvmppc_wait_for_nap(void) |
Paul Mackerras | 371fefd | 2011-06-29 00:23:08 +0000 | [diff] [blame] | 1936 | { |
Paul Mackerras | 5d5b99c | 2015-03-28 14:21:06 +1100 | [diff] [blame] | 1937 | int cpu = smp_processor_id(); |
| 1938 | int i, loops; |
Paul Mackerras | 371fefd | 2011-06-29 00:23:08 +0000 | [diff] [blame] | 1939 | |
Paul Mackerras | 5d5b99c | 2015-03-28 14:21:06 +1100 | [diff] [blame] | 1940 | for (loops = 0; loops < 1000000; ++loops) { |
| 1941 | /* |
| 1942 | * Check if all threads are finished. |
Paul Mackerras | b4deba5 | 2015-07-02 20:38:16 +1000 | [diff] [blame] | 1943 | * We set the vcore pointer when starting a thread |
Paul Mackerras | 5d5b99c | 2015-03-28 14:21:06 +1100 | [diff] [blame] | 1944 | * and the thread clears it when finished, so we look |
Paul Mackerras | b4deba5 | 2015-07-02 20:38:16 +1000 | [diff] [blame] | 1945 | * for any threads that still have a non-NULL vcore ptr. |
Paul Mackerras | 5d5b99c | 2015-03-28 14:21:06 +1100 | [diff] [blame] | 1946 | */ |
| 1947 | for (i = 1; i < threads_per_subcore; ++i) |
Paul Mackerras | b4deba5 | 2015-07-02 20:38:16 +1000 | [diff] [blame] | 1948 | if (paca[cpu + i].kvm_hstate.kvm_vcore) |
Paul Mackerras | 5d5b99c | 2015-03-28 14:21:06 +1100 | [diff] [blame] | 1949 | break; |
| 1950 | if (i == threads_per_subcore) { |
| 1951 | HMT_medium(); |
| 1952 | return; |
Paul Mackerras | 371fefd | 2011-06-29 00:23:08 +0000 | [diff] [blame] | 1953 | } |
Paul Mackerras | 5d5b99c | 2015-03-28 14:21:06 +1100 | [diff] [blame] | 1954 | HMT_low(); |
Paul Mackerras | 371fefd | 2011-06-29 00:23:08 +0000 | [diff] [blame] | 1955 | } |
| 1956 | HMT_medium(); |
Paul Mackerras | 5d5b99c | 2015-03-28 14:21:06 +1100 | [diff] [blame] | 1957 | for (i = 1; i < threads_per_subcore; ++i) |
Paul Mackerras | b4deba5 | 2015-07-02 20:38:16 +1000 | [diff] [blame] | 1958 | if (paca[cpu + i].kvm_hstate.kvm_vcore) |
Paul Mackerras | 5d5b99c | 2015-03-28 14:21:06 +1100 | [diff] [blame] | 1959 | pr_err("KVM: CPU %d seems to be stuck\n", cpu + i); |
Paul Mackerras | 371fefd | 2011-06-29 00:23:08 +0000 | [diff] [blame] | 1960 | } |
| 1961 | |
| 1962 | /* |
| 1963 | * Check that we are on thread 0 and that any other threads in |
Paul Mackerras | 7b444c6 | 2012-10-15 01:16:14 +0000 | [diff] [blame] | 1964 | * this core are off-line. Then grab the threads so they can't |
| 1965 | * enter the kernel. |
Paul Mackerras | 371fefd | 2011-06-29 00:23:08 +0000 | [diff] [blame] | 1966 | */ |
| 1967 | static int on_primary_thread(void) |
| 1968 | { |
| 1969 | int cpu = smp_processor_id(); |
Michael Ellerman | 3102f78 | 2014-05-23 18:15:29 +1000 | [diff] [blame] | 1970 | int thr; |
Paul Mackerras | 371fefd | 2011-06-29 00:23:08 +0000 | [diff] [blame] | 1971 | |
Michael Ellerman | 3102f78 | 2014-05-23 18:15:29 +1000 | [diff] [blame] | 1972 | /* Are we on a primary subcore? */ |
| 1973 | if (cpu_thread_in_subcore(cpu)) |
Paul Mackerras | 371fefd | 2011-06-29 00:23:08 +0000 | [diff] [blame] | 1974 | return 0; |
Michael Ellerman | 3102f78 | 2014-05-23 18:15:29 +1000 | [diff] [blame] | 1975 | |
| 1976 | thr = 0; |
| 1977 | while (++thr < threads_per_subcore) |
Paul Mackerras | 371fefd | 2011-06-29 00:23:08 +0000 | [diff] [blame] | 1978 | if (cpu_online(cpu + thr)) |
| 1979 | return 0; |
Paul Mackerras | 7b444c6 | 2012-10-15 01:16:14 +0000 | [diff] [blame] | 1980 | |
| 1981 | /* Grab all hw threads so they can't go into the kernel */ |
Michael Ellerman | 3102f78 | 2014-05-23 18:15:29 +1000 | [diff] [blame] | 1982 | for (thr = 1; thr < threads_per_subcore; ++thr) { |
Paul Mackerras | 7b444c6 | 2012-10-15 01:16:14 +0000 | [diff] [blame] | 1983 | if (kvmppc_grab_hwthread(cpu + thr)) { |
| 1984 | /* Couldn't grab one; let the others go */ |
| 1985 | do { |
| 1986 | kvmppc_release_hwthread(cpu + thr); |
| 1987 | } while (--thr > 0); |
| 1988 | return 0; |
| 1989 | } |
| 1990 | } |
Paul Mackerras | 371fefd | 2011-06-29 00:23:08 +0000 | [diff] [blame] | 1991 | return 1; |
| 1992 | } |
| 1993 | |
Paul Mackerras | ec25716 | 2015-06-24 21:18:03 +1000 | [diff] [blame] | 1994 | /* |
| 1995 | * A list of virtual cores for each physical CPU. |
| 1996 | * These are vcores that could run but their runner VCPU tasks are |
| 1997 | * (or may be) preempted. |
| 1998 | */ |
| 1999 | struct preempted_vcore_list { |
| 2000 | struct list_head list; |
| 2001 | spinlock_t lock; |
| 2002 | }; |
| 2003 | |
| 2004 | static DEFINE_PER_CPU(struct preempted_vcore_list, preempted_vcores); |
| 2005 | |
| 2006 | static void init_vcore_lists(void) |
| 2007 | { |
| 2008 | int cpu; |
| 2009 | |
| 2010 | for_each_possible_cpu(cpu) { |
| 2011 | struct preempted_vcore_list *lp = &per_cpu(preempted_vcores, cpu); |
| 2012 | spin_lock_init(&lp->lock); |
| 2013 | INIT_LIST_HEAD(&lp->list); |
| 2014 | } |
| 2015 | } |
| 2016 | |
| 2017 | static void kvmppc_vcore_preempt(struct kvmppc_vcore *vc) |
| 2018 | { |
| 2019 | struct preempted_vcore_list *lp = this_cpu_ptr(&preempted_vcores); |
| 2020 | |
| 2021 | vc->vcore_state = VCORE_PREEMPT; |
| 2022 | vc->pcpu = smp_processor_id(); |
| 2023 | if (vc->num_threads < threads_per_subcore) { |
| 2024 | spin_lock(&lp->lock); |
| 2025 | list_add_tail(&vc->preempt_list, &lp->list); |
| 2026 | spin_unlock(&lp->lock); |
| 2027 | } |
| 2028 | |
| 2029 | /* Start accumulating stolen time */ |
| 2030 | kvmppc_core_start_stolen(vc); |
| 2031 | } |
| 2032 | |
| 2033 | static void kvmppc_vcore_end_preempt(struct kvmppc_vcore *vc) |
| 2034 | { |
Paul Mackerras | 402813f | 2015-07-16 17:11:13 +1000 | [diff] [blame] | 2035 | struct preempted_vcore_list *lp; |
Paul Mackerras | ec25716 | 2015-06-24 21:18:03 +1000 | [diff] [blame] | 2036 | |
| 2037 | kvmppc_core_end_stolen(vc); |
| 2038 | if (!list_empty(&vc->preempt_list)) { |
Paul Mackerras | 402813f | 2015-07-16 17:11:13 +1000 | [diff] [blame] | 2039 | lp = &per_cpu(preempted_vcores, vc->pcpu); |
Paul Mackerras | ec25716 | 2015-06-24 21:18:03 +1000 | [diff] [blame] | 2040 | spin_lock(&lp->lock); |
| 2041 | list_del_init(&vc->preempt_list); |
| 2042 | spin_unlock(&lp->lock); |
| 2043 | } |
| 2044 | vc->vcore_state = VCORE_INACTIVE; |
| 2045 | } |
| 2046 | |
Paul Mackerras | b4deba5 | 2015-07-02 20:38:16 +1000 | [diff] [blame] | 2047 | /* |
| 2048 | * This stores information about the virtual cores currently |
| 2049 | * assigned to a physical core. |
| 2050 | */ |
Paul Mackerras | ec25716 | 2015-06-24 21:18:03 +1000 | [diff] [blame] | 2051 | struct core_info { |
Paul Mackerras | b4deba5 | 2015-07-02 20:38:16 +1000 | [diff] [blame] | 2052 | int n_subcores; |
| 2053 | int max_subcore_threads; |
Paul Mackerras | ec25716 | 2015-06-24 21:18:03 +1000 | [diff] [blame] | 2054 | int total_threads; |
Paul Mackerras | b4deba5 | 2015-07-02 20:38:16 +1000 | [diff] [blame] | 2055 | int subcore_threads[MAX_SUBCORES]; |
| 2056 | struct kvm *subcore_vm[MAX_SUBCORES]; |
| 2057 | struct list_head vcs[MAX_SUBCORES]; |
Paul Mackerras | ec25716 | 2015-06-24 21:18:03 +1000 | [diff] [blame] | 2058 | }; |
| 2059 | |
Paul Mackerras | b4deba5 | 2015-07-02 20:38:16 +1000 | [diff] [blame] | 2060 | /* |
| 2061 | * This mapping means subcores 0 and 1 can use threads 0-3 and 4-7 |
| 2062 | * respectively in 2-way micro-threading (split-core) mode. |
| 2063 | */ |
| 2064 | static int subcore_thread_map[MAX_SUBCORES] = { 0, 4, 2, 6 }; |
| 2065 | |
Paul Mackerras | ec25716 | 2015-06-24 21:18:03 +1000 | [diff] [blame] | 2066 | static void init_core_info(struct core_info *cip, struct kvmppc_vcore *vc) |
| 2067 | { |
Paul Mackerras | b4deba5 | 2015-07-02 20:38:16 +1000 | [diff] [blame] | 2068 | int sub; |
| 2069 | |
Paul Mackerras | ec25716 | 2015-06-24 21:18:03 +1000 | [diff] [blame] | 2070 | memset(cip, 0, sizeof(*cip)); |
Paul Mackerras | b4deba5 | 2015-07-02 20:38:16 +1000 | [diff] [blame] | 2071 | cip->n_subcores = 1; |
| 2072 | cip->max_subcore_threads = vc->num_threads; |
Paul Mackerras | ec25716 | 2015-06-24 21:18:03 +1000 | [diff] [blame] | 2073 | cip->total_threads = vc->num_threads; |
Paul Mackerras | b4deba5 | 2015-07-02 20:38:16 +1000 | [diff] [blame] | 2074 | cip->subcore_threads[0] = vc->num_threads; |
| 2075 | cip->subcore_vm[0] = vc->kvm; |
| 2076 | for (sub = 0; sub < MAX_SUBCORES; ++sub) |
| 2077 | INIT_LIST_HEAD(&cip->vcs[sub]); |
| 2078 | list_add_tail(&vc->preempt_list, &cip->vcs[0]); |
| 2079 | } |
| 2080 | |
| 2081 | static bool subcore_config_ok(int n_subcores, int n_threads) |
| 2082 | { |
| 2083 | /* Can only dynamically split if unsplit to begin with */ |
| 2084 | if (n_subcores > 1 && threads_per_subcore < MAX_SMT_THREADS) |
| 2085 | return false; |
| 2086 | if (n_subcores > MAX_SUBCORES) |
| 2087 | return false; |
| 2088 | if (n_subcores > 1) { |
| 2089 | if (!(dynamic_mt_modes & 2)) |
| 2090 | n_subcores = 4; |
| 2091 | if (n_subcores > 2 && !(dynamic_mt_modes & 4)) |
| 2092 | return false; |
| 2093 | } |
| 2094 | |
| 2095 | return n_subcores * roundup_pow_of_two(n_threads) <= MAX_SMT_THREADS; |
Paul Mackerras | ec25716 | 2015-06-24 21:18:03 +1000 | [diff] [blame] | 2096 | } |
| 2097 | |
| 2098 | static void init_master_vcore(struct kvmppc_vcore *vc) |
| 2099 | { |
| 2100 | vc->master_vcore = vc; |
| 2101 | vc->entry_exit_map = 0; |
| 2102 | vc->in_guest = 0; |
| 2103 | vc->napping_threads = 0; |
| 2104 | vc->conferring_threads = 0; |
| 2105 | } |
| 2106 | |
Paul Mackerras | b4deba5 | 2015-07-02 20:38:16 +1000 | [diff] [blame] | 2107 | static bool can_dynamic_split(struct kvmppc_vcore *vc, struct core_info *cip) |
| 2108 | { |
| 2109 | int n_threads = vc->num_threads; |
| 2110 | int sub; |
| 2111 | |
| 2112 | if (!cpu_has_feature(CPU_FTR_ARCH_207S)) |
| 2113 | return false; |
| 2114 | |
| 2115 | if (n_threads < cip->max_subcore_threads) |
| 2116 | n_threads = cip->max_subcore_threads; |
Paul Mackerras | b009031 | 2016-09-15 16:27:41 +1000 | [diff] [blame] | 2117 | if (!subcore_config_ok(cip->n_subcores + 1, n_threads)) |
Paul Mackerras | b4deba5 | 2015-07-02 20:38:16 +1000 | [diff] [blame] | 2118 | return false; |
Paul Mackerras | b009031 | 2016-09-15 16:27:41 +1000 | [diff] [blame] | 2119 | cip->max_subcore_threads = n_threads; |
Paul Mackerras | b4deba5 | 2015-07-02 20:38:16 +1000 | [diff] [blame] | 2120 | |
| 2121 | sub = cip->n_subcores; |
| 2122 | ++cip->n_subcores; |
| 2123 | cip->total_threads += vc->num_threads; |
| 2124 | cip->subcore_threads[sub] = vc->num_threads; |
| 2125 | cip->subcore_vm[sub] = vc->kvm; |
| 2126 | init_master_vcore(vc); |
| 2127 | list_del(&vc->preempt_list); |
| 2128 | list_add_tail(&vc->preempt_list, &cip->vcs[sub]); |
| 2129 | |
| 2130 | return true; |
| 2131 | } |
| 2132 | |
Paul Mackerras | b4deba5 | 2015-07-02 20:38:16 +1000 | [diff] [blame] | 2133 | /* |
| 2134 | * Work out whether it is possible to piggyback the execution of |
| 2135 | * vcore *pvc onto the execution of the other vcores described in *cip. |
| 2136 | */ |
| 2137 | static bool can_piggyback(struct kvmppc_vcore *pvc, struct core_info *cip, |
| 2138 | int target_threads) |
| 2139 | { |
Paul Mackerras | b4deba5 | 2015-07-02 20:38:16 +1000 | [diff] [blame] | 2140 | if (cip->total_threads + pvc->num_threads > target_threads) |
| 2141 | return false; |
Paul Mackerras | b4deba5 | 2015-07-02 20:38:16 +1000 | [diff] [blame] | 2142 | |
Paul Mackerras | b009031 | 2016-09-15 16:27:41 +1000 | [diff] [blame] | 2143 | return can_dynamic_split(pvc, cip); |
Paul Mackerras | b4deba5 | 2015-07-02 20:38:16 +1000 | [diff] [blame] | 2144 | } |
| 2145 | |
Paul Mackerras | d911f0b | 2015-03-28 14:21:03 +1100 | [diff] [blame] | 2146 | static void prepare_threads(struct kvmppc_vcore *vc) |
| 2147 | { |
Suraj Jitindar Singh | 7b5f827 | 2016-08-02 14:03:20 +1000 | [diff] [blame] | 2148 | int i; |
| 2149 | struct kvm_vcpu *vcpu; |
Paul Mackerras | d911f0b | 2015-03-28 14:21:03 +1100 | [diff] [blame] | 2150 | |
Suraj Jitindar Singh | 7b5f827 | 2016-08-02 14:03:20 +1000 | [diff] [blame] | 2151 | for_each_runnable_thread(i, vcpu, vc) { |
Paul Mackerras | d911f0b | 2015-03-28 14:21:03 +1100 | [diff] [blame] | 2152 | if (signal_pending(vcpu->arch.run_task)) |
| 2153 | vcpu->arch.ret = -EINTR; |
| 2154 | else if (vcpu->arch.vpa.update_pending || |
| 2155 | vcpu->arch.slb_shadow.update_pending || |
| 2156 | vcpu->arch.dtl.update_pending) |
| 2157 | vcpu->arch.ret = RESUME_GUEST; |
| 2158 | else |
| 2159 | continue; |
| 2160 | kvmppc_remove_runnable(vc, vcpu); |
| 2161 | wake_up(&vcpu->arch.cpu_run); |
| 2162 | } |
| 2163 | } |
| 2164 | |
Paul Mackerras | ec25716 | 2015-06-24 21:18:03 +1000 | [diff] [blame] | 2165 | static void collect_piggybacks(struct core_info *cip, int target_threads) |
Paul Mackerras | 25fedfc | 2015-03-28 14:21:05 +1100 | [diff] [blame] | 2166 | { |
Paul Mackerras | ec25716 | 2015-06-24 21:18:03 +1000 | [diff] [blame] | 2167 | struct preempted_vcore_list *lp = this_cpu_ptr(&preempted_vcores); |
| 2168 | struct kvmppc_vcore *pvc, *vcnext; |
| 2169 | |
| 2170 | spin_lock(&lp->lock); |
| 2171 | list_for_each_entry_safe(pvc, vcnext, &lp->list, preempt_list) { |
| 2172 | if (!spin_trylock(&pvc->lock)) |
| 2173 | continue; |
| 2174 | prepare_threads(pvc); |
| 2175 | if (!pvc->n_runnable) { |
| 2176 | list_del_init(&pvc->preempt_list); |
| 2177 | if (pvc->runner == NULL) { |
| 2178 | pvc->vcore_state = VCORE_INACTIVE; |
| 2179 | kvmppc_core_end_stolen(pvc); |
| 2180 | } |
| 2181 | spin_unlock(&pvc->lock); |
| 2182 | continue; |
| 2183 | } |
| 2184 | if (!can_piggyback(pvc, cip, target_threads)) { |
| 2185 | spin_unlock(&pvc->lock); |
| 2186 | continue; |
| 2187 | } |
| 2188 | kvmppc_core_end_stolen(pvc); |
| 2189 | pvc->vcore_state = VCORE_PIGGYBACK; |
| 2190 | if (cip->total_threads >= target_threads) |
| 2191 | break; |
| 2192 | } |
| 2193 | spin_unlock(&lp->lock); |
| 2194 | } |
| 2195 | |
| 2196 | static void post_guest_process(struct kvmppc_vcore *vc, bool is_master) |
| 2197 | { |
Suraj Jitindar Singh | 7b5f827 | 2016-08-02 14:03:20 +1000 | [diff] [blame] | 2198 | int still_running = 0, i; |
Paul Mackerras | 25fedfc | 2015-03-28 14:21:05 +1100 | [diff] [blame] | 2199 | u64 now; |
| 2200 | long ret; |
Suraj Jitindar Singh | 7b5f827 | 2016-08-02 14:03:20 +1000 | [diff] [blame] | 2201 | struct kvm_vcpu *vcpu; |
Paul Mackerras | 25fedfc | 2015-03-28 14:21:05 +1100 | [diff] [blame] | 2202 | |
Paul Mackerras | ec25716 | 2015-06-24 21:18:03 +1000 | [diff] [blame] | 2203 | spin_lock(&vc->lock); |
Paul Mackerras | 25fedfc | 2015-03-28 14:21:05 +1100 | [diff] [blame] | 2204 | now = get_tb(); |
Suraj Jitindar Singh | 7b5f827 | 2016-08-02 14:03:20 +1000 | [diff] [blame] | 2205 | for_each_runnable_thread(i, vcpu, vc) { |
Paul Mackerras | 25fedfc | 2015-03-28 14:21:05 +1100 | [diff] [blame] | 2206 | /* cancel pending dec exception if dec is positive */ |
| 2207 | if (now < vcpu->arch.dec_expires && |
| 2208 | kvmppc_core_pending_dec(vcpu)) |
| 2209 | kvmppc_core_dequeue_dec(vcpu); |
| 2210 | |
| 2211 | trace_kvm_guest_exit(vcpu); |
| 2212 | |
| 2213 | ret = RESUME_GUEST; |
| 2214 | if (vcpu->arch.trap) |
| 2215 | ret = kvmppc_handle_exit_hv(vcpu->arch.kvm_run, vcpu, |
| 2216 | vcpu->arch.run_task); |
| 2217 | |
| 2218 | vcpu->arch.ret = ret; |
| 2219 | vcpu->arch.trap = 0; |
| 2220 | |
Paul Mackerras | ec25716 | 2015-06-24 21:18:03 +1000 | [diff] [blame] | 2221 | if (is_kvmppc_resume_guest(vcpu->arch.ret)) { |
| 2222 | if (vcpu->arch.pending_exceptions) |
| 2223 | kvmppc_core_prepare_to_enter(vcpu); |
| 2224 | if (vcpu->arch.ceded) |
Paul Mackerras | 25fedfc | 2015-03-28 14:21:05 +1100 | [diff] [blame] | 2225 | kvmppc_set_timer(vcpu); |
Paul Mackerras | ec25716 | 2015-06-24 21:18:03 +1000 | [diff] [blame] | 2226 | else |
| 2227 | ++still_running; |
| 2228 | } else { |
Paul Mackerras | 25fedfc | 2015-03-28 14:21:05 +1100 | [diff] [blame] | 2229 | kvmppc_remove_runnable(vc, vcpu); |
| 2230 | wake_up(&vcpu->arch.cpu_run); |
| 2231 | } |
| 2232 | } |
Paul Mackerras | ec25716 | 2015-06-24 21:18:03 +1000 | [diff] [blame] | 2233 | list_del_init(&vc->preempt_list); |
| 2234 | if (!is_master) { |
Paul Mackerras | 563a1e9 | 2015-07-16 17:11:14 +1000 | [diff] [blame] | 2235 | if (still_running > 0) { |
Paul Mackerras | ec25716 | 2015-06-24 21:18:03 +1000 | [diff] [blame] | 2236 | kvmppc_vcore_preempt(vc); |
Paul Mackerras | 563a1e9 | 2015-07-16 17:11:14 +1000 | [diff] [blame] | 2237 | } else if (vc->runner) { |
| 2238 | vc->vcore_state = VCORE_PREEMPT; |
| 2239 | kvmppc_core_start_stolen(vc); |
| 2240 | } else { |
| 2241 | vc->vcore_state = VCORE_INACTIVE; |
| 2242 | } |
Paul Mackerras | ec25716 | 2015-06-24 21:18:03 +1000 | [diff] [blame] | 2243 | if (vc->n_runnable > 0 && vc->runner == NULL) { |
| 2244 | /* make sure there's a candidate runner awake */ |
Suraj Jitindar Singh | 7b5f827 | 2016-08-02 14:03:20 +1000 | [diff] [blame] | 2245 | i = -1; |
| 2246 | vcpu = next_runnable_thread(vc, &i); |
Paul Mackerras | ec25716 | 2015-06-24 21:18:03 +1000 | [diff] [blame] | 2247 | wake_up(&vcpu->arch.cpu_run); |
| 2248 | } |
| 2249 | } |
| 2250 | spin_unlock(&vc->lock); |
Paul Mackerras | 25fedfc | 2015-03-28 14:21:05 +1100 | [diff] [blame] | 2251 | } |
| 2252 | |
Paul Mackerras | 371fefd | 2011-06-29 00:23:08 +0000 | [diff] [blame] | 2253 | /* |
Suresh Warrier | b8e6a87 | 2015-12-17 14:59:07 -0600 | [diff] [blame] | 2254 | * Clear core from the list of active host cores as we are about to |
| 2255 | * enter the guest. Only do this if it is the primary thread of the |
| 2256 | * core (not if a subcore) that is entering the guest. |
| 2257 | */ |
| 2258 | static inline void kvmppc_clear_host_core(int cpu) |
| 2259 | { |
| 2260 | int core; |
| 2261 | |
| 2262 | if (!kvmppc_host_rm_ops_hv || cpu_thread_in_core(cpu)) |
| 2263 | return; |
| 2264 | /* |
| 2265 | * Memory barrier can be omitted here as we will do a smp_wmb() |
| 2266 | * later in kvmppc_start_thread and we need ensure that state is |
| 2267 | * visible to other CPUs only after we enter guest. |
| 2268 | */ |
| 2269 | core = cpu >> threads_shift; |
| 2270 | kvmppc_host_rm_ops_hv->rm_core[core].rm_state.in_host = 0; |
| 2271 | } |
| 2272 | |
| 2273 | /* |
| 2274 | * Advertise this core as an active host core since we exited the guest |
| 2275 | * Only need to do this if it is the primary thread of the core that is |
| 2276 | * exiting. |
| 2277 | */ |
| 2278 | static inline void kvmppc_set_host_core(int cpu) |
| 2279 | { |
| 2280 | int core; |
| 2281 | |
| 2282 | if (!kvmppc_host_rm_ops_hv || cpu_thread_in_core(cpu)) |
| 2283 | return; |
| 2284 | |
| 2285 | /* |
| 2286 | * Memory barrier can be omitted here because we do a spin_unlock |
| 2287 | * immediately after this which provides the memory barrier. |
| 2288 | */ |
| 2289 | core = cpu >> threads_shift; |
| 2290 | kvmppc_host_rm_ops_hv->rm_core[core].rm_state.in_host = 1; |
| 2291 | } |
| 2292 | |
| 2293 | /* |
Paul Mackerras | 371fefd | 2011-06-29 00:23:08 +0000 | [diff] [blame] | 2294 | * Run a set of guest threads on a physical core. |
| 2295 | * Called with vc->lock held. |
| 2296 | */ |
Paul Mackerras | 66feed6 | 2015-03-28 14:21:12 +1100 | [diff] [blame] | 2297 | static noinline void kvmppc_run_core(struct kvmppc_vcore *vc) |
Paul Mackerras | 371fefd | 2011-06-29 00:23:08 +0000 | [diff] [blame] | 2298 | { |
Suraj Jitindar Singh | 7b5f827 | 2016-08-02 14:03:20 +1000 | [diff] [blame] | 2299 | struct kvm_vcpu *vcpu; |
Paul Mackerras | d911f0b | 2015-03-28 14:21:03 +1100 | [diff] [blame] | 2300 | int i; |
Paul Mackerras | 2c9097e | 2012-09-11 13:27:01 +0000 | [diff] [blame] | 2301 | int srcu_idx; |
Paul Mackerras | ec25716 | 2015-06-24 21:18:03 +1000 | [diff] [blame] | 2302 | struct core_info core_info; |
| 2303 | struct kvmppc_vcore *pvc, *vcnext; |
Paul Mackerras | b4deba5 | 2015-07-02 20:38:16 +1000 | [diff] [blame] | 2304 | struct kvm_split_mode split_info, *sip; |
| 2305 | int split, subcore_size, active; |
| 2306 | int sub; |
| 2307 | bool thr0_done; |
| 2308 | unsigned long cmd_bit, stat_bit; |
Paul Mackerras | ec25716 | 2015-06-24 21:18:03 +1000 | [diff] [blame] | 2309 | int pcpu, thr; |
| 2310 | int target_threads; |
Paul Mackerras | 081f323 | 2012-06-01 20:20:24 +1000 | [diff] [blame] | 2311 | |
| 2312 | /* |
Paul Mackerras | d911f0b | 2015-03-28 14:21:03 +1100 | [diff] [blame] | 2313 | * Remove from the list any threads that have a signal pending |
| 2314 | * or need a VPA update done |
| 2315 | */ |
| 2316 | prepare_threads(vc); |
| 2317 | |
| 2318 | /* if the runner is no longer runnable, let the caller pick a new one */ |
| 2319 | if (vc->runner->arch.state != KVMPPC_VCPU_RUNNABLE) |
| 2320 | return; |
| 2321 | |
| 2322 | /* |
| 2323 | * Initialize *vc. |
Paul Mackerras | 081f323 | 2012-06-01 20:20:24 +1000 | [diff] [blame] | 2324 | */ |
Paul Mackerras | ec25716 | 2015-06-24 21:18:03 +1000 | [diff] [blame] | 2325 | init_master_vcore(vc); |
Paul Mackerras | 2711e24 | 2014-12-04 16:43:28 +1100 | [diff] [blame] | 2326 | vc->preempt_tb = TB_NIL; |
Paul Mackerras | 081f323 | 2012-06-01 20:20:24 +1000 | [diff] [blame] | 2327 | |
| 2328 | /* |
Michael Ellerman | 3102f78 | 2014-05-23 18:15:29 +1000 | [diff] [blame] | 2329 | * Make sure we are running on primary threads, and that secondary |
| 2330 | * threads are offline. Also check if the number of threads in this |
| 2331 | * guest are greater than the current system threads per guest. |
Paul Mackerras | 7b444c6 | 2012-10-15 01:16:14 +0000 | [diff] [blame] | 2332 | */ |
Michael Ellerman | 3102f78 | 2014-05-23 18:15:29 +1000 | [diff] [blame] | 2333 | if ((threads_per_core > 1) && |
| 2334 | ((vc->num_threads > threads_per_subcore) || !on_primary_thread())) { |
Suraj Jitindar Singh | 7b5f827 | 2016-08-02 14:03:20 +1000 | [diff] [blame] | 2335 | for_each_runnable_thread(i, vcpu, vc) { |
Paul Mackerras | 7b444c6 | 2012-10-15 01:16:14 +0000 | [diff] [blame] | 2336 | vcpu->arch.ret = -EBUSY; |
Paul Mackerras | 25fedfc | 2015-03-28 14:21:05 +1100 | [diff] [blame] | 2337 | kvmppc_remove_runnable(vc, vcpu); |
| 2338 | wake_up(&vcpu->arch.cpu_run); |
| 2339 | } |
Paul Mackerras | 7b444c6 | 2012-10-15 01:16:14 +0000 | [diff] [blame] | 2340 | goto out; |
| 2341 | } |
| 2342 | |
Paul Mackerras | ec25716 | 2015-06-24 21:18:03 +1000 | [diff] [blame] | 2343 | /* |
| 2344 | * See if we could run any other vcores on the physical core |
| 2345 | * along with this one. |
| 2346 | */ |
| 2347 | init_core_info(&core_info, vc); |
| 2348 | pcpu = smp_processor_id(); |
| 2349 | target_threads = threads_per_subcore; |
| 2350 | if (target_smt_mode && target_smt_mode < target_threads) |
| 2351 | target_threads = target_smt_mode; |
| 2352 | if (vc->num_threads < target_threads) |
| 2353 | collect_piggybacks(&core_info, target_threads); |
Michael Ellerman | 3102f78 | 2014-05-23 18:15:29 +1000 | [diff] [blame] | 2354 | |
Paul Mackerras | b4deba5 | 2015-07-02 20:38:16 +1000 | [diff] [blame] | 2355 | /* Decide on micro-threading (split-core) mode */ |
| 2356 | subcore_size = threads_per_subcore; |
| 2357 | cmd_bit = stat_bit = 0; |
| 2358 | split = core_info.n_subcores; |
| 2359 | sip = NULL; |
| 2360 | if (split > 1) { |
| 2361 | /* threads_per_subcore must be MAX_SMT_THREADS (8) here */ |
| 2362 | if (split == 2 && (dynamic_mt_modes & 2)) { |
| 2363 | cmd_bit = HID0_POWER8_1TO2LPAR; |
| 2364 | stat_bit = HID0_POWER8_2LPARMODE; |
| 2365 | } else { |
| 2366 | split = 4; |
| 2367 | cmd_bit = HID0_POWER8_1TO4LPAR; |
| 2368 | stat_bit = HID0_POWER8_4LPARMODE; |
Paul Mackerras | ec25716 | 2015-06-24 21:18:03 +1000 | [diff] [blame] | 2369 | } |
Paul Mackerras | b4deba5 | 2015-07-02 20:38:16 +1000 | [diff] [blame] | 2370 | subcore_size = MAX_SMT_THREADS / split; |
| 2371 | sip = &split_info; |
| 2372 | memset(&split_info, 0, sizeof(split_info)); |
| 2373 | split_info.rpr = mfspr(SPRN_RPR); |
| 2374 | split_info.pmmar = mfspr(SPRN_PMMAR); |
| 2375 | split_info.ldbar = mfspr(SPRN_LDBAR); |
| 2376 | split_info.subcore_size = subcore_size; |
| 2377 | for (sub = 0; sub < core_info.n_subcores; ++sub) |
| 2378 | split_info.master_vcs[sub] = |
| 2379 | list_first_entry(&core_info.vcs[sub], |
| 2380 | struct kvmppc_vcore, preempt_list); |
| 2381 | /* order writes to split_info before kvm_split_mode pointer */ |
| 2382 | smp_wmb(); |
| 2383 | } |
| 2384 | pcpu = smp_processor_id(); |
| 2385 | for (thr = 0; thr < threads_per_subcore; ++thr) |
| 2386 | paca[pcpu + thr].kvm_hstate.kvm_split_mode = sip; |
| 2387 | |
| 2388 | /* Initiate micro-threading (split-core) if required */ |
| 2389 | if (cmd_bit) { |
| 2390 | unsigned long hid0 = mfspr(SPRN_HID0); |
| 2391 | |
| 2392 | hid0 |= cmd_bit | HID0_POWER8_DYNLPARDIS; |
| 2393 | mb(); |
| 2394 | mtspr(SPRN_HID0, hid0); |
| 2395 | isync(); |
| 2396 | for (;;) { |
| 2397 | hid0 = mfspr(SPRN_HID0); |
| 2398 | if (hid0 & stat_bit) |
| 2399 | break; |
| 2400 | cpu_relax(); |
| 2401 | } |
Paul Mackerras | 2e25aa5 | 2012-02-19 17:46:32 +0000 | [diff] [blame] | 2402 | } |
Paul Mackerras | 371fefd | 2011-06-29 00:23:08 +0000 | [diff] [blame] | 2403 | |
Suresh Warrier | b8e6a87 | 2015-12-17 14:59:07 -0600 | [diff] [blame] | 2404 | kvmppc_clear_host_core(pcpu); |
| 2405 | |
Paul Mackerras | b4deba5 | 2015-07-02 20:38:16 +1000 | [diff] [blame] | 2406 | /* Start all the threads */ |
| 2407 | active = 0; |
| 2408 | for (sub = 0; sub < core_info.n_subcores; ++sub) { |
| 2409 | thr = subcore_thread_map[sub]; |
| 2410 | thr0_done = false; |
| 2411 | active |= 1 << thr; |
| 2412 | list_for_each_entry(pvc, &core_info.vcs[sub], preempt_list) { |
| 2413 | pvc->pcpu = pcpu + thr; |
Suraj Jitindar Singh | 7b5f827 | 2016-08-02 14:03:20 +1000 | [diff] [blame] | 2414 | for_each_runnable_thread(i, vcpu, pvc) { |
Paul Mackerras | b4deba5 | 2015-07-02 20:38:16 +1000 | [diff] [blame] | 2415 | kvmppc_start_thread(vcpu, pvc); |
| 2416 | kvmppc_create_dtl_entry(vcpu, pvc); |
| 2417 | trace_kvm_guest_enter(vcpu); |
| 2418 | if (!vcpu->arch.ptid) |
| 2419 | thr0_done = true; |
| 2420 | active |= 1 << (thr + vcpu->arch.ptid); |
| 2421 | } |
| 2422 | /* |
| 2423 | * We need to start the first thread of each subcore |
| 2424 | * even if it doesn't have a vcpu. |
| 2425 | */ |
| 2426 | if (pvc->master_vcore == pvc && !thr0_done) |
| 2427 | kvmppc_start_thread(NULL, pvc); |
| 2428 | thr += pvc->num_threads; |
| 2429 | } |
| 2430 | } |
Gautham R. Shenoy | 7f23532 | 2015-09-02 21:48:58 +0530 | [diff] [blame] | 2431 | |
| 2432 | /* |
| 2433 | * Ensure that split_info.do_nap is set after setting |
| 2434 | * the vcore pointer in the PACA of the secondaries. |
| 2435 | */ |
| 2436 | smp_mb(); |
| 2437 | if (cmd_bit) |
| 2438 | split_info.do_nap = 1; /* ask secondaries to nap when done */ |
| 2439 | |
Paul Mackerras | b4deba5 | 2015-07-02 20:38:16 +1000 | [diff] [blame] | 2440 | /* |
| 2441 | * When doing micro-threading, poke the inactive threads as well. |
| 2442 | * This gets them to the nap instruction after kvm_do_nap, |
| 2443 | * which reduces the time taken to unsplit later. |
| 2444 | */ |
| 2445 | if (split > 1) |
| 2446 | for (thr = 1; thr < threads_per_subcore; ++thr) |
| 2447 | if (!(active & (1 << thr))) |
| 2448 | kvmppc_ipi_thread(pcpu + thr); |
Paul Mackerras | e0b7ec0 | 2014-01-08 21:25:20 +1100 | [diff] [blame] | 2449 | |
Paul Mackerras | 2f12f03 | 2012-10-15 01:17:17 +0000 | [diff] [blame] | 2450 | vc->vcore_state = VCORE_RUNNING; |
Paul Mackerras | 371fefd | 2011-06-29 00:23:08 +0000 | [diff] [blame] | 2451 | preempt_disable(); |
Suresh E. Warrier | 3c78f78 | 2014-12-03 18:48:10 -0600 | [diff] [blame] | 2452 | |
| 2453 | trace_kvmppc_run_core(vc, 0); |
| 2454 | |
Paul Mackerras | b4deba5 | 2015-07-02 20:38:16 +1000 | [diff] [blame] | 2455 | for (sub = 0; sub < core_info.n_subcores; ++sub) |
| 2456 | list_for_each_entry(pvc, &core_info.vcs[sub], preempt_list) |
| 2457 | spin_unlock(&pvc->lock); |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 2458 | |
Paolo Bonzini | 6edaa53 | 2016-06-15 15:18:26 +0200 | [diff] [blame] | 2459 | guest_enter(); |
Paul Mackerras | 2c9097e | 2012-09-11 13:27:01 +0000 | [diff] [blame] | 2460 | |
Paul Mackerras | e0b7ec0 | 2014-01-08 21:25:20 +1100 | [diff] [blame] | 2461 | srcu_idx = srcu_read_lock(&vc->kvm->srcu); |
Paul Mackerras | 2c9097e | 2012-09-11 13:27:01 +0000 | [diff] [blame] | 2462 | |
Paul Mackerras | e0b7ec0 | 2014-01-08 21:25:20 +1100 | [diff] [blame] | 2463 | __kvmppc_vcore_entry(); |
Paul Mackerras | 19ccb76 | 2011-07-23 17:42:46 +1000 | [diff] [blame] | 2464 | |
Paul Mackerras | ec25716 | 2015-06-24 21:18:03 +1000 | [diff] [blame] | 2465 | srcu_read_unlock(&vc->kvm->srcu, srcu_idx); |
| 2466 | |
| 2467 | spin_lock(&vc->lock); |
Paul Mackerras | 371fefd | 2011-06-29 00:23:08 +0000 | [diff] [blame] | 2468 | /* prevent other vcpu threads from doing kvmppc_start_thread() now */ |
Paul Mackerras | 19ccb76 | 2011-07-23 17:42:46 +1000 | [diff] [blame] | 2469 | vc->vcore_state = VCORE_EXITING; |
Paul Mackerras | 371fefd | 2011-06-29 00:23:08 +0000 | [diff] [blame] | 2470 | |
Paul Mackerras | 371fefd | 2011-06-29 00:23:08 +0000 | [diff] [blame] | 2471 | /* wait for secondary threads to finish writing their state to memory */ |
Paul Mackerras | e0b7ec0 | 2014-01-08 21:25:20 +1100 | [diff] [blame] | 2472 | kvmppc_wait_for_nap(); |
Paul Mackerras | b4deba5 | 2015-07-02 20:38:16 +1000 | [diff] [blame] | 2473 | |
| 2474 | /* Return to whole-core mode if we split the core earlier */ |
| 2475 | if (split > 1) { |
| 2476 | unsigned long hid0 = mfspr(SPRN_HID0); |
| 2477 | unsigned long loops = 0; |
| 2478 | |
| 2479 | hid0 &= ~HID0_POWER8_DYNLPARDIS; |
| 2480 | stat_bit = HID0_POWER8_2LPARMODE | HID0_POWER8_4LPARMODE; |
| 2481 | mb(); |
| 2482 | mtspr(SPRN_HID0, hid0); |
| 2483 | isync(); |
| 2484 | for (;;) { |
| 2485 | hid0 = mfspr(SPRN_HID0); |
| 2486 | if (!(hid0 & stat_bit)) |
| 2487 | break; |
| 2488 | cpu_relax(); |
| 2489 | ++loops; |
| 2490 | } |
| 2491 | split_info.do_nap = 0; |
| 2492 | } |
| 2493 | |
| 2494 | /* Let secondaries go back to the offline loop */ |
| 2495 | for (i = 0; i < threads_per_subcore; ++i) { |
| 2496 | kvmppc_release_hwthread(pcpu + i); |
| 2497 | if (sip && sip->napped[i]) |
| 2498 | kvmppc_ipi_thread(pcpu + i); |
| 2499 | } |
| 2500 | |
Suresh Warrier | b8e6a87 | 2015-12-17 14:59:07 -0600 | [diff] [blame] | 2501 | kvmppc_set_host_core(pcpu); |
| 2502 | |
Paul Mackerras | 371fefd | 2011-06-29 00:23:08 +0000 | [diff] [blame] | 2503 | spin_unlock(&vc->lock); |
Paul Mackerras | 2c9097e | 2012-09-11 13:27:01 +0000 | [diff] [blame] | 2504 | |
Paul Mackerras | 371fefd | 2011-06-29 00:23:08 +0000 | [diff] [blame] | 2505 | /* make sure updates to secondary vcpu structs are visible now */ |
| 2506 | smp_mb(); |
Paolo Bonzini | 6edaa53 | 2016-06-15 15:18:26 +0200 | [diff] [blame] | 2507 | guest_exit(); |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 2508 | |
Paul Mackerras | b4deba5 | 2015-07-02 20:38:16 +1000 | [diff] [blame] | 2509 | for (sub = 0; sub < core_info.n_subcores; ++sub) |
| 2510 | list_for_each_entry_safe(pvc, vcnext, &core_info.vcs[sub], |
| 2511 | preempt_list) |
| 2512 | post_guest_process(pvc, pvc == vc); |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 2513 | |
Paul Mackerras | 913d3ff9a | 2012-10-15 01:16:48 +0000 | [diff] [blame] | 2514 | spin_lock(&vc->lock); |
Paul Mackerras | ec25716 | 2015-06-24 21:18:03 +1000 | [diff] [blame] | 2515 | preempt_enable(); |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 2516 | |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 2517 | out: |
Paul Mackerras | 19ccb76 | 2011-07-23 17:42:46 +1000 | [diff] [blame] | 2518 | vc->vcore_state = VCORE_INACTIVE; |
Suresh E. Warrier | 3c78f78 | 2014-12-03 18:48:10 -0600 | [diff] [blame] | 2519 | trace_kvmppc_run_core(vc, 1); |
Paul Mackerras | 371fefd | 2011-06-29 00:23:08 +0000 | [diff] [blame] | 2520 | } |
| 2521 | |
Paul Mackerras | 19ccb76 | 2011-07-23 17:42:46 +1000 | [diff] [blame] | 2522 | /* |
| 2523 | * Wait for some other vcpu thread to execute us, and |
| 2524 | * wake us up when we need to handle something in the host. |
| 2525 | */ |
Paul Mackerras | ec25716 | 2015-06-24 21:18:03 +1000 | [diff] [blame] | 2526 | static void kvmppc_wait_for_exec(struct kvmppc_vcore *vc, |
| 2527 | struct kvm_vcpu *vcpu, int wait_state) |
Paul Mackerras | 371fefd | 2011-06-29 00:23:08 +0000 | [diff] [blame] | 2528 | { |
Paul Mackerras | 371fefd | 2011-06-29 00:23:08 +0000 | [diff] [blame] | 2529 | DEFINE_WAIT(wait); |
| 2530 | |
Paul Mackerras | 19ccb76 | 2011-07-23 17:42:46 +1000 | [diff] [blame] | 2531 | prepare_to_wait(&vcpu->arch.cpu_run, &wait, wait_state); |
Paul Mackerras | ec25716 | 2015-06-24 21:18:03 +1000 | [diff] [blame] | 2532 | if (vcpu->arch.state == KVMPPC_VCPU_RUNNABLE) { |
| 2533 | spin_unlock(&vc->lock); |
Paul Mackerras | 19ccb76 | 2011-07-23 17:42:46 +1000 | [diff] [blame] | 2534 | schedule(); |
Paul Mackerras | ec25716 | 2015-06-24 21:18:03 +1000 | [diff] [blame] | 2535 | spin_lock(&vc->lock); |
| 2536 | } |
Paul Mackerras | 19ccb76 | 2011-07-23 17:42:46 +1000 | [diff] [blame] | 2537 | finish_wait(&vcpu->arch.cpu_run, &wait); |
| 2538 | } |
Paul Mackerras | 371fefd | 2011-06-29 00:23:08 +0000 | [diff] [blame] | 2539 | |
Suraj Jitindar Singh | 0cda69d | 2016-08-02 14:03:21 +1000 | [diff] [blame] | 2540 | static void grow_halt_poll_ns(struct kvmppc_vcore *vc) |
| 2541 | { |
| 2542 | /* 10us base */ |
| 2543 | if (vc->halt_poll_ns == 0 && halt_poll_ns_grow) |
| 2544 | vc->halt_poll_ns = 10000; |
| 2545 | else |
| 2546 | vc->halt_poll_ns *= halt_poll_ns_grow; |
| 2547 | |
| 2548 | if (vc->halt_poll_ns > halt_poll_max_ns) |
| 2549 | vc->halt_poll_ns = halt_poll_max_ns; |
| 2550 | } |
| 2551 | |
| 2552 | static void shrink_halt_poll_ns(struct kvmppc_vcore *vc) |
| 2553 | { |
| 2554 | if (halt_poll_ns_shrink == 0) |
| 2555 | vc->halt_poll_ns = 0; |
| 2556 | else |
| 2557 | vc->halt_poll_ns /= halt_poll_ns_shrink; |
| 2558 | } |
| 2559 | |
| 2560 | /* Check to see if any of the runnable vcpus on the vcore have pending |
| 2561 | * exceptions or are no longer ceded |
| 2562 | */ |
| 2563 | static int kvmppc_vcore_check_block(struct kvmppc_vcore *vc) |
| 2564 | { |
| 2565 | struct kvm_vcpu *vcpu; |
| 2566 | int i; |
| 2567 | |
| 2568 | for_each_runnable_thread(i, vcpu, vc) { |
| 2569 | if (vcpu->arch.pending_exceptions || !vcpu->arch.ceded) |
| 2570 | return 1; |
| 2571 | } |
| 2572 | |
| 2573 | return 0; |
| 2574 | } |
| 2575 | |
Paul Mackerras | 19ccb76 | 2011-07-23 17:42:46 +1000 | [diff] [blame] | 2576 | /* |
| 2577 | * All the vcpus in this vcore are idle, so wait for a decrementer |
| 2578 | * or external interrupt to one of the vcpus. vc->lock is held. |
| 2579 | */ |
| 2580 | static void kvmppc_vcore_blocked(struct kvmppc_vcore *vc) |
| 2581 | { |
Suraj Jitindar Singh | 2a27f51 | 2016-08-02 14:03:23 +1000 | [diff] [blame] | 2582 | ktime_t cur, start_poll, start_wait; |
Suraj Jitindar Singh | 0cda69d | 2016-08-02 14:03:21 +1000 | [diff] [blame] | 2583 | int do_sleep = 1; |
Suraj Jitindar Singh | 0cda69d | 2016-08-02 14:03:21 +1000 | [diff] [blame] | 2584 | u64 block_ns; |
Marcelo Tosatti | 8577370 | 2016-02-19 09:46:39 +0100 | [diff] [blame] | 2585 | DECLARE_SWAITQUEUE(wait); |
Suresh E. Warrier | 1bc5d59 | 2014-11-03 15:52:00 +1100 | [diff] [blame] | 2586 | |
Suraj Jitindar Singh | 0cda69d | 2016-08-02 14:03:21 +1000 | [diff] [blame] | 2587 | /* Poll for pending exceptions and ceded state */ |
Suraj Jitindar Singh | 2a27f51 | 2016-08-02 14:03:23 +1000 | [diff] [blame] | 2588 | cur = start_poll = ktime_get(); |
Suraj Jitindar Singh | 0cda69d | 2016-08-02 14:03:21 +1000 | [diff] [blame] | 2589 | if (vc->halt_poll_ns) { |
Suraj Jitindar Singh | 2a27f51 | 2016-08-02 14:03:23 +1000 | [diff] [blame] | 2590 | ktime_t stop = ktime_add_ns(start_poll, vc->halt_poll_ns); |
| 2591 | ++vc->runner->stat.halt_attempted_poll; |
Suraj Jitindar Singh | 0cda69d | 2016-08-02 14:03:21 +1000 | [diff] [blame] | 2592 | |
| 2593 | vc->vcore_state = VCORE_POLLING; |
| 2594 | spin_unlock(&vc->lock); |
| 2595 | |
| 2596 | do { |
| 2597 | if (kvmppc_vcore_check_block(vc)) { |
| 2598 | do_sleep = 0; |
| 2599 | break; |
| 2600 | } |
| 2601 | cur = ktime_get(); |
| 2602 | } while (single_task_running() && ktime_before(cur, stop)); |
| 2603 | |
| 2604 | spin_lock(&vc->lock); |
| 2605 | vc->vcore_state = VCORE_INACTIVE; |
| 2606 | |
Suraj Jitindar Singh | 2a27f51 | 2016-08-02 14:03:23 +1000 | [diff] [blame] | 2607 | if (!do_sleep) { |
| 2608 | ++vc->runner->stat.halt_successful_poll; |
Suraj Jitindar Singh | 0cda69d | 2016-08-02 14:03:21 +1000 | [diff] [blame] | 2609 | goto out; |
Suraj Jitindar Singh | 2a27f51 | 2016-08-02 14:03:23 +1000 | [diff] [blame] | 2610 | } |
Suraj Jitindar Singh | 0cda69d | 2016-08-02 14:03:21 +1000 | [diff] [blame] | 2611 | } |
| 2612 | |
Marcelo Tosatti | 8577370 | 2016-02-19 09:46:39 +0100 | [diff] [blame] | 2613 | prepare_to_swait(&vc->wq, &wait, TASK_INTERRUPTIBLE); |
Suresh E. Warrier | 1bc5d59 | 2014-11-03 15:52:00 +1100 | [diff] [blame] | 2614 | |
Suraj Jitindar Singh | 0cda69d | 2016-08-02 14:03:21 +1000 | [diff] [blame] | 2615 | if (kvmppc_vcore_check_block(vc)) { |
Marcelo Tosatti | 8577370 | 2016-02-19 09:46:39 +0100 | [diff] [blame] | 2616 | finish_swait(&vc->wq, &wait); |
Suraj Jitindar Singh | 0cda69d | 2016-08-02 14:03:21 +1000 | [diff] [blame] | 2617 | do_sleep = 0; |
Suraj Jitindar Singh | 2a27f51 | 2016-08-02 14:03:23 +1000 | [diff] [blame] | 2618 | /* If we polled, count this as a successful poll */ |
| 2619 | if (vc->halt_poll_ns) |
| 2620 | ++vc->runner->stat.halt_successful_poll; |
Suraj Jitindar Singh | 0cda69d | 2016-08-02 14:03:21 +1000 | [diff] [blame] | 2621 | goto out; |
Suresh E. Warrier | 1bc5d59 | 2014-11-03 15:52:00 +1100 | [diff] [blame] | 2622 | } |
| 2623 | |
Suraj Jitindar Singh | 2a27f51 | 2016-08-02 14:03:23 +1000 | [diff] [blame] | 2624 | start_wait = ktime_get(); |
| 2625 | |
Paul Mackerras | 19ccb76 | 2011-07-23 17:42:46 +1000 | [diff] [blame] | 2626 | vc->vcore_state = VCORE_SLEEPING; |
Suresh E. Warrier | 3c78f78 | 2014-12-03 18:48:10 -0600 | [diff] [blame] | 2627 | trace_kvmppc_vcore_blocked(vc, 0); |
Paul Mackerras | 19ccb76 | 2011-07-23 17:42:46 +1000 | [diff] [blame] | 2628 | spin_unlock(&vc->lock); |
Paul Mackerras | 913d3ff9a | 2012-10-15 01:16:48 +0000 | [diff] [blame] | 2629 | schedule(); |
Marcelo Tosatti | 8577370 | 2016-02-19 09:46:39 +0100 | [diff] [blame] | 2630 | finish_swait(&vc->wq, &wait); |
Paul Mackerras | 19ccb76 | 2011-07-23 17:42:46 +1000 | [diff] [blame] | 2631 | spin_lock(&vc->lock); |
| 2632 | vc->vcore_state = VCORE_INACTIVE; |
Suresh E. Warrier | 3c78f78 | 2014-12-03 18:48:10 -0600 | [diff] [blame] | 2633 | trace_kvmppc_vcore_blocked(vc, 1); |
Suraj Jitindar Singh | 2a27f51 | 2016-08-02 14:03:23 +1000 | [diff] [blame] | 2634 | ++vc->runner->stat.halt_successful_wait; |
Suraj Jitindar Singh | 0cda69d | 2016-08-02 14:03:21 +1000 | [diff] [blame] | 2635 | |
| 2636 | cur = ktime_get(); |
| 2637 | |
| 2638 | out: |
Suraj Jitindar Singh | 2a27f51 | 2016-08-02 14:03:23 +1000 | [diff] [blame] | 2639 | block_ns = ktime_to_ns(cur) - ktime_to_ns(start_poll); |
| 2640 | |
| 2641 | /* Attribute wait time */ |
| 2642 | if (do_sleep) { |
| 2643 | vc->runner->stat.halt_wait_ns += |
| 2644 | ktime_to_ns(cur) - ktime_to_ns(start_wait); |
| 2645 | /* Attribute failed poll time */ |
| 2646 | if (vc->halt_poll_ns) |
| 2647 | vc->runner->stat.halt_poll_fail_ns += |
| 2648 | ktime_to_ns(start_wait) - |
| 2649 | ktime_to_ns(start_poll); |
| 2650 | } else { |
| 2651 | /* Attribute successful poll time */ |
| 2652 | if (vc->halt_poll_ns) |
| 2653 | vc->runner->stat.halt_poll_success_ns += |
| 2654 | ktime_to_ns(cur) - |
| 2655 | ktime_to_ns(start_poll); |
| 2656 | } |
Suraj Jitindar Singh | 0cda69d | 2016-08-02 14:03:21 +1000 | [diff] [blame] | 2657 | |
| 2658 | /* Adjust poll time */ |
| 2659 | if (halt_poll_max_ns) { |
| 2660 | if (block_ns <= vc->halt_poll_ns) |
| 2661 | ; |
| 2662 | /* We slept and blocked for longer than the max halt time */ |
| 2663 | else if (vc->halt_poll_ns && block_ns > halt_poll_max_ns) |
| 2664 | shrink_halt_poll_ns(vc); |
| 2665 | /* We slept and our poll time is too small */ |
| 2666 | else if (vc->halt_poll_ns < halt_poll_max_ns && |
| 2667 | block_ns < halt_poll_max_ns) |
| 2668 | grow_halt_poll_ns(vc); |
| 2669 | } else |
| 2670 | vc->halt_poll_ns = 0; |
| 2671 | |
| 2672 | trace_kvmppc_vcore_wakeup(do_sleep, block_ns); |
Paul Mackerras | 19ccb76 | 2011-07-23 17:42:46 +1000 | [diff] [blame] | 2673 | } |
| 2674 | |
| 2675 | static int kvmppc_run_vcpu(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu) |
| 2676 | { |
Suraj Jitindar Singh | 7b5f827 | 2016-08-02 14:03:20 +1000 | [diff] [blame] | 2677 | int n_ceded, i; |
Paul Mackerras | 19ccb76 | 2011-07-23 17:42:46 +1000 | [diff] [blame] | 2678 | struct kvmppc_vcore *vc; |
Suraj Jitindar Singh | 7b5f827 | 2016-08-02 14:03:20 +1000 | [diff] [blame] | 2679 | struct kvm_vcpu *v; |
Paul Mackerras | 9e368f2 | 2011-06-29 00:40:08 +0000 | [diff] [blame] | 2680 | |
Suresh E. Warrier | 3c78f78 | 2014-12-03 18:48:10 -0600 | [diff] [blame] | 2681 | trace_kvmppc_run_vcpu_enter(vcpu); |
| 2682 | |
Paul Mackerras | 371fefd | 2011-06-29 00:23:08 +0000 | [diff] [blame] | 2683 | kvm_run->exit_reason = 0; |
| 2684 | vcpu->arch.ret = RESUME_GUEST; |
| 2685 | vcpu->arch.trap = 0; |
Paul Mackerras | 2f12f03 | 2012-10-15 01:17:17 +0000 | [diff] [blame] | 2686 | kvmppc_update_vpas(vcpu); |
Paul Mackerras | 371fefd | 2011-06-29 00:23:08 +0000 | [diff] [blame] | 2687 | |
Paul Mackerras | 371fefd | 2011-06-29 00:23:08 +0000 | [diff] [blame] | 2688 | /* |
| 2689 | * Synchronize with other threads in this virtual core |
| 2690 | */ |
| 2691 | vc = vcpu->arch.vcore; |
| 2692 | spin_lock(&vc->lock); |
Paul Mackerras | 19ccb76 | 2011-07-23 17:42:46 +1000 | [diff] [blame] | 2693 | vcpu->arch.ceded = 0; |
Paul Mackerras | 371fefd | 2011-06-29 00:23:08 +0000 | [diff] [blame] | 2694 | vcpu->arch.run_task = current; |
| 2695 | vcpu->arch.kvm_run = kvm_run; |
Paul Mackerras | c7b6767 | 2012-10-15 01:18:07 +0000 | [diff] [blame] | 2696 | vcpu->arch.stolen_logged = vcore_stolen_time(vc, mftb()); |
Paul Mackerras | 19ccb76 | 2011-07-23 17:42:46 +1000 | [diff] [blame] | 2697 | vcpu->arch.state = KVMPPC_VCPU_RUNNABLE; |
Paul Mackerras | c7b6767 | 2012-10-15 01:18:07 +0000 | [diff] [blame] | 2698 | vcpu->arch.busy_preempt = TB_NIL; |
Suraj Jitindar Singh | 7b5f827 | 2016-08-02 14:03:20 +1000 | [diff] [blame] | 2699 | WRITE_ONCE(vc->runnable_threads[vcpu->arch.ptid], vcpu); |
Paul Mackerras | 371fefd | 2011-06-29 00:23:08 +0000 | [diff] [blame] | 2700 | ++vc->n_runnable; |
| 2701 | |
Paul Mackerras | 19ccb76 | 2011-07-23 17:42:46 +1000 | [diff] [blame] | 2702 | /* |
| 2703 | * This happens the first time this is called for a vcpu. |
| 2704 | * If the vcore is already running, we may be able to start |
| 2705 | * this thread straight away and have it join in. |
| 2706 | */ |
Paul Mackerras | 8455d79 | 2012-10-15 01:17:42 +0000 | [diff] [blame] | 2707 | if (!signal_pending(current)) { |
Paul Mackerras | ec25716 | 2015-06-24 21:18:03 +1000 | [diff] [blame] | 2708 | if (vc->vcore_state == VCORE_PIGGYBACK) { |
| 2709 | struct kvmppc_vcore *mvc = vc->master_vcore; |
| 2710 | if (spin_trylock(&mvc->lock)) { |
| 2711 | if (mvc->vcore_state == VCORE_RUNNING && |
| 2712 | !VCORE_IS_EXITING(mvc)) { |
| 2713 | kvmppc_create_dtl_entry(vcpu, vc); |
Paul Mackerras | b4deba5 | 2015-07-02 20:38:16 +1000 | [diff] [blame] | 2714 | kvmppc_start_thread(vcpu, vc); |
Paul Mackerras | ec25716 | 2015-06-24 21:18:03 +1000 | [diff] [blame] | 2715 | trace_kvm_guest_enter(vcpu); |
| 2716 | } |
| 2717 | spin_unlock(&mvc->lock); |
| 2718 | } |
| 2719 | } else if (vc->vcore_state == VCORE_RUNNING && |
| 2720 | !VCORE_IS_EXITING(vc)) { |
Paul Mackerras | 2f12f03 | 2012-10-15 01:17:17 +0000 | [diff] [blame] | 2721 | kvmppc_create_dtl_entry(vcpu, vc); |
Paul Mackerras | b4deba5 | 2015-07-02 20:38:16 +1000 | [diff] [blame] | 2722 | kvmppc_start_thread(vcpu, vc); |
Suresh E. Warrier | 3c78f78 | 2014-12-03 18:48:10 -0600 | [diff] [blame] | 2723 | trace_kvm_guest_enter(vcpu); |
Paul Mackerras | 8455d79 | 2012-10-15 01:17:42 +0000 | [diff] [blame] | 2724 | } else if (vc->vcore_state == VCORE_SLEEPING) { |
Marcelo Tosatti | 8577370 | 2016-02-19 09:46:39 +0100 | [diff] [blame] | 2725 | swake_up(&vc->wq); |
Paul Mackerras | 19ccb76 | 2011-07-23 17:42:46 +1000 | [diff] [blame] | 2726 | } |
Paul Mackerras | 371fefd | 2011-06-29 00:23:08 +0000 | [diff] [blame] | 2727 | |
Paul Mackerras | 8455d79 | 2012-10-15 01:17:42 +0000 | [diff] [blame] | 2728 | } |
Paul Mackerras | 19ccb76 | 2011-07-23 17:42:46 +1000 | [diff] [blame] | 2729 | |
| 2730 | while (vcpu->arch.state == KVMPPC_VCPU_RUNNABLE && |
| 2731 | !signal_pending(current)) { |
Paul Mackerras | ec25716 | 2015-06-24 21:18:03 +1000 | [diff] [blame] | 2732 | if (vc->vcore_state == VCORE_PREEMPT && vc->runner == NULL) |
| 2733 | kvmppc_vcore_end_preempt(vc); |
| 2734 | |
Paul Mackerras | 8455d79 | 2012-10-15 01:17:42 +0000 | [diff] [blame] | 2735 | if (vc->vcore_state != VCORE_INACTIVE) { |
Paul Mackerras | ec25716 | 2015-06-24 21:18:03 +1000 | [diff] [blame] | 2736 | kvmppc_wait_for_exec(vc, vcpu, TASK_INTERRUPTIBLE); |
Paul Mackerras | 19ccb76 | 2011-07-23 17:42:46 +1000 | [diff] [blame] | 2737 | continue; |
| 2738 | } |
Suraj Jitindar Singh | 7b5f827 | 2016-08-02 14:03:20 +1000 | [diff] [blame] | 2739 | for_each_runnable_thread(i, v, vc) { |
Scott Wood | 7e28e60e | 2011-11-08 18:23:20 -0600 | [diff] [blame] | 2740 | kvmppc_core_prepare_to_enter(v); |
Paul Mackerras | 19ccb76 | 2011-07-23 17:42:46 +1000 | [diff] [blame] | 2741 | if (signal_pending(v->arch.run_task)) { |
| 2742 | kvmppc_remove_runnable(vc, v); |
| 2743 | v->stat.signal_exits++; |
| 2744 | v->arch.kvm_run->exit_reason = KVM_EXIT_INTR; |
| 2745 | v->arch.ret = -EINTR; |
| 2746 | wake_up(&v->arch.cpu_run); |
| 2747 | } |
| 2748 | } |
Paul Mackerras | 8455d79 | 2012-10-15 01:17:42 +0000 | [diff] [blame] | 2749 | if (!vc->n_runnable || vcpu->arch.state != KVMPPC_VCPU_RUNNABLE) |
| 2750 | break; |
Paul Mackerras | 8455d79 | 2012-10-15 01:17:42 +0000 | [diff] [blame] | 2751 | n_ceded = 0; |
Suraj Jitindar Singh | 7b5f827 | 2016-08-02 14:03:20 +1000 | [diff] [blame] | 2752 | for_each_runnable_thread(i, v, vc) { |
Paul Mackerras | 8455d79 | 2012-10-15 01:17:42 +0000 | [diff] [blame] | 2753 | if (!v->arch.pending_exceptions) |
| 2754 | n_ceded += v->arch.ceded; |
Paul Mackerras | 4619ac8 | 2013-04-17 20:31:41 +0000 | [diff] [blame] | 2755 | else |
| 2756 | v->arch.ceded = 0; |
| 2757 | } |
Paul Mackerras | 25fedfc | 2015-03-28 14:21:05 +1100 | [diff] [blame] | 2758 | vc->runner = vcpu; |
| 2759 | if (n_ceded == vc->n_runnable) { |
Paul Mackerras | 8455d79 | 2012-10-15 01:17:42 +0000 | [diff] [blame] | 2760 | kvmppc_vcore_blocked(vc); |
Konstantin Khlebnikov | c56dadf | 2015-07-15 12:52:03 +0300 | [diff] [blame] | 2761 | } else if (need_resched()) { |
Paul Mackerras | ec25716 | 2015-06-24 21:18:03 +1000 | [diff] [blame] | 2762 | kvmppc_vcore_preempt(vc); |
Paul Mackerras | 25fedfc | 2015-03-28 14:21:05 +1100 | [diff] [blame] | 2763 | /* Let something else run */ |
| 2764 | cond_resched_lock(&vc->lock); |
Paul Mackerras | ec25716 | 2015-06-24 21:18:03 +1000 | [diff] [blame] | 2765 | if (vc->vcore_state == VCORE_PREEMPT) |
| 2766 | kvmppc_vcore_end_preempt(vc); |
Paul Mackerras | 25fedfc | 2015-03-28 14:21:05 +1100 | [diff] [blame] | 2767 | } else { |
Paul Mackerras | 8455d79 | 2012-10-15 01:17:42 +0000 | [diff] [blame] | 2768 | kvmppc_run_core(vc); |
Paul Mackerras | 25fedfc | 2015-03-28 14:21:05 +1100 | [diff] [blame] | 2769 | } |
Paul Mackerras | 0456ec4 | 2012-02-03 00:56:21 +0000 | [diff] [blame] | 2770 | vc->runner = NULL; |
Paul Mackerras | 371fefd | 2011-06-29 00:23:08 +0000 | [diff] [blame] | 2771 | } |
| 2772 | |
Paul Mackerras | 8455d79 | 2012-10-15 01:17:42 +0000 | [diff] [blame] | 2773 | while (vcpu->arch.state == KVMPPC_VCPU_RUNNABLE && |
| 2774 | (vc->vcore_state == VCORE_RUNNING || |
Paul Mackerras | 5fc3e64 | 2015-09-18 13:13:44 +1000 | [diff] [blame] | 2775 | vc->vcore_state == VCORE_EXITING || |
| 2776 | vc->vcore_state == VCORE_PIGGYBACK)) |
Paul Mackerras | ec25716 | 2015-06-24 21:18:03 +1000 | [diff] [blame] | 2777 | kvmppc_wait_for_exec(vc, vcpu, TASK_UNINTERRUPTIBLE); |
Paul Mackerras | 8455d79 | 2012-10-15 01:17:42 +0000 | [diff] [blame] | 2778 | |
Paul Mackerras | 5fc3e64 | 2015-09-18 13:13:44 +1000 | [diff] [blame] | 2779 | if (vc->vcore_state == VCORE_PREEMPT && vc->runner == NULL) |
| 2780 | kvmppc_vcore_end_preempt(vc); |
| 2781 | |
Paul Mackerras | 8455d79 | 2012-10-15 01:17:42 +0000 | [diff] [blame] | 2782 | if (vcpu->arch.state == KVMPPC_VCPU_RUNNABLE) { |
| 2783 | kvmppc_remove_runnable(vc, vcpu); |
| 2784 | vcpu->stat.signal_exits++; |
| 2785 | kvm_run->exit_reason = KVM_EXIT_INTR; |
| 2786 | vcpu->arch.ret = -EINTR; |
| 2787 | } |
| 2788 | |
| 2789 | if (vc->n_runnable && vc->vcore_state == VCORE_INACTIVE) { |
| 2790 | /* Wake up some vcpu to run the core */ |
Suraj Jitindar Singh | 7b5f827 | 2016-08-02 14:03:20 +1000 | [diff] [blame] | 2791 | i = -1; |
| 2792 | v = next_runnable_thread(vc, &i); |
Paul Mackerras | 8455d79 | 2012-10-15 01:17:42 +0000 | [diff] [blame] | 2793 | wake_up(&v->arch.cpu_run); |
Paul Mackerras | 19ccb76 | 2011-07-23 17:42:46 +1000 | [diff] [blame] | 2794 | } |
Paul Mackerras | 371fefd | 2011-06-29 00:23:08 +0000 | [diff] [blame] | 2795 | |
Suresh E. Warrier | 3c78f78 | 2014-12-03 18:48:10 -0600 | [diff] [blame] | 2796 | trace_kvmppc_run_vcpu_exit(vcpu, kvm_run); |
Paul Mackerras | 19ccb76 | 2011-07-23 17:42:46 +1000 | [diff] [blame] | 2797 | spin_unlock(&vc->lock); |
Paul Mackerras | 371fefd | 2011-06-29 00:23:08 +0000 | [diff] [blame] | 2798 | return vcpu->arch.ret; |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 2799 | } |
| 2800 | |
Aneesh Kumar K.V | 3a167bea | 2013-10-07 22:17:53 +0530 | [diff] [blame] | 2801 | static int kvmppc_vcpu_run_hv(struct kvm_run *run, struct kvm_vcpu *vcpu) |
Paul Mackerras | a8606e2 | 2011-06-29 00:22:05 +0000 | [diff] [blame] | 2802 | { |
| 2803 | int r; |
Paul Mackerras | 913d3ff9a | 2012-10-15 01:16:48 +0000 | [diff] [blame] | 2804 | int srcu_idx; |
Paul Mackerras | 2f1527e | 2017-06-06 16:47:22 +1000 | [diff] [blame] | 2805 | unsigned long ebb_regs[3] = {}; /* shut up GCC */ |
Paul Mackerras | e5cd34d | 2017-06-15 15:43:17 +1000 | [diff] [blame] | 2806 | unsigned long user_tar = 0; |
| 2807 | unsigned int user_vrsave; |
Paul Mackerras | a8606e2 | 2011-06-29 00:22:05 +0000 | [diff] [blame] | 2808 | |
Alexander Graf | af8f38b | 2011-08-10 13:57:08 +0200 | [diff] [blame] | 2809 | if (!vcpu->arch.sane) { |
| 2810 | run->exit_reason = KVM_EXIT_INTERNAL_ERROR; |
| 2811 | return -EINVAL; |
| 2812 | } |
| 2813 | |
Paul Mackerras | 468aa93 | 2017-06-15 16:10:27 +1000 | [diff] [blame] | 2814 | /* |
| 2815 | * Don't allow entry with a suspended transaction, because |
| 2816 | * the guest entry/exit code will lose it. |
| 2817 | * If the guest has TM enabled, save away their TM-related SPRs |
| 2818 | * (they will get restored by the TM unavailable interrupt). |
| 2819 | */ |
| 2820 | #ifdef CONFIG_PPC_TRANSACTIONAL_MEM |
| 2821 | if (cpu_has_feature(CPU_FTR_TM) && current->thread.regs && |
| 2822 | (current->thread.regs->msr & MSR_TM)) { |
| 2823 | if (MSR_TM_ACTIVE(current->thread.regs->msr)) { |
| 2824 | run->exit_reason = KVM_EXIT_FAIL_ENTRY; |
| 2825 | run->fail_entry.hardware_entry_failure_reason = 0; |
| 2826 | return -EINVAL; |
| 2827 | } |
Paul Mackerras | d745f0f | 2017-07-21 13:57:14 +1000 | [diff] [blame] | 2828 | /* Enable TM so we can read the TM SPRs */ |
| 2829 | mtmsr(mfmsr() | MSR_TM); |
Paul Mackerras | 468aa93 | 2017-06-15 16:10:27 +1000 | [diff] [blame] | 2830 | current->thread.tm_tfhar = mfspr(SPRN_TFHAR); |
| 2831 | current->thread.tm_tfiar = mfspr(SPRN_TFIAR); |
| 2832 | current->thread.tm_texasr = mfspr(SPRN_TEXASR); |
| 2833 | current->thread.regs->msr &= ~MSR_TM; |
| 2834 | } |
| 2835 | #endif |
| 2836 | |
Scott Wood | 25051b5 | 2011-11-08 18:23:23 -0600 | [diff] [blame] | 2837 | kvmppc_core_prepare_to_enter(vcpu); |
| 2838 | |
Paul Mackerras | 19ccb76 | 2011-07-23 17:42:46 +1000 | [diff] [blame] | 2839 | /* No need to go into the guest when all we'll do is come back out */ |
| 2840 | if (signal_pending(current)) { |
| 2841 | run->exit_reason = KVM_EXIT_INTR; |
| 2842 | return -EINTR; |
| 2843 | } |
| 2844 | |
Paul Mackerras | 32fad28 | 2012-05-04 02:32:53 +0000 | [diff] [blame] | 2845 | atomic_inc(&vcpu->kvm->arch.vcpus_running); |
Aneesh Kumar K.V | 31037ec | 2015-03-20 20:39:42 +1100 | [diff] [blame] | 2846 | /* Order vcpus_running vs. hpte_setup_done, see kvmppc_alloc_reset_hpt */ |
Paul Mackerras | 32fad28 | 2012-05-04 02:32:53 +0000 | [diff] [blame] | 2847 | smp_mb(); |
| 2848 | |
Paul Mackerras | c17b98c | 2014-12-03 13:30:38 +1100 | [diff] [blame] | 2849 | /* On the first time here, set up HTAB and VRMA */ |
Aneesh Kumar K.V | 31037ec | 2015-03-20 20:39:42 +1100 | [diff] [blame] | 2850 | if (!vcpu->kvm->arch.hpte_setup_done) { |
Paul Mackerras | 32fad28 | 2012-05-04 02:32:53 +0000 | [diff] [blame] | 2851 | r = kvmppc_hv_setup_htab_rma(vcpu); |
Paul Mackerras | c77162d | 2011-12-12 12:31:00 +0000 | [diff] [blame] | 2852 | if (r) |
Paul Mackerras | 32fad28 | 2012-05-04 02:32:53 +0000 | [diff] [blame] | 2853 | goto out; |
Paul Mackerras | c77162d | 2011-12-12 12:31:00 +0000 | [diff] [blame] | 2854 | } |
Paul Mackerras | 19ccb76 | 2011-07-23 17:42:46 +1000 | [diff] [blame] | 2855 | |
Anton Blanchard | 579e633 | 2015-10-29 11:44:09 +1100 | [diff] [blame] | 2856 | flush_all_to_thread(current); |
| 2857 | |
Paul Mackerras | e5cd34d | 2017-06-15 15:43:17 +1000 | [diff] [blame] | 2858 | /* Save userspace EBB and other register values */ |
Paul Mackerras | 2f1527e | 2017-06-06 16:47:22 +1000 | [diff] [blame] | 2859 | if (cpu_has_feature(CPU_FTR_ARCH_207S)) { |
| 2860 | ebb_regs[0] = mfspr(SPRN_EBBHR); |
| 2861 | ebb_regs[1] = mfspr(SPRN_EBBRR); |
| 2862 | ebb_regs[2] = mfspr(SPRN_BESCR); |
Paul Mackerras | e5cd34d | 2017-06-15 15:43:17 +1000 | [diff] [blame] | 2863 | user_tar = mfspr(SPRN_TAR); |
Paul Mackerras | 2f1527e | 2017-06-06 16:47:22 +1000 | [diff] [blame] | 2864 | } |
Paul Mackerras | e5cd34d | 2017-06-15 15:43:17 +1000 | [diff] [blame] | 2865 | user_vrsave = mfspr(SPRN_VRSAVE); |
Paul Mackerras | 2f1527e | 2017-06-06 16:47:22 +1000 | [diff] [blame] | 2866 | |
Paul Mackerras | 19ccb76 | 2011-07-23 17:42:46 +1000 | [diff] [blame] | 2867 | vcpu->arch.wqp = &vcpu->arch.vcore->wq; |
Paul Mackerras | 342d3db | 2011-12-12 12:38:05 +0000 | [diff] [blame] | 2868 | vcpu->arch.pgdir = current->mm->pgd; |
Paul Mackerras | c7b6767 | 2012-10-15 01:18:07 +0000 | [diff] [blame] | 2869 | vcpu->arch.state = KVMPPC_VCPU_BUSY_IN_HOST; |
Paul Mackerras | 19ccb76 | 2011-07-23 17:42:46 +1000 | [diff] [blame] | 2870 | |
Paul Mackerras | a8606e2 | 2011-06-29 00:22:05 +0000 | [diff] [blame] | 2871 | do { |
| 2872 | r = kvmppc_run_vcpu(run, vcpu); |
| 2873 | |
| 2874 | if (run->exit_reason == KVM_EXIT_PAPR_HCALL && |
| 2875 | !(vcpu->arch.shregs.msr & MSR_PR)) { |
Suresh E. Warrier | 3c78f78 | 2014-12-03 18:48:10 -0600 | [diff] [blame] | 2876 | trace_kvm_hcall_enter(vcpu); |
Paul Mackerras | a8606e2 | 2011-06-29 00:22:05 +0000 | [diff] [blame] | 2877 | r = kvmppc_pseries_do_hcall(vcpu); |
Suresh E. Warrier | 3c78f78 | 2014-12-03 18:48:10 -0600 | [diff] [blame] | 2878 | trace_kvm_hcall_exit(vcpu, r); |
Scott Wood | 7e28e60e | 2011-11-08 18:23:20 -0600 | [diff] [blame] | 2879 | kvmppc_core_prepare_to_enter(vcpu); |
Paul Mackerras | 913d3ff9a | 2012-10-15 01:16:48 +0000 | [diff] [blame] | 2880 | } else if (r == RESUME_PAGE_FAULT) { |
| 2881 | srcu_idx = srcu_read_lock(&vcpu->kvm->srcu); |
| 2882 | r = kvmppc_book3s_hv_page_fault(run, vcpu, |
| 2883 | vcpu->arch.fault_dar, vcpu->arch.fault_dsisr); |
| 2884 | srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx); |
Suresh Warrier | f7af520 | 2016-08-19 15:35:52 +1000 | [diff] [blame] | 2885 | } else if (r == RESUME_PASSTHROUGH) |
| 2886 | r = kvmppc_xics_rm_complete(vcpu, 0); |
Greg Kurz | e59d24e | 2014-02-06 17:36:56 +0100 | [diff] [blame] | 2887 | } while (is_kvmppc_resume_guest(r)); |
Paul Mackerras | 32fad28 | 2012-05-04 02:32:53 +0000 | [diff] [blame] | 2888 | |
Paul Mackerras | e5cd34d | 2017-06-15 15:43:17 +1000 | [diff] [blame] | 2889 | /* Restore userspace EBB and other register values */ |
Paul Mackerras | 2f1527e | 2017-06-06 16:47:22 +1000 | [diff] [blame] | 2890 | if (cpu_has_feature(CPU_FTR_ARCH_207S)) { |
| 2891 | mtspr(SPRN_EBBHR, ebb_regs[0]); |
| 2892 | mtspr(SPRN_EBBRR, ebb_regs[1]); |
| 2893 | mtspr(SPRN_BESCR, ebb_regs[2]); |
Paul Mackerras | e5cd34d | 2017-06-15 15:43:17 +1000 | [diff] [blame] | 2894 | mtspr(SPRN_TAR, user_tar); |
| 2895 | mtspr(SPRN_FSCR, current->thread.fscr); |
Paul Mackerras | 2f1527e | 2017-06-06 16:47:22 +1000 | [diff] [blame] | 2896 | } |
Paul Mackerras | e5cd34d | 2017-06-15 15:43:17 +1000 | [diff] [blame] | 2897 | mtspr(SPRN_VRSAVE, user_vrsave); |
Paul Mackerras | 2f1527e | 2017-06-06 16:47:22 +1000 | [diff] [blame] | 2898 | |
Paul Mackerras | 32fad28 | 2012-05-04 02:32:53 +0000 | [diff] [blame] | 2899 | out: |
Paul Mackerras | c7b6767 | 2012-10-15 01:18:07 +0000 | [diff] [blame] | 2900 | vcpu->arch.state = KVMPPC_VCPU_NOTREADY; |
Paul Mackerras | 32fad28 | 2012-05-04 02:32:53 +0000 | [diff] [blame] | 2901 | atomic_dec(&vcpu->kvm->arch.vcpus_running); |
Paul Mackerras | a8606e2 | 2011-06-29 00:22:05 +0000 | [diff] [blame] | 2902 | return r; |
| 2903 | } |
| 2904 | |
Benjamin Herrenschmidt | 5b74716 | 2012-04-26 19:43:42 +0000 | [diff] [blame] | 2905 | static void kvmppc_add_seg_page_size(struct kvm_ppc_one_seg_page_size **sps, |
| 2906 | int linux_psize) |
| 2907 | { |
| 2908 | struct mmu_psize_def *def = &mmu_psize_defs[linux_psize]; |
| 2909 | |
| 2910 | if (!def->shift) |
| 2911 | return; |
| 2912 | (*sps)->page_shift = def->shift; |
| 2913 | (*sps)->slb_enc = def->sllp; |
| 2914 | (*sps)->enc[0].page_shift = def->shift; |
Aneesh Kumar K.V | b1022fb | 2013-04-28 09:37:35 +0000 | [diff] [blame] | 2915 | (*sps)->enc[0].pte_enc = def->penc[linux_psize]; |
Aneesh Kumar K.V | 1f365bb | 2014-05-06 23:31:36 +0530 | [diff] [blame] | 2916 | /* |
| 2917 | * Add 16MB MPSS support if host supports it |
| 2918 | */ |
| 2919 | if (linux_psize != MMU_PAGE_16M && def->penc[MMU_PAGE_16M] != -1) { |
| 2920 | (*sps)->enc[1].page_shift = 24; |
| 2921 | (*sps)->enc[1].pte_enc = def->penc[MMU_PAGE_16M]; |
| 2922 | } |
Benjamin Herrenschmidt | 5b74716 | 2012-04-26 19:43:42 +0000 | [diff] [blame] | 2923 | (*sps)++; |
| 2924 | } |
| 2925 | |
Aneesh Kumar K.V | 3a167bea | 2013-10-07 22:17:53 +0530 | [diff] [blame] | 2926 | static int kvm_vm_ioctl_get_smmu_info_hv(struct kvm *kvm, |
| 2927 | struct kvm_ppc_smmu_info *info) |
Benjamin Herrenschmidt | 5b74716 | 2012-04-26 19:43:42 +0000 | [diff] [blame] | 2928 | { |
| 2929 | struct kvm_ppc_one_seg_page_size *sps; |
| 2930 | |
| 2931 | info->flags = KVM_PPC_PAGE_SIZES_REAL; |
| 2932 | if (mmu_has_feature(MMU_FTR_1T_SEGMENT)) |
| 2933 | info->flags |= KVM_PPC_1T_SEGMENTS; |
| 2934 | info->slb_size = mmu_slb_size; |
| 2935 | |
| 2936 | /* We only support these sizes for now, and no muti-size segments */ |
| 2937 | sps = &info->sps[0]; |
| 2938 | kvmppc_add_seg_page_size(&sps, MMU_PAGE_4K); |
| 2939 | kvmppc_add_seg_page_size(&sps, MMU_PAGE_64K); |
| 2940 | kvmppc_add_seg_page_size(&sps, MMU_PAGE_16M); |
| 2941 | |
| 2942 | return 0; |
| 2943 | } |
| 2944 | |
Paul Mackerras | 82ed361 | 2011-12-15 02:03:22 +0000 | [diff] [blame] | 2945 | /* |
| 2946 | * Get (and clear) the dirty memory log for a memory slot. |
| 2947 | */ |
Aneesh Kumar K.V | 3a167bea | 2013-10-07 22:17:53 +0530 | [diff] [blame] | 2948 | static int kvm_vm_ioctl_get_dirty_log_hv(struct kvm *kvm, |
| 2949 | struct kvm_dirty_log *log) |
Paul Mackerras | 82ed361 | 2011-12-15 02:03:22 +0000 | [diff] [blame] | 2950 | { |
Paolo Bonzini | 9f6b802 | 2015-05-17 16:20:07 +0200 | [diff] [blame] | 2951 | struct kvm_memslots *slots; |
Paul Mackerras | 82ed361 | 2011-12-15 02:03:22 +0000 | [diff] [blame] | 2952 | struct kvm_memory_slot *memslot; |
| 2953 | int r; |
| 2954 | unsigned long n; |
| 2955 | |
| 2956 | mutex_lock(&kvm->slots_lock); |
| 2957 | |
| 2958 | r = -EINVAL; |
Alex Williamson | bbacc0c | 2012-12-10 10:33:09 -0700 | [diff] [blame] | 2959 | if (log->slot >= KVM_USER_MEM_SLOTS) |
Paul Mackerras | 82ed361 | 2011-12-15 02:03:22 +0000 | [diff] [blame] | 2960 | goto out; |
| 2961 | |
Paolo Bonzini | 9f6b802 | 2015-05-17 16:20:07 +0200 | [diff] [blame] | 2962 | slots = kvm_memslots(kvm); |
| 2963 | memslot = id_to_memslot(slots, log->slot); |
Paul Mackerras | 82ed361 | 2011-12-15 02:03:22 +0000 | [diff] [blame] | 2964 | r = -ENOENT; |
| 2965 | if (!memslot->dirty_bitmap) |
| 2966 | goto out; |
| 2967 | |
| 2968 | n = kvm_dirty_bitmap_bytes(memslot); |
| 2969 | memset(memslot->dirty_bitmap, 0, n); |
| 2970 | |
Paul Mackerras | dfe49db | 2012-09-11 13:28:18 +0000 | [diff] [blame] | 2971 | r = kvmppc_hv_get_dirty_log(kvm, memslot, memslot->dirty_bitmap); |
Paul Mackerras | 82ed361 | 2011-12-15 02:03:22 +0000 | [diff] [blame] | 2972 | if (r) |
| 2973 | goto out; |
| 2974 | |
| 2975 | r = -EFAULT; |
| 2976 | if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n)) |
| 2977 | goto out; |
| 2978 | |
| 2979 | r = 0; |
| 2980 | out: |
| 2981 | mutex_unlock(&kvm->slots_lock); |
| 2982 | return r; |
| 2983 | } |
| 2984 | |
Aneesh Kumar K.V | 3a167bea | 2013-10-07 22:17:53 +0530 | [diff] [blame] | 2985 | static void kvmppc_core_free_memslot_hv(struct kvm_memory_slot *free, |
| 2986 | struct kvm_memory_slot *dont) |
Paul Mackerras | a66b48c | 2012-09-11 13:27:46 +0000 | [diff] [blame] | 2987 | { |
| 2988 | if (!dont || free->arch.rmap != dont->arch.rmap) { |
| 2989 | vfree(free->arch.rmap); |
| 2990 | free->arch.rmap = NULL; |
| 2991 | } |
Paul Mackerras | a66b48c | 2012-09-11 13:27:46 +0000 | [diff] [blame] | 2992 | } |
| 2993 | |
Aneesh Kumar K.V | 3a167bea | 2013-10-07 22:17:53 +0530 | [diff] [blame] | 2994 | static int kvmppc_core_create_memslot_hv(struct kvm_memory_slot *slot, |
| 2995 | unsigned long npages) |
Paul Mackerras | a66b48c | 2012-09-11 13:27:46 +0000 | [diff] [blame] | 2996 | { |
| 2997 | slot->arch.rmap = vzalloc(npages * sizeof(*slot->arch.rmap)); |
| 2998 | if (!slot->arch.rmap) |
| 2999 | return -ENOMEM; |
Paul Mackerras | a66b48c | 2012-09-11 13:27:46 +0000 | [diff] [blame] | 3000 | |
| 3001 | return 0; |
| 3002 | } |
| 3003 | |
Aneesh Kumar K.V | 3a167bea | 2013-10-07 22:17:53 +0530 | [diff] [blame] | 3004 | static int kvmppc_core_prepare_memory_region_hv(struct kvm *kvm, |
| 3005 | struct kvm_memory_slot *memslot, |
Paolo Bonzini | 09170a4 | 2015-05-18 13:59:39 +0200 | [diff] [blame] | 3006 | const struct kvm_userspace_memory_region *mem) |
Paul Mackerras | a66b48c | 2012-09-11 13:27:46 +0000 | [diff] [blame] | 3007 | { |
Paul Mackerras | a66b48c | 2012-09-11 13:27:46 +0000 | [diff] [blame] | 3008 | return 0; |
| 3009 | } |
| 3010 | |
Aneesh Kumar K.V | 3a167bea | 2013-10-07 22:17:53 +0530 | [diff] [blame] | 3011 | static void kvmppc_core_commit_memory_region_hv(struct kvm *kvm, |
Paolo Bonzini | 09170a4 | 2015-05-18 13:59:39 +0200 | [diff] [blame] | 3012 | const struct kvm_userspace_memory_region *mem, |
Paolo Bonzini | f36f3f2 | 2015-05-18 13:20:23 +0200 | [diff] [blame] | 3013 | const struct kvm_memory_slot *old, |
| 3014 | const struct kvm_memory_slot *new) |
Paul Mackerras | c77162d | 2011-12-12 12:31:00 +0000 | [diff] [blame] | 3015 | { |
Paul Mackerras | dfe49db | 2012-09-11 13:28:18 +0000 | [diff] [blame] | 3016 | unsigned long npages = mem->memory_size >> PAGE_SHIFT; |
Paolo Bonzini | 9f6b802 | 2015-05-17 16:20:07 +0200 | [diff] [blame] | 3017 | struct kvm_memslots *slots; |
Paul Mackerras | dfe49db | 2012-09-11 13:28:18 +0000 | [diff] [blame] | 3018 | struct kvm_memory_slot *memslot; |
| 3019 | |
Takuya Yoshikawa | 8482644 | 2013-02-27 19:45:25 +0900 | [diff] [blame] | 3020 | if (npages && old->npages) { |
Paul Mackerras | dfe49db | 2012-09-11 13:28:18 +0000 | [diff] [blame] | 3021 | /* |
| 3022 | * If modifying a memslot, reset all the rmap dirty bits. |
| 3023 | * If this is a new memslot, we don't need to do anything |
| 3024 | * since the rmap array starts out as all zeroes, |
| 3025 | * i.e. no pages are dirty. |
| 3026 | */ |
Paolo Bonzini | 9f6b802 | 2015-05-17 16:20:07 +0200 | [diff] [blame] | 3027 | slots = kvm_memslots(kvm); |
| 3028 | memslot = id_to_memslot(slots, mem->slot); |
Paul Mackerras | dfe49db | 2012-09-11 13:28:18 +0000 | [diff] [blame] | 3029 | kvmppc_hv_get_dirty_log(kvm, memslot, NULL); |
| 3030 | } |
Paul Mackerras | c77162d | 2011-12-12 12:31:00 +0000 | [diff] [blame] | 3031 | } |
| 3032 | |
Paul Mackerras | a0144e2 | 2013-09-20 14:52:38 +1000 | [diff] [blame] | 3033 | /* |
| 3034 | * Update LPCR values in kvm->arch and in vcores. |
| 3035 | * Caller must hold kvm->lock. |
| 3036 | */ |
| 3037 | void kvmppc_update_lpcr(struct kvm *kvm, unsigned long lpcr, unsigned long mask) |
| 3038 | { |
| 3039 | long int i; |
| 3040 | u32 cores_done = 0; |
| 3041 | |
| 3042 | if ((kvm->arch.lpcr & mask) == lpcr) |
| 3043 | return; |
| 3044 | |
| 3045 | kvm->arch.lpcr = (kvm->arch.lpcr & ~mask) | lpcr; |
| 3046 | |
| 3047 | for (i = 0; i < KVM_MAX_VCORES; ++i) { |
| 3048 | struct kvmppc_vcore *vc = kvm->arch.vcores[i]; |
| 3049 | if (!vc) |
| 3050 | continue; |
| 3051 | spin_lock(&vc->lock); |
| 3052 | vc->lpcr = (vc->lpcr & ~mask) | lpcr; |
| 3053 | spin_unlock(&vc->lock); |
| 3054 | if (++cores_done >= kvm->arch.online_vcores) |
| 3055 | break; |
| 3056 | } |
| 3057 | } |
| 3058 | |
Aneesh Kumar K.V | 3a167bea | 2013-10-07 22:17:53 +0530 | [diff] [blame] | 3059 | static void kvmppc_mmu_destroy_hv(struct kvm_vcpu *vcpu) |
| 3060 | { |
| 3061 | return; |
| 3062 | } |
| 3063 | |
Paul Mackerras | 32fad28 | 2012-05-04 02:32:53 +0000 | [diff] [blame] | 3064 | static int kvmppc_hv_setup_htab_rma(struct kvm_vcpu *vcpu) |
Paul Mackerras | c77162d | 2011-12-12 12:31:00 +0000 | [diff] [blame] | 3065 | { |
| 3066 | int err = 0; |
| 3067 | struct kvm *kvm = vcpu->kvm; |
Paul Mackerras | c77162d | 2011-12-12 12:31:00 +0000 | [diff] [blame] | 3068 | unsigned long hva; |
| 3069 | struct kvm_memory_slot *memslot; |
| 3070 | struct vm_area_struct *vma; |
Paul Mackerras | a0144e2 | 2013-09-20 14:52:38 +1000 | [diff] [blame] | 3071 | unsigned long lpcr = 0, senc; |
Paul Mackerras | c77162d | 2011-12-12 12:31:00 +0000 | [diff] [blame] | 3072 | unsigned long psize, porder; |
Paul Mackerras | 2c9097e | 2012-09-11 13:27:01 +0000 | [diff] [blame] | 3073 | int srcu_idx; |
Paul Mackerras | c77162d | 2011-12-12 12:31:00 +0000 | [diff] [blame] | 3074 | |
| 3075 | mutex_lock(&kvm->lock); |
Aneesh Kumar K.V | 31037ec | 2015-03-20 20:39:42 +1100 | [diff] [blame] | 3076 | if (kvm->arch.hpte_setup_done) |
Paul Mackerras | c77162d | 2011-12-12 12:31:00 +0000 | [diff] [blame] | 3077 | goto out; /* another vcpu beat us to it */ |
| 3078 | |
Paul Mackerras | 32fad28 | 2012-05-04 02:32:53 +0000 | [diff] [blame] | 3079 | /* Allocate hashed page table (if not done already) and reset it */ |
| 3080 | if (!kvm->arch.hpt_virt) { |
| 3081 | err = kvmppc_alloc_hpt(kvm, NULL); |
| 3082 | if (err) { |
| 3083 | pr_err("KVM: Couldn't alloc HPT\n"); |
| 3084 | goto out; |
| 3085 | } |
| 3086 | } |
| 3087 | |
Paul Mackerras | c77162d | 2011-12-12 12:31:00 +0000 | [diff] [blame] | 3088 | /* Look up the memslot for guest physical address 0 */ |
Paul Mackerras | 2c9097e | 2012-09-11 13:27:01 +0000 | [diff] [blame] | 3089 | srcu_idx = srcu_read_lock(&kvm->srcu); |
Paul Mackerras | c77162d | 2011-12-12 12:31:00 +0000 | [diff] [blame] | 3090 | memslot = gfn_to_memslot(kvm, 0); |
| 3091 | |
| 3092 | /* We must have some memory at 0 by now */ |
| 3093 | err = -EINVAL; |
| 3094 | if (!memslot || (memslot->flags & KVM_MEMSLOT_INVALID)) |
Paul Mackerras | 2c9097e | 2012-09-11 13:27:01 +0000 | [diff] [blame] | 3095 | goto out_srcu; |
Paul Mackerras | c77162d | 2011-12-12 12:31:00 +0000 | [diff] [blame] | 3096 | |
| 3097 | /* Look up the VMA for the start of this memory slot */ |
| 3098 | hva = memslot->userspace_addr; |
| 3099 | down_read(¤t->mm->mmap_sem); |
| 3100 | vma = find_vma(current->mm, hva); |
| 3101 | if (!vma || vma->vm_start > hva || (vma->vm_flags & VM_IO)) |
| 3102 | goto up_out; |
| 3103 | |
| 3104 | psize = vma_kernel_pagesize(vma); |
Paul Mackerras | aa04b4c | 2011-06-29 00:25:44 +0000 | [diff] [blame] | 3105 | |
Paul Mackerras | c77162d | 2011-12-12 12:31:00 +0000 | [diff] [blame] | 3106 | up_read(¤t->mm->mmap_sem); |
| 3107 | |
Paul Mackerras | c17b98c | 2014-12-03 13:30:38 +1100 | [diff] [blame] | 3108 | /* We can handle 4k, 64k or 16M pages in the VRMA */ |
Paul Mackerras | ec12bb5 | 2018-03-02 15:38:04 +1100 | [diff] [blame] | 3109 | if (psize >= 0x1000000) |
| 3110 | psize = 0x1000000; |
| 3111 | else if (psize >= 0x10000) |
| 3112 | psize = 0x10000; |
| 3113 | else |
| 3114 | psize = 0x1000; |
| 3115 | porder = __ilog2(psize); |
Paul Mackerras | aa04b4c | 2011-06-29 00:25:44 +0000 | [diff] [blame] | 3116 | |
Paul Mackerras | c17b98c | 2014-12-03 13:30:38 +1100 | [diff] [blame] | 3117 | /* Update VRMASD field in the LPCR */ |
| 3118 | senc = slb_pgsize_encoding(psize); |
| 3119 | kvm->arch.vrma_slb_v = senc | SLB_VSID_B_1T | |
| 3120 | (VRMA_VSID << SLB_VSID_SHIFT_1T); |
| 3121 | /* the -4 is to account for senc values starting at 0x10 */ |
| 3122 | lpcr = senc << (LPCR_VRMASD_SH - 4); |
Paul Mackerras | da9d1d7 | 2011-12-12 12:31:41 +0000 | [diff] [blame] | 3123 | |
Paul Mackerras | c17b98c | 2014-12-03 13:30:38 +1100 | [diff] [blame] | 3124 | /* Create HPTEs in the hash page table for the VRMA */ |
| 3125 | kvmppc_map_vrma(vcpu, memslot, porder); |
Paul Mackerras | aa04b4c | 2011-06-29 00:25:44 +0000 | [diff] [blame] | 3126 | |
Paul Mackerras | c17b98c | 2014-12-03 13:30:38 +1100 | [diff] [blame] | 3127 | kvmppc_update_lpcr(kvm, lpcr, LPCR_VRMASD); |
Paul Mackerras | a0144e2 | 2013-09-20 14:52:38 +1000 | [diff] [blame] | 3128 | |
Aneesh Kumar K.V | 31037ec | 2015-03-20 20:39:42 +1100 | [diff] [blame] | 3129 | /* Order updates to kvm->arch.lpcr etc. vs. hpte_setup_done */ |
Paul Mackerras | c77162d | 2011-12-12 12:31:00 +0000 | [diff] [blame] | 3130 | smp_wmb(); |
Aneesh Kumar K.V | 31037ec | 2015-03-20 20:39:42 +1100 | [diff] [blame] | 3131 | kvm->arch.hpte_setup_done = 1; |
Paul Mackerras | c77162d | 2011-12-12 12:31:00 +0000 | [diff] [blame] | 3132 | err = 0; |
Paul Mackerras | 2c9097e | 2012-09-11 13:27:01 +0000 | [diff] [blame] | 3133 | out_srcu: |
| 3134 | srcu_read_unlock(&kvm->srcu, srcu_idx); |
Paul Mackerras | c77162d | 2011-12-12 12:31:00 +0000 | [diff] [blame] | 3135 | out: |
| 3136 | mutex_unlock(&kvm->lock); |
| 3137 | return err; |
Paul Mackerras | aa04b4c | 2011-06-29 00:25:44 +0000 | [diff] [blame] | 3138 | |
Paul Mackerras | c77162d | 2011-12-12 12:31:00 +0000 | [diff] [blame] | 3139 | up_out: |
| 3140 | up_read(¤t->mm->mmap_sem); |
Lai Jiangshan | 505d642 | 2013-03-16 00:50:49 +0800 | [diff] [blame] | 3141 | goto out_srcu; |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 3142 | } |
| 3143 | |
Suresh Warrier | 79b6c24 | 2015-12-17 14:59:06 -0600 | [diff] [blame] | 3144 | #ifdef CONFIG_KVM_XICS |
Suresh Warrier | 6f3bb80 | 2015-12-17 14:59:08 -0600 | [diff] [blame] | 3145 | static int kvmppc_cpu_notify(struct notifier_block *self, unsigned long action, |
| 3146 | void *hcpu) |
| 3147 | { |
| 3148 | unsigned long cpu = (long)hcpu; |
| 3149 | |
| 3150 | switch (action) { |
| 3151 | case CPU_UP_PREPARE: |
| 3152 | case CPU_UP_PREPARE_FROZEN: |
| 3153 | kvmppc_set_host_core(cpu); |
| 3154 | break; |
| 3155 | |
| 3156 | #ifdef CONFIG_HOTPLUG_CPU |
| 3157 | case CPU_DEAD: |
| 3158 | case CPU_DEAD_FROZEN: |
| 3159 | case CPU_UP_CANCELED: |
| 3160 | case CPU_UP_CANCELED_FROZEN: |
| 3161 | kvmppc_clear_host_core(cpu); |
| 3162 | break; |
| 3163 | #endif |
| 3164 | default: |
| 3165 | break; |
| 3166 | } |
| 3167 | |
| 3168 | return NOTIFY_OK; |
| 3169 | } |
| 3170 | |
| 3171 | static struct notifier_block kvmppc_cpu_notifier = { |
| 3172 | .notifier_call = kvmppc_cpu_notify, |
| 3173 | }; |
| 3174 | |
Suresh Warrier | 79b6c24 | 2015-12-17 14:59:06 -0600 | [diff] [blame] | 3175 | /* |
| 3176 | * Allocate a per-core structure for managing state about which cores are |
| 3177 | * running in the host versus the guest and for exchanging data between |
| 3178 | * real mode KVM and CPU running in the host. |
| 3179 | * This is only done for the first VM. |
| 3180 | * The allocated structure stays even if all VMs have stopped. |
| 3181 | * It is only freed when the kvm-hv module is unloaded. |
| 3182 | * It's OK for this routine to fail, we just don't support host |
| 3183 | * core operations like redirecting H_IPI wakeups. |
| 3184 | */ |
| 3185 | void kvmppc_alloc_host_rm_ops(void) |
| 3186 | { |
| 3187 | struct kvmppc_host_rm_ops *ops; |
| 3188 | unsigned long l_ops; |
| 3189 | int cpu, core; |
| 3190 | int size; |
| 3191 | |
| 3192 | /* Not the first time here ? */ |
| 3193 | if (kvmppc_host_rm_ops_hv != NULL) |
| 3194 | return; |
| 3195 | |
| 3196 | ops = kzalloc(sizeof(struct kvmppc_host_rm_ops), GFP_KERNEL); |
| 3197 | if (!ops) |
| 3198 | return; |
| 3199 | |
| 3200 | size = cpu_nr_cores() * sizeof(struct kvmppc_host_rm_core); |
| 3201 | ops->rm_core = kzalloc(size, GFP_KERNEL); |
| 3202 | |
| 3203 | if (!ops->rm_core) { |
| 3204 | kfree(ops); |
| 3205 | return; |
| 3206 | } |
| 3207 | |
Suresh Warrier | 6f3bb80 | 2015-12-17 14:59:08 -0600 | [diff] [blame] | 3208 | get_online_cpus(); |
| 3209 | |
Suresh Warrier | 79b6c24 | 2015-12-17 14:59:06 -0600 | [diff] [blame] | 3210 | for (cpu = 0; cpu < nr_cpu_ids; cpu += threads_per_core) { |
| 3211 | if (!cpu_online(cpu)) |
| 3212 | continue; |
| 3213 | |
| 3214 | core = cpu >> threads_shift; |
| 3215 | ops->rm_core[core].rm_state.in_host = 1; |
| 3216 | } |
| 3217 | |
Suresh Warrier | 0c2a660 | 2015-12-17 14:59:09 -0600 | [diff] [blame] | 3218 | ops->vcpu_kick = kvmppc_fast_vcpu_kick_hv; |
| 3219 | |
Suresh Warrier | 79b6c24 | 2015-12-17 14:59:06 -0600 | [diff] [blame] | 3220 | /* |
| 3221 | * Make the contents of the kvmppc_host_rm_ops structure visible |
| 3222 | * to other CPUs before we assign it to the global variable. |
| 3223 | * Do an atomic assignment (no locks used here), but if someone |
| 3224 | * beats us to it, just free our copy and return. |
| 3225 | */ |
| 3226 | smp_wmb(); |
| 3227 | l_ops = (unsigned long) ops; |
| 3228 | |
| 3229 | if (cmpxchg64((unsigned long *)&kvmppc_host_rm_ops_hv, 0, l_ops)) { |
Suresh Warrier | 6f3bb80 | 2015-12-17 14:59:08 -0600 | [diff] [blame] | 3230 | put_online_cpus(); |
Suresh Warrier | 79b6c24 | 2015-12-17 14:59:06 -0600 | [diff] [blame] | 3231 | kfree(ops->rm_core); |
| 3232 | kfree(ops); |
Suresh Warrier | 6f3bb80 | 2015-12-17 14:59:08 -0600 | [diff] [blame] | 3233 | return; |
Suresh Warrier | 79b6c24 | 2015-12-17 14:59:06 -0600 | [diff] [blame] | 3234 | } |
Suresh Warrier | 6f3bb80 | 2015-12-17 14:59:08 -0600 | [diff] [blame] | 3235 | |
| 3236 | register_cpu_notifier(&kvmppc_cpu_notifier); |
| 3237 | |
| 3238 | put_online_cpus(); |
Suresh Warrier | 79b6c24 | 2015-12-17 14:59:06 -0600 | [diff] [blame] | 3239 | } |
| 3240 | |
| 3241 | void kvmppc_free_host_rm_ops(void) |
| 3242 | { |
| 3243 | if (kvmppc_host_rm_ops_hv) { |
Suresh Warrier | 6f3bb80 | 2015-12-17 14:59:08 -0600 | [diff] [blame] | 3244 | unregister_cpu_notifier(&kvmppc_cpu_notifier); |
Suresh Warrier | 79b6c24 | 2015-12-17 14:59:06 -0600 | [diff] [blame] | 3245 | kfree(kvmppc_host_rm_ops_hv->rm_core); |
| 3246 | kfree(kvmppc_host_rm_ops_hv); |
| 3247 | kvmppc_host_rm_ops_hv = NULL; |
| 3248 | } |
| 3249 | } |
| 3250 | #endif |
| 3251 | |
Aneesh Kumar K.V | 3a167bea | 2013-10-07 22:17:53 +0530 | [diff] [blame] | 3252 | static int kvmppc_core_init_vm_hv(struct kvm *kvm) |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 3253 | { |
Paul Mackerras | 32fad28 | 2012-05-04 02:32:53 +0000 | [diff] [blame] | 3254 | unsigned long lpcr, lpid; |
Paul Mackerras | e23a808 | 2015-03-28 14:21:01 +1100 | [diff] [blame] | 3255 | char buf[32]; |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 3256 | |
Paul Mackerras | 32fad28 | 2012-05-04 02:32:53 +0000 | [diff] [blame] | 3257 | /* Allocate the guest's logical partition ID */ |
| 3258 | |
| 3259 | lpid = kvmppc_alloc_lpid(); |
Chen Gang | 5d226ae | 2013-07-22 14:32:35 +0800 | [diff] [blame] | 3260 | if ((long)lpid < 0) |
Paul Mackerras | 32fad28 | 2012-05-04 02:32:53 +0000 | [diff] [blame] | 3261 | return -ENOMEM; |
| 3262 | kvm->arch.lpid = lpid; |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 3263 | |
Suresh Warrier | 79b6c24 | 2015-12-17 14:59:06 -0600 | [diff] [blame] | 3264 | kvmppc_alloc_host_rm_ops(); |
| 3265 | |
Paul Mackerras | 1b400ba | 2012-11-21 23:28:08 +0000 | [diff] [blame] | 3266 | /* |
| 3267 | * Since we don't flush the TLB when tearing down a VM, |
| 3268 | * and this lpid might have previously been used, |
| 3269 | * make sure we flush on each core before running the new VM. |
| 3270 | */ |
| 3271 | cpumask_setall(&kvm->arch.need_tlb_flush); |
| 3272 | |
Paul Mackerras | 699a0ea | 2014-06-02 11:02:59 +1000 | [diff] [blame] | 3273 | /* Start out with the default set of hcalls enabled */ |
| 3274 | memcpy(kvm->arch.enabled_hcalls, default_enabled_hcalls, |
| 3275 | sizeof(kvm->arch.enabled_hcalls)); |
| 3276 | |
Paul Mackerras | 9e368f2 | 2011-06-29 00:40:08 +0000 | [diff] [blame] | 3277 | kvm->arch.host_sdr1 = mfspr(SPRN_SDR1); |
Paul Mackerras | aa04b4c | 2011-06-29 00:25:44 +0000 | [diff] [blame] | 3278 | |
Paul Mackerras | c17b98c | 2014-12-03 13:30:38 +1100 | [diff] [blame] | 3279 | /* Init LPCR for virtual RMA mode */ |
| 3280 | kvm->arch.host_lpid = mfspr(SPRN_LPID); |
| 3281 | kvm->arch.host_lpcr = lpcr = mfspr(SPRN_LPCR); |
| 3282 | lpcr &= LPCR_PECE | LPCR_LPES; |
| 3283 | lpcr |= (4UL << LPCR_DPFD_SH) | LPCR_HDICE | |
| 3284 | LPCR_VPM0 | LPCR_VPM1; |
| 3285 | kvm->arch.vrma_slb_v = SLB_VSID_B_1T | |
| 3286 | (VRMA_VSID << SLB_VSID_SHIFT_1T); |
| 3287 | /* On POWER8 turn on online bit to enable PURR/SPURR */ |
| 3288 | if (cpu_has_feature(CPU_FTR_ARCH_207S)) |
| 3289 | lpcr |= LPCR_ONL; |
Paul Mackerras | 9e368f2 | 2011-06-29 00:40:08 +0000 | [diff] [blame] | 3290 | kvm->arch.lpcr = lpcr; |
Paul Mackerras | aa04b4c | 2011-06-29 00:25:44 +0000 | [diff] [blame] | 3291 | |
Paul Mackerras | 512691d | 2012-10-15 01:15:41 +0000 | [diff] [blame] | 3292 | /* |
Michael Ellerman | 441c19c | 2014-05-23 18:15:25 +1000 | [diff] [blame] | 3293 | * Track that we now have a HV mode VM active. This blocks secondary |
| 3294 | * CPU threads from coming online. |
Paul Mackerras | 512691d | 2012-10-15 01:15:41 +0000 | [diff] [blame] | 3295 | */ |
Michael Ellerman | 441c19c | 2014-05-23 18:15:25 +1000 | [diff] [blame] | 3296 | kvm_hv_vm_activated(); |
Paul Mackerras | 512691d | 2012-10-15 01:15:41 +0000 | [diff] [blame] | 3297 | |
Paul Mackerras | e23a808 | 2015-03-28 14:21:01 +1100 | [diff] [blame] | 3298 | /* |
| 3299 | * Create a debugfs directory for the VM |
| 3300 | */ |
| 3301 | snprintf(buf, sizeof(buf), "vm%d", current->pid); |
| 3302 | kvm->arch.debugfs_dir = debugfs_create_dir(buf, kvm_debugfs_dir); |
| 3303 | if (!IS_ERR_OR_NULL(kvm->arch.debugfs_dir)) |
| 3304 | kvmppc_mmu_debugfs_init(kvm); |
| 3305 | |
David Gibson | 54738c0 | 2011-06-29 00:22:41 +0000 | [diff] [blame] | 3306 | return 0; |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 3307 | } |
| 3308 | |
Paul Mackerras | f1378b1 | 2013-09-27 15:33:43 +0530 | [diff] [blame] | 3309 | static void kvmppc_free_vcores(struct kvm *kvm) |
| 3310 | { |
| 3311 | long int i; |
| 3312 | |
Paul Mackerras | 2331631 | 2015-10-21 16:03:14 +1100 | [diff] [blame] | 3313 | for (i = 0; i < KVM_MAX_VCORES; ++i) |
Paul Mackerras | f1378b1 | 2013-09-27 15:33:43 +0530 | [diff] [blame] | 3314 | kfree(kvm->arch.vcores[i]); |
| 3315 | kvm->arch.online_vcores = 0; |
| 3316 | } |
| 3317 | |
Aneesh Kumar K.V | 3a167bea | 2013-10-07 22:17:53 +0530 | [diff] [blame] | 3318 | static void kvmppc_core_destroy_vm_hv(struct kvm *kvm) |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 3319 | { |
Paul Mackerras | e23a808 | 2015-03-28 14:21:01 +1100 | [diff] [blame] | 3320 | debugfs_remove_recursive(kvm->arch.debugfs_dir); |
| 3321 | |
Michael Ellerman | 441c19c | 2014-05-23 18:15:25 +1000 | [diff] [blame] | 3322 | kvm_hv_vm_deactivated(); |
Paul Mackerras | 512691d | 2012-10-15 01:15:41 +0000 | [diff] [blame] | 3323 | |
Paul Mackerras | f1378b1 | 2013-09-27 15:33:43 +0530 | [diff] [blame] | 3324 | kvmppc_free_vcores(kvm); |
Paul Mackerras | aa04b4c | 2011-06-29 00:25:44 +0000 | [diff] [blame] | 3325 | |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 3326 | kvmppc_free_hpt(kvm); |
Suresh Warrier | c57875f | 2016-08-19 15:35:50 +1000 | [diff] [blame] | 3327 | |
| 3328 | kvmppc_free_pimap(kvm); |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 3329 | } |
| 3330 | |
| 3331 | /* We don't need to emulate any privileged instructions or dcbz */ |
Aneesh Kumar K.V | 3a167bea | 2013-10-07 22:17:53 +0530 | [diff] [blame] | 3332 | static int kvmppc_core_emulate_op_hv(struct kvm_run *run, struct kvm_vcpu *vcpu, |
| 3333 | unsigned int inst, int *advance) |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 3334 | { |
| 3335 | return EMULATE_FAIL; |
| 3336 | } |
| 3337 | |
Aneesh Kumar K.V | 3a167bea | 2013-10-07 22:17:53 +0530 | [diff] [blame] | 3338 | static int kvmppc_core_emulate_mtspr_hv(struct kvm_vcpu *vcpu, int sprn, |
| 3339 | ulong spr_val) |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 3340 | { |
| 3341 | return EMULATE_FAIL; |
| 3342 | } |
| 3343 | |
Aneesh Kumar K.V | 3a167bea | 2013-10-07 22:17:53 +0530 | [diff] [blame] | 3344 | static int kvmppc_core_emulate_mfspr_hv(struct kvm_vcpu *vcpu, int sprn, |
| 3345 | ulong *spr_val) |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 3346 | { |
| 3347 | return EMULATE_FAIL; |
| 3348 | } |
| 3349 | |
Aneesh Kumar K.V | 3a167bea | 2013-10-07 22:17:53 +0530 | [diff] [blame] | 3350 | static int kvmppc_core_check_processor_compat_hv(void) |
| 3351 | { |
Paul Mackerras | c17b98c | 2014-12-03 13:30:38 +1100 | [diff] [blame] | 3352 | if (!cpu_has_feature(CPU_FTR_HVMODE) || |
| 3353 | !cpu_has_feature(CPU_FTR_ARCH_206)) |
Aneesh Kumar K.V | 3a167bea | 2013-10-07 22:17:53 +0530 | [diff] [blame] | 3354 | return -EIO; |
Aneesh Kumar K.V | 50de596 | 2016-04-29 23:25:43 +1000 | [diff] [blame] | 3355 | /* |
| 3356 | * Disable KVM for Power9, untill the required bits merged. |
| 3357 | */ |
| 3358 | if (cpu_has_feature(CPU_FTR_ARCH_300)) |
| 3359 | return -EIO; |
| 3360 | |
Aneesh Kumar K.V | 3a167bea | 2013-10-07 22:17:53 +0530 | [diff] [blame] | 3361 | return 0; |
| 3362 | } |
| 3363 | |
Suresh Warrier | 8daaafc | 2016-08-19 15:35:48 +1000 | [diff] [blame] | 3364 | #ifdef CONFIG_KVM_XICS |
| 3365 | |
| 3366 | void kvmppc_free_pimap(struct kvm *kvm) |
| 3367 | { |
| 3368 | kfree(kvm->arch.pimap); |
| 3369 | } |
| 3370 | |
Suresh Warrier | c57875f | 2016-08-19 15:35:50 +1000 | [diff] [blame] | 3371 | static struct kvmppc_passthru_irqmap *kvmppc_alloc_pimap(void) |
Suresh Warrier | 8daaafc | 2016-08-19 15:35:48 +1000 | [diff] [blame] | 3372 | { |
| 3373 | return kzalloc(sizeof(struct kvmppc_passthru_irqmap), GFP_KERNEL); |
| 3374 | } |
Suresh Warrier | c57875f | 2016-08-19 15:35:50 +1000 | [diff] [blame] | 3375 | |
| 3376 | static int kvmppc_set_passthru_irq(struct kvm *kvm, int host_irq, int guest_gsi) |
| 3377 | { |
| 3378 | struct irq_desc *desc; |
| 3379 | struct kvmppc_irq_map *irq_map; |
| 3380 | struct kvmppc_passthru_irqmap *pimap; |
| 3381 | struct irq_chip *chip; |
| 3382 | int i; |
| 3383 | |
Suresh Warrier | 644abbb | 2016-08-19 15:35:54 +1000 | [diff] [blame] | 3384 | if (!kvm_irq_bypass) |
| 3385 | return 1; |
| 3386 | |
Suresh Warrier | c57875f | 2016-08-19 15:35:50 +1000 | [diff] [blame] | 3387 | desc = irq_to_desc(host_irq); |
| 3388 | if (!desc) |
| 3389 | return -EIO; |
| 3390 | |
| 3391 | mutex_lock(&kvm->lock); |
| 3392 | |
| 3393 | pimap = kvm->arch.pimap; |
| 3394 | if (pimap == NULL) { |
| 3395 | /* First call, allocate structure to hold IRQ map */ |
| 3396 | pimap = kvmppc_alloc_pimap(); |
| 3397 | if (pimap == NULL) { |
| 3398 | mutex_unlock(&kvm->lock); |
| 3399 | return -ENOMEM; |
| 3400 | } |
| 3401 | kvm->arch.pimap = pimap; |
| 3402 | } |
| 3403 | |
| 3404 | /* |
| 3405 | * For now, we only support interrupts for which the EOI operation |
| 3406 | * is an OPAL call followed by a write to XIRR, since that's |
| 3407 | * what our real-mode EOI code does. |
| 3408 | */ |
| 3409 | chip = irq_data_get_irq_chip(&desc->irq_data); |
| 3410 | if (!chip || !is_pnv_opal_msi(chip)) { |
| 3411 | pr_warn("kvmppc_set_passthru_irq_hv: Could not assign IRQ map for (%d,%d)\n", |
| 3412 | host_irq, guest_gsi); |
| 3413 | mutex_unlock(&kvm->lock); |
| 3414 | return -ENOENT; |
| 3415 | } |
| 3416 | |
| 3417 | /* |
| 3418 | * See if we already have an entry for this guest IRQ number. |
| 3419 | * If it's mapped to a hardware IRQ number, that's an error, |
| 3420 | * otherwise re-use this entry. |
| 3421 | */ |
| 3422 | for (i = 0; i < pimap->n_mapped; i++) { |
| 3423 | if (guest_gsi == pimap->mapped[i].v_hwirq) { |
| 3424 | if (pimap->mapped[i].r_hwirq) { |
| 3425 | mutex_unlock(&kvm->lock); |
| 3426 | return -EINVAL; |
| 3427 | } |
| 3428 | break; |
| 3429 | } |
| 3430 | } |
| 3431 | |
| 3432 | if (i == KVMPPC_PIRQ_MAPPED) { |
| 3433 | mutex_unlock(&kvm->lock); |
| 3434 | return -EAGAIN; /* table is full */ |
| 3435 | } |
| 3436 | |
| 3437 | irq_map = &pimap->mapped[i]; |
| 3438 | |
| 3439 | irq_map->v_hwirq = guest_gsi; |
Suresh Warrier | c57875f | 2016-08-19 15:35:50 +1000 | [diff] [blame] | 3440 | irq_map->desc = desc; |
| 3441 | |
Suresh Warrier | e3c13e5 | 2016-08-19 15:35:51 +1000 | [diff] [blame] | 3442 | /* |
| 3443 | * Order the above two stores before the next to serialize with |
| 3444 | * the KVM real mode handler. |
| 3445 | */ |
| 3446 | smp_wmb(); |
| 3447 | irq_map->r_hwirq = desc->irq_data.hwirq; |
| 3448 | |
Suresh Warrier | c57875f | 2016-08-19 15:35:50 +1000 | [diff] [blame] | 3449 | if (i == pimap->n_mapped) |
| 3450 | pimap->n_mapped++; |
| 3451 | |
Paul Mackerras | 5d37519 | 2016-08-19 15:35:56 +1000 | [diff] [blame] | 3452 | kvmppc_xics_set_mapped(kvm, guest_gsi, desc->irq_data.hwirq); |
| 3453 | |
Suresh Warrier | c57875f | 2016-08-19 15:35:50 +1000 | [diff] [blame] | 3454 | mutex_unlock(&kvm->lock); |
| 3455 | |
| 3456 | return 0; |
| 3457 | } |
| 3458 | |
| 3459 | static int kvmppc_clr_passthru_irq(struct kvm *kvm, int host_irq, int guest_gsi) |
| 3460 | { |
| 3461 | struct irq_desc *desc; |
| 3462 | struct kvmppc_passthru_irqmap *pimap; |
| 3463 | int i; |
| 3464 | |
Suresh Warrier | 644abbb | 2016-08-19 15:35:54 +1000 | [diff] [blame] | 3465 | if (!kvm_irq_bypass) |
| 3466 | return 0; |
| 3467 | |
Suresh Warrier | c57875f | 2016-08-19 15:35:50 +1000 | [diff] [blame] | 3468 | desc = irq_to_desc(host_irq); |
| 3469 | if (!desc) |
| 3470 | return -EIO; |
| 3471 | |
| 3472 | mutex_lock(&kvm->lock); |
| 3473 | |
| 3474 | if (kvm->arch.pimap == NULL) { |
| 3475 | mutex_unlock(&kvm->lock); |
| 3476 | return 0; |
| 3477 | } |
| 3478 | pimap = kvm->arch.pimap; |
| 3479 | |
| 3480 | for (i = 0; i < pimap->n_mapped; i++) { |
| 3481 | if (guest_gsi == pimap->mapped[i].v_hwirq) |
| 3482 | break; |
| 3483 | } |
| 3484 | |
| 3485 | if (i == pimap->n_mapped) { |
| 3486 | mutex_unlock(&kvm->lock); |
| 3487 | return -ENODEV; |
| 3488 | } |
| 3489 | |
Paul Mackerras | 5d37519 | 2016-08-19 15:35:56 +1000 | [diff] [blame] | 3490 | kvmppc_xics_clr_mapped(kvm, guest_gsi, pimap->mapped[i].r_hwirq); |
| 3491 | |
Suresh Warrier | c57875f | 2016-08-19 15:35:50 +1000 | [diff] [blame] | 3492 | /* invalidate the entry */ |
| 3493 | pimap->mapped[i].r_hwirq = 0; |
| 3494 | |
| 3495 | /* |
| 3496 | * We don't free this structure even when the count goes to |
| 3497 | * zero. The structure is freed when we destroy the VM. |
| 3498 | */ |
| 3499 | |
| 3500 | mutex_unlock(&kvm->lock); |
| 3501 | return 0; |
| 3502 | } |
| 3503 | |
| 3504 | static int kvmppc_irq_bypass_add_producer_hv(struct irq_bypass_consumer *cons, |
| 3505 | struct irq_bypass_producer *prod) |
| 3506 | { |
| 3507 | int ret = 0; |
| 3508 | struct kvm_kernel_irqfd *irqfd = |
| 3509 | container_of(cons, struct kvm_kernel_irqfd, consumer); |
| 3510 | |
| 3511 | irqfd->producer = prod; |
| 3512 | |
| 3513 | ret = kvmppc_set_passthru_irq(irqfd->kvm, prod->irq, irqfd->gsi); |
| 3514 | if (ret) |
| 3515 | pr_info("kvmppc_set_passthru_irq (irq %d, gsi %d) fails: %d\n", |
| 3516 | prod->irq, irqfd->gsi, ret); |
| 3517 | |
| 3518 | return ret; |
| 3519 | } |
| 3520 | |
| 3521 | static void kvmppc_irq_bypass_del_producer_hv(struct irq_bypass_consumer *cons, |
| 3522 | struct irq_bypass_producer *prod) |
| 3523 | { |
| 3524 | int ret; |
| 3525 | struct kvm_kernel_irqfd *irqfd = |
| 3526 | container_of(cons, struct kvm_kernel_irqfd, consumer); |
| 3527 | |
| 3528 | irqfd->producer = NULL; |
| 3529 | |
| 3530 | /* |
| 3531 | * When producer of consumer is unregistered, we change back to |
| 3532 | * default external interrupt handling mode - KVM real mode |
| 3533 | * will switch back to host. |
| 3534 | */ |
| 3535 | ret = kvmppc_clr_passthru_irq(irqfd->kvm, prod->irq, irqfd->gsi); |
| 3536 | if (ret) |
| 3537 | pr_warn("kvmppc_clr_passthru_irq (irq %d, gsi %d) fails: %d\n", |
| 3538 | prod->irq, irqfd->gsi, ret); |
| 3539 | } |
Suresh Warrier | 8daaafc | 2016-08-19 15:35:48 +1000 | [diff] [blame] | 3540 | #endif |
| 3541 | |
Aneesh Kumar K.V | 3a167bea | 2013-10-07 22:17:53 +0530 | [diff] [blame] | 3542 | static long kvm_arch_vm_ioctl_hv(struct file *filp, |
| 3543 | unsigned int ioctl, unsigned long arg) |
| 3544 | { |
| 3545 | struct kvm *kvm __maybe_unused = filp->private_data; |
| 3546 | void __user *argp = (void __user *)arg; |
| 3547 | long r; |
| 3548 | |
| 3549 | switch (ioctl) { |
| 3550 | |
Aneesh Kumar K.V | 3a167bea | 2013-10-07 22:17:53 +0530 | [diff] [blame] | 3551 | case KVM_PPC_ALLOCATE_HTAB: { |
| 3552 | u32 htab_order; |
| 3553 | |
| 3554 | r = -EFAULT; |
| 3555 | if (get_user(htab_order, (u32 __user *)argp)) |
| 3556 | break; |
| 3557 | r = kvmppc_alloc_reset_hpt(kvm, &htab_order); |
| 3558 | if (r) |
| 3559 | break; |
| 3560 | r = -EFAULT; |
| 3561 | if (put_user(htab_order, (u32 __user *)argp)) |
| 3562 | break; |
| 3563 | r = 0; |
| 3564 | break; |
| 3565 | } |
| 3566 | |
| 3567 | case KVM_PPC_GET_HTAB_FD: { |
| 3568 | struct kvm_get_htab_fd ghf; |
| 3569 | |
| 3570 | r = -EFAULT; |
| 3571 | if (copy_from_user(&ghf, argp, sizeof(ghf))) |
| 3572 | break; |
| 3573 | r = kvm_vm_ioctl_get_htab_fd(kvm, &ghf); |
| 3574 | break; |
| 3575 | } |
| 3576 | |
| 3577 | default: |
| 3578 | r = -ENOTTY; |
| 3579 | } |
| 3580 | |
| 3581 | return r; |
| 3582 | } |
| 3583 | |
Paul Mackerras | 699a0ea | 2014-06-02 11:02:59 +1000 | [diff] [blame] | 3584 | /* |
| 3585 | * List of hcall numbers to enable by default. |
| 3586 | * For compatibility with old userspace, we enable by default |
| 3587 | * all hcalls that were implemented before the hcall-enabling |
| 3588 | * facility was added. Note this list should not include H_RTAS. |
| 3589 | */ |
| 3590 | static unsigned int default_hcall_list[] = { |
| 3591 | H_REMOVE, |
| 3592 | H_ENTER, |
| 3593 | H_READ, |
| 3594 | H_PROTECT, |
| 3595 | H_BULK_REMOVE, |
| 3596 | H_GET_TCE, |
| 3597 | H_PUT_TCE, |
| 3598 | H_SET_DABR, |
| 3599 | H_SET_XDABR, |
| 3600 | H_CEDE, |
| 3601 | H_PROD, |
| 3602 | H_CONFER, |
| 3603 | H_REGISTER_VPA, |
| 3604 | #ifdef CONFIG_KVM_XICS |
| 3605 | H_EOI, |
| 3606 | H_CPPR, |
| 3607 | H_IPI, |
| 3608 | H_IPOLL, |
| 3609 | H_XIRR, |
| 3610 | H_XIRR_X, |
| 3611 | #endif |
| 3612 | 0 |
| 3613 | }; |
| 3614 | |
| 3615 | static void init_default_hcalls(void) |
| 3616 | { |
| 3617 | int i; |
Paul Mackerras | ae2113a | 2014-06-02 11:03:00 +1000 | [diff] [blame] | 3618 | unsigned int hcall; |
Paul Mackerras | 699a0ea | 2014-06-02 11:02:59 +1000 | [diff] [blame] | 3619 | |
Paul Mackerras | ae2113a | 2014-06-02 11:03:00 +1000 | [diff] [blame] | 3620 | for (i = 0; default_hcall_list[i]; ++i) { |
| 3621 | hcall = default_hcall_list[i]; |
| 3622 | WARN_ON(!kvmppc_hcall_impl_hv(hcall)); |
| 3623 | __set_bit(hcall / 4, default_enabled_hcalls); |
| 3624 | } |
Paul Mackerras | 699a0ea | 2014-06-02 11:02:59 +1000 | [diff] [blame] | 3625 | } |
| 3626 | |
Aneesh Kumar K.V | cbbc58d | 2013-10-07 22:18:01 +0530 | [diff] [blame] | 3627 | static struct kvmppc_ops kvm_ops_hv = { |
Aneesh Kumar K.V | 3a167bea | 2013-10-07 22:17:53 +0530 | [diff] [blame] | 3628 | .get_sregs = kvm_arch_vcpu_ioctl_get_sregs_hv, |
| 3629 | .set_sregs = kvm_arch_vcpu_ioctl_set_sregs_hv, |
| 3630 | .get_one_reg = kvmppc_get_one_reg_hv, |
| 3631 | .set_one_reg = kvmppc_set_one_reg_hv, |
| 3632 | .vcpu_load = kvmppc_core_vcpu_load_hv, |
| 3633 | .vcpu_put = kvmppc_core_vcpu_put_hv, |
| 3634 | .set_msr = kvmppc_set_msr_hv, |
| 3635 | .vcpu_run = kvmppc_vcpu_run_hv, |
| 3636 | .vcpu_create = kvmppc_core_vcpu_create_hv, |
| 3637 | .vcpu_free = kvmppc_core_vcpu_free_hv, |
| 3638 | .check_requests = kvmppc_core_check_requests_hv, |
| 3639 | .get_dirty_log = kvm_vm_ioctl_get_dirty_log_hv, |
| 3640 | .flush_memslot = kvmppc_core_flush_memslot_hv, |
| 3641 | .prepare_memory_region = kvmppc_core_prepare_memory_region_hv, |
| 3642 | .commit_memory_region = kvmppc_core_commit_memory_region_hv, |
| 3643 | .unmap_hva = kvm_unmap_hva_hv, |
| 3644 | .unmap_hva_range = kvm_unmap_hva_range_hv, |
| 3645 | .age_hva = kvm_age_hva_hv, |
| 3646 | .test_age_hva = kvm_test_age_hva_hv, |
| 3647 | .set_spte_hva = kvm_set_spte_hva_hv, |
| 3648 | .mmu_destroy = kvmppc_mmu_destroy_hv, |
| 3649 | .free_memslot = kvmppc_core_free_memslot_hv, |
| 3650 | .create_memslot = kvmppc_core_create_memslot_hv, |
| 3651 | .init_vm = kvmppc_core_init_vm_hv, |
| 3652 | .destroy_vm = kvmppc_core_destroy_vm_hv, |
Aneesh Kumar K.V | 3a167bea | 2013-10-07 22:17:53 +0530 | [diff] [blame] | 3653 | .get_smmu_info = kvm_vm_ioctl_get_smmu_info_hv, |
| 3654 | .emulate_op = kvmppc_core_emulate_op_hv, |
| 3655 | .emulate_mtspr = kvmppc_core_emulate_mtspr_hv, |
| 3656 | .emulate_mfspr = kvmppc_core_emulate_mfspr_hv, |
| 3657 | .fast_vcpu_kick = kvmppc_fast_vcpu_kick_hv, |
| 3658 | .arch_vm_ioctl = kvm_arch_vm_ioctl_hv, |
Paul Mackerras | ae2113a | 2014-06-02 11:03:00 +1000 | [diff] [blame] | 3659 | .hcall_implemented = kvmppc_hcall_impl_hv, |
Suresh Warrier | c57875f | 2016-08-19 15:35:50 +1000 | [diff] [blame] | 3660 | #ifdef CONFIG_KVM_XICS |
| 3661 | .irq_bypass_add_producer = kvmppc_irq_bypass_add_producer_hv, |
| 3662 | .irq_bypass_del_producer = kvmppc_irq_bypass_del_producer_hv, |
| 3663 | #endif |
Aneesh Kumar K.V | 3a167bea | 2013-10-07 22:17:53 +0530 | [diff] [blame] | 3664 | }; |
| 3665 | |
Mahesh Salgaonkar | fd7bacb | 2016-05-15 09:44:26 +0530 | [diff] [blame] | 3666 | static int kvm_init_subcore_bitmap(void) |
| 3667 | { |
| 3668 | int i, j; |
| 3669 | int nr_cores = cpu_nr_cores(); |
| 3670 | struct sibling_subcore_state *sibling_subcore_state; |
| 3671 | |
| 3672 | for (i = 0; i < nr_cores; i++) { |
| 3673 | int first_cpu = i * threads_per_core; |
| 3674 | int node = cpu_to_node(first_cpu); |
| 3675 | |
| 3676 | /* Ignore if it is already allocated. */ |
| 3677 | if (paca[first_cpu].sibling_subcore_state) |
| 3678 | continue; |
| 3679 | |
| 3680 | sibling_subcore_state = |
| 3681 | kmalloc_node(sizeof(struct sibling_subcore_state), |
| 3682 | GFP_KERNEL, node); |
| 3683 | if (!sibling_subcore_state) |
| 3684 | return -ENOMEM; |
| 3685 | |
| 3686 | memset(sibling_subcore_state, 0, |
| 3687 | sizeof(struct sibling_subcore_state)); |
| 3688 | |
| 3689 | for (j = 0; j < threads_per_core; j++) { |
| 3690 | int cpu = first_cpu + j; |
| 3691 | |
| 3692 | paca[cpu].sibling_subcore_state = sibling_subcore_state; |
| 3693 | } |
| 3694 | } |
| 3695 | return 0; |
| 3696 | } |
| 3697 | |
Aneesh Kumar K.V | 3a167bea | 2013-10-07 22:17:53 +0530 | [diff] [blame] | 3698 | static int kvmppc_book3s_init_hv(void) |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 3699 | { |
| 3700 | int r; |
Aneesh Kumar K.V | cbbc58d | 2013-10-07 22:18:01 +0530 | [diff] [blame] | 3701 | /* |
| 3702 | * FIXME!! Do we need to check on all cpus ? |
| 3703 | */ |
| 3704 | r = kvmppc_core_check_processor_compat_hv(); |
| 3705 | if (r < 0) |
Paul Mackerras | 739e242 | 2014-03-25 10:47:05 +1100 | [diff] [blame] | 3706 | return -ENODEV; |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 3707 | |
Mahesh Salgaonkar | fd7bacb | 2016-05-15 09:44:26 +0530 | [diff] [blame] | 3708 | r = kvm_init_subcore_bitmap(); |
| 3709 | if (r) |
| 3710 | return r; |
| 3711 | |
Aneesh Kumar K.V | cbbc58d | 2013-10-07 22:18:01 +0530 | [diff] [blame] | 3712 | kvm_ops_hv.owner = THIS_MODULE; |
| 3713 | kvmppc_hv_ops = &kvm_ops_hv; |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 3714 | |
Paul Mackerras | 699a0ea | 2014-06-02 11:02:59 +1000 | [diff] [blame] | 3715 | init_default_hcalls(); |
| 3716 | |
Paul Mackerras | ec25716 | 2015-06-24 21:18:03 +1000 | [diff] [blame] | 3717 | init_vcore_lists(); |
| 3718 | |
Aneesh Kumar K.V | cbbc58d | 2013-10-07 22:18:01 +0530 | [diff] [blame] | 3719 | r = kvmppc_mmu_hv_init(); |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 3720 | return r; |
| 3721 | } |
| 3722 | |
Aneesh Kumar K.V | 3a167bea | 2013-10-07 22:17:53 +0530 | [diff] [blame] | 3723 | static void kvmppc_book3s_exit_hv(void) |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 3724 | { |
Suresh Warrier | 79b6c24 | 2015-12-17 14:59:06 -0600 | [diff] [blame] | 3725 | kvmppc_free_host_rm_ops(); |
Aneesh Kumar K.V | cbbc58d | 2013-10-07 22:18:01 +0530 | [diff] [blame] | 3726 | kvmppc_hv_ops = NULL; |
Paul Mackerras | de56a94 | 2011-06-29 00:21:34 +0000 | [diff] [blame] | 3727 | } |
| 3728 | |
Aneesh Kumar K.V | 3a167bea | 2013-10-07 22:17:53 +0530 | [diff] [blame] | 3729 | module_init(kvmppc_book3s_init_hv); |
| 3730 | module_exit(kvmppc_book3s_exit_hv); |
Aneesh Kumar K.V | 2ba9f0d | 2013-10-07 22:17:59 +0530 | [diff] [blame] | 3731 | MODULE_LICENSE("GPL"); |
Alexander Graf | 398a76c | 2013-12-09 13:53:42 +0100 | [diff] [blame] | 3732 | MODULE_ALIAS_MISCDEV(KVM_MINOR); |
| 3733 | MODULE_ALIAS("devname:kvm"); |