blob: 77dec0f8a03040dd614aef5535acaa2505529172 [file] [log] [blame]
Paul Mackerrasde56a942011-06-29 00:21:34 +00001/*
2 * Copyright 2011 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
3 * Copyright (C) 2009. SUSE Linux Products GmbH. All rights reserved.
4 *
5 * Authors:
6 * Paul Mackerras <paulus@au1.ibm.com>
7 * Alexander Graf <agraf@suse.de>
8 * Kevin Wolf <mail@kevin-wolf.de>
9 *
10 * Description: KVM functions specific to running on Book 3S
11 * processors in hypervisor mode (specifically POWER7 and later).
12 *
13 * This file is derived from arch/powerpc/kvm/book3s.c,
14 * by Alexander Graf <agraf@suse.de>.
15 *
16 * This program is free software; you can redistribute it and/or modify
17 * it under the terms of the GNU General Public License, version 2, as
18 * published by the Free Software Foundation.
19 */
20
21#include <linux/kvm_host.h>
22#include <linux/err.h>
23#include <linux/slab.h>
24#include <linux/preempt.h>
25#include <linux/sched.h>
26#include <linux/delay.h>
Paul Gortmaker66b15db2011-05-27 10:46:24 -040027#include <linux/export.h>
Paul Mackerrasde56a942011-06-29 00:21:34 +000028#include <linux/fs.h>
29#include <linux/anon_inodes.h>
30#include <linux/cpumask.h>
Paul Mackerrasaa04b4c2011-06-29 00:25:44 +000031#include <linux/spinlock.h>
32#include <linux/page-flags.h>
Paul Mackerras2c9097e2012-09-11 13:27:01 +000033#include <linux/srcu.h>
Paul Mackerrasde56a942011-06-29 00:21:34 +000034
35#include <asm/reg.h>
36#include <asm/cputable.h>
37#include <asm/cacheflush.h>
38#include <asm/tlbflush.h>
39#include <asm/uaccess.h>
40#include <asm/io.h>
41#include <asm/kvm_ppc.h>
42#include <asm/kvm_book3s.h>
43#include <asm/mmu_context.h>
44#include <asm/lppaca.h>
45#include <asm/processor.h>
Paul Mackerras371fefd2011-06-29 00:23:08 +000046#include <asm/cputhreads.h>
Paul Mackerrasaa04b4c2011-06-29 00:25:44 +000047#include <asm/page.h>
Michael Neulingde1d9242011-11-09 20:39:49 +000048#include <asm/hvcall.h>
David Howellsae3a1972012-03-28 18:30:02 +010049#include <asm/switch_to.h>
Paul Mackerras512691d2012-10-15 01:15:41 +000050#include <asm/smp.h>
Paul Mackerrasde56a942011-06-29 00:21:34 +000051#include <linux/gfp.h>
Paul Mackerrasde56a942011-06-29 00:21:34 +000052#include <linux/vmalloc.h>
53#include <linux/highmem.h>
Paul Mackerrasc77162d2011-12-12 12:31:00 +000054#include <linux/hugetlb.h>
Paul Mackerrasde56a942011-06-29 00:21:34 +000055
56/* #define EXIT_DEBUG */
57/* #define EXIT_DEBUG_SIMPLE */
58/* #define EXIT_DEBUG_INT */
59
Paul Mackerras19ccb762011-07-23 17:42:46 +100060static void kvmppc_end_cede(struct kvm_vcpu *vcpu);
Paul Mackerras32fad282012-05-04 02:32:53 +000061static int kvmppc_hv_setup_htab_rma(struct kvm_vcpu *vcpu);
Paul Mackerras19ccb762011-07-23 17:42:46 +100062
Paul Mackerrasde56a942011-06-29 00:21:34 +000063void kvmppc_core_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
64{
Paul Mackerras0456ec42012-02-03 00:56:21 +000065 struct kvmppc_vcore *vc = vcpu->arch.vcore;
66
Paul Mackerras0456ec42012-02-03 00:56:21 +000067 if (vc->runner == vcpu && vc->vcore_state != VCORE_INACTIVE)
68 vc->stolen_tb += mftb() - vc->preempt_tb;
Paul Mackerrasde56a942011-06-29 00:21:34 +000069}
70
71void kvmppc_core_vcpu_put(struct kvm_vcpu *vcpu)
72{
Paul Mackerras0456ec42012-02-03 00:56:21 +000073 struct kvmppc_vcore *vc = vcpu->arch.vcore;
74
75 if (vc->runner == vcpu && vc->vcore_state != VCORE_INACTIVE)
76 vc->preempt_tb = mftb();
Paul Mackerrasde56a942011-06-29 00:21:34 +000077}
78
Paul Mackerrasde56a942011-06-29 00:21:34 +000079void kvmppc_set_msr(struct kvm_vcpu *vcpu, u64 msr)
80{
81 vcpu->arch.shregs.msr = msr;
Paul Mackerras19ccb762011-07-23 17:42:46 +100082 kvmppc_end_cede(vcpu);
Paul Mackerrasde56a942011-06-29 00:21:34 +000083}
84
85void kvmppc_set_pvr(struct kvm_vcpu *vcpu, u32 pvr)
86{
87 vcpu->arch.pvr = pvr;
88}
89
90void kvmppc_dump_regs(struct kvm_vcpu *vcpu)
91{
92 int r;
93
94 pr_err("vcpu %p (%d):\n", vcpu, vcpu->vcpu_id);
95 pr_err("pc = %.16lx msr = %.16llx trap = %x\n",
96 vcpu->arch.pc, vcpu->arch.shregs.msr, vcpu->arch.trap);
97 for (r = 0; r < 16; ++r)
98 pr_err("r%2d = %.16lx r%d = %.16lx\n",
99 r, kvmppc_get_gpr(vcpu, r),
100 r+16, kvmppc_get_gpr(vcpu, r+16));
101 pr_err("ctr = %.16lx lr = %.16lx\n",
102 vcpu->arch.ctr, vcpu->arch.lr);
103 pr_err("srr0 = %.16llx srr1 = %.16llx\n",
104 vcpu->arch.shregs.srr0, vcpu->arch.shregs.srr1);
105 pr_err("sprg0 = %.16llx sprg1 = %.16llx\n",
106 vcpu->arch.shregs.sprg0, vcpu->arch.shregs.sprg1);
107 pr_err("sprg2 = %.16llx sprg3 = %.16llx\n",
108 vcpu->arch.shregs.sprg2, vcpu->arch.shregs.sprg3);
109 pr_err("cr = %.8x xer = %.16lx dsisr = %.8x\n",
110 vcpu->arch.cr, vcpu->arch.xer, vcpu->arch.shregs.dsisr);
111 pr_err("dar = %.16llx\n", vcpu->arch.shregs.dar);
112 pr_err("fault dar = %.16lx dsisr = %.8x\n",
113 vcpu->arch.fault_dar, vcpu->arch.fault_dsisr);
114 pr_err("SLB (%d entries):\n", vcpu->arch.slb_max);
115 for (r = 0; r < vcpu->arch.slb_max; ++r)
116 pr_err(" ESID = %.16llx VSID = %.16llx\n",
117 vcpu->arch.slb[r].orige, vcpu->arch.slb[r].origv);
118 pr_err("lpcr = %.16lx sdr1 = %.16lx last_inst = %.8x\n",
Paul Mackerrasaa04b4c2011-06-29 00:25:44 +0000119 vcpu->kvm->arch.lpcr, vcpu->kvm->arch.sdr1,
Paul Mackerrasde56a942011-06-29 00:21:34 +0000120 vcpu->arch.last_inst);
121}
122
Paul Mackerrasa8606e22011-06-29 00:22:05 +0000123struct kvm_vcpu *kvmppc_find_vcpu(struct kvm *kvm, int id)
124{
125 int r;
126 struct kvm_vcpu *v, *ret = NULL;
127
128 mutex_lock(&kvm->lock);
129 kvm_for_each_vcpu(r, v, kvm) {
130 if (v->vcpu_id == id) {
131 ret = v;
132 break;
133 }
134 }
135 mutex_unlock(&kvm->lock);
136 return ret;
137}
138
139static void init_vpa(struct kvm_vcpu *vcpu, struct lppaca *vpa)
140{
141 vpa->shared_proc = 1;
142 vpa->yield_count = 1;
143}
144
Paul Mackerras55b665b2012-09-25 20:33:06 +0000145static int set_vpa(struct kvm_vcpu *vcpu, struct kvmppc_vpa *v,
146 unsigned long addr, unsigned long len)
147{
148 /* check address is cacheline aligned */
149 if (addr & (L1_CACHE_BYTES - 1))
150 return -EINVAL;
151 spin_lock(&vcpu->arch.vpa_update_lock);
152 if (v->next_gpa != addr || v->len != len) {
153 v->next_gpa = addr;
154 v->len = addr ? len : 0;
155 v->update_pending = 1;
156 }
157 spin_unlock(&vcpu->arch.vpa_update_lock);
158 return 0;
159}
160
Paul Mackerras2e25aa52012-02-19 17:46:32 +0000161/* Length for a per-processor buffer is passed in at offset 4 in the buffer */
162struct reg_vpa {
163 u32 dummy;
164 union {
165 u16 hword;
166 u32 word;
167 } length;
168};
169
170static int vpa_is_registered(struct kvmppc_vpa *vpap)
171{
172 if (vpap->update_pending)
173 return vpap->next_gpa != 0;
174 return vpap->pinned_addr != NULL;
175}
176
Paul Mackerrasa8606e22011-06-29 00:22:05 +0000177static unsigned long do_h_register_vpa(struct kvm_vcpu *vcpu,
178 unsigned long flags,
179 unsigned long vcpuid, unsigned long vpa)
180{
181 struct kvm *kvm = vcpu->kvm;
Paul Mackerras93e60242011-12-12 12:28:55 +0000182 unsigned long len, nb;
Paul Mackerrasa8606e22011-06-29 00:22:05 +0000183 void *va;
184 struct kvm_vcpu *tvcpu;
Paul Mackerras2e25aa52012-02-19 17:46:32 +0000185 int err;
186 int subfunc;
187 struct kvmppc_vpa *vpap;
Paul Mackerrasa8606e22011-06-29 00:22:05 +0000188
189 tvcpu = kvmppc_find_vcpu(kvm, vcpuid);
190 if (!tvcpu)
191 return H_PARAMETER;
192
Paul Mackerras2e25aa52012-02-19 17:46:32 +0000193 subfunc = (flags >> H_VPA_FUNC_SHIFT) & H_VPA_FUNC_MASK;
194 if (subfunc == H_VPA_REG_VPA || subfunc == H_VPA_REG_DTL ||
195 subfunc == H_VPA_REG_SLB) {
196 /* Registering new area - address must be cache-line aligned */
197 if ((vpa & (L1_CACHE_BYTES - 1)) || !vpa)
Paul Mackerrasa8606e22011-06-29 00:22:05 +0000198 return H_PARAMETER;
Paul Mackerras2e25aa52012-02-19 17:46:32 +0000199
200 /* convert logical addr to kernel addr and read length */
Paul Mackerras93e60242011-12-12 12:28:55 +0000201 va = kvmppc_pin_guest_page(kvm, vpa, &nb);
202 if (va == NULL)
Paul Mackerrasb2b2f162011-12-12 12:28:21 +0000203 return H_PARAMETER;
Paul Mackerras2e25aa52012-02-19 17:46:32 +0000204 if (subfunc == H_VPA_REG_VPA)
205 len = ((struct reg_vpa *)va)->length.hword;
Paul Mackerrasa8606e22011-06-29 00:22:05 +0000206 else
Paul Mackerras2e25aa52012-02-19 17:46:32 +0000207 len = ((struct reg_vpa *)va)->length.word;
208 kvmppc_unpin_guest_page(kvm, va);
209
210 /* Check length */
211 if (len > nb || len < sizeof(struct reg_vpa))
212 return H_PARAMETER;
Paul Mackerrasa8606e22011-06-29 00:22:05 +0000213 } else {
Paul Mackerras2e25aa52012-02-19 17:46:32 +0000214 vpa = 0;
215 len = 0;
216 }
217
218 err = H_PARAMETER;
219 vpap = NULL;
220 spin_lock(&tvcpu->arch.vpa_update_lock);
221
222 switch (subfunc) {
223 case H_VPA_REG_VPA: /* register VPA */
224 if (len < sizeof(struct lppaca))
Paul Mackerrasa8606e22011-06-29 00:22:05 +0000225 break;
Paul Mackerras2e25aa52012-02-19 17:46:32 +0000226 vpap = &tvcpu->arch.vpa;
227 err = 0;
228 break;
229
230 case H_VPA_REG_DTL: /* register DTL */
231 if (len < sizeof(struct dtl_entry))
Paul Mackerrasa8606e22011-06-29 00:22:05 +0000232 break;
Paul Mackerras2e25aa52012-02-19 17:46:32 +0000233 len -= len % sizeof(struct dtl_entry);
234
235 /* Check that they have previously registered a VPA */
236 err = H_RESOURCE;
237 if (!vpa_is_registered(&tvcpu->arch.vpa))
Paul Mackerrasa8606e22011-06-29 00:22:05 +0000238 break;
Paul Mackerras2e25aa52012-02-19 17:46:32 +0000239
240 vpap = &tvcpu->arch.dtl;
241 err = 0;
242 break;
243
244 case H_VPA_REG_SLB: /* register SLB shadow buffer */
245 /* Check that they have previously registered a VPA */
246 err = H_RESOURCE;
247 if (!vpa_is_registered(&tvcpu->arch.vpa))
248 break;
249
250 vpap = &tvcpu->arch.slb_shadow;
251 err = 0;
252 break;
253
254 case H_VPA_DEREG_VPA: /* deregister VPA */
255 /* Check they don't still have a DTL or SLB buf registered */
256 err = H_RESOURCE;
257 if (vpa_is_registered(&tvcpu->arch.dtl) ||
258 vpa_is_registered(&tvcpu->arch.slb_shadow))
259 break;
260
261 vpap = &tvcpu->arch.vpa;
262 err = 0;
263 break;
264
265 case H_VPA_DEREG_DTL: /* deregister DTL */
266 vpap = &tvcpu->arch.dtl;
267 err = 0;
268 break;
269
270 case H_VPA_DEREG_SLB: /* deregister SLB shadow buffer */
271 vpap = &tvcpu->arch.slb_shadow;
272 err = 0;
273 break;
274 }
275
276 if (vpap) {
277 vpap->next_gpa = vpa;
278 vpap->len = len;
279 vpap->update_pending = 1;
280 }
281
282 spin_unlock(&tvcpu->arch.vpa_update_lock);
283
284 return err;
285}
286
Paul Mackerras081f3232012-06-01 20:20:24 +1000287static void kvmppc_update_vpa(struct kvm_vcpu *vcpu, struct kvmppc_vpa *vpap)
Paul Mackerras2e25aa52012-02-19 17:46:32 +0000288{
Paul Mackerras081f3232012-06-01 20:20:24 +1000289 struct kvm *kvm = vcpu->kvm;
Paul Mackerras2e25aa52012-02-19 17:46:32 +0000290 void *va;
291 unsigned long nb;
Paul Mackerras081f3232012-06-01 20:20:24 +1000292 unsigned long gpa;
293
294 /*
295 * We need to pin the page pointed to by vpap->next_gpa,
296 * but we can't call kvmppc_pin_guest_page under the lock
297 * as it does get_user_pages() and down_read(). So we
298 * have to drop the lock, pin the page, then get the lock
299 * again and check that a new area didn't get registered
300 * in the meantime.
301 */
302 for (;;) {
303 gpa = vpap->next_gpa;
304 spin_unlock(&vcpu->arch.vpa_update_lock);
305 va = NULL;
306 nb = 0;
307 if (gpa)
308 va = kvmppc_pin_guest_page(kvm, vpap->next_gpa, &nb);
309 spin_lock(&vcpu->arch.vpa_update_lock);
310 if (gpa == vpap->next_gpa)
311 break;
312 /* sigh... unpin that one and try again */
313 if (va)
314 kvmppc_unpin_guest_page(kvm, va);
315 }
Paul Mackerras2e25aa52012-02-19 17:46:32 +0000316
317 vpap->update_pending = 0;
Paul Mackerras081f3232012-06-01 20:20:24 +1000318 if (va && nb < vpap->len) {
319 /*
320 * If it's now too short, it must be that userspace
321 * has changed the mappings underlying guest memory,
322 * so unregister the region.
323 */
324 kvmppc_unpin_guest_page(kvm, va);
325 va = NULL;
Paul Mackerrasa8606e22011-06-29 00:22:05 +0000326 }
Paul Mackerras2e25aa52012-02-19 17:46:32 +0000327 if (vpap->pinned_addr)
328 kvmppc_unpin_guest_page(kvm, vpap->pinned_addr);
329 vpap->pinned_addr = va;
330 if (va)
331 vpap->pinned_end = va + vpap->len;
332}
Paul Mackerras93e60242011-12-12 12:28:55 +0000333
Paul Mackerras2e25aa52012-02-19 17:46:32 +0000334static void kvmppc_update_vpas(struct kvm_vcpu *vcpu)
335{
Paul Mackerras2e25aa52012-02-19 17:46:32 +0000336 spin_lock(&vcpu->arch.vpa_update_lock);
337 if (vcpu->arch.vpa.update_pending) {
Paul Mackerras081f3232012-06-01 20:20:24 +1000338 kvmppc_update_vpa(vcpu, &vcpu->arch.vpa);
Paul Mackerras55b665b2012-09-25 20:33:06 +0000339 if (vcpu->arch.vpa.pinned_addr)
340 init_vpa(vcpu, vcpu->arch.vpa.pinned_addr);
Paul Mackerras2e25aa52012-02-19 17:46:32 +0000341 }
342 if (vcpu->arch.dtl.update_pending) {
Paul Mackerras081f3232012-06-01 20:20:24 +1000343 kvmppc_update_vpa(vcpu, &vcpu->arch.dtl);
Paul Mackerras2e25aa52012-02-19 17:46:32 +0000344 vcpu->arch.dtl_ptr = vcpu->arch.dtl.pinned_addr;
345 vcpu->arch.dtl_index = 0;
346 }
347 if (vcpu->arch.slb_shadow.update_pending)
Paul Mackerras081f3232012-06-01 20:20:24 +1000348 kvmppc_update_vpa(vcpu, &vcpu->arch.slb_shadow);
Paul Mackerras2e25aa52012-02-19 17:46:32 +0000349 spin_unlock(&vcpu->arch.vpa_update_lock);
Paul Mackerrasa8606e22011-06-29 00:22:05 +0000350}
351
Paul Mackerras0456ec42012-02-03 00:56:21 +0000352static void kvmppc_create_dtl_entry(struct kvm_vcpu *vcpu,
353 struct kvmppc_vcore *vc)
354{
355 struct dtl_entry *dt;
356 struct lppaca *vpa;
357 unsigned long old_stolen;
358
359 dt = vcpu->arch.dtl_ptr;
360 vpa = vcpu->arch.vpa.pinned_addr;
361 old_stolen = vcpu->arch.stolen_logged;
362 vcpu->arch.stolen_logged = vc->stolen_tb;
363 if (!dt || !vpa)
364 return;
365 memset(dt, 0, sizeof(struct dtl_entry));
366 dt->dispatch_reason = 7;
367 dt->processor_id = vc->pcpu + vcpu->arch.ptid;
368 dt->timebase = mftb();
369 dt->enqueue_to_dispatch_time = vc->stolen_tb - old_stolen;
370 dt->srr0 = kvmppc_get_pc(vcpu);
371 dt->srr1 = vcpu->arch.shregs.msr;
372 ++dt;
373 if (dt == vcpu->arch.dtl.pinned_end)
374 dt = vcpu->arch.dtl.pinned_addr;
375 vcpu->arch.dtl_ptr = dt;
376 /* order writing *dt vs. writing vpa->dtl_idx */
377 smp_wmb();
378 vpa->dtl_idx = ++vcpu->arch.dtl_index;
Paul Mackerrasa8606e22011-06-29 00:22:05 +0000379}
380
381int kvmppc_pseries_do_hcall(struct kvm_vcpu *vcpu)
382{
383 unsigned long req = kvmppc_get_gpr(vcpu, 3);
384 unsigned long target, ret = H_SUCCESS;
385 struct kvm_vcpu *tvcpu;
Paul Mackerras2c9097e2012-09-11 13:27:01 +0000386 int idx;
Paul Mackerrasa8606e22011-06-29 00:22:05 +0000387
388 switch (req) {
Paul Mackerrasc77162d2011-12-12 12:31:00 +0000389 case H_ENTER:
Paul Mackerras2c9097e2012-09-11 13:27:01 +0000390 idx = srcu_read_lock(&vcpu->kvm->srcu);
Paul Mackerrasc77162d2011-12-12 12:31:00 +0000391 ret = kvmppc_virtmode_h_enter(vcpu, kvmppc_get_gpr(vcpu, 4),
392 kvmppc_get_gpr(vcpu, 5),
393 kvmppc_get_gpr(vcpu, 6),
394 kvmppc_get_gpr(vcpu, 7));
Paul Mackerras2c9097e2012-09-11 13:27:01 +0000395 srcu_read_unlock(&vcpu->kvm->srcu, idx);
Paul Mackerrasc77162d2011-12-12 12:31:00 +0000396 break;
Paul Mackerrasa8606e22011-06-29 00:22:05 +0000397 case H_CEDE:
Paul Mackerrasa8606e22011-06-29 00:22:05 +0000398 break;
399 case H_PROD:
400 target = kvmppc_get_gpr(vcpu, 4);
401 tvcpu = kvmppc_find_vcpu(vcpu->kvm, target);
402 if (!tvcpu) {
403 ret = H_PARAMETER;
404 break;
405 }
406 tvcpu->arch.prodded = 1;
407 smp_mb();
408 if (vcpu->arch.ceded) {
409 if (waitqueue_active(&vcpu->wq)) {
410 wake_up_interruptible(&vcpu->wq);
411 vcpu->stat.halt_wakeup++;
412 }
413 }
414 break;
415 case H_CONFER:
416 break;
417 case H_REGISTER_VPA:
418 ret = do_h_register_vpa(vcpu, kvmppc_get_gpr(vcpu, 4),
419 kvmppc_get_gpr(vcpu, 5),
420 kvmppc_get_gpr(vcpu, 6));
421 break;
422 default:
423 return RESUME_HOST;
424 }
425 kvmppc_set_gpr(vcpu, 3, ret);
426 vcpu->arch.hcall_needed = 0;
427 return RESUME_GUEST;
428}
429
Paul Mackerrasde56a942011-06-29 00:21:34 +0000430static int kvmppc_handle_exit(struct kvm_run *run, struct kvm_vcpu *vcpu,
431 struct task_struct *tsk)
432{
433 int r = RESUME_HOST;
Paul Mackerras2c9097e2012-09-11 13:27:01 +0000434 int srcu_idx;
Paul Mackerrasde56a942011-06-29 00:21:34 +0000435
436 vcpu->stat.sum_exits++;
437
438 run->exit_reason = KVM_EXIT_UNKNOWN;
439 run->ready_for_interrupt_injection = 1;
440 switch (vcpu->arch.trap) {
441 /* We're good on these - the host merely wanted to get our attention */
442 case BOOK3S_INTERRUPT_HV_DECREMENTER:
443 vcpu->stat.dec_exits++;
444 r = RESUME_GUEST;
445 break;
446 case BOOK3S_INTERRUPT_EXTERNAL:
447 vcpu->stat.ext_intr_exits++;
448 r = RESUME_GUEST;
449 break;
450 case BOOK3S_INTERRUPT_PERFMON:
451 r = RESUME_GUEST;
452 break;
453 case BOOK3S_INTERRUPT_PROGRAM:
454 {
455 ulong flags;
456 /*
457 * Normally program interrupts are delivered directly
458 * to the guest by the hardware, but we can get here
459 * as a result of a hypervisor emulation interrupt
460 * (e40) getting turned into a 700 by BML RTAS.
461 */
462 flags = vcpu->arch.shregs.msr & 0x1f0000ull;
463 kvmppc_core_queue_program(vcpu, flags);
464 r = RESUME_GUEST;
465 break;
466 }
467 case BOOK3S_INTERRUPT_SYSCALL:
468 {
469 /* hcall - punt to userspace */
470 int i;
471
472 if (vcpu->arch.shregs.msr & MSR_PR) {
473 /* sc 1 from userspace - reflect to guest syscall */
474 kvmppc_book3s_queue_irqprio(vcpu, BOOK3S_INTERRUPT_SYSCALL);
475 r = RESUME_GUEST;
476 break;
477 }
478 run->papr_hcall.nr = kvmppc_get_gpr(vcpu, 3);
479 for (i = 0; i < 9; ++i)
480 run->papr_hcall.args[i] = kvmppc_get_gpr(vcpu, 4 + i);
481 run->exit_reason = KVM_EXIT_PAPR_HCALL;
482 vcpu->arch.hcall_needed = 1;
483 r = RESUME_HOST;
484 break;
485 }
486 /*
Paul Mackerras342d3db2011-12-12 12:38:05 +0000487 * We get these next two if the guest accesses a page which it thinks
488 * it has mapped but which is not actually present, either because
489 * it is for an emulated I/O device or because the corresonding
490 * host page has been paged out. Any other HDSI/HISI interrupts
491 * have been handled already.
Paul Mackerrasde56a942011-06-29 00:21:34 +0000492 */
493 case BOOK3S_INTERRUPT_H_DATA_STORAGE:
Paul Mackerras2c9097e2012-09-11 13:27:01 +0000494 srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
Paul Mackerras697d3892011-12-12 12:36:37 +0000495 r = kvmppc_book3s_hv_page_fault(run, vcpu,
496 vcpu->arch.fault_dar, vcpu->arch.fault_dsisr);
Paul Mackerras2c9097e2012-09-11 13:27:01 +0000497 srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
Paul Mackerrasde56a942011-06-29 00:21:34 +0000498 break;
499 case BOOK3S_INTERRUPT_H_INST_STORAGE:
Paul Mackerras2c9097e2012-09-11 13:27:01 +0000500 srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
Paul Mackerras342d3db2011-12-12 12:38:05 +0000501 r = kvmppc_book3s_hv_page_fault(run, vcpu,
502 kvmppc_get_pc(vcpu), 0);
Paul Mackerras2c9097e2012-09-11 13:27:01 +0000503 srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
Paul Mackerrasde56a942011-06-29 00:21:34 +0000504 break;
505 /*
506 * This occurs if the guest executes an illegal instruction.
507 * We just generate a program interrupt to the guest, since
508 * we don't emulate any guest instructions at this stage.
509 */
510 case BOOK3S_INTERRUPT_H_EMUL_ASSIST:
511 kvmppc_core_queue_program(vcpu, 0x80000);
512 r = RESUME_GUEST;
513 break;
514 default:
515 kvmppc_dump_regs(vcpu);
516 printk(KERN_EMERG "trap=0x%x | pc=0x%lx | msr=0x%llx\n",
517 vcpu->arch.trap, kvmppc_get_pc(vcpu),
518 vcpu->arch.shregs.msr);
519 r = RESUME_HOST;
520 BUG();
521 break;
522 }
523
Paul Mackerrasde56a942011-06-29 00:21:34 +0000524 return r;
525}
526
527int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
528 struct kvm_sregs *sregs)
529{
530 int i;
531
532 sregs->pvr = vcpu->arch.pvr;
533
534 memset(sregs, 0, sizeof(struct kvm_sregs));
535 for (i = 0; i < vcpu->arch.slb_max; i++) {
536 sregs->u.s.ppc64.slb[i].slbe = vcpu->arch.slb[i].orige;
537 sregs->u.s.ppc64.slb[i].slbv = vcpu->arch.slb[i].origv;
538 }
539
540 return 0;
541}
542
543int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
544 struct kvm_sregs *sregs)
545{
546 int i, j;
547
548 kvmppc_set_pvr(vcpu, sregs->pvr);
549
550 j = 0;
551 for (i = 0; i < vcpu->arch.slb_nr; i++) {
552 if (sregs->u.s.ppc64.slb[i].slbe & SLB_ESID_V) {
553 vcpu->arch.slb[j].orige = sregs->u.s.ppc64.slb[i].slbe;
554 vcpu->arch.slb[j].origv = sregs->u.s.ppc64.slb[i].slbv;
555 ++j;
556 }
557 }
558 vcpu->arch.slb_max = j;
559
560 return 0;
561}
562
Paul Mackerrasa136a8b2012-09-25 20:31:56 +0000563int kvmppc_get_one_reg(struct kvm_vcpu *vcpu, u64 id, union kvmppc_one_reg *val)
Paul Mackerras31f34382011-12-12 12:26:50 +0000564{
Paul Mackerrasa136a8b2012-09-25 20:31:56 +0000565 int r = 0;
566 long int i;
Paul Mackerras31f34382011-12-12 12:26:50 +0000567
Paul Mackerrasa136a8b2012-09-25 20:31:56 +0000568 switch (id) {
Paul Mackerras31f34382011-12-12 12:26:50 +0000569 case KVM_REG_PPC_HIOR:
Paul Mackerrasa136a8b2012-09-25 20:31:56 +0000570 *val = get_reg_val(id, 0);
571 break;
572 case KVM_REG_PPC_DABR:
573 *val = get_reg_val(id, vcpu->arch.dabr);
574 break;
575 case KVM_REG_PPC_DSCR:
576 *val = get_reg_val(id, vcpu->arch.dscr);
577 break;
578 case KVM_REG_PPC_PURR:
579 *val = get_reg_val(id, vcpu->arch.purr);
580 break;
581 case KVM_REG_PPC_SPURR:
582 *val = get_reg_val(id, vcpu->arch.spurr);
583 break;
584 case KVM_REG_PPC_AMR:
585 *val = get_reg_val(id, vcpu->arch.amr);
586 break;
587 case KVM_REG_PPC_UAMOR:
588 *val = get_reg_val(id, vcpu->arch.uamor);
589 break;
590 case KVM_REG_PPC_MMCR0 ... KVM_REG_PPC_MMCRA:
591 i = id - KVM_REG_PPC_MMCR0;
592 *val = get_reg_val(id, vcpu->arch.mmcr[i]);
593 break;
594 case KVM_REG_PPC_PMC1 ... KVM_REG_PPC_PMC8:
595 i = id - KVM_REG_PPC_PMC1;
596 *val = get_reg_val(id, vcpu->arch.pmc[i]);
Paul Mackerras31f34382011-12-12 12:26:50 +0000597 break;
Paul Mackerrasa8bd19e2012-09-25 20:32:30 +0000598#ifdef CONFIG_VSX
599 case KVM_REG_PPC_FPR0 ... KVM_REG_PPC_FPR31:
600 if (cpu_has_feature(CPU_FTR_VSX)) {
601 /* VSX => FP reg i is stored in arch.vsr[2*i] */
602 long int i = id - KVM_REG_PPC_FPR0;
603 *val = get_reg_val(id, vcpu->arch.vsr[2 * i]);
604 } else {
605 /* let generic code handle it */
606 r = -EINVAL;
607 }
608 break;
609 case KVM_REG_PPC_VSR0 ... KVM_REG_PPC_VSR31:
610 if (cpu_has_feature(CPU_FTR_VSX)) {
611 long int i = id - KVM_REG_PPC_VSR0;
612 val->vsxval[0] = vcpu->arch.vsr[2 * i];
613 val->vsxval[1] = vcpu->arch.vsr[2 * i + 1];
614 } else {
615 r = -ENXIO;
616 }
617 break;
618#endif /* CONFIG_VSX */
Paul Mackerras55b665b2012-09-25 20:33:06 +0000619 case KVM_REG_PPC_VPA_ADDR:
620 spin_lock(&vcpu->arch.vpa_update_lock);
621 *val = get_reg_val(id, vcpu->arch.vpa.next_gpa);
622 spin_unlock(&vcpu->arch.vpa_update_lock);
623 break;
624 case KVM_REG_PPC_VPA_SLB:
625 spin_lock(&vcpu->arch.vpa_update_lock);
626 val->vpaval.addr = vcpu->arch.slb_shadow.next_gpa;
627 val->vpaval.length = vcpu->arch.slb_shadow.len;
628 spin_unlock(&vcpu->arch.vpa_update_lock);
629 break;
630 case KVM_REG_PPC_VPA_DTL:
631 spin_lock(&vcpu->arch.vpa_update_lock);
632 val->vpaval.addr = vcpu->arch.dtl.next_gpa;
633 val->vpaval.length = vcpu->arch.dtl.len;
634 spin_unlock(&vcpu->arch.vpa_update_lock);
635 break;
Paul Mackerras31f34382011-12-12 12:26:50 +0000636 default:
Paul Mackerrasa136a8b2012-09-25 20:31:56 +0000637 r = -EINVAL;
Paul Mackerras31f34382011-12-12 12:26:50 +0000638 break;
639 }
640
641 return r;
642}
643
Paul Mackerrasa136a8b2012-09-25 20:31:56 +0000644int kvmppc_set_one_reg(struct kvm_vcpu *vcpu, u64 id, union kvmppc_one_reg *val)
Paul Mackerras31f34382011-12-12 12:26:50 +0000645{
Paul Mackerrasa136a8b2012-09-25 20:31:56 +0000646 int r = 0;
647 long int i;
Paul Mackerras55b665b2012-09-25 20:33:06 +0000648 unsigned long addr, len;
Paul Mackerras31f34382011-12-12 12:26:50 +0000649
Paul Mackerrasa136a8b2012-09-25 20:31:56 +0000650 switch (id) {
Paul Mackerras31f34382011-12-12 12:26:50 +0000651 case KVM_REG_PPC_HIOR:
Paul Mackerras31f34382011-12-12 12:26:50 +0000652 /* Only allow this to be set to zero */
Paul Mackerrasa136a8b2012-09-25 20:31:56 +0000653 if (set_reg_val(id, *val))
Paul Mackerras31f34382011-12-12 12:26:50 +0000654 r = -EINVAL;
655 break;
Paul Mackerrasa136a8b2012-09-25 20:31:56 +0000656 case KVM_REG_PPC_DABR:
657 vcpu->arch.dabr = set_reg_val(id, *val);
658 break;
659 case KVM_REG_PPC_DSCR:
660 vcpu->arch.dscr = set_reg_val(id, *val);
661 break;
662 case KVM_REG_PPC_PURR:
663 vcpu->arch.purr = set_reg_val(id, *val);
664 break;
665 case KVM_REG_PPC_SPURR:
666 vcpu->arch.spurr = set_reg_val(id, *val);
667 break;
668 case KVM_REG_PPC_AMR:
669 vcpu->arch.amr = set_reg_val(id, *val);
670 break;
671 case KVM_REG_PPC_UAMOR:
672 vcpu->arch.uamor = set_reg_val(id, *val);
673 break;
674 case KVM_REG_PPC_MMCR0 ... KVM_REG_PPC_MMCRA:
675 i = id - KVM_REG_PPC_MMCR0;
676 vcpu->arch.mmcr[i] = set_reg_val(id, *val);
677 break;
678 case KVM_REG_PPC_PMC1 ... KVM_REG_PPC_PMC8:
679 i = id - KVM_REG_PPC_PMC1;
680 vcpu->arch.pmc[i] = set_reg_val(id, *val);
681 break;
Paul Mackerrasa8bd19e2012-09-25 20:32:30 +0000682#ifdef CONFIG_VSX
683 case KVM_REG_PPC_FPR0 ... KVM_REG_PPC_FPR31:
684 if (cpu_has_feature(CPU_FTR_VSX)) {
685 /* VSX => FP reg i is stored in arch.vsr[2*i] */
686 long int i = id - KVM_REG_PPC_FPR0;
687 vcpu->arch.vsr[2 * i] = set_reg_val(id, *val);
688 } else {
689 /* let generic code handle it */
690 r = -EINVAL;
691 }
692 break;
693 case KVM_REG_PPC_VSR0 ... KVM_REG_PPC_VSR31:
694 if (cpu_has_feature(CPU_FTR_VSX)) {
695 long int i = id - KVM_REG_PPC_VSR0;
696 vcpu->arch.vsr[2 * i] = val->vsxval[0];
697 vcpu->arch.vsr[2 * i + 1] = val->vsxval[1];
698 } else {
699 r = -ENXIO;
700 }
701 break;
702#endif /* CONFIG_VSX */
Paul Mackerras55b665b2012-09-25 20:33:06 +0000703 case KVM_REG_PPC_VPA_ADDR:
704 addr = set_reg_val(id, *val);
705 r = -EINVAL;
706 if (!addr && (vcpu->arch.slb_shadow.next_gpa ||
707 vcpu->arch.dtl.next_gpa))
708 break;
709 r = set_vpa(vcpu, &vcpu->arch.vpa, addr, sizeof(struct lppaca));
710 break;
711 case KVM_REG_PPC_VPA_SLB:
712 addr = val->vpaval.addr;
713 len = val->vpaval.length;
714 r = -EINVAL;
715 if (addr && !vcpu->arch.vpa.next_gpa)
716 break;
717 r = set_vpa(vcpu, &vcpu->arch.slb_shadow, addr, len);
718 break;
719 case KVM_REG_PPC_VPA_DTL:
720 addr = val->vpaval.addr;
721 len = val->vpaval.length;
722 r = -EINVAL;
723 if (len < sizeof(struct dtl_entry))
724 break;
725 if (addr && !vcpu->arch.vpa.next_gpa)
726 break;
727 len -= len % sizeof(struct dtl_entry);
728 r = set_vpa(vcpu, &vcpu->arch.dtl, addr, len);
729 break;
Paul Mackerras31f34382011-12-12 12:26:50 +0000730 default:
Paul Mackerrasa136a8b2012-09-25 20:31:56 +0000731 r = -EINVAL;
Paul Mackerras31f34382011-12-12 12:26:50 +0000732 break;
733 }
734
735 return r;
736}
737
Paul Mackerrasde56a942011-06-29 00:21:34 +0000738int kvmppc_core_check_processor_compat(void)
739{
Paul Mackerras9e368f22011-06-29 00:40:08 +0000740 if (cpu_has_feature(CPU_FTR_HVMODE))
Paul Mackerrasde56a942011-06-29 00:21:34 +0000741 return 0;
742 return -EIO;
743}
744
745struct kvm_vcpu *kvmppc_core_vcpu_create(struct kvm *kvm, unsigned int id)
746{
747 struct kvm_vcpu *vcpu;
Paul Mackerras371fefd2011-06-29 00:23:08 +0000748 int err = -EINVAL;
749 int core;
750 struct kvmppc_vcore *vcore;
Paul Mackerrasde56a942011-06-29 00:21:34 +0000751
Paul Mackerras371fefd2011-06-29 00:23:08 +0000752 core = id / threads_per_core;
753 if (core >= KVM_MAX_VCORES)
754 goto out;
755
756 err = -ENOMEM;
Sasha Levin6b75e6b2011-12-07 10:24:56 +0200757 vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
Paul Mackerrasde56a942011-06-29 00:21:34 +0000758 if (!vcpu)
759 goto out;
760
761 err = kvm_vcpu_init(vcpu, kvm, id);
762 if (err)
763 goto free_vcpu;
764
765 vcpu->arch.shared = &vcpu->arch.shregs;
766 vcpu->arch.last_cpu = -1;
767 vcpu->arch.mmcr[0] = MMCR0_FC;
768 vcpu->arch.ctrl = CTRL_RUNLATCH;
769 /* default to host PVR, since we can't spoof it */
770 vcpu->arch.pvr = mfspr(SPRN_PVR);
771 kvmppc_set_pvr(vcpu, vcpu->arch.pvr);
Paul Mackerras2e25aa52012-02-19 17:46:32 +0000772 spin_lock_init(&vcpu->arch.vpa_update_lock);
Paul Mackerrasde56a942011-06-29 00:21:34 +0000773
Paul Mackerrasde56a942011-06-29 00:21:34 +0000774 kvmppc_mmu_book3s_hv_init(vcpu);
775
Paul Mackerras371fefd2011-06-29 00:23:08 +0000776 /*
Paul Mackerras19ccb762011-07-23 17:42:46 +1000777 * We consider the vcpu stopped until we see the first run ioctl for it.
Paul Mackerras371fefd2011-06-29 00:23:08 +0000778 */
Paul Mackerras19ccb762011-07-23 17:42:46 +1000779 vcpu->arch.state = KVMPPC_VCPU_STOPPED;
Paul Mackerras371fefd2011-06-29 00:23:08 +0000780
781 init_waitqueue_head(&vcpu->arch.cpu_run);
782
783 mutex_lock(&kvm->lock);
784 vcore = kvm->arch.vcores[core];
785 if (!vcore) {
786 vcore = kzalloc(sizeof(struct kvmppc_vcore), GFP_KERNEL);
787 if (vcore) {
788 INIT_LIST_HEAD(&vcore->runnable_threads);
789 spin_lock_init(&vcore->lock);
Paul Mackerras19ccb762011-07-23 17:42:46 +1000790 init_waitqueue_head(&vcore->wq);
Paul Mackerras0456ec42012-02-03 00:56:21 +0000791 vcore->preempt_tb = mftb();
Paul Mackerras371fefd2011-06-29 00:23:08 +0000792 }
793 kvm->arch.vcores[core] = vcore;
794 }
795 mutex_unlock(&kvm->lock);
796
797 if (!vcore)
798 goto free_vcpu;
799
800 spin_lock(&vcore->lock);
801 ++vcore->num_threads;
Paul Mackerras371fefd2011-06-29 00:23:08 +0000802 spin_unlock(&vcore->lock);
803 vcpu->arch.vcore = vcore;
Paul Mackerras0456ec42012-02-03 00:56:21 +0000804 vcpu->arch.stolen_logged = vcore->stolen_tb;
Paul Mackerras371fefd2011-06-29 00:23:08 +0000805
Alexander Grafaf8f38b2011-08-10 13:57:08 +0200806 vcpu->arch.cpu_type = KVM_CPU_3S_64;
807 kvmppc_sanity_check(vcpu);
808
Paul Mackerrasde56a942011-06-29 00:21:34 +0000809 return vcpu;
810
811free_vcpu:
Sasha Levin6b75e6b2011-12-07 10:24:56 +0200812 kmem_cache_free(kvm_vcpu_cache, vcpu);
Paul Mackerrasde56a942011-06-29 00:21:34 +0000813out:
814 return ERR_PTR(err);
815}
816
817void kvmppc_core_vcpu_free(struct kvm_vcpu *vcpu)
818{
Paul Mackerras2e25aa52012-02-19 17:46:32 +0000819 spin_lock(&vcpu->arch.vpa_update_lock);
820 if (vcpu->arch.dtl.pinned_addr)
821 kvmppc_unpin_guest_page(vcpu->kvm, vcpu->arch.dtl.pinned_addr);
822 if (vcpu->arch.slb_shadow.pinned_addr)
823 kvmppc_unpin_guest_page(vcpu->kvm, vcpu->arch.slb_shadow.pinned_addr);
824 if (vcpu->arch.vpa.pinned_addr)
825 kvmppc_unpin_guest_page(vcpu->kvm, vcpu->arch.vpa.pinned_addr);
826 spin_unlock(&vcpu->arch.vpa_update_lock);
Paul Mackerrasde56a942011-06-29 00:21:34 +0000827 kvm_vcpu_uninit(vcpu);
Sasha Levin6b75e6b2011-12-07 10:24:56 +0200828 kmem_cache_free(kvm_vcpu_cache, vcpu);
Paul Mackerrasde56a942011-06-29 00:21:34 +0000829}
830
Paul Mackerras19ccb762011-07-23 17:42:46 +1000831static void kvmppc_set_timer(struct kvm_vcpu *vcpu)
Paul Mackerrasde56a942011-06-29 00:21:34 +0000832{
Paul Mackerras19ccb762011-07-23 17:42:46 +1000833 unsigned long dec_nsec, now;
Paul Mackerras371fefd2011-06-29 00:23:08 +0000834
Paul Mackerras19ccb762011-07-23 17:42:46 +1000835 now = get_tb();
836 if (now > vcpu->arch.dec_expires) {
837 /* decrementer has already gone negative */
838 kvmppc_core_queue_dec(vcpu);
Scott Wood7e28e60e2011-11-08 18:23:20 -0600839 kvmppc_core_prepare_to_enter(vcpu);
Paul Mackerras19ccb762011-07-23 17:42:46 +1000840 return;
Paul Mackerras371fefd2011-06-29 00:23:08 +0000841 }
Paul Mackerras19ccb762011-07-23 17:42:46 +1000842 dec_nsec = (vcpu->arch.dec_expires - now) * NSEC_PER_SEC
843 / tb_ticks_per_sec;
844 hrtimer_start(&vcpu->arch.dec_timer, ktime_set(0, dec_nsec),
845 HRTIMER_MODE_REL);
846 vcpu->arch.timer_running = 1;
Paul Mackerras371fefd2011-06-29 00:23:08 +0000847}
848
Paul Mackerras19ccb762011-07-23 17:42:46 +1000849static void kvmppc_end_cede(struct kvm_vcpu *vcpu)
Paul Mackerras371fefd2011-06-29 00:23:08 +0000850{
Paul Mackerras19ccb762011-07-23 17:42:46 +1000851 vcpu->arch.ceded = 0;
852 if (vcpu->arch.timer_running) {
853 hrtimer_try_to_cancel(&vcpu->arch.dec_timer);
854 vcpu->arch.timer_running = 0;
855 }
Paul Mackerras371fefd2011-06-29 00:23:08 +0000856}
857
858extern int __kvmppc_vcore_entry(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu);
859extern void xics_wake_cpu(int cpu);
860
861static void kvmppc_remove_runnable(struct kvmppc_vcore *vc,
862 struct kvm_vcpu *vcpu)
863{
Paul Mackerras371fefd2011-06-29 00:23:08 +0000864 if (vcpu->arch.state != KVMPPC_VCPU_RUNNABLE)
865 return;
866 vcpu->arch.state = KVMPPC_VCPU_BUSY_IN_HOST;
867 --vc->n_runnable;
Paul Mackerras19ccb762011-07-23 17:42:46 +1000868 ++vc->n_busy;
Paul Mackerras371fefd2011-06-29 00:23:08 +0000869 list_del(&vcpu->arch.run_list);
870}
871
Paul Mackerrasf0888f72012-02-03 00:54:17 +0000872static int kvmppc_grab_hwthread(int cpu)
873{
874 struct paca_struct *tpaca;
875 long timeout = 1000;
876
877 tpaca = &paca[cpu];
878
879 /* Ensure the thread won't go into the kernel if it wakes */
880 tpaca->kvm_hstate.hwthread_req = 1;
Paul Mackerras7b444c62012-10-15 01:16:14 +0000881 tpaca->kvm_hstate.kvm_vcpu = NULL;
Paul Mackerrasf0888f72012-02-03 00:54:17 +0000882
883 /*
884 * If the thread is already executing in the kernel (e.g. handling
885 * a stray interrupt), wait for it to get back to nap mode.
886 * The smp_mb() is to ensure that our setting of hwthread_req
887 * is visible before we look at hwthread_state, so if this
888 * races with the code at system_reset_pSeries and the thread
889 * misses our setting of hwthread_req, we are sure to see its
890 * setting of hwthread_state, and vice versa.
891 */
892 smp_mb();
893 while (tpaca->kvm_hstate.hwthread_state == KVM_HWTHREAD_IN_KERNEL) {
894 if (--timeout <= 0) {
895 pr_err("KVM: couldn't grab cpu %d\n", cpu);
896 return -EBUSY;
897 }
898 udelay(1);
899 }
900 return 0;
901}
902
903static void kvmppc_release_hwthread(int cpu)
904{
905 struct paca_struct *tpaca;
906
907 tpaca = &paca[cpu];
908 tpaca->kvm_hstate.hwthread_req = 0;
909 tpaca->kvm_hstate.kvm_vcpu = NULL;
910}
911
Paul Mackerras371fefd2011-06-29 00:23:08 +0000912static void kvmppc_start_thread(struct kvm_vcpu *vcpu)
913{
914 int cpu;
915 struct paca_struct *tpaca;
916 struct kvmppc_vcore *vc = vcpu->arch.vcore;
917
Paul Mackerras19ccb762011-07-23 17:42:46 +1000918 if (vcpu->arch.timer_running) {
919 hrtimer_try_to_cancel(&vcpu->arch.dec_timer);
920 vcpu->arch.timer_running = 0;
921 }
Paul Mackerras371fefd2011-06-29 00:23:08 +0000922 cpu = vc->pcpu + vcpu->arch.ptid;
923 tpaca = &paca[cpu];
924 tpaca->kvm_hstate.kvm_vcpu = vcpu;
925 tpaca->kvm_hstate.kvm_vcore = vc;
Paul Mackerras19ccb762011-07-23 17:42:46 +1000926 tpaca->kvm_hstate.napping = 0;
927 vcpu->cpu = vc->pcpu;
Paul Mackerras371fefd2011-06-29 00:23:08 +0000928 smp_wmb();
Michael Neuling251da032011-11-10 16:03:20 +0000929#if defined(CONFIG_PPC_ICP_NATIVE) && defined(CONFIG_SMP)
Paul Mackerras371fefd2011-06-29 00:23:08 +0000930 if (vcpu->arch.ptid) {
Paul Mackerras371fefd2011-06-29 00:23:08 +0000931 xics_wake_cpu(cpu);
932 ++vc->n_woken;
933 }
934#endif
935}
936
937static void kvmppc_wait_for_nap(struct kvmppc_vcore *vc)
938{
939 int i;
940
941 HMT_low();
942 i = 0;
943 while (vc->nap_count < vc->n_woken) {
944 if (++i >= 1000000) {
945 pr_err("kvmppc_wait_for_nap timeout %d %d\n",
946 vc->nap_count, vc->n_woken);
947 break;
948 }
949 cpu_relax();
950 }
951 HMT_medium();
952}
953
954/*
955 * Check that we are on thread 0 and that any other threads in
Paul Mackerras7b444c62012-10-15 01:16:14 +0000956 * this core are off-line. Then grab the threads so they can't
957 * enter the kernel.
Paul Mackerras371fefd2011-06-29 00:23:08 +0000958 */
959static int on_primary_thread(void)
960{
961 int cpu = smp_processor_id();
962 int thr = cpu_thread_in_core(cpu);
963
964 if (thr)
965 return 0;
966 while (++thr < threads_per_core)
967 if (cpu_online(cpu + thr))
968 return 0;
Paul Mackerras7b444c62012-10-15 01:16:14 +0000969
970 /* Grab all hw threads so they can't go into the kernel */
971 for (thr = 1; thr < threads_per_core; ++thr) {
972 if (kvmppc_grab_hwthread(cpu + thr)) {
973 /* Couldn't grab one; let the others go */
974 do {
975 kvmppc_release_hwthread(cpu + thr);
976 } while (--thr > 0);
977 return 0;
978 }
979 }
Paul Mackerras371fefd2011-06-29 00:23:08 +0000980 return 1;
981}
982
983/*
984 * Run a set of guest threads on a physical core.
985 * Called with vc->lock held.
986 */
987static int kvmppc_run_core(struct kvmppc_vcore *vc)
988{
Paul Mackerras19ccb762011-07-23 17:42:46 +1000989 struct kvm_vcpu *vcpu, *vcpu0, *vnext;
Paul Mackerras371fefd2011-06-29 00:23:08 +0000990 long ret;
Paul Mackerrasde56a942011-06-29 00:21:34 +0000991 u64 now;
Paul Mackerras081f3232012-06-01 20:20:24 +1000992 int ptid, i, need_vpa_update;
Paul Mackerras2c9097e2012-09-11 13:27:01 +0000993 int srcu_idx;
Paul Mackerrasde56a942011-06-29 00:21:34 +0000994
Paul Mackerras371fefd2011-06-29 00:23:08 +0000995 /* don't start if any threads have a signal pending */
Paul Mackerras081f3232012-06-01 20:20:24 +1000996 need_vpa_update = 0;
997 list_for_each_entry(vcpu, &vc->runnable_threads, arch.run_list) {
Paul Mackerras371fefd2011-06-29 00:23:08 +0000998 if (signal_pending(vcpu->arch.run_task))
999 return 0;
Paul Mackerras081f3232012-06-01 20:20:24 +10001000 need_vpa_update |= vcpu->arch.vpa.update_pending |
1001 vcpu->arch.slb_shadow.update_pending |
1002 vcpu->arch.dtl.update_pending;
1003 }
1004
1005 /*
1006 * Initialize *vc, in particular vc->vcore_state, so we can
1007 * drop the vcore lock if necessary.
1008 */
1009 vc->n_woken = 0;
1010 vc->nap_count = 0;
1011 vc->entry_exit_count = 0;
1012 vc->vcore_state = VCORE_RUNNING;
1013 vc->in_guest = 0;
1014 vc->napping_threads = 0;
1015
1016 /*
1017 * Updating any of the vpas requires calling kvmppc_pin_guest_page,
1018 * which can't be called with any spinlocks held.
1019 */
1020 if (need_vpa_update) {
1021 spin_unlock(&vc->lock);
1022 list_for_each_entry(vcpu, &vc->runnable_threads, arch.run_list)
1023 kvmppc_update_vpas(vcpu);
1024 spin_lock(&vc->lock);
1025 }
Paul Mackerrasde56a942011-06-29 00:21:34 +00001026
1027 /*
Paul Mackerras19ccb762011-07-23 17:42:46 +10001028 * Assign physical thread IDs, first to non-ceded vcpus
1029 * and then to ceded ones.
1030 */
1031 ptid = 0;
1032 vcpu0 = NULL;
1033 list_for_each_entry(vcpu, &vc->runnable_threads, arch.run_list) {
1034 if (!vcpu->arch.ceded) {
1035 if (!ptid)
1036 vcpu0 = vcpu;
1037 vcpu->arch.ptid = ptid++;
1038 }
1039 }
1040 if (!vcpu0)
1041 return 0; /* nothing to run */
1042 list_for_each_entry(vcpu, &vc->runnable_threads, arch.run_list)
1043 if (vcpu->arch.ceded)
1044 vcpu->arch.ptid = ptid++;
1045
Paul Mackerras7b444c62012-10-15 01:16:14 +00001046 /*
1047 * Make sure we are running on thread 0, and that
1048 * secondary threads are offline.
1049 */
1050 if (threads_per_core > 1 && !on_primary_thread()) {
1051 list_for_each_entry(vcpu, &vc->runnable_threads, arch.run_list)
1052 vcpu->arch.ret = -EBUSY;
1053 goto out;
1054 }
1055
Paul Mackerras0456ec42012-02-03 00:56:21 +00001056 vc->stolen_tb += mftb() - vc->preempt_tb;
Paul Mackerras371fefd2011-06-29 00:23:08 +00001057 vc->pcpu = smp_processor_id();
Paul Mackerras2e25aa52012-02-19 17:46:32 +00001058 list_for_each_entry(vcpu, &vc->runnable_threads, arch.run_list) {
Paul Mackerras371fefd2011-06-29 00:23:08 +00001059 kvmppc_start_thread(vcpu);
Paul Mackerras0456ec42012-02-03 00:56:21 +00001060 kvmppc_create_dtl_entry(vcpu, vc);
Paul Mackerras2e25aa52012-02-19 17:46:32 +00001061 }
Paul Mackerras371fefd2011-06-29 00:23:08 +00001062
1063 preempt_disable();
Paul Mackerras19ccb762011-07-23 17:42:46 +10001064 spin_unlock(&vc->lock);
Paul Mackerrasde56a942011-06-29 00:21:34 +00001065
Paul Mackerras19ccb762011-07-23 17:42:46 +10001066 kvm_guest_enter();
Paul Mackerras2c9097e2012-09-11 13:27:01 +00001067
1068 srcu_idx = srcu_read_lock(&vcpu0->kvm->srcu);
1069
Paul Mackerras19ccb762011-07-23 17:42:46 +10001070 __kvmppc_vcore_entry(NULL, vcpu0);
Paul Mackerrasf0888f72012-02-03 00:54:17 +00001071 for (i = 0; i < threads_per_core; ++i)
1072 kvmppc_release_hwthread(vc->pcpu + i);
Paul Mackerras19ccb762011-07-23 17:42:46 +10001073
Paul Mackerras371fefd2011-06-29 00:23:08 +00001074 spin_lock(&vc->lock);
Paul Mackerras19ccb762011-07-23 17:42:46 +10001075 /* disable sending of IPIs on virtual external irqs */
1076 list_for_each_entry(vcpu, &vc->runnable_threads, arch.run_list)
1077 vcpu->cpu = -1;
1078 /* wait for secondary threads to finish writing their state to memory */
Paul Mackerras371fefd2011-06-29 00:23:08 +00001079 if (vc->nap_count < vc->n_woken)
1080 kvmppc_wait_for_nap(vc);
1081 /* prevent other vcpu threads from doing kvmppc_start_thread() now */
Paul Mackerras19ccb762011-07-23 17:42:46 +10001082 vc->vcore_state = VCORE_EXITING;
Paul Mackerras371fefd2011-06-29 00:23:08 +00001083 spin_unlock(&vc->lock);
1084
Paul Mackerras2c9097e2012-09-11 13:27:01 +00001085 srcu_read_unlock(&vcpu0->kvm->srcu, srcu_idx);
1086
Paul Mackerras371fefd2011-06-29 00:23:08 +00001087 /* make sure updates to secondary vcpu structs are visible now */
1088 smp_mb();
Paul Mackerrasde56a942011-06-29 00:21:34 +00001089 kvm_guest_exit();
1090
1091 preempt_enable();
1092 kvm_resched(vcpu);
1093
1094 now = get_tb();
Paul Mackerras371fefd2011-06-29 00:23:08 +00001095 list_for_each_entry(vcpu, &vc->runnable_threads, arch.run_list) {
1096 /* cancel pending dec exception if dec is positive */
1097 if (now < vcpu->arch.dec_expires &&
1098 kvmppc_core_pending_dec(vcpu))
1099 kvmppc_core_dequeue_dec(vcpu);
Paul Mackerras19ccb762011-07-23 17:42:46 +10001100
1101 ret = RESUME_GUEST;
1102 if (vcpu->arch.trap)
1103 ret = kvmppc_handle_exit(vcpu->arch.kvm_run, vcpu,
1104 vcpu->arch.run_task);
1105
Paul Mackerras371fefd2011-06-29 00:23:08 +00001106 vcpu->arch.ret = ret;
1107 vcpu->arch.trap = 0;
Paul Mackerras19ccb762011-07-23 17:42:46 +10001108
1109 if (vcpu->arch.ceded) {
1110 if (ret != RESUME_GUEST)
1111 kvmppc_end_cede(vcpu);
1112 else
1113 kvmppc_set_timer(vcpu);
1114 }
Paul Mackerras371fefd2011-06-29 00:23:08 +00001115 }
Paul Mackerrasde56a942011-06-29 00:21:34 +00001116
Paul Mackerras371fefd2011-06-29 00:23:08 +00001117 spin_lock(&vc->lock);
Paul Mackerrasde56a942011-06-29 00:21:34 +00001118 out:
Paul Mackerras19ccb762011-07-23 17:42:46 +10001119 vc->vcore_state = VCORE_INACTIVE;
Paul Mackerras0456ec42012-02-03 00:56:21 +00001120 vc->preempt_tb = mftb();
Paul Mackerras371fefd2011-06-29 00:23:08 +00001121 list_for_each_entry_safe(vcpu, vnext, &vc->runnable_threads,
1122 arch.run_list) {
1123 if (vcpu->arch.ret != RESUME_GUEST) {
1124 kvmppc_remove_runnable(vc, vcpu);
1125 wake_up(&vcpu->arch.cpu_run);
1126 }
1127 }
1128
1129 return 1;
1130}
1131
Paul Mackerras19ccb762011-07-23 17:42:46 +10001132/*
1133 * Wait for some other vcpu thread to execute us, and
1134 * wake us up when we need to handle something in the host.
1135 */
1136static void kvmppc_wait_for_exec(struct kvm_vcpu *vcpu, int wait_state)
Paul Mackerras371fefd2011-06-29 00:23:08 +00001137{
Paul Mackerras371fefd2011-06-29 00:23:08 +00001138 DEFINE_WAIT(wait);
1139
Paul Mackerras19ccb762011-07-23 17:42:46 +10001140 prepare_to_wait(&vcpu->arch.cpu_run, &wait, wait_state);
1141 if (vcpu->arch.state == KVMPPC_VCPU_RUNNABLE)
1142 schedule();
1143 finish_wait(&vcpu->arch.cpu_run, &wait);
1144}
Paul Mackerras371fefd2011-06-29 00:23:08 +00001145
Paul Mackerras19ccb762011-07-23 17:42:46 +10001146/*
1147 * All the vcpus in this vcore are idle, so wait for a decrementer
1148 * or external interrupt to one of the vcpus. vc->lock is held.
1149 */
1150static void kvmppc_vcore_blocked(struct kvmppc_vcore *vc)
1151{
1152 DEFINE_WAIT(wait);
1153 struct kvm_vcpu *v;
1154 int all_idle = 1;
1155
1156 prepare_to_wait(&vc->wq, &wait, TASK_INTERRUPTIBLE);
1157 vc->vcore_state = VCORE_SLEEPING;
1158 spin_unlock(&vc->lock);
1159 list_for_each_entry(v, &vc->runnable_threads, arch.run_list) {
1160 if (!v->arch.ceded || v->arch.pending_exceptions) {
1161 all_idle = 0;
1162 break;
1163 }
1164 }
1165 if (all_idle)
1166 schedule();
1167 finish_wait(&vc->wq, &wait);
1168 spin_lock(&vc->lock);
1169 vc->vcore_state = VCORE_INACTIVE;
1170}
1171
1172static int kvmppc_run_vcpu(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu)
1173{
1174 int n_ceded;
1175 int prev_state;
1176 struct kvmppc_vcore *vc;
1177 struct kvm_vcpu *v, *vn;
Paul Mackerras9e368f22011-06-29 00:40:08 +00001178
Paul Mackerras371fefd2011-06-29 00:23:08 +00001179 kvm_run->exit_reason = 0;
1180 vcpu->arch.ret = RESUME_GUEST;
1181 vcpu->arch.trap = 0;
1182
Paul Mackerras371fefd2011-06-29 00:23:08 +00001183 /*
1184 * Synchronize with other threads in this virtual core
1185 */
1186 vc = vcpu->arch.vcore;
1187 spin_lock(&vc->lock);
Paul Mackerras19ccb762011-07-23 17:42:46 +10001188 vcpu->arch.ceded = 0;
Paul Mackerras371fefd2011-06-29 00:23:08 +00001189 vcpu->arch.run_task = current;
1190 vcpu->arch.kvm_run = kvm_run;
Paul Mackerras19ccb762011-07-23 17:42:46 +10001191 prev_state = vcpu->arch.state;
1192 vcpu->arch.state = KVMPPC_VCPU_RUNNABLE;
Paul Mackerras371fefd2011-06-29 00:23:08 +00001193 list_add_tail(&vcpu->arch.run_list, &vc->runnable_threads);
1194 ++vc->n_runnable;
1195
Paul Mackerras19ccb762011-07-23 17:42:46 +10001196 /*
1197 * This happens the first time this is called for a vcpu.
1198 * If the vcore is already running, we may be able to start
1199 * this thread straight away and have it join in.
1200 */
1201 if (prev_state == KVMPPC_VCPU_STOPPED) {
1202 if (vc->vcore_state == VCORE_RUNNING &&
1203 VCORE_EXIT_COUNT(vc) == 0) {
1204 vcpu->arch.ptid = vc->n_runnable - 1;
Paul Mackerras371fefd2011-06-29 00:23:08 +00001205 kvmppc_start_thread(vcpu);
Paul Mackerras19ccb762011-07-23 17:42:46 +10001206 }
Paul Mackerras371fefd2011-06-29 00:23:08 +00001207
Paul Mackerras19ccb762011-07-23 17:42:46 +10001208 } else if (prev_state == KVMPPC_VCPU_BUSY_IN_HOST)
1209 --vc->n_busy;
1210
1211 while (vcpu->arch.state == KVMPPC_VCPU_RUNNABLE &&
1212 !signal_pending(current)) {
1213 if (vc->n_busy || vc->vcore_state != VCORE_INACTIVE) {
1214 spin_unlock(&vc->lock);
1215 kvmppc_wait_for_exec(vcpu, TASK_INTERRUPTIBLE);
1216 spin_lock(&vc->lock);
1217 continue;
1218 }
Paul Mackerras0456ec42012-02-03 00:56:21 +00001219 vc->runner = vcpu;
Paul Mackerras19ccb762011-07-23 17:42:46 +10001220 n_ceded = 0;
1221 list_for_each_entry(v, &vc->runnable_threads, arch.run_list)
1222 n_ceded += v->arch.ceded;
1223 if (n_ceded == vc->n_runnable)
1224 kvmppc_vcore_blocked(vc);
1225 else
1226 kvmppc_run_core(vc);
1227
1228 list_for_each_entry_safe(v, vn, &vc->runnable_threads,
1229 arch.run_list) {
Scott Wood7e28e60e2011-11-08 18:23:20 -06001230 kvmppc_core_prepare_to_enter(v);
Paul Mackerras19ccb762011-07-23 17:42:46 +10001231 if (signal_pending(v->arch.run_task)) {
1232 kvmppc_remove_runnable(vc, v);
1233 v->stat.signal_exits++;
1234 v->arch.kvm_run->exit_reason = KVM_EXIT_INTR;
1235 v->arch.ret = -EINTR;
1236 wake_up(&v->arch.cpu_run);
1237 }
1238 }
Paul Mackerras0456ec42012-02-03 00:56:21 +00001239 vc->runner = NULL;
Paul Mackerras371fefd2011-06-29 00:23:08 +00001240 }
1241
Paul Mackerras19ccb762011-07-23 17:42:46 +10001242 if (signal_pending(current)) {
1243 if (vc->vcore_state == VCORE_RUNNING ||
1244 vc->vcore_state == VCORE_EXITING) {
1245 spin_unlock(&vc->lock);
1246 kvmppc_wait_for_exec(vcpu, TASK_UNINTERRUPTIBLE);
1247 spin_lock(&vc->lock);
1248 }
1249 if (vcpu->arch.state == KVMPPC_VCPU_RUNNABLE) {
1250 kvmppc_remove_runnable(vc, vcpu);
1251 vcpu->stat.signal_exits++;
1252 kvm_run->exit_reason = KVM_EXIT_INTR;
1253 vcpu->arch.ret = -EINTR;
1254 }
1255 }
Paul Mackerras371fefd2011-06-29 00:23:08 +00001256
Paul Mackerras19ccb762011-07-23 17:42:46 +10001257 spin_unlock(&vc->lock);
Paul Mackerras371fefd2011-06-29 00:23:08 +00001258 return vcpu->arch.ret;
Paul Mackerrasde56a942011-06-29 00:21:34 +00001259}
1260
Paul Mackerrasa8606e22011-06-29 00:22:05 +00001261int kvmppc_vcpu_run(struct kvm_run *run, struct kvm_vcpu *vcpu)
1262{
1263 int r;
1264
Alexander Grafaf8f38b2011-08-10 13:57:08 +02001265 if (!vcpu->arch.sane) {
1266 run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
1267 return -EINVAL;
1268 }
1269
Scott Wood25051b52011-11-08 18:23:23 -06001270 kvmppc_core_prepare_to_enter(vcpu);
1271
Paul Mackerras19ccb762011-07-23 17:42:46 +10001272 /* No need to go into the guest when all we'll do is come back out */
1273 if (signal_pending(current)) {
1274 run->exit_reason = KVM_EXIT_INTR;
1275 return -EINTR;
1276 }
1277
Paul Mackerras32fad282012-05-04 02:32:53 +00001278 atomic_inc(&vcpu->kvm->arch.vcpus_running);
1279 /* Order vcpus_running vs. rma_setup_done, see kvmppc_alloc_reset_hpt */
1280 smp_mb();
1281
1282 /* On the first time here, set up HTAB and VRMA or RMA */
Paul Mackerrasc77162d2011-12-12 12:31:00 +00001283 if (!vcpu->kvm->arch.rma_setup_done) {
Paul Mackerras32fad282012-05-04 02:32:53 +00001284 r = kvmppc_hv_setup_htab_rma(vcpu);
Paul Mackerrasc77162d2011-12-12 12:31:00 +00001285 if (r)
Paul Mackerras32fad282012-05-04 02:32:53 +00001286 goto out;
Paul Mackerrasc77162d2011-12-12 12:31:00 +00001287 }
Paul Mackerras19ccb762011-07-23 17:42:46 +10001288
1289 flush_fp_to_thread(current);
1290 flush_altivec_to_thread(current);
1291 flush_vsx_to_thread(current);
1292 vcpu->arch.wqp = &vcpu->arch.vcore->wq;
Paul Mackerras342d3db2011-12-12 12:38:05 +00001293 vcpu->arch.pgdir = current->mm->pgd;
Paul Mackerras19ccb762011-07-23 17:42:46 +10001294
Paul Mackerrasa8606e22011-06-29 00:22:05 +00001295 do {
1296 r = kvmppc_run_vcpu(run, vcpu);
1297
1298 if (run->exit_reason == KVM_EXIT_PAPR_HCALL &&
1299 !(vcpu->arch.shregs.msr & MSR_PR)) {
1300 r = kvmppc_pseries_do_hcall(vcpu);
Scott Wood7e28e60e2011-11-08 18:23:20 -06001301 kvmppc_core_prepare_to_enter(vcpu);
Paul Mackerrasa8606e22011-06-29 00:22:05 +00001302 }
1303 } while (r == RESUME_GUEST);
Paul Mackerras32fad282012-05-04 02:32:53 +00001304
1305 out:
1306 atomic_dec(&vcpu->kvm->arch.vcpus_running);
Paul Mackerrasa8606e22011-06-29 00:22:05 +00001307 return r;
1308}
1309
David Gibson54738c02011-06-29 00:22:41 +00001310
Paul Mackerrasaa04b4c2011-06-29 00:25:44 +00001311/* Work out RMLS (real mode limit selector) field value for a given RMA size.
Paul Mackerras9e368f22011-06-29 00:40:08 +00001312 Assumes POWER7 or PPC970. */
Paul Mackerrasaa04b4c2011-06-29 00:25:44 +00001313static inline int lpcr_rmls(unsigned long rma_size)
1314{
1315 switch (rma_size) {
1316 case 32ul << 20: /* 32 MB */
Paul Mackerras9e368f22011-06-29 00:40:08 +00001317 if (cpu_has_feature(CPU_FTR_ARCH_206))
1318 return 8; /* only supported on POWER7 */
1319 return -1;
Paul Mackerrasaa04b4c2011-06-29 00:25:44 +00001320 case 64ul << 20: /* 64 MB */
1321 return 3;
1322 case 128ul << 20: /* 128 MB */
1323 return 7;
1324 case 256ul << 20: /* 256 MB */
1325 return 4;
1326 case 1ul << 30: /* 1 GB */
1327 return 2;
1328 case 16ul << 30: /* 16 GB */
1329 return 1;
1330 case 256ul << 30: /* 256 GB */
1331 return 0;
1332 default:
1333 return -1;
1334 }
1335}
1336
1337static int kvm_rma_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1338{
Alexander Grafb4e70612012-01-16 16:50:10 +01001339 struct kvmppc_linear_info *ri = vma->vm_file->private_data;
Paul Mackerrasaa04b4c2011-06-29 00:25:44 +00001340 struct page *page;
1341
1342 if (vmf->pgoff >= ri->npages)
1343 return VM_FAULT_SIGBUS;
1344
1345 page = pfn_to_page(ri->base_pfn + vmf->pgoff);
1346 get_page(page);
1347 vmf->page = page;
1348 return 0;
1349}
1350
1351static const struct vm_operations_struct kvm_rma_vm_ops = {
1352 .fault = kvm_rma_fault,
1353};
1354
1355static int kvm_rma_mmap(struct file *file, struct vm_area_struct *vma)
1356{
1357 vma->vm_flags |= VM_RESERVED;
1358 vma->vm_ops = &kvm_rma_vm_ops;
1359 return 0;
1360}
1361
1362static int kvm_rma_release(struct inode *inode, struct file *filp)
1363{
Alexander Grafb4e70612012-01-16 16:50:10 +01001364 struct kvmppc_linear_info *ri = filp->private_data;
Paul Mackerrasaa04b4c2011-06-29 00:25:44 +00001365
1366 kvm_release_rma(ri);
1367 return 0;
1368}
1369
1370static struct file_operations kvm_rma_fops = {
1371 .mmap = kvm_rma_mmap,
1372 .release = kvm_rma_release,
1373};
1374
1375long kvm_vm_ioctl_allocate_rma(struct kvm *kvm, struct kvm_allocate_rma *ret)
1376{
Alexander Grafb4e70612012-01-16 16:50:10 +01001377 struct kvmppc_linear_info *ri;
Paul Mackerrasaa04b4c2011-06-29 00:25:44 +00001378 long fd;
1379
1380 ri = kvm_alloc_rma();
1381 if (!ri)
1382 return -ENOMEM;
1383
1384 fd = anon_inode_getfd("kvm-rma", &kvm_rma_fops, ri, O_RDWR);
1385 if (fd < 0)
1386 kvm_release_rma(ri);
1387
1388 ret->rma_size = ri->npages << PAGE_SHIFT;
1389 return fd;
1390}
1391
Benjamin Herrenschmidt5b747162012-04-26 19:43:42 +00001392static void kvmppc_add_seg_page_size(struct kvm_ppc_one_seg_page_size **sps,
1393 int linux_psize)
1394{
1395 struct mmu_psize_def *def = &mmu_psize_defs[linux_psize];
1396
1397 if (!def->shift)
1398 return;
1399 (*sps)->page_shift = def->shift;
1400 (*sps)->slb_enc = def->sllp;
1401 (*sps)->enc[0].page_shift = def->shift;
1402 (*sps)->enc[0].pte_enc = def->penc;
1403 (*sps)++;
1404}
1405
1406int kvm_vm_ioctl_get_smmu_info(struct kvm *kvm, struct kvm_ppc_smmu_info *info)
1407{
1408 struct kvm_ppc_one_seg_page_size *sps;
1409
1410 info->flags = KVM_PPC_PAGE_SIZES_REAL;
1411 if (mmu_has_feature(MMU_FTR_1T_SEGMENT))
1412 info->flags |= KVM_PPC_1T_SEGMENTS;
1413 info->slb_size = mmu_slb_size;
1414
1415 /* We only support these sizes for now, and no muti-size segments */
1416 sps = &info->sps[0];
1417 kvmppc_add_seg_page_size(&sps, MMU_PAGE_4K);
1418 kvmppc_add_seg_page_size(&sps, MMU_PAGE_64K);
1419 kvmppc_add_seg_page_size(&sps, MMU_PAGE_16M);
1420
1421 return 0;
1422}
1423
Paul Mackerras82ed3612011-12-15 02:03:22 +00001424/*
1425 * Get (and clear) the dirty memory log for a memory slot.
1426 */
1427int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, struct kvm_dirty_log *log)
1428{
1429 struct kvm_memory_slot *memslot;
1430 int r;
1431 unsigned long n;
1432
1433 mutex_lock(&kvm->slots_lock);
1434
1435 r = -EINVAL;
1436 if (log->slot >= KVM_MEMORY_SLOTS)
1437 goto out;
1438
1439 memslot = id_to_memslot(kvm->memslots, log->slot);
1440 r = -ENOENT;
1441 if (!memslot->dirty_bitmap)
1442 goto out;
1443
1444 n = kvm_dirty_bitmap_bytes(memslot);
1445 memset(memslot->dirty_bitmap, 0, n);
1446
Paul Mackerrasdfe49db2012-09-11 13:28:18 +00001447 r = kvmppc_hv_get_dirty_log(kvm, memslot, memslot->dirty_bitmap);
Paul Mackerras82ed3612011-12-15 02:03:22 +00001448 if (r)
1449 goto out;
1450
1451 r = -EFAULT;
1452 if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n))
1453 goto out;
1454
1455 r = 0;
1456out:
1457 mutex_unlock(&kvm->slots_lock);
1458 return r;
1459}
1460
Paul Mackerrasda9d1d72011-12-12 12:31:41 +00001461static unsigned long slb_pgsize_encoding(unsigned long psize)
1462{
1463 unsigned long senc = 0;
1464
1465 if (psize > 0x1000) {
1466 senc = SLB_VSID_L;
1467 if (psize == 0x10000)
1468 senc |= SLB_VSID_LP_01;
1469 }
1470 return senc;
1471}
1472
Paul Mackerrasa66b48c2012-09-11 13:27:46 +00001473static void unpin_slot(struct kvm_memory_slot *memslot)
Paul Mackerrasc77162d2011-12-12 12:31:00 +00001474{
1475 unsigned long *physp;
1476 unsigned long j, npages, pfn;
1477 struct page *page;
1478
Paul Mackerrasa66b48c2012-09-11 13:27:46 +00001479 physp = memslot->arch.slot_phys;
1480 npages = memslot->npages;
1481 if (!physp)
1482 return;
1483 for (j = 0; j < npages; j++) {
1484 if (!(physp[j] & KVMPPC_GOT_PAGE))
1485 continue;
1486 pfn = physp[j] >> PAGE_SHIFT;
1487 page = pfn_to_page(pfn);
1488 SetPageDirty(page);
1489 put_page(page);
Paul Mackerrasc77162d2011-12-12 12:31:00 +00001490 }
1491}
1492
Paul Mackerrasa66b48c2012-09-11 13:27:46 +00001493void kvmppc_core_free_memslot(struct kvm_memory_slot *free,
1494 struct kvm_memory_slot *dont)
1495{
1496 if (!dont || free->arch.rmap != dont->arch.rmap) {
1497 vfree(free->arch.rmap);
1498 free->arch.rmap = NULL;
1499 }
1500 if (!dont || free->arch.slot_phys != dont->arch.slot_phys) {
1501 unpin_slot(free);
1502 vfree(free->arch.slot_phys);
1503 free->arch.slot_phys = NULL;
1504 }
1505}
1506
1507int kvmppc_core_create_memslot(struct kvm_memory_slot *slot,
1508 unsigned long npages)
1509{
1510 slot->arch.rmap = vzalloc(npages * sizeof(*slot->arch.rmap));
1511 if (!slot->arch.rmap)
1512 return -ENOMEM;
1513 slot->arch.slot_phys = NULL;
1514
1515 return 0;
1516}
1517
1518int kvmppc_core_prepare_memory_region(struct kvm *kvm,
1519 struct kvm_memory_slot *memslot,
1520 struct kvm_userspace_memory_region *mem)
1521{
1522 unsigned long *phys;
1523
1524 /* Allocate a slot_phys array if needed */
1525 phys = memslot->arch.slot_phys;
1526 if (!kvm->arch.using_mmu_notifiers && !phys && memslot->npages) {
1527 phys = vzalloc(memslot->npages * sizeof(unsigned long));
1528 if (!phys)
1529 return -ENOMEM;
1530 memslot->arch.slot_phys = phys;
1531 }
1532
1533 return 0;
1534}
1535
Paul Mackerrasc77162d2011-12-12 12:31:00 +00001536void kvmppc_core_commit_memory_region(struct kvm *kvm,
Paul Mackerrasdfe49db2012-09-11 13:28:18 +00001537 struct kvm_userspace_memory_region *mem,
1538 struct kvm_memory_slot old)
Paul Mackerrasc77162d2011-12-12 12:31:00 +00001539{
Paul Mackerrasdfe49db2012-09-11 13:28:18 +00001540 unsigned long npages = mem->memory_size >> PAGE_SHIFT;
1541 struct kvm_memory_slot *memslot;
1542
1543 if (npages && old.npages) {
1544 /*
1545 * If modifying a memslot, reset all the rmap dirty bits.
1546 * If this is a new memslot, we don't need to do anything
1547 * since the rmap array starts out as all zeroes,
1548 * i.e. no pages are dirty.
1549 */
1550 memslot = id_to_memslot(kvm->memslots, mem->slot);
1551 kvmppc_hv_get_dirty_log(kvm, memslot, NULL);
1552 }
Paul Mackerrasc77162d2011-12-12 12:31:00 +00001553}
1554
Paul Mackerras32fad282012-05-04 02:32:53 +00001555static int kvmppc_hv_setup_htab_rma(struct kvm_vcpu *vcpu)
Paul Mackerrasc77162d2011-12-12 12:31:00 +00001556{
1557 int err = 0;
1558 struct kvm *kvm = vcpu->kvm;
Alexander Grafb4e70612012-01-16 16:50:10 +01001559 struct kvmppc_linear_info *ri = NULL;
Paul Mackerrasc77162d2011-12-12 12:31:00 +00001560 unsigned long hva;
1561 struct kvm_memory_slot *memslot;
1562 struct vm_area_struct *vma;
Paul Mackerrasda9d1d72011-12-12 12:31:41 +00001563 unsigned long lpcr, senc;
Paul Mackerrasc77162d2011-12-12 12:31:00 +00001564 unsigned long psize, porder;
1565 unsigned long rma_size;
1566 unsigned long rmls;
1567 unsigned long *physp;
Paul Mackerrasda9d1d72011-12-12 12:31:41 +00001568 unsigned long i, npages;
Paul Mackerras2c9097e2012-09-11 13:27:01 +00001569 int srcu_idx;
Paul Mackerrasc77162d2011-12-12 12:31:00 +00001570
1571 mutex_lock(&kvm->lock);
1572 if (kvm->arch.rma_setup_done)
1573 goto out; /* another vcpu beat us to it */
1574
Paul Mackerras32fad282012-05-04 02:32:53 +00001575 /* Allocate hashed page table (if not done already) and reset it */
1576 if (!kvm->arch.hpt_virt) {
1577 err = kvmppc_alloc_hpt(kvm, NULL);
1578 if (err) {
1579 pr_err("KVM: Couldn't alloc HPT\n");
1580 goto out;
1581 }
1582 }
1583
Paul Mackerrasc77162d2011-12-12 12:31:00 +00001584 /* Look up the memslot for guest physical address 0 */
Paul Mackerras2c9097e2012-09-11 13:27:01 +00001585 srcu_idx = srcu_read_lock(&kvm->srcu);
Paul Mackerrasc77162d2011-12-12 12:31:00 +00001586 memslot = gfn_to_memslot(kvm, 0);
1587
1588 /* We must have some memory at 0 by now */
1589 err = -EINVAL;
1590 if (!memslot || (memslot->flags & KVM_MEMSLOT_INVALID))
Paul Mackerras2c9097e2012-09-11 13:27:01 +00001591 goto out_srcu;
Paul Mackerrasc77162d2011-12-12 12:31:00 +00001592
1593 /* Look up the VMA for the start of this memory slot */
1594 hva = memslot->userspace_addr;
1595 down_read(&current->mm->mmap_sem);
1596 vma = find_vma(current->mm, hva);
1597 if (!vma || vma->vm_start > hva || (vma->vm_flags & VM_IO))
1598 goto up_out;
1599
1600 psize = vma_kernel_pagesize(vma);
Paul Mackerrasda9d1d72011-12-12 12:31:41 +00001601 porder = __ilog2(psize);
Paul Mackerrasaa04b4c2011-06-29 00:25:44 +00001602
Paul Mackerrasaa04b4c2011-06-29 00:25:44 +00001603 /* Is this one of our preallocated RMAs? */
Paul Mackerrasc77162d2011-12-12 12:31:00 +00001604 if (vma->vm_file && vma->vm_file->f_op == &kvm_rma_fops &&
1605 hva == vma->vm_start)
1606 ri = vma->vm_file->private_data;
Paul Mackerrasaa04b4c2011-06-29 00:25:44 +00001607
Paul Mackerrasc77162d2011-12-12 12:31:00 +00001608 up_read(&current->mm->mmap_sem);
1609
1610 if (!ri) {
1611 /* On POWER7, use VRMA; on PPC970, give up */
1612 err = -EPERM;
1613 if (cpu_has_feature(CPU_FTR_ARCH_201)) {
1614 pr_err("KVM: CPU requires an RMO\n");
Paul Mackerras2c9097e2012-09-11 13:27:01 +00001615 goto out_srcu;
Paul Mackerras9e368f22011-06-29 00:40:08 +00001616 }
Paul Mackerrasaa04b4c2011-06-29 00:25:44 +00001617
Paul Mackerrasda9d1d72011-12-12 12:31:41 +00001618 /* We can handle 4k, 64k or 16M pages in the VRMA */
1619 err = -EINVAL;
1620 if (!(psize == 0x1000 || psize == 0x10000 ||
1621 psize == 0x1000000))
Paul Mackerras2c9097e2012-09-11 13:27:01 +00001622 goto out_srcu;
Paul Mackerrasda9d1d72011-12-12 12:31:41 +00001623
Paul Mackerrasc77162d2011-12-12 12:31:00 +00001624 /* Update VRMASD field in the LPCR */
Paul Mackerrasda9d1d72011-12-12 12:31:41 +00001625 senc = slb_pgsize_encoding(psize);
Paul Mackerras697d3892011-12-12 12:36:37 +00001626 kvm->arch.vrma_slb_v = senc | SLB_VSID_B_1T |
1627 (VRMA_VSID << SLB_VSID_SHIFT_1T);
Paul Mackerrasda9d1d72011-12-12 12:31:41 +00001628 lpcr = kvm->arch.lpcr & ~LPCR_VRMASD;
1629 lpcr |= senc << (LPCR_VRMASD_SH - 4);
Paul Mackerrasc77162d2011-12-12 12:31:00 +00001630 kvm->arch.lpcr = lpcr;
Paul Mackerrasaa04b4c2011-06-29 00:25:44 +00001631
Paul Mackerrasc77162d2011-12-12 12:31:00 +00001632 /* Create HPTEs in the hash page table for the VRMA */
Paul Mackerrasda9d1d72011-12-12 12:31:41 +00001633 kvmppc_map_vrma(vcpu, memslot, porder);
Paul Mackerrasc77162d2011-12-12 12:31:00 +00001634
1635 } else {
1636 /* Set up to use an RMO region */
1637 rma_size = ri->npages;
1638 if (rma_size > memslot->npages)
1639 rma_size = memslot->npages;
1640 rma_size <<= PAGE_SHIFT;
Paul Mackerrasaa04b4c2011-06-29 00:25:44 +00001641 rmls = lpcr_rmls(rma_size);
Paul Mackerrasc77162d2011-12-12 12:31:00 +00001642 err = -EINVAL;
Paul Mackerrasaa04b4c2011-06-29 00:25:44 +00001643 if (rmls < 0) {
Paul Mackerrasc77162d2011-12-12 12:31:00 +00001644 pr_err("KVM: Can't use RMA of 0x%lx bytes\n", rma_size);
Paul Mackerras2c9097e2012-09-11 13:27:01 +00001645 goto out_srcu;
Paul Mackerrasaa04b4c2011-06-29 00:25:44 +00001646 }
1647 atomic_inc(&ri->use_count);
1648 kvm->arch.rma = ri;
Paul Mackerras9e368f22011-06-29 00:40:08 +00001649
1650 /* Update LPCR and RMOR */
1651 lpcr = kvm->arch.lpcr;
1652 if (cpu_has_feature(CPU_FTR_ARCH_201)) {
1653 /* PPC970; insert RMLS value (split field) in HID4 */
1654 lpcr &= ~((1ul << HID4_RMLS0_SH) |
1655 (3ul << HID4_RMLS2_SH));
1656 lpcr |= ((rmls >> 2) << HID4_RMLS0_SH) |
1657 ((rmls & 3) << HID4_RMLS2_SH);
1658 /* RMOR is also in HID4 */
1659 lpcr |= ((ri->base_pfn >> (26 - PAGE_SHIFT)) & 0xffff)
1660 << HID4_RMOR_SH;
1661 } else {
1662 /* POWER7 */
1663 lpcr &= ~(LPCR_VPM0 | LPCR_VRMA_L);
1664 lpcr |= rmls << LPCR_RMLS_SH;
1665 kvm->arch.rmor = kvm->arch.rma->base_pfn << PAGE_SHIFT;
1666 }
Paul Mackerrasaa04b4c2011-06-29 00:25:44 +00001667 kvm->arch.lpcr = lpcr;
Paul Mackerrasc77162d2011-12-12 12:31:00 +00001668 pr_info("KVM: Using RMO at %lx size %lx (LPCR = %lx)\n",
Paul Mackerrasaa04b4c2011-06-29 00:25:44 +00001669 ri->base_pfn << PAGE_SHIFT, rma_size, lpcr);
Paul Mackerrasc77162d2011-12-12 12:31:00 +00001670
1671 /* Initialize phys addrs of pages in RMO */
Paul Mackerrasda9d1d72011-12-12 12:31:41 +00001672 npages = ri->npages;
1673 porder = __ilog2(npages);
Paul Mackerrasa66b48c2012-09-11 13:27:46 +00001674 physp = memslot->arch.slot_phys;
1675 if (physp) {
1676 if (npages > memslot->npages)
1677 npages = memslot->npages;
1678 spin_lock(&kvm->arch.slot_phys_lock);
1679 for (i = 0; i < npages; ++i)
1680 physp[i] = ((ri->base_pfn + i) << PAGE_SHIFT) +
1681 porder;
1682 spin_unlock(&kvm->arch.slot_phys_lock);
1683 }
Paul Mackerrasaa04b4c2011-06-29 00:25:44 +00001684 }
1685
Paul Mackerrasc77162d2011-12-12 12:31:00 +00001686 /* Order updates to kvm->arch.lpcr etc. vs. rma_setup_done */
1687 smp_wmb();
1688 kvm->arch.rma_setup_done = 1;
1689 err = 0;
Paul Mackerras2c9097e2012-09-11 13:27:01 +00001690 out_srcu:
1691 srcu_read_unlock(&kvm->srcu, srcu_idx);
Paul Mackerrasc77162d2011-12-12 12:31:00 +00001692 out:
1693 mutex_unlock(&kvm->lock);
1694 return err;
Paul Mackerrasaa04b4c2011-06-29 00:25:44 +00001695
Paul Mackerrasc77162d2011-12-12 12:31:00 +00001696 up_out:
1697 up_read(&current->mm->mmap_sem);
1698 goto out;
Paul Mackerrasde56a942011-06-29 00:21:34 +00001699}
1700
1701int kvmppc_core_init_vm(struct kvm *kvm)
1702{
Paul Mackerras32fad282012-05-04 02:32:53 +00001703 unsigned long lpcr, lpid;
Paul Mackerrasde56a942011-06-29 00:21:34 +00001704
Paul Mackerras32fad282012-05-04 02:32:53 +00001705 /* Allocate the guest's logical partition ID */
1706
1707 lpid = kvmppc_alloc_lpid();
1708 if (lpid < 0)
1709 return -ENOMEM;
1710 kvm->arch.lpid = lpid;
Paul Mackerrasde56a942011-06-29 00:21:34 +00001711
David Gibson54738c02011-06-29 00:22:41 +00001712 INIT_LIST_HEAD(&kvm->arch.spapr_tce_tables);
Paul Mackerrasaa04b4c2011-06-29 00:25:44 +00001713
Paul Mackerrasaa04b4c2011-06-29 00:25:44 +00001714 kvm->arch.rma = NULL;
Paul Mackerrasaa04b4c2011-06-29 00:25:44 +00001715
Paul Mackerras9e368f22011-06-29 00:40:08 +00001716 kvm->arch.host_sdr1 = mfspr(SPRN_SDR1);
Paul Mackerrasaa04b4c2011-06-29 00:25:44 +00001717
Paul Mackerras9e368f22011-06-29 00:40:08 +00001718 if (cpu_has_feature(CPU_FTR_ARCH_201)) {
1719 /* PPC970; HID4 is effectively the LPCR */
Paul Mackerras9e368f22011-06-29 00:40:08 +00001720 kvm->arch.host_lpid = 0;
1721 kvm->arch.host_lpcr = lpcr = mfspr(SPRN_HID4);
1722 lpcr &= ~((3 << HID4_LPID1_SH) | (0xful << HID4_LPID5_SH));
1723 lpcr |= ((lpid >> 4) << HID4_LPID1_SH) |
1724 ((lpid & 0xf) << HID4_LPID5_SH);
1725 } else {
1726 /* POWER7; init LPCR for virtual RMA mode */
1727 kvm->arch.host_lpid = mfspr(SPRN_LPID);
1728 kvm->arch.host_lpcr = lpcr = mfspr(SPRN_LPCR);
1729 lpcr &= LPCR_PECE | LPCR_LPES;
1730 lpcr |= (4UL << LPCR_DPFD_SH) | LPCR_HDICE |
Paul Mackerras697d3892011-12-12 12:36:37 +00001731 LPCR_VPM0 | LPCR_VPM1;
1732 kvm->arch.vrma_slb_v = SLB_VSID_B_1T |
1733 (VRMA_VSID << SLB_VSID_SHIFT_1T);
Paul Mackerras9e368f22011-06-29 00:40:08 +00001734 }
1735 kvm->arch.lpcr = lpcr;
Paul Mackerrasaa04b4c2011-06-29 00:25:44 +00001736
Paul Mackerras342d3db2011-12-12 12:38:05 +00001737 kvm->arch.using_mmu_notifiers = !!cpu_has_feature(CPU_FTR_ARCH_206);
Paul Mackerrasc77162d2011-12-12 12:31:00 +00001738 spin_lock_init(&kvm->arch.slot_phys_lock);
Paul Mackerras512691d2012-10-15 01:15:41 +00001739
1740 /*
1741 * Don't allow secondary CPU threads to come online
1742 * while any KVM VMs exist.
1743 */
1744 inhibit_secondary_onlining();
1745
David Gibson54738c02011-06-29 00:22:41 +00001746 return 0;
Paul Mackerrasde56a942011-06-29 00:21:34 +00001747}
1748
1749void kvmppc_core_destroy_vm(struct kvm *kvm)
1750{
Paul Mackerras512691d2012-10-15 01:15:41 +00001751 uninhibit_secondary_onlining();
1752
Paul Mackerrasaa04b4c2011-06-29 00:25:44 +00001753 if (kvm->arch.rma) {
1754 kvm_release_rma(kvm->arch.rma);
1755 kvm->arch.rma = NULL;
1756 }
1757
Paul Mackerrasde56a942011-06-29 00:21:34 +00001758 kvmppc_free_hpt(kvm);
David Gibson54738c02011-06-29 00:22:41 +00001759 WARN_ON(!list_empty(&kvm->arch.spapr_tce_tables));
Paul Mackerrasde56a942011-06-29 00:21:34 +00001760}
1761
1762/* These are stubs for now */
1763void kvmppc_mmu_pte_pflush(struct kvm_vcpu *vcpu, ulong pa_start, ulong pa_end)
1764{
1765}
1766
1767/* We don't need to emulate any privileged instructions or dcbz */
1768int kvmppc_core_emulate_op(struct kvm_run *run, struct kvm_vcpu *vcpu,
1769 unsigned int inst, int *advance)
1770{
1771 return EMULATE_FAIL;
1772}
1773
Alexander Graf54771e62012-05-04 14:55:12 +02001774int kvmppc_core_emulate_mtspr(struct kvm_vcpu *vcpu, int sprn, ulong spr_val)
Paul Mackerrasde56a942011-06-29 00:21:34 +00001775{
1776 return EMULATE_FAIL;
1777}
1778
Alexander Graf54771e62012-05-04 14:55:12 +02001779int kvmppc_core_emulate_mfspr(struct kvm_vcpu *vcpu, int sprn, ulong *spr_val)
Paul Mackerrasde56a942011-06-29 00:21:34 +00001780{
1781 return EMULATE_FAIL;
1782}
1783
1784static int kvmppc_book3s_hv_init(void)
1785{
1786 int r;
1787
1788 r = kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
1789
1790 if (r)
1791 return r;
1792
1793 r = kvmppc_mmu_hv_init();
1794
1795 return r;
1796}
1797
1798static void kvmppc_book3s_hv_exit(void)
1799{
1800 kvm_exit();
1801}
1802
1803module_init(kvmppc_book3s_hv_init);
1804module_exit(kvmppc_book3s_hv_exit);