blob: 6c972d229ace8ec84ed711c3a804ac6ec1031ab8 [file] [log] [blame]
Heiko Carstensb0c632d2008-03-25 18:47:20 +01001/*
Heiko Carstensa53c8fa2012-07-20 11:15:04 +02002 * hosting zSeries kernel virtual machines
Heiko Carstensb0c632d2008-03-25 18:47:20 +01003 *
Heiko Carstensa53c8fa2012-07-20 11:15:04 +02004 * Copyright IBM Corp. 2008, 2009
Heiko Carstensb0c632d2008-03-25 18:47:20 +01005 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License (version 2 only)
8 * as published by the Free Software Foundation.
9 *
10 * Author(s): Carsten Otte <cotte@de.ibm.com>
11 * Christian Borntraeger <borntraeger@de.ibm.com>
12 * Heiko Carstens <heiko.carstens@de.ibm.com>
Christian Ehrhardt628eb9b2009-05-25 13:40:51 +020013 * Christian Ehrhardt <ehrhardt@de.ibm.com>
Jason J. Herne15f36eb2012-08-02 10:10:17 -040014 * Jason J. Herne <jjherne@us.ibm.com>
Heiko Carstensb0c632d2008-03-25 18:47:20 +010015 */
16
17#include <linux/compiler.h>
18#include <linux/err.h>
19#include <linux/fs.h>
Christian Borntraegerca872302009-05-12 17:21:49 +020020#include <linux/hrtimer.h>
Heiko Carstensb0c632d2008-03-25 18:47:20 +010021#include <linux/init.h>
22#include <linux/kvm.h>
23#include <linux/kvm_host.h>
24#include <linux/module.h>
25#include <linux/slab.h>
Carsten Otteba5c1e92008-03-25 18:47:26 +010026#include <linux/timer.h>
Heiko Carstenscbb870c2010-02-26 22:37:43 +010027#include <asm/asm-offsets.h>
Heiko Carstensb0c632d2008-03-25 18:47:20 +010028#include <asm/lowcore.h>
29#include <asm/pgtable.h>
Heiko Carstensf5daba12009-03-26 15:24:01 +010030#include <asm/nmi.h>
David Howellsa0616cd2012-03-28 18:30:02 +010031#include <asm/switch_to.h>
Michael Mueller78c4b592013-07-26 15:04:04 +020032#include <asm/facility.h>
Christian Borntraeger1526bf92012-05-15 14:15:25 +020033#include <asm/sclp.h>
Christian Borntraeger8f2abe62008-03-25 18:47:23 +010034#include "kvm-s390.h"
Heiko Carstensb0c632d2008-03-25 18:47:20 +010035#include "gaccess.h"
36
Cornelia Huck5786fff2012-07-23 17:20:29 +020037#define CREATE_TRACE_POINTS
38#include "trace.h"
Cornelia Huckade38c32012-07-23 17:20:30 +020039#include "trace-s390.h"
Cornelia Huck5786fff2012-07-23 17:20:29 +020040
Heiko Carstensb0c632d2008-03-25 18:47:20 +010041#define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
42
43struct kvm_stats_debugfs_item debugfs_entries[] = {
44 { "userspace_handled", VCPU_STAT(exit_userspace) },
Christian Borntraeger0eaeafa2008-05-07 09:22:53 +020045 { "exit_null", VCPU_STAT(exit_null) },
Christian Borntraeger8f2abe62008-03-25 18:47:23 +010046 { "exit_validity", VCPU_STAT(exit_validity) },
47 { "exit_stop_request", VCPU_STAT(exit_stop_request) },
48 { "exit_external_request", VCPU_STAT(exit_external_request) },
49 { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) },
Carsten Otteba5c1e92008-03-25 18:47:26 +010050 { "exit_instruction", VCPU_STAT(exit_instruction) },
51 { "exit_program_interruption", VCPU_STAT(exit_program_interruption) },
52 { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) },
Christian Borntraegerf5e10b02008-07-25 15:52:44 +020053 { "instruction_lctlg", VCPU_STAT(instruction_lctlg) },
Carsten Otteba5c1e92008-03-25 18:47:26 +010054 { "instruction_lctl", VCPU_STAT(instruction_lctl) },
David Hildenbrandaba07502014-01-23 10:47:13 +010055 { "instruction_stctl", VCPU_STAT(instruction_stctl) },
56 { "instruction_stctg", VCPU_STAT(instruction_stctg) },
Carsten Otteba5c1e92008-03-25 18:47:26 +010057 { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) },
Christian Ehrhardt7697e71f2011-10-18 12:27:15 +020058 { "deliver_external_call", VCPU_STAT(deliver_external_call) },
Carsten Otteba5c1e92008-03-25 18:47:26 +010059 { "deliver_service_signal", VCPU_STAT(deliver_service_signal) },
60 { "deliver_virtio_interrupt", VCPU_STAT(deliver_virtio_interrupt) },
61 { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) },
62 { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) },
63 { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) },
64 { "deliver_program_interruption", VCPU_STAT(deliver_program_int) },
65 { "exit_wait_state", VCPU_STAT(exit_wait_state) },
Christian Borntraeger69d0d3a2013-06-12 13:54:53 +020066 { "instruction_pfmf", VCPU_STAT(instruction_pfmf) },
Christian Borntraeger453423d2008-03-25 18:47:29 +010067 { "instruction_stidp", VCPU_STAT(instruction_stidp) },
68 { "instruction_spx", VCPU_STAT(instruction_spx) },
69 { "instruction_stpx", VCPU_STAT(instruction_stpx) },
70 { "instruction_stap", VCPU_STAT(instruction_stap) },
71 { "instruction_storage_key", VCPU_STAT(instruction_storage_key) },
Heiko Carstens8a2422342014-01-10 14:33:28 +010072 { "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock) },
Christian Borntraeger453423d2008-03-25 18:47:29 +010073 { "instruction_stsch", VCPU_STAT(instruction_stsch) },
74 { "instruction_chsc", VCPU_STAT(instruction_chsc) },
Konstantin Weitzb31288f2013-04-17 17:36:29 +020075 { "instruction_essa", VCPU_STAT(instruction_essa) },
Christian Borntraeger453423d2008-03-25 18:47:29 +010076 { "instruction_stsi", VCPU_STAT(instruction_stsi) },
77 { "instruction_stfl", VCPU_STAT(instruction_stfl) },
Christian Borntraegerbb25b9b2011-07-24 10:48:17 +020078 { "instruction_tprot", VCPU_STAT(instruction_tprot) },
Christian Borntraeger5288fbf2008-03-25 18:47:31 +010079 { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) },
Cornelia Huckbd59d3a2011-11-17 11:00:42 +010080 { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) },
Christian Ehrhardt7697e71f2011-10-18 12:27:15 +020081 { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) },
Christian Borntraeger5288fbf2008-03-25 18:47:31 +010082 { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) },
83 { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) },
84 { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) },
85 { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) },
86 { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) },
Christian Borntraeger388186b2011-10-30 15:17:03 +010087 { "diagnose_10", VCPU_STAT(diagnose_10) },
Christian Borntraegere28acfe2008-03-25 18:47:34 +010088 { "diagnose_44", VCPU_STAT(diagnose_44) },
Konstantin Weitz41628d32012-04-25 15:30:38 +020089 { "diagnose_9c", VCPU_STAT(diagnose_9c) },
Heiko Carstensb0c632d2008-03-25 18:47:20 +010090 { NULL }
91};
92
Michael Mueller78c4b592013-07-26 15:04:04 +020093unsigned long *vfacilities;
Christian Borntraeger2c70fe42013-05-17 14:41:36 +020094static struct gmap_notifier gmap_notifier;
Heiko Carstensb0c632d2008-03-25 18:47:20 +010095
Michael Mueller78c4b592013-07-26 15:04:04 +020096/* test availability of vfacility */
Heiko Carstens280ef0f2013-12-17 09:08:28 +010097int test_vfacility(unsigned long nr)
Michael Mueller78c4b592013-07-26 15:04:04 +020098{
99 return __test_facility(nr, (void *) vfacilities);
100}
101
Heiko Carstensb0c632d2008-03-25 18:47:20 +0100102/* Section: not file related */
Alexander Graf10474ae2009-09-15 11:37:46 +0200103int kvm_arch_hardware_enable(void *garbage)
Heiko Carstensb0c632d2008-03-25 18:47:20 +0100104{
105 /* every s390 is virtualization enabled ;-) */
Alexander Graf10474ae2009-09-15 11:37:46 +0200106 return 0;
Heiko Carstensb0c632d2008-03-25 18:47:20 +0100107}
108
109void kvm_arch_hardware_disable(void *garbage)
110{
111}
112
Christian Borntraeger2c70fe42013-05-17 14:41:36 +0200113static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address);
114
Heiko Carstensb0c632d2008-03-25 18:47:20 +0100115int kvm_arch_hardware_setup(void)
116{
Christian Borntraeger2c70fe42013-05-17 14:41:36 +0200117 gmap_notifier.notifier_call = kvm_gmap_notifier;
118 gmap_register_ipte_notifier(&gmap_notifier);
Heiko Carstensb0c632d2008-03-25 18:47:20 +0100119 return 0;
120}
121
122void kvm_arch_hardware_unsetup(void)
123{
Christian Borntraeger2c70fe42013-05-17 14:41:36 +0200124 gmap_unregister_ipte_notifier(&gmap_notifier);
Heiko Carstensb0c632d2008-03-25 18:47:20 +0100125}
126
127void kvm_arch_check_processor_compat(void *rtn)
128{
129}
130
131int kvm_arch_init(void *opaque)
132{
133 return 0;
134}
135
136void kvm_arch_exit(void)
137{
138}
139
140/* Section: device related */
141long kvm_arch_dev_ioctl(struct file *filp,
142 unsigned int ioctl, unsigned long arg)
143{
144 if (ioctl == KVM_S390_ENABLE_SIE)
145 return s390_enable_sie();
146 return -EINVAL;
147}
148
149int kvm_dev_ioctl_check_extension(long ext)
150{
Carsten Otted7b0b5e2009-11-19 14:21:16 +0100151 int r;
152
Carsten Otte2bd0ac42008-07-25 15:49:13 +0200153 switch (ext) {
Carsten Otted7b0b5e2009-11-19 14:21:16 +0100154 case KVM_CAP_S390_PSW:
Christian Borntraegerb6cf8782011-09-20 17:07:29 +0200155 case KVM_CAP_S390_GMAP:
Christian Borntraeger52e16b12011-11-17 11:00:44 +0100156 case KVM_CAP_SYNC_MMU:
Carsten Otte1efd0f52012-01-04 10:25:29 +0100157#ifdef CONFIG_KVM_S390_UCONTROL
158 case KVM_CAP_S390_UCONTROL:
159#endif
Dominik Dingel3c038e62013-10-07 17:11:48 +0200160 case KVM_CAP_ASYNC_PF:
Christian Borntraeger60b413c2012-01-11 11:20:31 +0100161 case KVM_CAP_SYNC_REGS:
Carsten Otte14eebd92012-05-15 14:15:26 +0200162 case KVM_CAP_ONE_REG:
Cornelia Huckd6712df2012-12-20 15:32:11 +0100163 case KVM_CAP_ENABLE_CAP:
Cornelia Huckfa6b7fe2012-12-20 15:32:12 +0100164 case KVM_CAP_S390_CSS_SUPPORT:
Cornelia Huck10ccaa12013-02-28 12:33:21 +0100165 case KVM_CAP_IOEVENTFD:
Jens Freimannc05c4182013-10-07 16:13:45 +0200166 case KVM_CAP_DEVICE_CTRL:
Cornelia Huckd938dc52013-10-23 18:26:34 +0200167 case KVM_CAP_ENABLE_CAP_VM:
Dominik Dingelf2061652014-04-09 13:13:00 +0200168 case KVM_CAP_VM_ATTRIBUTES:
Carsten Otted7b0b5e2009-11-19 14:21:16 +0100169 r = 1;
170 break;
Christian Borntraegere726b1b2012-05-02 10:50:38 +0200171 case KVM_CAP_NR_VCPUS:
172 case KVM_CAP_MAX_VCPUS:
173 r = KVM_MAX_VCPUS;
174 break;
Nick Wange1e2e602013-03-25 17:22:58 +0100175 case KVM_CAP_NR_MEMSLOTS:
176 r = KVM_USER_MEM_SLOTS;
177 break;
Christian Borntraeger1526bf92012-05-15 14:15:25 +0200178 case KVM_CAP_S390_COW:
Martin Schwidefskyabf09be2012-11-07 13:17:37 +0100179 r = MACHINE_HAS_ESOP;
Christian Borntraeger1526bf92012-05-15 14:15:25 +0200180 break;
Carsten Otte2bd0ac42008-07-25 15:49:13 +0200181 default:
Carsten Otted7b0b5e2009-11-19 14:21:16 +0100182 r = 0;
Carsten Otte2bd0ac42008-07-25 15:49:13 +0200183 }
Carsten Otted7b0b5e2009-11-19 14:21:16 +0100184 return r;
Heiko Carstensb0c632d2008-03-25 18:47:20 +0100185}
186
Jason J. Herne15f36eb2012-08-02 10:10:17 -0400187static void kvm_s390_sync_dirty_log(struct kvm *kvm,
188 struct kvm_memory_slot *memslot)
189{
190 gfn_t cur_gfn, last_gfn;
191 unsigned long address;
192 struct gmap *gmap = kvm->arch.gmap;
193
194 down_read(&gmap->mm->mmap_sem);
195 /* Loop over all guest pages */
196 last_gfn = memslot->base_gfn + memslot->npages;
197 for (cur_gfn = memslot->base_gfn; cur_gfn <= last_gfn; cur_gfn++) {
198 address = gfn_to_hva_memslot(memslot, cur_gfn);
199
200 if (gmap_test_and_clear_dirty(address, gmap))
201 mark_page_dirty(kvm, cur_gfn);
202 }
203 up_read(&gmap->mm->mmap_sem);
204}
205
Heiko Carstensb0c632d2008-03-25 18:47:20 +0100206/* Section: vm related */
207/*
208 * Get (and clear) the dirty memory log for a memory slot.
209 */
210int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
211 struct kvm_dirty_log *log)
212{
Jason J. Herne15f36eb2012-08-02 10:10:17 -0400213 int r;
214 unsigned long n;
215 struct kvm_memory_slot *memslot;
216 int is_dirty = 0;
217
218 mutex_lock(&kvm->slots_lock);
219
220 r = -EINVAL;
221 if (log->slot >= KVM_USER_MEM_SLOTS)
222 goto out;
223
224 memslot = id_to_memslot(kvm->memslots, log->slot);
225 r = -ENOENT;
226 if (!memslot->dirty_bitmap)
227 goto out;
228
229 kvm_s390_sync_dirty_log(kvm, memslot);
230 r = kvm_get_dirty_log(kvm, log, &is_dirty);
231 if (r)
232 goto out;
233
234 /* Clear the dirty log */
235 if (is_dirty) {
236 n = kvm_dirty_bitmap_bytes(memslot);
237 memset(memslot->dirty_bitmap, 0, n);
238 }
239 r = 0;
240out:
241 mutex_unlock(&kvm->slots_lock);
242 return r;
Heiko Carstensb0c632d2008-03-25 18:47:20 +0100243}
244
Cornelia Huckd938dc52013-10-23 18:26:34 +0200245static int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
246{
247 int r;
248
249 if (cap->flags)
250 return -EINVAL;
251
252 switch (cap->cap) {
Cornelia Huck84223592013-07-15 13:36:01 +0200253 case KVM_CAP_S390_IRQCHIP:
254 kvm->arch.use_irqchip = 1;
255 r = 0;
256 break;
Cornelia Huckd938dc52013-10-23 18:26:34 +0200257 default:
258 r = -EINVAL;
259 break;
260 }
261 return r;
262}
263
Dominik Dingel4f718ea2014-04-09 13:13:00 +0200264static int kvm_s390_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
265{
266 int ret;
267 unsigned int idx;
268 switch (attr->attr) {
269 case KVM_S390_VM_MEM_ENABLE_CMMA:
270 ret = -EBUSY;
271 mutex_lock(&kvm->lock);
272 if (atomic_read(&kvm->online_vcpus) == 0) {
273 kvm->arch.use_cmma = 1;
274 ret = 0;
275 }
276 mutex_unlock(&kvm->lock);
277 break;
278 case KVM_S390_VM_MEM_CLR_CMMA:
279 mutex_lock(&kvm->lock);
280 idx = srcu_read_lock(&kvm->srcu);
281 page_table_reset_pgste(kvm->arch.gmap->mm, 0, TASK_SIZE, false);
282 srcu_read_unlock(&kvm->srcu, idx);
283 mutex_unlock(&kvm->lock);
284 ret = 0;
285 break;
286 default:
287 ret = -ENXIO;
288 break;
289 }
290 return ret;
291}
292
Dominik Dingelf2061652014-04-09 13:13:00 +0200293static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
294{
295 int ret;
296
297 switch (attr->group) {
Dominik Dingel4f718ea2014-04-09 13:13:00 +0200298 case KVM_S390_VM_MEM_CTRL:
299 ret = kvm_s390_mem_control(kvm, attr);
300 break;
Dominik Dingelf2061652014-04-09 13:13:00 +0200301 default:
302 ret = -ENXIO;
303 break;
304 }
305
306 return ret;
307}
308
309static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr)
310{
311 return -ENXIO;
312}
313
314static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
315{
316 int ret;
317
318 switch (attr->group) {
Dominik Dingel4f718ea2014-04-09 13:13:00 +0200319 case KVM_S390_VM_MEM_CTRL:
320 switch (attr->attr) {
321 case KVM_S390_VM_MEM_ENABLE_CMMA:
322 case KVM_S390_VM_MEM_CLR_CMMA:
323 ret = 0;
324 break;
325 default:
326 ret = -ENXIO;
327 break;
328 }
329 break;
Dominik Dingelf2061652014-04-09 13:13:00 +0200330 default:
331 ret = -ENXIO;
332 break;
333 }
334
335 return ret;
336}
337
Heiko Carstensb0c632d2008-03-25 18:47:20 +0100338long kvm_arch_vm_ioctl(struct file *filp,
339 unsigned int ioctl, unsigned long arg)
340{
341 struct kvm *kvm = filp->private_data;
342 void __user *argp = (void __user *)arg;
Dominik Dingelf2061652014-04-09 13:13:00 +0200343 struct kvm_device_attr attr;
Heiko Carstensb0c632d2008-03-25 18:47:20 +0100344 int r;
345
346 switch (ioctl) {
Carsten Otteba5c1e92008-03-25 18:47:26 +0100347 case KVM_S390_INTERRUPT: {
348 struct kvm_s390_interrupt s390int;
349
350 r = -EFAULT;
351 if (copy_from_user(&s390int, argp, sizeof(s390int)))
352 break;
353 r = kvm_s390_inject_vm(kvm, &s390int);
354 break;
355 }
Cornelia Huckd938dc52013-10-23 18:26:34 +0200356 case KVM_ENABLE_CAP: {
357 struct kvm_enable_cap cap;
358 r = -EFAULT;
359 if (copy_from_user(&cap, argp, sizeof(cap)))
360 break;
361 r = kvm_vm_ioctl_enable_cap(kvm, &cap);
362 break;
363 }
Cornelia Huck84223592013-07-15 13:36:01 +0200364 case KVM_CREATE_IRQCHIP: {
365 struct kvm_irq_routing_entry routing;
366
367 r = -EINVAL;
368 if (kvm->arch.use_irqchip) {
369 /* Set up dummy routing. */
370 memset(&routing, 0, sizeof(routing));
371 kvm_set_irq_routing(kvm, &routing, 0, 0);
372 r = 0;
373 }
374 break;
375 }
Dominik Dingelf2061652014-04-09 13:13:00 +0200376 case KVM_SET_DEVICE_ATTR: {
377 r = -EFAULT;
378 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
379 break;
380 r = kvm_s390_vm_set_attr(kvm, &attr);
381 break;
382 }
383 case KVM_GET_DEVICE_ATTR: {
384 r = -EFAULT;
385 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
386 break;
387 r = kvm_s390_vm_get_attr(kvm, &attr);
388 break;
389 }
390 case KVM_HAS_DEVICE_ATTR: {
391 r = -EFAULT;
392 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
393 break;
394 r = kvm_s390_vm_has_attr(kvm, &attr);
395 break;
396 }
Heiko Carstensb0c632d2008-03-25 18:47:20 +0100397 default:
Avi Kivity367e1312009-08-26 14:57:07 +0300398 r = -ENOTTY;
Heiko Carstensb0c632d2008-03-25 18:47:20 +0100399 }
400
401 return r;
402}
403
Carsten Ottee08b9632012-01-04 10:25:20 +0100404int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
Heiko Carstensb0c632d2008-03-25 18:47:20 +0100405{
Heiko Carstensb0c632d2008-03-25 18:47:20 +0100406 int rc;
407 char debug_name[16];
Christian Borntraegerf6c137f2014-03-19 11:18:29 +0100408 static unsigned long sca_offset;
Heiko Carstensb0c632d2008-03-25 18:47:20 +0100409
Carsten Ottee08b9632012-01-04 10:25:20 +0100410 rc = -EINVAL;
411#ifdef CONFIG_KVM_S390_UCONTROL
412 if (type & ~KVM_VM_S390_UCONTROL)
413 goto out_err;
414 if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN)))
415 goto out_err;
416#else
417 if (type)
418 goto out_err;
419#endif
420
Heiko Carstensb0c632d2008-03-25 18:47:20 +0100421 rc = s390_enable_sie();
422 if (rc)
Jan Kiszkad89f5ef2010-11-09 17:02:49 +0100423 goto out_err;
Heiko Carstensb0c632d2008-03-25 18:47:20 +0100424
Carsten Otteb2904112011-10-18 12:27:13 +0200425 rc = -ENOMEM;
426
Heiko Carstensb0c632d2008-03-25 18:47:20 +0100427 kvm->arch.sca = (struct sca_block *) get_zeroed_page(GFP_KERNEL);
428 if (!kvm->arch.sca)
Jan Kiszkad89f5ef2010-11-09 17:02:49 +0100429 goto out_err;
Christian Borntraegerf6c137f2014-03-19 11:18:29 +0100430 spin_lock(&kvm_lock);
431 sca_offset = (sca_offset + 16) & 0x7f0;
432 kvm->arch.sca = (struct sca_block *) ((char *) kvm->arch.sca + sca_offset);
433 spin_unlock(&kvm_lock);
Heiko Carstensb0c632d2008-03-25 18:47:20 +0100434
435 sprintf(debug_name, "kvm-%u", current->pid);
436
437 kvm->arch.dbf = debug_register(debug_name, 8, 2, 8 * sizeof(long));
438 if (!kvm->arch.dbf)
439 goto out_nodbf;
440
Carsten Otteba5c1e92008-03-25 18:47:26 +0100441 spin_lock_init(&kvm->arch.float_int.lock);
442 INIT_LIST_HEAD(&kvm->arch.float_int.list);
Heiko Carstens8a2422342014-01-10 14:33:28 +0100443 init_waitqueue_head(&kvm->arch.ipte_wq);
Carsten Otteba5c1e92008-03-25 18:47:26 +0100444
Heiko Carstensb0c632d2008-03-25 18:47:20 +0100445 debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
446 VM_EVENT(kvm, 3, "%s", "vm created");
447
Carsten Ottee08b9632012-01-04 10:25:20 +0100448 if (type & KVM_VM_S390_UCONTROL) {
449 kvm->arch.gmap = NULL;
450 } else {
451 kvm->arch.gmap = gmap_alloc(current->mm);
452 if (!kvm->arch.gmap)
453 goto out_nogmap;
Christian Borntraeger2c70fe42013-05-17 14:41:36 +0200454 kvm->arch.gmap->private = kvm;
Dominik Dingel24eb3a82013-06-17 16:25:18 +0200455 kvm->arch.gmap->pfault_enabled = 0;
Carsten Ottee08b9632012-01-04 10:25:20 +0100456 }
Cornelia Huckfa6b7fe2012-12-20 15:32:12 +0100457
458 kvm->arch.css_support = 0;
Cornelia Huck84223592013-07-15 13:36:01 +0200459 kvm->arch.use_irqchip = 0;
Cornelia Huckfa6b7fe2012-12-20 15:32:12 +0100460
Jan Kiszkad89f5ef2010-11-09 17:02:49 +0100461 return 0;
Carsten Otte598841c2011-07-24 10:48:21 +0200462out_nogmap:
463 debug_unregister(kvm->arch.dbf);
Heiko Carstensb0c632d2008-03-25 18:47:20 +0100464out_nodbf:
465 free_page((unsigned long)(kvm->arch.sca));
Jan Kiszkad89f5ef2010-11-09 17:02:49 +0100466out_err:
467 return rc;
Heiko Carstensb0c632d2008-03-25 18:47:20 +0100468}
469
Christian Borntraegerd329c032008-11-26 14:50:27 +0100470void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
471{
472 VCPU_EVENT(vcpu, 3, "%s", "free cpu");
Cornelia Huckade38c32012-07-23 17:20:30 +0200473 trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
Christian Borntraeger67335e62014-03-25 17:09:08 +0100474 kvm_s390_clear_local_irqs(vcpu);
Dominik Dingel3c038e62013-10-07 17:11:48 +0200475 kvm_clear_async_pf_completion_queue(vcpu);
Carsten Otte58f94602012-01-04 10:25:27 +0100476 if (!kvm_is_ucontrol(vcpu->kvm)) {
477 clear_bit(63 - vcpu->vcpu_id,
478 (unsigned long *) &vcpu->kvm->arch.sca->mcn);
479 if (vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sda ==
480 (__u64) vcpu->arch.sie_block)
481 vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sda = 0;
482 }
Carsten Otteabf4a712009-05-12 17:21:51 +0200483 smp_mb();
Carsten Otte27e03932012-01-04 10:25:21 +0100484
485 if (kvm_is_ucontrol(vcpu->kvm))
486 gmap_free(vcpu->arch.gmap);
487
Dominik Dingelb31605c2014-03-25 13:47:11 +0100488 if (kvm_s390_cmma_enabled(vcpu->kvm))
489 kvm_s390_vcpu_unsetup_cmma(vcpu);
Christian Borntraegerd329c032008-11-26 14:50:27 +0100490 free_page((unsigned long)(vcpu->arch.sie_block));
Konstantin Weitzb31288f2013-04-17 17:36:29 +0200491
Christian Borntraeger6692cef2008-11-26 14:51:08 +0100492 kvm_vcpu_uninit(vcpu);
Michael Muellerb110fea2013-06-12 13:54:54 +0200493 kmem_cache_free(kvm_vcpu_cache, vcpu);
Christian Borntraegerd329c032008-11-26 14:50:27 +0100494}
495
496static void kvm_free_vcpus(struct kvm *kvm)
497{
498 unsigned int i;
Gleb Natapov988a2ca2009-06-09 15:56:29 +0300499 struct kvm_vcpu *vcpu;
Christian Borntraegerd329c032008-11-26 14:50:27 +0100500
Gleb Natapov988a2ca2009-06-09 15:56:29 +0300501 kvm_for_each_vcpu(i, vcpu, kvm)
502 kvm_arch_vcpu_destroy(vcpu);
503
504 mutex_lock(&kvm->lock);
505 for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
506 kvm->vcpus[i] = NULL;
507
508 atomic_set(&kvm->online_vcpus, 0);
509 mutex_unlock(&kvm->lock);
Christian Borntraegerd329c032008-11-26 14:50:27 +0100510}
511
Sheng Yangad8ba2c2009-01-06 10:03:02 +0800512void kvm_arch_sync_events(struct kvm *kvm)
513{
514}
515
Heiko Carstensb0c632d2008-03-25 18:47:20 +0100516void kvm_arch_destroy_vm(struct kvm *kvm)
517{
Christian Borntraegerd329c032008-11-26 14:50:27 +0100518 kvm_free_vcpus(kvm);
Heiko Carstensb0c632d2008-03-25 18:47:20 +0100519 free_page((unsigned long)(kvm->arch.sca));
Christian Borntraegerd329c032008-11-26 14:50:27 +0100520 debug_unregister(kvm->arch.dbf);
Carsten Otte27e03932012-01-04 10:25:21 +0100521 if (!kvm_is_ucontrol(kvm))
522 gmap_free(kvm->arch.gmap);
Cornelia Huck841b91c2013-07-15 13:36:01 +0200523 kvm_s390_destroy_adapters(kvm);
Christian Borntraeger67335e62014-03-25 17:09:08 +0100524 kvm_s390_clear_float_irqs(kvm);
Heiko Carstensb0c632d2008-03-25 18:47:20 +0100525}
526
527/* Section: vcpu related */
528int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
529{
Dominik Dingel3c038e62013-10-07 17:11:48 +0200530 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
531 kvm_clear_async_pf_completion_queue(vcpu);
Carsten Otte27e03932012-01-04 10:25:21 +0100532 if (kvm_is_ucontrol(vcpu->kvm)) {
533 vcpu->arch.gmap = gmap_alloc(current->mm);
534 if (!vcpu->arch.gmap)
535 return -ENOMEM;
Christian Borntraeger2c70fe42013-05-17 14:41:36 +0200536 vcpu->arch.gmap->private = vcpu->kvm;
Carsten Otte27e03932012-01-04 10:25:21 +0100537 return 0;
538 }
539
Carsten Otte598841c2011-07-24 10:48:21 +0200540 vcpu->arch.gmap = vcpu->kvm->arch.gmap;
Christian Borntraeger59674c12012-01-11 11:20:33 +0100541 vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX |
542 KVM_SYNC_GPRS |
Christian Borntraeger9eed07352012-02-06 10:59:07 +0100543 KVM_SYNC_ACRS |
544 KVM_SYNC_CRS;
Heiko Carstensb0c632d2008-03-25 18:47:20 +0100545 return 0;
546}
547
548void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu)
549{
Christian Borntraeger6692cef2008-11-26 14:51:08 +0100550 /* Nothing todo */
Heiko Carstensb0c632d2008-03-25 18:47:20 +0100551}
552
553void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
554{
Martin Schwidefsky4725c862013-10-15 16:08:34 +0200555 save_fp_ctl(&vcpu->arch.host_fpregs.fpc);
556 save_fp_regs(vcpu->arch.host_fpregs.fprs);
Heiko Carstensb0c632d2008-03-25 18:47:20 +0100557 save_access_regs(vcpu->arch.host_acrs);
Martin Schwidefsky4725c862013-10-15 16:08:34 +0200558 restore_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
559 restore_fp_regs(vcpu->arch.guest_fpregs.fprs);
Christian Borntraeger59674c12012-01-11 11:20:33 +0100560 restore_access_regs(vcpu->run->s.regs.acrs);
Christian Borntraeger480e5922011-09-20 17:07:28 +0200561 gmap_enable(vcpu->arch.gmap);
Cornelia Huck9e6dabe2011-11-17 11:00:41 +0100562 atomic_set_mask(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
Heiko Carstensb0c632d2008-03-25 18:47:20 +0100563}
564
565void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
566{
Cornelia Huck9e6dabe2011-11-17 11:00:41 +0100567 atomic_clear_mask(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
Christian Borntraeger480e5922011-09-20 17:07:28 +0200568 gmap_disable(vcpu->arch.gmap);
Martin Schwidefsky4725c862013-10-15 16:08:34 +0200569 save_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
570 save_fp_regs(vcpu->arch.guest_fpregs.fprs);
Christian Borntraeger59674c12012-01-11 11:20:33 +0100571 save_access_regs(vcpu->run->s.regs.acrs);
Martin Schwidefsky4725c862013-10-15 16:08:34 +0200572 restore_fp_ctl(&vcpu->arch.host_fpregs.fpc);
573 restore_fp_regs(vcpu->arch.host_fpregs.fprs);
Heiko Carstensb0c632d2008-03-25 18:47:20 +0100574 restore_access_regs(vcpu->arch.host_acrs);
575}
576
577static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu)
578{
579 /* this equals initial cpu reset in pop, but we don't switch to ESA */
580 vcpu->arch.sie_block->gpsw.mask = 0UL;
581 vcpu->arch.sie_block->gpsw.addr = 0UL;
Christian Borntraeger8d26cf72012-01-11 11:19:32 +0100582 kvm_s390_set_prefix(vcpu, 0);
Heiko Carstensb0c632d2008-03-25 18:47:20 +0100583 vcpu->arch.sie_block->cputm = 0UL;
584 vcpu->arch.sie_block->ckc = 0UL;
585 vcpu->arch.sie_block->todpr = 0;
586 memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64));
587 vcpu->arch.sie_block->gcr[0] = 0xE0UL;
588 vcpu->arch.sie_block->gcr[14] = 0xC2000000UL;
589 vcpu->arch.guest_fpregs.fpc = 0;
590 asm volatile("lfpc %0" : : "Q" (vcpu->arch.guest_fpregs.fpc));
591 vcpu->arch.sie_block->gbea = 1;
Christian Borntraeger672550f2014-02-10 15:32:19 +0100592 vcpu->arch.sie_block->pp = 0;
Dominik Dingel3c038e62013-10-07 17:11:48 +0200593 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
594 kvm_clear_async_pf_completion_queue(vcpu);
David Hildenbrand6852d7b2014-03-14 10:59:29 +0100595 kvm_s390_vcpu_stop(vcpu);
Jens Freimann2ed10cc2014-02-11 13:48:07 +0100596 kvm_s390_clear_local_irqs(vcpu);
Heiko Carstensb0c632d2008-03-25 18:47:20 +0100597}
598
Marcelo Tosatti42897d82012-11-27 23:29:02 -0200599int kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
600{
601 return 0;
602}
603
Dominik Dingelb31605c2014-03-25 13:47:11 +0100604void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
605{
606 free_page(vcpu->arch.sie_block->cbrlo);
607 vcpu->arch.sie_block->cbrlo = 0;
608}
609
610int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu)
611{
612 vcpu->arch.sie_block->cbrlo = get_zeroed_page(GFP_KERNEL);
613 if (!vcpu->arch.sie_block->cbrlo)
614 return -ENOMEM;
615
616 vcpu->arch.sie_block->ecb2 |= 0x80;
617 vcpu->arch.sie_block->ecb2 &= ~0x08;
618 return 0;
619}
620
Heiko Carstensb0c632d2008-03-25 18:47:20 +0100621int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
622{
Dominik Dingelb31605c2014-03-25 13:47:11 +0100623 int rc = 0;
Konstantin Weitzb31288f2013-04-17 17:36:29 +0200624
Cornelia Huck9e6dabe2011-11-17 11:00:41 +0100625 atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH |
626 CPUSTAT_SM |
Christian Borntraeger69d0d3a2013-06-12 13:54:53 +0200627 CPUSTAT_STOPPED |
628 CPUSTAT_GED);
Christian Borntraegerfc345312010-06-17 23:16:20 +0200629 vcpu->arch.sie_block->ecb = 6;
Michael Mueller7feb6bb2013-06-28 13:30:24 +0200630 if (test_vfacility(50) && test_vfacility(73))
631 vcpu->arch.sie_block->ecb |= 0x10;
632
Christian Borntraeger69d0d3a2013-06-12 13:54:53 +0200633 vcpu->arch.sie_block->ecb2 = 8;
Heiko Carstens217a4402013-12-30 12:54:14 +0100634 vcpu->arch.sie_block->eca = 0xC1002000U;
635 if (sclp_has_siif())
636 vcpu->arch.sie_block->eca |= 1;
Michael Mueller78c4b592013-07-26 15:04:04 +0200637 vcpu->arch.sie_block->fac = (int) (long) vfacilities;
Dominik Dingel693ffc02014-01-14 18:11:14 +0100638 vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE;
Dominik Dingelb31605c2014-03-25 13:47:11 +0100639 if (kvm_s390_cmma_enabled(vcpu->kvm)) {
640 rc = kvm_s390_vcpu_setup_cmma(vcpu);
641 if (rc)
642 return rc;
Konstantin Weitzb31288f2013-04-17 17:36:29 +0200643 }
Christian Borntraegerca872302009-05-12 17:21:49 +0200644 hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_REALTIME, HRTIMER_MODE_ABS);
645 tasklet_init(&vcpu->arch.tasklet, kvm_s390_tasklet,
646 (unsigned long) vcpu);
647 vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
Christian Borntraeger453423d2008-03-25 18:47:29 +0100648 get_cpu_id(&vcpu->arch.cpu_id);
Christian Borntraeger92e6ecf2009-03-26 15:23:58 +0100649 vcpu->arch.cpu_id.version = 0xff;
Dominik Dingelb31605c2014-03-25 13:47:11 +0100650 return rc;
Heiko Carstensb0c632d2008-03-25 18:47:20 +0100651}
652
653struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
654 unsigned int id)
655{
Carsten Otte4d475552011-10-18 12:27:12 +0200656 struct kvm_vcpu *vcpu;
Michael Mueller7feb6bb2013-06-28 13:30:24 +0200657 struct sie_page *sie_page;
Carsten Otte4d475552011-10-18 12:27:12 +0200658 int rc = -EINVAL;
Heiko Carstensb0c632d2008-03-25 18:47:20 +0100659
Carsten Otte4d475552011-10-18 12:27:12 +0200660 if (id >= KVM_MAX_VCPUS)
661 goto out;
662
663 rc = -ENOMEM;
664
Michael Muellerb110fea2013-06-12 13:54:54 +0200665 vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
Heiko Carstensb0c632d2008-03-25 18:47:20 +0100666 if (!vcpu)
Carsten Otte4d475552011-10-18 12:27:12 +0200667 goto out;
Heiko Carstensb0c632d2008-03-25 18:47:20 +0100668
Michael Mueller7feb6bb2013-06-28 13:30:24 +0200669 sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL);
670 if (!sie_page)
Heiko Carstensb0c632d2008-03-25 18:47:20 +0100671 goto out_free_cpu;
672
Michael Mueller7feb6bb2013-06-28 13:30:24 +0200673 vcpu->arch.sie_block = &sie_page->sie_block;
674 vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb;
675
Heiko Carstensb0c632d2008-03-25 18:47:20 +0100676 vcpu->arch.sie_block->icpua = id;
Carsten Otte58f94602012-01-04 10:25:27 +0100677 if (!kvm_is_ucontrol(kvm)) {
678 if (!kvm->arch.sca) {
679 WARN_ON_ONCE(1);
680 goto out_free_cpu;
681 }
682 if (!kvm->arch.sca->cpu[id].sda)
683 kvm->arch.sca->cpu[id].sda =
684 (__u64) vcpu->arch.sie_block;
685 vcpu->arch.sie_block->scaoh =
686 (__u32)(((__u64)kvm->arch.sca) >> 32);
687 vcpu->arch.sie_block->scaol = (__u32)(__u64)kvm->arch.sca;
688 set_bit(63 - id, (unsigned long *) &kvm->arch.sca->mcn);
689 }
Heiko Carstensb0c632d2008-03-25 18:47:20 +0100690
Carsten Otteba5c1e92008-03-25 18:47:26 +0100691 spin_lock_init(&vcpu->arch.local_int.lock);
692 INIT_LIST_HEAD(&vcpu->arch.local_int.list);
693 vcpu->arch.local_int.float_int = &kvm->arch.float_int;
Christian Borntraegerd0321a22013-06-12 13:54:55 +0200694 vcpu->arch.local_int.wq = &vcpu->wq;
Christian Borntraeger5288fbf2008-03-25 18:47:31 +0100695 vcpu->arch.local_int.cpuflags = &vcpu->arch.sie_block->cpuflags;
Carsten Otteba5c1e92008-03-25 18:47:26 +0100696
Heiko Carstensb0c632d2008-03-25 18:47:20 +0100697 rc = kvm_vcpu_init(vcpu, kvm, id);
698 if (rc)
Wei Yongjun7b06bf22010-03-09 14:37:53 +0800699 goto out_free_sie_block;
Heiko Carstensb0c632d2008-03-25 18:47:20 +0100700 VM_EVENT(kvm, 3, "create cpu %d at %p, sie block at %p", id, vcpu,
701 vcpu->arch.sie_block);
Cornelia Huckade38c32012-07-23 17:20:30 +0200702 trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block);
Heiko Carstensb0c632d2008-03-25 18:47:20 +0100703
Heiko Carstensb0c632d2008-03-25 18:47:20 +0100704 return vcpu;
Wei Yongjun7b06bf22010-03-09 14:37:53 +0800705out_free_sie_block:
706 free_page((unsigned long)(vcpu->arch.sie_block));
Heiko Carstensb0c632d2008-03-25 18:47:20 +0100707out_free_cpu:
Michael Muellerb110fea2013-06-12 13:54:54 +0200708 kmem_cache_free(kvm_vcpu_cache, vcpu);
Carsten Otte4d475552011-10-18 12:27:12 +0200709out:
Heiko Carstensb0c632d2008-03-25 18:47:20 +0100710 return ERR_PTR(rc);
711}
712
Heiko Carstensb0c632d2008-03-25 18:47:20 +0100713int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
714{
Michael Muellerf87618e2014-02-26 16:14:17 +0100715 return kvm_cpu_has_interrupt(vcpu);
Heiko Carstensb0c632d2008-03-25 18:47:20 +0100716}
717
Christian Borntraeger49b99e12013-05-17 14:41:35 +0200718void s390_vcpu_block(struct kvm_vcpu *vcpu)
719{
720 atomic_set_mask(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
721}
722
723void s390_vcpu_unblock(struct kvm_vcpu *vcpu)
724{
725 atomic_clear_mask(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
726}
727
728/*
729 * Kick a guest cpu out of SIE and wait until SIE is not running.
730 * If the CPU is not running (e.g. waiting as idle) the function will
731 * return immediately. */
732void exit_sie(struct kvm_vcpu *vcpu)
733{
734 atomic_set_mask(CPUSTAT_STOP_INT, &vcpu->arch.sie_block->cpuflags);
735 while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
736 cpu_relax();
737}
738
739/* Kick a guest cpu out of SIE and prevent SIE-reentry */
740void exit_sie_sync(struct kvm_vcpu *vcpu)
741{
742 s390_vcpu_block(vcpu);
743 exit_sie(vcpu);
744}
745
Christian Borntraeger2c70fe42013-05-17 14:41:36 +0200746static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address)
747{
748 int i;
749 struct kvm *kvm = gmap->private;
750 struct kvm_vcpu *vcpu;
751
752 kvm_for_each_vcpu(i, vcpu, kvm) {
753 /* match against both prefix pages */
754 if (vcpu->arch.sie_block->prefix == (address & ~0x1000UL)) {
755 VCPU_EVENT(vcpu, 2, "gmap notifier for %lx", address);
756 kvm_make_request(KVM_REQ_MMU_RELOAD, vcpu);
757 exit_sie_sync(vcpu);
758 }
759 }
760}
761
Christoffer Dallb6d33832012-03-08 16:44:24 -0500762int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
763{
764 /* kvm common code refers to this, but never calls it */
765 BUG();
766 return 0;
767}
768
Carsten Otte14eebd92012-05-15 14:15:26 +0200769static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu,
770 struct kvm_one_reg *reg)
771{
772 int r = -EINVAL;
773
774 switch (reg->id) {
Carsten Otte29b7c712012-05-15 14:15:27 +0200775 case KVM_REG_S390_TODPR:
776 r = put_user(vcpu->arch.sie_block->todpr,
777 (u32 __user *)reg->addr);
778 break;
779 case KVM_REG_S390_EPOCHDIFF:
780 r = put_user(vcpu->arch.sie_block->epoch,
781 (u64 __user *)reg->addr);
782 break;
Jason J. herne46a6dd12012-05-15 14:15:28 +0200783 case KVM_REG_S390_CPU_TIMER:
784 r = put_user(vcpu->arch.sie_block->cputm,
785 (u64 __user *)reg->addr);
786 break;
787 case KVM_REG_S390_CLOCK_COMP:
788 r = put_user(vcpu->arch.sie_block->ckc,
789 (u64 __user *)reg->addr);
790 break;
Dominik Dingel536336c2013-09-30 10:55:33 +0200791 case KVM_REG_S390_PFTOKEN:
792 r = put_user(vcpu->arch.pfault_token,
793 (u64 __user *)reg->addr);
794 break;
795 case KVM_REG_S390_PFCOMPARE:
796 r = put_user(vcpu->arch.pfault_compare,
797 (u64 __user *)reg->addr);
798 break;
799 case KVM_REG_S390_PFSELECT:
800 r = put_user(vcpu->arch.pfault_select,
801 (u64 __user *)reg->addr);
802 break;
Christian Borntraeger672550f2014-02-10 15:32:19 +0100803 case KVM_REG_S390_PP:
804 r = put_user(vcpu->arch.sie_block->pp,
805 (u64 __user *)reg->addr);
806 break;
Christian Borntraegerafa45ff2014-02-10 15:39:23 +0100807 case KVM_REG_S390_GBEA:
808 r = put_user(vcpu->arch.sie_block->gbea,
809 (u64 __user *)reg->addr);
810 break;
Carsten Otte14eebd92012-05-15 14:15:26 +0200811 default:
812 break;
813 }
814
815 return r;
816}
817
818static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
819 struct kvm_one_reg *reg)
820{
821 int r = -EINVAL;
822
823 switch (reg->id) {
Carsten Otte29b7c712012-05-15 14:15:27 +0200824 case KVM_REG_S390_TODPR:
825 r = get_user(vcpu->arch.sie_block->todpr,
826 (u32 __user *)reg->addr);
827 break;
828 case KVM_REG_S390_EPOCHDIFF:
829 r = get_user(vcpu->arch.sie_block->epoch,
830 (u64 __user *)reg->addr);
831 break;
Jason J. herne46a6dd12012-05-15 14:15:28 +0200832 case KVM_REG_S390_CPU_TIMER:
833 r = get_user(vcpu->arch.sie_block->cputm,
834 (u64 __user *)reg->addr);
835 break;
836 case KVM_REG_S390_CLOCK_COMP:
837 r = get_user(vcpu->arch.sie_block->ckc,
838 (u64 __user *)reg->addr);
839 break;
Dominik Dingel536336c2013-09-30 10:55:33 +0200840 case KVM_REG_S390_PFTOKEN:
841 r = get_user(vcpu->arch.pfault_token,
842 (u64 __user *)reg->addr);
843 break;
844 case KVM_REG_S390_PFCOMPARE:
845 r = get_user(vcpu->arch.pfault_compare,
846 (u64 __user *)reg->addr);
847 break;
848 case KVM_REG_S390_PFSELECT:
849 r = get_user(vcpu->arch.pfault_select,
850 (u64 __user *)reg->addr);
851 break;
Christian Borntraeger672550f2014-02-10 15:32:19 +0100852 case KVM_REG_S390_PP:
853 r = get_user(vcpu->arch.sie_block->pp,
854 (u64 __user *)reg->addr);
855 break;
Christian Borntraegerafa45ff2014-02-10 15:39:23 +0100856 case KVM_REG_S390_GBEA:
857 r = get_user(vcpu->arch.sie_block->gbea,
858 (u64 __user *)reg->addr);
859 break;
Carsten Otte14eebd92012-05-15 14:15:26 +0200860 default:
861 break;
862 }
863
864 return r;
865}
Christoffer Dallb6d33832012-03-08 16:44:24 -0500866
Heiko Carstensb0c632d2008-03-25 18:47:20 +0100867static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
868{
Heiko Carstensb0c632d2008-03-25 18:47:20 +0100869 kvm_s390_vcpu_initial_reset(vcpu);
Heiko Carstensb0c632d2008-03-25 18:47:20 +0100870 return 0;
871}
872
873int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
874{
Christian Borntraeger5a32c1a2012-01-11 11:20:32 +0100875 memcpy(&vcpu->run->s.regs.gprs, &regs->gprs, sizeof(regs->gprs));
Heiko Carstensb0c632d2008-03-25 18:47:20 +0100876 return 0;
877}
878
879int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
880{
Christian Borntraeger5a32c1a2012-01-11 11:20:32 +0100881 memcpy(&regs->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs));
Heiko Carstensb0c632d2008-03-25 18:47:20 +0100882 return 0;
883}
884
885int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
886 struct kvm_sregs *sregs)
887{
Christian Borntraeger59674c12012-01-11 11:20:33 +0100888 memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
Heiko Carstensb0c632d2008-03-25 18:47:20 +0100889 memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
Christian Borntraeger59674c12012-01-11 11:20:33 +0100890 restore_access_regs(vcpu->run->s.regs.acrs);
Heiko Carstensb0c632d2008-03-25 18:47:20 +0100891 return 0;
892}
893
894int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
895 struct kvm_sregs *sregs)
896{
Christian Borntraeger59674c12012-01-11 11:20:33 +0100897 memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs));
Heiko Carstensb0c632d2008-03-25 18:47:20 +0100898 memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
Heiko Carstensb0c632d2008-03-25 18:47:20 +0100899 return 0;
900}
901
902int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
903{
Martin Schwidefsky4725c862013-10-15 16:08:34 +0200904 if (test_fp_ctl(fpu->fpc))
905 return -EINVAL;
Heiko Carstensb0c632d2008-03-25 18:47:20 +0100906 memcpy(&vcpu->arch.guest_fpregs.fprs, &fpu->fprs, sizeof(fpu->fprs));
Martin Schwidefsky4725c862013-10-15 16:08:34 +0200907 vcpu->arch.guest_fpregs.fpc = fpu->fpc;
908 restore_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
909 restore_fp_regs(vcpu->arch.guest_fpregs.fprs);
Heiko Carstensb0c632d2008-03-25 18:47:20 +0100910 return 0;
911}
912
913int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
914{
Heiko Carstensb0c632d2008-03-25 18:47:20 +0100915 memcpy(&fpu->fprs, &vcpu->arch.guest_fpregs.fprs, sizeof(fpu->fprs));
916 fpu->fpc = vcpu->arch.guest_fpregs.fpc;
Heiko Carstensb0c632d2008-03-25 18:47:20 +0100917 return 0;
918}
919
920static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw)
921{
922 int rc = 0;
923
Cornelia Huck9e6dabe2011-11-17 11:00:41 +0100924 if (!(atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_STOPPED))
Heiko Carstensb0c632d2008-03-25 18:47:20 +0100925 rc = -EBUSY;
Carsten Otted7b0b5e2009-11-19 14:21:16 +0100926 else {
927 vcpu->run->psw_mask = psw.mask;
928 vcpu->run->psw_addr = psw.addr;
929 }
Heiko Carstensb0c632d2008-03-25 18:47:20 +0100930 return rc;
931}
932
933int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
934 struct kvm_translation *tr)
935{
936 return -EINVAL; /* not implemented yet */
937}
938
David Hildenbrand27291e22014-01-23 12:26:52 +0100939#define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \
940 KVM_GUESTDBG_USE_HW_BP | \
941 KVM_GUESTDBG_ENABLE)
942
Jan Kiszkad0bfb942008-12-15 13:52:10 +0100943int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
944 struct kvm_guest_debug *dbg)
Heiko Carstensb0c632d2008-03-25 18:47:20 +0100945{
David Hildenbrand27291e22014-01-23 12:26:52 +0100946 int rc = 0;
947
948 vcpu->guest_debug = 0;
949 kvm_s390_clear_bp_data(vcpu);
950
951 if (vcpu->guest_debug & ~VALID_GUESTDBG_FLAGS)
952 return -EINVAL;
953
954 if (dbg->control & KVM_GUESTDBG_ENABLE) {
955 vcpu->guest_debug = dbg->control;
956 /* enforce guest PER */
957 atomic_set_mask(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
958
959 if (dbg->control & KVM_GUESTDBG_USE_HW_BP)
960 rc = kvm_s390_import_bp_data(vcpu, dbg);
961 } else {
962 atomic_clear_mask(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
963 vcpu->arch.guestdbg.last_bp = 0;
964 }
965
966 if (rc) {
967 vcpu->guest_debug = 0;
968 kvm_s390_clear_bp_data(vcpu);
969 atomic_clear_mask(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
970 }
971
972 return rc;
Heiko Carstensb0c632d2008-03-25 18:47:20 +0100973}
974
Marcelo Tosatti62d9f0d2008-04-11 13:24:45 -0300975int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
976 struct kvm_mp_state *mp_state)
977{
978 return -EINVAL; /* not implemented yet */
979}
980
981int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
982 struct kvm_mp_state *mp_state)
983{
984 return -EINVAL; /* not implemented yet */
985}
986
Dominik Dingelb31605c2014-03-25 13:47:11 +0100987bool kvm_s390_cmma_enabled(struct kvm *kvm)
988{
989 if (!MACHINE_IS_LPAR)
990 return false;
991 /* only enable for z10 and later */
992 if (!MACHINE_HAS_EDAT1)
993 return false;
994 if (!kvm->arch.use_cmma)
995 return false;
996 return true;
997}
998
Christian Borntraeger2c70fe42013-05-17 14:41:36 +0200999static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
1000{
1001 /*
1002 * We use MMU_RELOAD just to re-arm the ipte notifier for the
1003 * guest prefix page. gmap_ipte_notify will wait on the ptl lock.
1004 * This ensures that the ipte instruction for this request has
1005 * already finished. We might race against a second unmapper that
1006 * wants to set the blocking bit. Lets just retry the request loop.
1007 */
1008 while (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) {
1009 int rc;
1010 rc = gmap_ipte_notify(vcpu->arch.gmap,
1011 vcpu->arch.sie_block->prefix,
1012 PAGE_SIZE * 2);
1013 if (rc)
1014 return rc;
1015 s390_vcpu_unblock(vcpu);
1016 }
1017 return 0;
1018}
1019
Dominik Dingel24eb3a82013-06-17 16:25:18 +02001020static long kvm_arch_fault_in_sync(struct kvm_vcpu *vcpu)
1021{
1022 long rc;
1023 hva_t fault = gmap_fault(current->thread.gmap_addr, vcpu->arch.gmap);
1024 struct mm_struct *mm = current->mm;
1025 down_read(&mm->mmap_sem);
1026 rc = get_user_pages(current, mm, fault, 1, 1, 0, NULL, NULL);
1027 up_read(&mm->mmap_sem);
1028 return rc;
1029}
1030
Dominik Dingel3c038e62013-10-07 17:11:48 +02001031static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token,
1032 unsigned long token)
1033{
1034 struct kvm_s390_interrupt inti;
1035 inti.parm64 = token;
1036
1037 if (start_token) {
1038 inti.type = KVM_S390_INT_PFAULT_INIT;
1039 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &inti));
1040 } else {
1041 inti.type = KVM_S390_INT_PFAULT_DONE;
1042 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti));
1043 }
1044}
1045
1046void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
1047 struct kvm_async_pf *work)
1048{
1049 trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token);
1050 __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token);
1051}
1052
1053void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
1054 struct kvm_async_pf *work)
1055{
1056 trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token);
1057 __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token);
1058}
1059
1060void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
1061 struct kvm_async_pf *work)
1062{
1063 /* s390 will always inject the page directly */
1064}
1065
1066bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
1067{
1068 /*
1069 * s390 will always inject the page directly,
1070 * but we still want check_async_completion to cleanup
1071 */
1072 return true;
1073}
1074
1075static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
1076{
1077 hva_t hva;
1078 struct kvm_arch_async_pf arch;
1079 int rc;
1080
1081 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
1082 return 0;
1083 if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) !=
1084 vcpu->arch.pfault_compare)
1085 return 0;
1086 if (psw_extint_disabled(vcpu))
1087 return 0;
1088 if (kvm_cpu_has_interrupt(vcpu))
1089 return 0;
1090 if (!(vcpu->arch.sie_block->gcr[0] & 0x200ul))
1091 return 0;
1092 if (!vcpu->arch.gmap->pfault_enabled)
1093 return 0;
1094
Heiko Carstens81480cc2014-01-01 16:36:07 +01001095 hva = gfn_to_hva(vcpu->kvm, gpa_to_gfn(current->thread.gmap_addr));
1096 hva += current->thread.gmap_addr & ~PAGE_MASK;
1097 if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8))
Dominik Dingel3c038e62013-10-07 17:11:48 +02001098 return 0;
1099
1100 rc = kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch);
1101 return rc;
1102}
1103
Thomas Huth3fb4c402013-09-12 10:33:43 +02001104static int vcpu_pre_run(struct kvm_vcpu *vcpu)
Heiko Carstensb0c632d2008-03-25 18:47:20 +01001105{
Thomas Huth3fb4c402013-09-12 10:33:43 +02001106 int rc, cpuflags;
Carsten Ottee168bf82012-01-04 10:25:22 +01001107
Dominik Dingel3c038e62013-10-07 17:11:48 +02001108 /*
1109 * On s390 notifications for arriving pages will be delivered directly
1110 * to the guest but the house keeping for completed pfaults is
1111 * handled outside the worker.
1112 */
1113 kvm_check_async_pf_completion(vcpu);
1114
Christian Borntraeger5a32c1a2012-01-11 11:20:32 +01001115 memcpy(&vcpu->arch.sie_block->gg14, &vcpu->run->s.regs.gprs[14], 16);
Heiko Carstensb0c632d2008-03-25 18:47:20 +01001116
1117 if (need_resched())
1118 schedule();
1119
Christian Borntraeger71cde582008-05-21 13:37:34 +02001120 if (test_thread_flag(TIF_MCCK_PENDING))
1121 s390_handle_mcck();
1122
Carsten Otted6b6d162012-01-04 10:25:25 +01001123 if (!kvm_is_ucontrol(vcpu->kvm))
1124 kvm_s390_deliver_pending_interrupts(vcpu);
Carsten Otte0ff31862008-05-21 13:37:37 +02001125
Christian Borntraeger2c70fe42013-05-17 14:41:36 +02001126 rc = kvm_s390_handle_requests(vcpu);
1127 if (rc)
1128 return rc;
1129
David Hildenbrand27291e22014-01-23 12:26:52 +01001130 if (guestdbg_enabled(vcpu)) {
1131 kvm_s390_backup_guest_per_regs(vcpu);
1132 kvm_s390_patch_guest_per_regs(vcpu);
1133 }
1134
Heiko Carstensb0c632d2008-03-25 18:47:20 +01001135 vcpu->arch.sie_block->icptcode = 0;
Thomas Huth3fb4c402013-09-12 10:33:43 +02001136 cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags);
1137 VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags);
1138 trace_kvm_s390_sie_enter(vcpu, cpuflags);
Dominik Dingel2b29a9f2013-07-26 15:04:00 +02001139
Thomas Huth3fb4c402013-09-12 10:33:43 +02001140 return 0;
1141}
1142
1143static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
1144{
Dominik Dingel24eb3a82013-06-17 16:25:18 +02001145 int rc = -1;
Dominik Dingel2b29a9f2013-07-26 15:04:00 +02001146
1147 VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
1148 vcpu->arch.sie_block->icptcode);
1149 trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);
1150
David Hildenbrand27291e22014-01-23 12:26:52 +01001151 if (guestdbg_enabled(vcpu))
1152 kvm_s390_restore_guest_per_regs(vcpu);
1153
Thomas Huth3fb4c402013-09-12 10:33:43 +02001154 if (exit_reason >= 0) {
Martin Schwidefsky7c470532013-05-17 14:41:37 +02001155 rc = 0;
Thomas Huth210b16072013-09-19 16:26:18 +02001156 } else if (kvm_is_ucontrol(vcpu->kvm)) {
1157 vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL;
1158 vcpu->run->s390_ucontrol.trans_exc_code =
1159 current->thread.gmap_addr;
1160 vcpu->run->s390_ucontrol.pgm_code = 0x10;
1161 rc = -EREMOTE;
Dominik Dingel24eb3a82013-06-17 16:25:18 +02001162
1163 } else if (current->thread.gmap_pfault) {
Dominik Dingel3c038e62013-10-07 17:11:48 +02001164 trace_kvm_s390_major_guest_pfault(vcpu);
Dominik Dingel24eb3a82013-06-17 16:25:18 +02001165 current->thread.gmap_pfault = 0;
Dominik Dingel3c038e62013-10-07 17:11:48 +02001166 if (kvm_arch_setup_async_pf(vcpu) ||
1167 (kvm_arch_fault_in_sync(vcpu) >= 0))
Dominik Dingel24eb3a82013-06-17 16:25:18 +02001168 rc = 0;
1169 }
1170
1171 if (rc == -1) {
Christian Borntraeger699bde32014-01-20 12:34:13 +01001172 VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
1173 trace_kvm_s390_sie_fault(vcpu);
1174 rc = kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
Carsten Otte1f0d0f02008-05-21 13:37:40 +02001175 }
Heiko Carstensb0c632d2008-03-25 18:47:20 +01001176
Christian Borntraeger5a32c1a2012-01-11 11:20:32 +01001177 memcpy(&vcpu->run->s.regs.gprs[14], &vcpu->arch.sie_block->gg14, 16);
Thomas Huth3fb4c402013-09-12 10:33:43 +02001178
Thomas Hutha76ccff2013-09-12 10:33:44 +02001179 if (rc == 0) {
1180 if (kvm_is_ucontrol(vcpu->kvm))
Christian Borntraeger2955c832014-03-06 16:01:38 +01001181 /* Don't exit for host interrupts. */
1182 rc = vcpu->arch.sie_block->icptcode ? -EOPNOTSUPP : 0;
Thomas Hutha76ccff2013-09-12 10:33:44 +02001183 else
1184 rc = kvm_handle_sie_intercept(vcpu);
1185 }
1186
Thomas Huth3fb4c402013-09-12 10:33:43 +02001187 return rc;
1188}
1189
1190static int __vcpu_run(struct kvm_vcpu *vcpu)
1191{
1192 int rc, exit_reason;
1193
Thomas Huth800c1062013-09-12 10:33:45 +02001194 /*
1195 * We try to hold kvm->srcu during most of vcpu_run (except when run-
1196 * ning the guest), so that memslots (and other stuff) are protected
1197 */
1198 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
1199
Thomas Hutha76ccff2013-09-12 10:33:44 +02001200 do {
1201 rc = vcpu_pre_run(vcpu);
1202 if (rc)
1203 break;
Thomas Huth3fb4c402013-09-12 10:33:43 +02001204
Thomas Huth800c1062013-09-12 10:33:45 +02001205 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
Thomas Hutha76ccff2013-09-12 10:33:44 +02001206 /*
1207 * As PF_VCPU will be used in fault handler, between
1208 * guest_enter and guest_exit should be no uaccess.
1209 */
1210 preempt_disable();
1211 kvm_guest_enter();
1212 preempt_enable();
1213 exit_reason = sie64a(vcpu->arch.sie_block,
1214 vcpu->run->s.regs.gprs);
1215 kvm_guest_exit();
Thomas Huth800c1062013-09-12 10:33:45 +02001216 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
Thomas Huth3fb4c402013-09-12 10:33:43 +02001217
Thomas Hutha76ccff2013-09-12 10:33:44 +02001218 rc = vcpu_post_run(vcpu, exit_reason);
David Hildenbrand27291e22014-01-23 12:26:52 +01001219 } while (!signal_pending(current) && !guestdbg_exit_pending(vcpu) && !rc);
Thomas Huth3fb4c402013-09-12 10:33:43 +02001220
Thomas Huth800c1062013-09-12 10:33:45 +02001221 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
Carsten Ottee168bf82012-01-04 10:25:22 +01001222 return rc;
Heiko Carstensb0c632d2008-03-25 18:47:20 +01001223}
1224
1225int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
1226{
Christian Borntraeger8f2abe62008-03-25 18:47:23 +01001227 int rc;
Heiko Carstensb0c632d2008-03-25 18:47:20 +01001228 sigset_t sigsaved;
1229
David Hildenbrand27291e22014-01-23 12:26:52 +01001230 if (guestdbg_exit_pending(vcpu)) {
1231 kvm_s390_prepare_debug_exit(vcpu);
1232 return 0;
1233 }
1234
Heiko Carstensb0c632d2008-03-25 18:47:20 +01001235 if (vcpu->sigset_active)
1236 sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
1237
David Hildenbrand6852d7b2014-03-14 10:59:29 +01001238 kvm_s390_vcpu_start(vcpu);
Heiko Carstensb0c632d2008-03-25 18:47:20 +01001239
Christian Borntraeger8f2abe62008-03-25 18:47:23 +01001240 switch (kvm_run->exit_reason) {
1241 case KVM_EXIT_S390_SIEIC:
Christian Borntraeger8f2abe62008-03-25 18:47:23 +01001242 case KVM_EXIT_UNKNOWN:
Christian Ehrhardt9ace9032009-05-20 15:34:55 +02001243 case KVM_EXIT_INTR:
Christian Borntraeger8f2abe62008-03-25 18:47:23 +01001244 case KVM_EXIT_S390_RESET:
Carsten Ottee168bf82012-01-04 10:25:22 +01001245 case KVM_EXIT_S390_UCONTROL:
Cornelia Huckfa6b7fe2012-12-20 15:32:12 +01001246 case KVM_EXIT_S390_TSCH:
David Hildenbrand27291e22014-01-23 12:26:52 +01001247 case KVM_EXIT_DEBUG:
Christian Borntraeger8f2abe62008-03-25 18:47:23 +01001248 break;
1249 default:
1250 BUG();
1251 }
1252
Carsten Otted7b0b5e2009-11-19 14:21:16 +01001253 vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
1254 vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
Christian Borntraeger60b413c2012-01-11 11:20:31 +01001255 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX) {
1256 kvm_run->kvm_dirty_regs &= ~KVM_SYNC_PREFIX;
1257 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
1258 }
Christian Borntraeger9eed07352012-02-06 10:59:07 +01001259 if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
1260 kvm_run->kvm_dirty_regs &= ~KVM_SYNC_CRS;
1261 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
1262 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
1263 }
Carsten Otted7b0b5e2009-11-19 14:21:16 +01001264
Heiko Carstensdab4079d2009-06-12 10:26:32 +02001265 might_fault();
Thomas Hutha76ccff2013-09-12 10:33:44 +02001266 rc = __vcpu_run(vcpu);
Christian Ehrhardt9ace9032009-05-20 15:34:55 +02001267
Christian Ehrhardtb1d16c42009-05-20 15:34:56 +02001268 if (signal_pending(current) && !rc) {
1269 kvm_run->exit_reason = KVM_EXIT_INTR;
Christian Borntraeger8f2abe62008-03-25 18:47:23 +01001270 rc = -EINTR;
Christian Ehrhardtb1d16c42009-05-20 15:34:56 +02001271 }
Christian Borntraeger8f2abe62008-03-25 18:47:23 +01001272
David Hildenbrand27291e22014-01-23 12:26:52 +01001273 if (guestdbg_exit_pending(vcpu) && !rc) {
1274 kvm_s390_prepare_debug_exit(vcpu);
1275 rc = 0;
1276 }
1277
Heiko Carstensb8e660b2010-02-26 22:37:41 +01001278 if (rc == -EOPNOTSUPP) {
Christian Borntraeger8f2abe62008-03-25 18:47:23 +01001279 /* intercept cannot be handled in-kernel, prepare kvm-run */
1280 kvm_run->exit_reason = KVM_EXIT_S390_SIEIC;
1281 kvm_run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
Christian Borntraeger8f2abe62008-03-25 18:47:23 +01001282 kvm_run->s390_sieic.ipa = vcpu->arch.sie_block->ipa;
1283 kvm_run->s390_sieic.ipb = vcpu->arch.sie_block->ipb;
1284 rc = 0;
1285 }
1286
1287 if (rc == -EREMOTE) {
1288 /* intercept was handled, but userspace support is needed
1289 * kvm_run has been prepared by the handler */
1290 rc = 0;
1291 }
Heiko Carstensb0c632d2008-03-25 18:47:20 +01001292
Carsten Otted7b0b5e2009-11-19 14:21:16 +01001293 kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask;
1294 kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr;
Christian Borntraeger60b413c2012-01-11 11:20:31 +01001295 kvm_run->s.regs.prefix = vcpu->arch.sie_block->prefix;
Christian Borntraeger9eed07352012-02-06 10:59:07 +01001296 memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
Carsten Otted7b0b5e2009-11-19 14:21:16 +01001297
Heiko Carstensb0c632d2008-03-25 18:47:20 +01001298 if (vcpu->sigset_active)
1299 sigprocmask(SIG_SETMASK, &sigsaved, NULL);
1300
Heiko Carstensb0c632d2008-03-25 18:47:20 +01001301 vcpu->stat.exit_userspace++;
Heiko Carstens7e8e6ab2008-04-04 15:12:35 +02001302 return rc;
Heiko Carstensb0c632d2008-03-25 18:47:20 +01001303}
1304
Heiko Carstensb0c632d2008-03-25 18:47:20 +01001305/*
1306 * store status at address
1307 * we use have two special cases:
1308 * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
1309 * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
1310 */
Heiko Carstensd0bce602014-01-01 16:45:58 +01001311int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa)
Heiko Carstensb0c632d2008-03-25 18:47:20 +01001312{
Carsten Otte092670c2011-07-24 10:48:22 +02001313 unsigned char archmode = 1;
Thomas Huth178bd782013-11-13 20:28:18 +01001314 u64 clkcomp;
Heiko Carstensd0bce602014-01-01 16:45:58 +01001315 int rc;
Heiko Carstensb0c632d2008-03-25 18:47:20 +01001316
Heiko Carstensd0bce602014-01-01 16:45:58 +01001317 if (gpa == KVM_S390_STORE_STATUS_NOADDR) {
1318 if (write_guest_abs(vcpu, 163, &archmode, 1))
Heiko Carstensb0c632d2008-03-25 18:47:20 +01001319 return -EFAULT;
Heiko Carstensd0bce602014-01-01 16:45:58 +01001320 gpa = SAVE_AREA_BASE;
1321 } else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) {
1322 if (write_guest_real(vcpu, 163, &archmode, 1))
Heiko Carstensb0c632d2008-03-25 18:47:20 +01001323 return -EFAULT;
Heiko Carstensd0bce602014-01-01 16:45:58 +01001324 gpa = kvm_s390_real_to_abs(vcpu, SAVE_AREA_BASE);
1325 }
1326 rc = write_guest_abs(vcpu, gpa + offsetof(struct save_area, fp_regs),
1327 vcpu->arch.guest_fpregs.fprs, 128);
1328 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, gp_regs),
1329 vcpu->run->s.regs.gprs, 128);
1330 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, psw),
1331 &vcpu->arch.sie_block->gpsw, 16);
1332 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, pref_reg),
1333 &vcpu->arch.sie_block->prefix, 4);
1334 rc |= write_guest_abs(vcpu,
1335 gpa + offsetof(struct save_area, fp_ctrl_reg),
1336 &vcpu->arch.guest_fpregs.fpc, 4);
1337 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, tod_reg),
1338 &vcpu->arch.sie_block->todpr, 4);
1339 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, timer),
1340 &vcpu->arch.sie_block->cputm, 8);
Thomas Huth178bd782013-11-13 20:28:18 +01001341 clkcomp = vcpu->arch.sie_block->ckc >> 8;
Heiko Carstensd0bce602014-01-01 16:45:58 +01001342 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, clk_cmp),
1343 &clkcomp, 8);
1344 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, acc_regs),
1345 &vcpu->run->s.regs.acrs, 64);
1346 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, ctrl_regs),
1347 &vcpu->arch.sie_block->gcr, 128);
1348 return rc ? -EFAULT : 0;
Heiko Carstensb0c632d2008-03-25 18:47:20 +01001349}
1350
Thomas Huthe8798922013-11-06 15:46:33 +01001351int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
1352{
1353 /*
1354 * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
1355 * copying in vcpu load/put. Lets update our copies before we save
1356 * it into the save area
1357 */
1358 save_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
1359 save_fp_regs(vcpu->arch.guest_fpregs.fprs);
1360 save_access_regs(vcpu->run->s.regs.acrs);
1361
1362 return kvm_s390_store_status_unloaded(vcpu, addr);
1363}
1364
David Hildenbrand6852d7b2014-03-14 10:59:29 +01001365void kvm_s390_vcpu_start(struct kvm_vcpu *vcpu)
1366{
1367 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 1);
1368 atomic_clear_mask(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
1369}
1370
1371void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu)
1372{
1373 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 0);
1374 atomic_set_mask(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
1375}
1376
Cornelia Huckd6712df2012-12-20 15:32:11 +01001377static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
1378 struct kvm_enable_cap *cap)
1379{
1380 int r;
1381
1382 if (cap->flags)
1383 return -EINVAL;
1384
1385 switch (cap->cap) {
Cornelia Huckfa6b7fe2012-12-20 15:32:12 +01001386 case KVM_CAP_S390_CSS_SUPPORT:
1387 if (!vcpu->kvm->arch.css_support) {
1388 vcpu->kvm->arch.css_support = 1;
1389 trace_kvm_s390_enable_css(vcpu->kvm);
1390 }
1391 r = 0;
1392 break;
Cornelia Huckd6712df2012-12-20 15:32:11 +01001393 default:
1394 r = -EINVAL;
1395 break;
1396 }
1397 return r;
1398}
1399
Heiko Carstensb0c632d2008-03-25 18:47:20 +01001400long kvm_arch_vcpu_ioctl(struct file *filp,
1401 unsigned int ioctl, unsigned long arg)
1402{
1403 struct kvm_vcpu *vcpu = filp->private_data;
1404 void __user *argp = (void __user *)arg;
Thomas Huth800c1062013-09-12 10:33:45 +02001405 int idx;
Avi Kivitybc923cc2010-05-13 12:21:46 +03001406 long r;
Heiko Carstensb0c632d2008-03-25 18:47:20 +01001407
Avi Kivity93736622010-05-13 12:35:17 +03001408 switch (ioctl) {
1409 case KVM_S390_INTERRUPT: {
Carsten Otteba5c1e92008-03-25 18:47:26 +01001410 struct kvm_s390_interrupt s390int;
1411
Avi Kivity93736622010-05-13 12:35:17 +03001412 r = -EFAULT;
Carsten Otteba5c1e92008-03-25 18:47:26 +01001413 if (copy_from_user(&s390int, argp, sizeof(s390int)))
Avi Kivity93736622010-05-13 12:35:17 +03001414 break;
1415 r = kvm_s390_inject_vcpu(vcpu, &s390int);
1416 break;
Carsten Otteba5c1e92008-03-25 18:47:26 +01001417 }
Heiko Carstensb0c632d2008-03-25 18:47:20 +01001418 case KVM_S390_STORE_STATUS:
Thomas Huth800c1062013-09-12 10:33:45 +02001419 idx = srcu_read_lock(&vcpu->kvm->srcu);
Avi Kivitybc923cc2010-05-13 12:21:46 +03001420 r = kvm_s390_vcpu_store_status(vcpu, arg);
Thomas Huth800c1062013-09-12 10:33:45 +02001421 srcu_read_unlock(&vcpu->kvm->srcu, idx);
Avi Kivitybc923cc2010-05-13 12:21:46 +03001422 break;
Heiko Carstensb0c632d2008-03-25 18:47:20 +01001423 case KVM_S390_SET_INITIAL_PSW: {
1424 psw_t psw;
1425
Avi Kivitybc923cc2010-05-13 12:21:46 +03001426 r = -EFAULT;
Heiko Carstensb0c632d2008-03-25 18:47:20 +01001427 if (copy_from_user(&psw, argp, sizeof(psw)))
Avi Kivitybc923cc2010-05-13 12:21:46 +03001428 break;
1429 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
1430 break;
Heiko Carstensb0c632d2008-03-25 18:47:20 +01001431 }
1432 case KVM_S390_INITIAL_RESET:
Avi Kivitybc923cc2010-05-13 12:21:46 +03001433 r = kvm_arch_vcpu_ioctl_initial_reset(vcpu);
1434 break;
Carsten Otte14eebd92012-05-15 14:15:26 +02001435 case KVM_SET_ONE_REG:
1436 case KVM_GET_ONE_REG: {
1437 struct kvm_one_reg reg;
1438 r = -EFAULT;
1439 if (copy_from_user(&reg, argp, sizeof(reg)))
1440 break;
1441 if (ioctl == KVM_SET_ONE_REG)
1442 r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, &reg);
1443 else
1444 r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, &reg);
1445 break;
1446 }
Carsten Otte27e03932012-01-04 10:25:21 +01001447#ifdef CONFIG_KVM_S390_UCONTROL
1448 case KVM_S390_UCAS_MAP: {
1449 struct kvm_s390_ucas_mapping ucasmap;
1450
1451 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
1452 r = -EFAULT;
1453 break;
1454 }
1455
1456 if (!kvm_is_ucontrol(vcpu->kvm)) {
1457 r = -EINVAL;
1458 break;
1459 }
1460
1461 r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr,
1462 ucasmap.vcpu_addr, ucasmap.length);
1463 break;
1464 }
1465 case KVM_S390_UCAS_UNMAP: {
1466 struct kvm_s390_ucas_mapping ucasmap;
1467
1468 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
1469 r = -EFAULT;
1470 break;
1471 }
1472
1473 if (!kvm_is_ucontrol(vcpu->kvm)) {
1474 r = -EINVAL;
1475 break;
1476 }
1477
1478 r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr,
1479 ucasmap.length);
1480 break;
1481 }
1482#endif
Carsten Otteccc79102012-01-04 10:25:26 +01001483 case KVM_S390_VCPU_FAULT: {
1484 r = gmap_fault(arg, vcpu->arch.gmap);
1485 if (!IS_ERR_VALUE(r))
1486 r = 0;
1487 break;
1488 }
Cornelia Huckd6712df2012-12-20 15:32:11 +01001489 case KVM_ENABLE_CAP:
1490 {
1491 struct kvm_enable_cap cap;
1492 r = -EFAULT;
1493 if (copy_from_user(&cap, argp, sizeof(cap)))
1494 break;
1495 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
1496 break;
1497 }
Heiko Carstensb0c632d2008-03-25 18:47:20 +01001498 default:
Carsten Otte3e6afcf2012-01-04 10:25:30 +01001499 r = -ENOTTY;
Heiko Carstensb0c632d2008-03-25 18:47:20 +01001500 }
Avi Kivitybc923cc2010-05-13 12:21:46 +03001501 return r;
Heiko Carstensb0c632d2008-03-25 18:47:20 +01001502}
1503
Carsten Otte5b1c1492012-01-04 10:25:23 +01001504int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
1505{
1506#ifdef CONFIG_KVM_S390_UCONTROL
1507 if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET)
1508 && (kvm_is_ucontrol(vcpu->kvm))) {
1509 vmf->page = virt_to_page(vcpu->arch.sie_block);
1510 get_page(vmf->page);
1511 return 0;
1512 }
1513#endif
1514 return VM_FAULT_SIGBUS;
1515}
1516
Aneesh Kumar K.V55870272013-10-07 22:18:00 +05301517void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *free,
Takuya Yoshikawadb3fe4e2012-02-08 13:02:18 +09001518 struct kvm_memory_slot *dont)
1519{
1520}
1521
Aneesh Kumar K.V55870272013-10-07 22:18:00 +05301522int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
1523 unsigned long npages)
Takuya Yoshikawadb3fe4e2012-02-08 13:02:18 +09001524{
1525 return 0;
1526}
1527
Takuya Yoshikawae59dbe02013-07-04 13:40:29 +09001528void kvm_arch_memslots_updated(struct kvm *kvm)
1529{
1530}
1531
Heiko Carstensb0c632d2008-03-25 18:47:20 +01001532/* Section: memory related */
Marcelo Tosattif7784b82009-12-23 14:35:18 -02001533int kvm_arch_prepare_memory_region(struct kvm *kvm,
1534 struct kvm_memory_slot *memslot,
Takuya Yoshikawa7b6195a2013-02-27 19:44:34 +09001535 struct kvm_userspace_memory_region *mem,
1536 enum kvm_mr_change change)
Heiko Carstensb0c632d2008-03-25 18:47:20 +01001537{
Nick Wangdd2887e2013-03-25 17:22:57 +01001538 /* A few sanity checks. We can have memory slots which have to be
1539 located/ended at a segment boundary (1MB). The memory in userland is
1540 ok to be fragmented into various different vmas. It is okay to mmap()
1541 and munmap() stuff in this slot after doing this call at any time */
Heiko Carstensb0c632d2008-03-25 18:47:20 +01001542
Carsten Otte598841c2011-07-24 10:48:21 +02001543 if (mem->userspace_addr & 0xffffful)
Heiko Carstensb0c632d2008-03-25 18:47:20 +01001544 return -EINVAL;
1545
Carsten Otte598841c2011-07-24 10:48:21 +02001546 if (mem->memory_size & 0xffffful)
Heiko Carstensb0c632d2008-03-25 18:47:20 +01001547 return -EINVAL;
1548
Marcelo Tosattif7784b82009-12-23 14:35:18 -02001549 return 0;
1550}
1551
1552void kvm_arch_commit_memory_region(struct kvm *kvm,
1553 struct kvm_userspace_memory_region *mem,
Takuya Yoshikawa84826442013-02-27 19:45:25 +09001554 const struct kvm_memory_slot *old,
1555 enum kvm_mr_change change)
Marcelo Tosattif7784b82009-12-23 14:35:18 -02001556{
Carsten Ottef7850c92011-07-24 10:48:23 +02001557 int rc;
Marcelo Tosattif7784b82009-12-23 14:35:18 -02001558
Christian Borntraeger2cef4de2013-03-25 17:22:48 +01001559 /* If the basics of the memslot do not change, we do not want
1560 * to update the gmap. Every update causes several unnecessary
1561 * segment translation exceptions. This is usually handled just
1562 * fine by the normal fault handler + gmap, but it will also
1563 * cause faults on the prefix page of running guest CPUs.
1564 */
1565 if (old->userspace_addr == mem->userspace_addr &&
1566 old->base_gfn * PAGE_SIZE == mem->guest_phys_addr &&
1567 old->npages * PAGE_SIZE == mem->memory_size)
1568 return;
Carsten Otte598841c2011-07-24 10:48:21 +02001569
1570 rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr,
1571 mem->guest_phys_addr, mem->memory_size);
1572 if (rc)
Carsten Ottef7850c92011-07-24 10:48:23 +02001573 printk(KERN_WARNING "kvm-s390: failed to commit memory region\n");
Carsten Otte598841c2011-07-24 10:48:21 +02001574 return;
Heiko Carstensb0c632d2008-03-25 18:47:20 +01001575}
1576
Marcelo Tosatti2df72e92012-08-24 15:54:57 -03001577void kvm_arch_flush_shadow_all(struct kvm *kvm)
1578{
1579}
1580
1581void kvm_arch_flush_shadow_memslot(struct kvm *kvm,
1582 struct kvm_memory_slot *slot)
Marcelo Tosatti34d4cb82008-07-10 20:49:31 -03001583{
1584}
1585
Heiko Carstensb0c632d2008-03-25 18:47:20 +01001586static int __init kvm_s390_init(void)
1587{
Christian Borntraegeref50f7a2009-06-23 17:24:07 +02001588 int ret;
Avi Kivity0ee75be2010-04-28 15:39:01 +03001589 ret = kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
Christian Borntraegeref50f7a2009-06-23 17:24:07 +02001590 if (ret)
1591 return ret;
1592
1593 /*
1594 * guests can ask for up to 255+1 double words, we need a full page
Lucas De Marchi25985ed2011-03-30 22:57:33 -03001595 * to hold the maximum amount of facilities. On the other hand, we
Christian Borntraegeref50f7a2009-06-23 17:24:07 +02001596 * only set facilities that are known to work in KVM.
1597 */
Michael Mueller78c4b592013-07-26 15:04:04 +02001598 vfacilities = (unsigned long *) get_zeroed_page(GFP_KERNEL|GFP_DMA);
1599 if (!vfacilities) {
Christian Borntraegeref50f7a2009-06-23 17:24:07 +02001600 kvm_exit();
1601 return -ENOMEM;
1602 }
Michael Mueller78c4b592013-07-26 15:04:04 +02001603 memcpy(vfacilities, S390_lowcore.stfle_fac_list, 16);
Thomas Huthd208c792013-12-12 13:40:40 +01001604 vfacilities[0] &= 0xff82fff3f4fc2000UL;
Michael Mueller7feb6bb2013-06-28 13:30:24 +02001605 vfacilities[1] &= 0x005c000000000000UL;
Christian Borntraegeref50f7a2009-06-23 17:24:07 +02001606 return 0;
Heiko Carstensb0c632d2008-03-25 18:47:20 +01001607}
1608
1609static void __exit kvm_s390_exit(void)
1610{
Michael Mueller78c4b592013-07-26 15:04:04 +02001611 free_page((unsigned long) vfacilities);
Heiko Carstensb0c632d2008-03-25 18:47:20 +01001612 kvm_exit();
1613}
1614
1615module_init(kvm_s390_init);
1616module_exit(kvm_s390_exit);
Cornelia Huck566af942013-05-27 18:42:33 +02001617
1618/*
1619 * Enable autoloading of the kvm module.
1620 * Note that we add the module alias here instead of virt/kvm/kvm_main.c
1621 * since x86 takes a different approach.
1622 */
1623#include <linux/miscdevice.h>
1624MODULE_ALIAS_MISCDEV(KVM_MINOR);
1625MODULE_ALIAS("devname:kvm");