blob: a87f45edfae848e07c295c7623097e235ffe8c78 [file] [log] [blame]
Avi Kivity6aa8b732006-12-10 02:21:36 -08001/*
2 * Kernel-based Virtual Machine driver for Linux
3 *
4 * This module enables machines with Intel VT-x extensions to run virtual
5 * machines without emulation or binary translation.
6 *
7 * Copyright (C) 2006 Qumranet, Inc.
8 *
9 * Authors:
10 * Avi Kivity <avi@qumranet.com>
11 * Yaniv Kamay <yaniv@qumranet.com>
12 *
13 * This work is licensed under the terms of the GNU GPL, version 2. See
14 * the COPYING file in the top-level directory.
15 *
16 */
17
Hollis Blancharde2174022007-12-03 15:30:24 -060018#include "iodev.h"
Avi Kivity6aa8b732006-12-10 02:21:36 -080019
Avi Kivityedf88412007-12-16 11:02:48 +020020#include <linux/kvm_host.h>
Avi Kivity6aa8b732006-12-10 02:21:36 -080021#include <linux/kvm.h>
22#include <linux/module.h>
23#include <linux/errno.h>
Avi Kivity6aa8b732006-12-10 02:21:36 -080024#include <linux/percpu.h>
25#include <linux/gfp.h>
Avi Kivity6aa8b732006-12-10 02:21:36 -080026#include <linux/mm.h>
27#include <linux/miscdevice.h>
28#include <linux/vmalloc.h>
Avi Kivity6aa8b732006-12-10 02:21:36 -080029#include <linux/reboot.h>
Avi Kivity6aa8b732006-12-10 02:21:36 -080030#include <linux/debugfs.h>
31#include <linux/highmem.h>
32#include <linux/file.h>
Avi Kivity59ae6c62007-02-12 00:54:48 -080033#include <linux/sysdev.h>
Avi Kivity774c47f2007-02-12 00:54:47 -080034#include <linux/cpu.h>
Alexey Dobriyane8edc6e2007-05-21 01:22:52 +040035#include <linux/sched.h>
Avi Kivityd9e368d2007-06-07 19:18:30 +030036#include <linux/cpumask.h>
37#include <linux/smp.h>
Avi Kivityd6d28162007-06-28 08:38:16 -040038#include <linux/anon_inodes.h>
Avi Kivity04d2cc72007-09-10 18:10:54 +030039#include <linux/profile.h>
Anthony Liguori7aa81cc2007-09-17 14:57:50 -050040#include <linux/kvm_para.h>
Izik Eidus6fc138d2007-10-09 19:20:39 +020041#include <linux/pagemap.h>
Anthony Liguori8d4e1282007-10-18 09:59:34 -050042#include <linux/mman.h>
Anthony Liguori35149e22008-04-02 14:46:56 -050043#include <linux/swap.h>
Avi Kivity6aa8b732006-12-10 02:21:36 -080044
Avi Kivitye4956062007-06-28 14:15:57 -040045#include <asm/processor.h>
Avi Kivitye4956062007-06-28 14:15:57 -040046#include <asm/io.h>
47#include <asm/uaccess.h>
Izik Eidus3e021bf2007-11-19 11:16:57 +020048#include <asm/pgtable.h>
Avi Kivity6aa8b732006-12-10 02:21:36 -080049
Laurent Vivier5f94c172008-05-30 16:05:54 +020050#ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
51#include "coalesced_mmio.h"
52#endif
53
Xiantao Zhang8a98f662008-10-06 13:47:38 +080054#ifdef KVM_CAP_DEVICE_ASSIGNMENT
55#include <linux/pci.h>
56#include <linux/interrupt.h>
57#include "irq.h"
58#endif
59
Avi Kivity6aa8b732006-12-10 02:21:36 -080060MODULE_AUTHOR("Qumranet");
61MODULE_LICENSE("GPL");
62
Zhang Xiantaoe9b11c12007-11-14 20:38:21 +080063DEFINE_SPINLOCK(kvm_lock);
64LIST_HEAD(vm_list);
Avi Kivity133de902007-02-12 00:54:44 -080065
Avi Kivity1b6c0162007-05-24 13:03:52 +030066static cpumask_t cpus_hardware_enabled;
67
Rusty Russellc16f8622007-07-30 21:12:19 +100068struct kmem_cache *kvm_vcpu_cache;
69EXPORT_SYMBOL_GPL(kvm_vcpu_cache);
Avi Kivity1165f5f2007-04-19 17:27:43 +030070
Avi Kivity15ad7142007-07-11 18:17:21 +030071static __read_mostly struct preempt_ops kvm_preempt_ops;
72
Hollis Blanchard76f7c872008-04-15 16:05:42 -050073struct dentry *kvm_debugfs_dir;
Avi Kivity6aa8b732006-12-10 02:21:36 -080074
Avi Kivitybccf2152007-02-21 18:04:26 +020075static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl,
76 unsigned long arg);
77
Avi Kivity4ecac3f2008-05-13 13:23:38 +030078bool kvm_rebooting;
79
Xiantao Zhang8a98f662008-10-06 13:47:38 +080080#ifdef KVM_CAP_DEVICE_ASSIGNMENT
81static struct kvm_assigned_dev_kernel *kvm_find_assigned_dev(struct list_head *head,
82 int assigned_dev_id)
83{
84 struct list_head *ptr;
85 struct kvm_assigned_dev_kernel *match;
86
87 list_for_each(ptr, head) {
88 match = list_entry(ptr, struct kvm_assigned_dev_kernel, list);
89 if (match->assigned_dev_id == assigned_dev_id)
90 return match;
91 }
92 return NULL;
93}
94
95static void kvm_assigned_dev_interrupt_work_handler(struct work_struct *work)
96{
97 struct kvm_assigned_dev_kernel *assigned_dev;
98
99 assigned_dev = container_of(work, struct kvm_assigned_dev_kernel,
100 interrupt_work);
101
102 /* This is taken to safely inject irq inside the guest. When
103 * the interrupt injection (or the ioapic code) uses a
104 * finer-grained lock, update this
105 */
106 mutex_lock(&assigned_dev->kvm->lock);
107 kvm_set_irq(assigned_dev->kvm,
Sheng Yang5550af42008-10-15 20:15:06 +0800108 assigned_dev->irq_source_id,
Xiantao Zhang8a98f662008-10-06 13:47:38 +0800109 assigned_dev->guest_irq, 1);
110 mutex_unlock(&assigned_dev->kvm->lock);
111 kvm_put_kvm(assigned_dev->kvm);
112}
113
Xiantao Zhang8a98f662008-10-06 13:47:38 +0800114static irqreturn_t kvm_assigned_dev_intr(int irq, void *dev_id)
115{
116 struct kvm_assigned_dev_kernel *assigned_dev =
117 (struct kvm_assigned_dev_kernel *) dev_id;
118
119 kvm_get_kvm(assigned_dev->kvm);
120 schedule_work(&assigned_dev->interrupt_work);
121 disable_irq_nosync(irq);
122 return IRQ_HANDLED;
123}
124
125/* Ack the irq line for an assigned device */
126static void kvm_assigned_dev_ack_irq(struct kvm_irq_ack_notifier *kian)
127{
128 struct kvm_assigned_dev_kernel *dev;
129
130 if (kian->gsi == -1)
131 return;
132
133 dev = container_of(kian, struct kvm_assigned_dev_kernel,
134 ack_notifier);
Sheng Yang5550af42008-10-15 20:15:06 +0800135 kvm_set_irq(dev->kvm, dev->irq_source_id, dev->guest_irq, 0);
Xiantao Zhang8a98f662008-10-06 13:47:38 +0800136 enable_irq(dev->host_irq);
137}
138
139static void kvm_free_assigned_device(struct kvm *kvm,
140 struct kvm_assigned_dev_kernel
141 *assigned_dev)
142{
143 if (irqchip_in_kernel(kvm) && assigned_dev->irq_requested)
144 free_irq(assigned_dev->host_irq, (void *)assigned_dev);
145
146 kvm_unregister_irq_ack_notifier(kvm, &assigned_dev->ack_notifier);
Sheng Yang5550af42008-10-15 20:15:06 +0800147 kvm_free_irq_source_id(kvm, assigned_dev->irq_source_id);
Xiantao Zhang8a98f662008-10-06 13:47:38 +0800148
149 if (cancel_work_sync(&assigned_dev->interrupt_work))
150 /* We had pending work. That means we will have to take
151 * care of kvm_put_kvm.
152 */
153 kvm_put_kvm(kvm);
154
155 pci_release_regions(assigned_dev->dev);
156 pci_disable_device(assigned_dev->dev);
157 pci_dev_put(assigned_dev->dev);
158
159 list_del(&assigned_dev->list);
160 kfree(assigned_dev);
161}
162
163void kvm_free_all_assigned_devices(struct kvm *kvm)
164{
165 struct list_head *ptr, *ptr2;
166 struct kvm_assigned_dev_kernel *assigned_dev;
167
168 list_for_each_safe(ptr, ptr2, &kvm->arch.assigned_dev_head) {
169 assigned_dev = list_entry(ptr,
170 struct kvm_assigned_dev_kernel,
171 list);
172
173 kvm_free_assigned_device(kvm, assigned_dev);
174 }
175}
176
177static int kvm_vm_ioctl_assign_irq(struct kvm *kvm,
178 struct kvm_assigned_irq
179 *assigned_irq)
180{
181 int r = 0;
182 struct kvm_assigned_dev_kernel *match;
183
184 mutex_lock(&kvm->lock);
185
186 match = kvm_find_assigned_dev(&kvm->arch.assigned_dev_head,
187 assigned_irq->assigned_dev_id);
188 if (!match) {
189 mutex_unlock(&kvm->lock);
190 return -EINVAL;
191 }
192
193 if (match->irq_requested) {
194 match->guest_irq = assigned_irq->guest_irq;
195 match->ack_notifier.gsi = assigned_irq->guest_irq;
196 mutex_unlock(&kvm->lock);
197 return 0;
198 }
199
200 INIT_WORK(&match->interrupt_work,
201 kvm_assigned_dev_interrupt_work_handler);
202
203 if (irqchip_in_kernel(kvm)) {
204 if (!capable(CAP_SYS_RAWIO)) {
205 r = -EPERM;
206 goto out_release;
207 }
208
209 if (assigned_irq->host_irq)
210 match->host_irq = assigned_irq->host_irq;
211 else
212 match->host_irq = match->dev->irq;
213 match->guest_irq = assigned_irq->guest_irq;
214 match->ack_notifier.gsi = assigned_irq->guest_irq;
215 match->ack_notifier.irq_acked = kvm_assigned_dev_ack_irq;
216 kvm_register_irq_ack_notifier(kvm, &match->ack_notifier);
Sheng Yang5550af42008-10-15 20:15:06 +0800217 r = kvm_request_irq_source_id(kvm);
218 if (r < 0)
219 goto out_release;
220 else
221 match->irq_source_id = r;
Xiantao Zhang8a98f662008-10-06 13:47:38 +0800222
223 /* Even though this is PCI, we don't want to use shared
224 * interrupts. Sharing host devices with guest-assigned devices
225 * on the same interrupt line is not a happy situation: there
226 * are going to be long delays in accepting, acking, etc.
227 */
228 if (request_irq(match->host_irq, kvm_assigned_dev_intr, 0,
229 "kvm_assigned_device", (void *)match)) {
230 r = -EIO;
231 goto out_release;
232 }
233 }
234
235 match->irq_requested = true;
236 mutex_unlock(&kvm->lock);
237 return r;
238out_release:
239 mutex_unlock(&kvm->lock);
240 kvm_free_assigned_device(kvm, match);
241 return r;
242}
243
244static int kvm_vm_ioctl_assign_device(struct kvm *kvm,
245 struct kvm_assigned_pci_dev *assigned_dev)
246{
247 int r = 0;
248 struct kvm_assigned_dev_kernel *match;
249 struct pci_dev *dev;
250
251 mutex_lock(&kvm->lock);
252
253 match = kvm_find_assigned_dev(&kvm->arch.assigned_dev_head,
254 assigned_dev->assigned_dev_id);
255 if (match) {
256 /* device already assigned */
257 r = -EINVAL;
258 goto out;
259 }
260
261 match = kzalloc(sizeof(struct kvm_assigned_dev_kernel), GFP_KERNEL);
262 if (match == NULL) {
263 printk(KERN_INFO "%s: Couldn't allocate memory\n",
264 __func__);
265 r = -ENOMEM;
266 goto out;
267 }
268 dev = pci_get_bus_and_slot(assigned_dev->busnr,
269 assigned_dev->devfn);
270 if (!dev) {
271 printk(KERN_INFO "%s: host device not found\n", __func__);
272 r = -EINVAL;
273 goto out_free;
274 }
275 if (pci_enable_device(dev)) {
276 printk(KERN_INFO "%s: Could not enable PCI device\n", __func__);
277 r = -EBUSY;
278 goto out_put;
279 }
280 r = pci_request_regions(dev, "kvm_assigned_device");
281 if (r) {
282 printk(KERN_INFO "%s: Could not get access to device regions\n",
283 __func__);
284 goto out_disable;
285 }
286 match->assigned_dev_id = assigned_dev->assigned_dev_id;
287 match->host_busnr = assigned_dev->busnr;
288 match->host_devfn = assigned_dev->devfn;
289 match->dev = dev;
290
291 match->kvm = kvm;
292
293 list_add(&match->list, &kvm->arch.assigned_dev_head);
294
295 if (assigned_dev->flags & KVM_DEV_ASSIGN_ENABLE_IOMMU) {
296 r = kvm_iommu_map_guest(kvm, match);
297 if (r)
298 goto out_list_del;
299 }
300
301out:
302 mutex_unlock(&kvm->lock);
303 return r;
304out_list_del:
305 list_del(&match->list);
306 pci_release_regions(dev);
307out_disable:
308 pci_disable_device(dev);
309out_put:
310 pci_dev_put(dev);
311out_free:
312 kfree(match);
313 mutex_unlock(&kvm->lock);
314 return r;
315}
316#endif
317
James Morris5aacf0c2006-12-22 01:04:55 -0800318static inline int valid_vcpu(int n)
319{
320 return likely(n >= 0 && n < KVM_MAX_VCPUS);
321}
322
Xiantao Zhangc77fb9d2008-09-27 10:55:40 +0800323inline int kvm_is_mmio_pfn(pfn_t pfn)
Ben-Ami Yassourcbff90a2008-07-28 19:26:24 +0300324{
325 if (pfn_valid(pfn))
326 return PageReserved(pfn_to_page(pfn));
327
328 return true;
329}
330
Avi Kivity6aa8b732006-12-10 02:21:36 -0800331/*
332 * Switches to specified vcpu, until a matching vcpu_put()
333 */
Carsten Otte313a3dc2007-10-11 19:16:52 +0200334void vcpu_load(struct kvm_vcpu *vcpu)
Avi Kivity6aa8b732006-12-10 02:21:36 -0800335{
Avi Kivity15ad7142007-07-11 18:17:21 +0300336 int cpu;
337
Avi Kivitybccf2152007-02-21 18:04:26 +0200338 mutex_lock(&vcpu->mutex);
Avi Kivity15ad7142007-07-11 18:17:21 +0300339 cpu = get_cpu();
340 preempt_notifier_register(&vcpu->preempt_notifier);
Carsten Otte313a3dc2007-10-11 19:16:52 +0200341 kvm_arch_vcpu_load(vcpu, cpu);
Avi Kivity15ad7142007-07-11 18:17:21 +0300342 put_cpu();
Avi Kivitybccf2152007-02-21 18:04:26 +0200343}
344
Carsten Otte313a3dc2007-10-11 19:16:52 +0200345void vcpu_put(struct kvm_vcpu *vcpu)
Avi Kivity6aa8b732006-12-10 02:21:36 -0800346{
Avi Kivity15ad7142007-07-11 18:17:21 +0300347 preempt_disable();
Carsten Otte313a3dc2007-10-11 19:16:52 +0200348 kvm_arch_vcpu_put(vcpu);
Avi Kivity15ad7142007-07-11 18:17:21 +0300349 preempt_notifier_unregister(&vcpu->preempt_notifier);
350 preempt_enable();
Avi Kivity6aa8b732006-12-10 02:21:36 -0800351 mutex_unlock(&vcpu->mutex);
352}
353
Avi Kivityd9e368d2007-06-07 19:18:30 +0300354static void ack_flush(void *_completed)
355{
Avi Kivityd9e368d2007-06-07 19:18:30 +0300356}
357
358void kvm_flush_remote_tlbs(struct kvm *kvm)
359{
Avi Kivity597a5f52008-07-20 14:24:22 +0300360 int i, cpu, me;
Avi Kivityd9e368d2007-06-07 19:18:30 +0300361 cpumask_t cpus;
362 struct kvm_vcpu *vcpu;
Avi Kivityd9e368d2007-06-07 19:18:30 +0300363
Avi Kivity597a5f52008-07-20 14:24:22 +0300364 me = get_cpu();
Avi Kivityd9e368d2007-06-07 19:18:30 +0300365 cpus_clear(cpus);
Rusty Russellfb3f0f52007-07-27 17:16:56 +1000366 for (i = 0; i < KVM_MAX_VCPUS; ++i) {
367 vcpu = kvm->vcpus[i];
368 if (!vcpu)
369 continue;
Avi Kivity3176bc32007-10-16 17:22:08 +0200370 if (test_and_set_bit(KVM_REQ_TLB_FLUSH, &vcpu->requests))
Avi Kivityd9e368d2007-06-07 19:18:30 +0300371 continue;
372 cpu = vcpu->cpu;
Avi Kivity597a5f52008-07-20 14:24:22 +0300373 if (cpu != -1 && cpu != me)
Laurent Vivier49d3bd72007-10-22 16:33:07 +0200374 cpu_set(cpu, cpus);
Avi Kivityd9e368d2007-06-07 19:18:30 +0300375 }
Avi Kivity0f74a242007-11-20 23:01:14 +0200376 if (cpus_empty(cpus))
Avi Kivity597a5f52008-07-20 14:24:22 +0300377 goto out;
Avi Kivity0f74a242007-11-20 23:01:14 +0200378 ++kvm->stat.remote_tlb_flush;
Laurent Vivier49d3bd72007-10-22 16:33:07 +0200379 smp_call_function_mask(cpus, ack_flush, NULL, 1);
Avi Kivity597a5f52008-07-20 14:24:22 +0300380out:
381 put_cpu();
Avi Kivityd9e368d2007-06-07 19:18:30 +0300382}
383
Marcelo Tosatti2e53d632008-02-20 14:47:24 -0500384void kvm_reload_remote_mmus(struct kvm *kvm)
385{
Avi Kivity597a5f52008-07-20 14:24:22 +0300386 int i, cpu, me;
Marcelo Tosatti2e53d632008-02-20 14:47:24 -0500387 cpumask_t cpus;
388 struct kvm_vcpu *vcpu;
389
Avi Kivity597a5f52008-07-20 14:24:22 +0300390 me = get_cpu();
Marcelo Tosatti2e53d632008-02-20 14:47:24 -0500391 cpus_clear(cpus);
392 for (i = 0; i < KVM_MAX_VCPUS; ++i) {
393 vcpu = kvm->vcpus[i];
394 if (!vcpu)
395 continue;
396 if (test_and_set_bit(KVM_REQ_MMU_RELOAD, &vcpu->requests))
397 continue;
398 cpu = vcpu->cpu;
Avi Kivity597a5f52008-07-20 14:24:22 +0300399 if (cpu != -1 && cpu != me)
Marcelo Tosatti2e53d632008-02-20 14:47:24 -0500400 cpu_set(cpu, cpus);
401 }
402 if (cpus_empty(cpus))
Avi Kivity597a5f52008-07-20 14:24:22 +0300403 goto out;
Marcelo Tosatti2e53d632008-02-20 14:47:24 -0500404 smp_call_function_mask(cpus, ack_flush, NULL, 1);
Avi Kivity597a5f52008-07-20 14:24:22 +0300405out:
406 put_cpu();
Marcelo Tosatti2e53d632008-02-20 14:47:24 -0500407}
408
409
Rusty Russellfb3f0f52007-07-27 17:16:56 +1000410int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id)
411{
412 struct page *page;
413 int r;
414
415 mutex_init(&vcpu->mutex);
416 vcpu->cpu = -1;
Rusty Russellfb3f0f52007-07-27 17:16:56 +1000417 vcpu->kvm = kvm;
418 vcpu->vcpu_id = id;
Eddie Dongb6958ce2007-07-18 12:15:21 +0300419 init_waitqueue_head(&vcpu->wq);
Rusty Russellfb3f0f52007-07-27 17:16:56 +1000420
421 page = alloc_page(GFP_KERNEL | __GFP_ZERO);
422 if (!page) {
423 r = -ENOMEM;
424 goto fail;
425 }
426 vcpu->run = page_address(page);
427
Zhang Xiantaoe9b11c12007-11-14 20:38:21 +0800428 r = kvm_arch_vcpu_init(vcpu);
Rusty Russellfb3f0f52007-07-27 17:16:56 +1000429 if (r < 0)
Zhang Xiantaoe9b11c12007-11-14 20:38:21 +0800430 goto fail_free_run;
Rusty Russellfb3f0f52007-07-27 17:16:56 +1000431 return 0;
432
Rusty Russellfb3f0f52007-07-27 17:16:56 +1000433fail_free_run:
434 free_page((unsigned long)vcpu->run);
435fail:
Rusty Russell76fafa52007-10-08 10:50:48 +1000436 return r;
Rusty Russellfb3f0f52007-07-27 17:16:56 +1000437}
438EXPORT_SYMBOL_GPL(kvm_vcpu_init);
439
440void kvm_vcpu_uninit(struct kvm_vcpu *vcpu)
441{
Zhang Xiantaoe9b11c12007-11-14 20:38:21 +0800442 kvm_arch_vcpu_uninit(vcpu);
Rusty Russellfb3f0f52007-07-27 17:16:56 +1000443 free_page((unsigned long)vcpu->run);
444}
445EXPORT_SYMBOL_GPL(kvm_vcpu_uninit);
446
Andrea Arcangelie930bff2008-07-25 16:24:52 +0200447#if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
448static inline struct kvm *mmu_notifier_to_kvm(struct mmu_notifier *mn)
449{
450 return container_of(mn, struct kvm, mmu_notifier);
451}
452
453static void kvm_mmu_notifier_invalidate_page(struct mmu_notifier *mn,
454 struct mm_struct *mm,
455 unsigned long address)
456{
457 struct kvm *kvm = mmu_notifier_to_kvm(mn);
458 int need_tlb_flush;
459
460 /*
461 * When ->invalidate_page runs, the linux pte has been zapped
462 * already but the page is still allocated until
463 * ->invalidate_page returns. So if we increase the sequence
464 * here the kvm page fault will notice if the spte can't be
465 * established because the page is going to be freed. If
466 * instead the kvm page fault establishes the spte before
467 * ->invalidate_page runs, kvm_unmap_hva will release it
468 * before returning.
469 *
470 * The sequence increase only need to be seen at spin_unlock
471 * time, and not at spin_lock time.
472 *
473 * Increasing the sequence after the spin_unlock would be
474 * unsafe because the kvm page fault could then establish the
475 * pte after kvm_unmap_hva returned, without noticing the page
476 * is going to be freed.
477 */
478 spin_lock(&kvm->mmu_lock);
479 kvm->mmu_notifier_seq++;
480 need_tlb_flush = kvm_unmap_hva(kvm, address);
481 spin_unlock(&kvm->mmu_lock);
482
483 /* we've to flush the tlb before the pages can be freed */
484 if (need_tlb_flush)
485 kvm_flush_remote_tlbs(kvm);
486
487}
488
489static void kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn,
490 struct mm_struct *mm,
491 unsigned long start,
492 unsigned long end)
493{
494 struct kvm *kvm = mmu_notifier_to_kvm(mn);
495 int need_tlb_flush = 0;
496
497 spin_lock(&kvm->mmu_lock);
498 /*
499 * The count increase must become visible at unlock time as no
500 * spte can be established without taking the mmu_lock and
501 * count is also read inside the mmu_lock critical section.
502 */
503 kvm->mmu_notifier_count++;
504 for (; start < end; start += PAGE_SIZE)
505 need_tlb_flush |= kvm_unmap_hva(kvm, start);
506 spin_unlock(&kvm->mmu_lock);
507
508 /* we've to flush the tlb before the pages can be freed */
509 if (need_tlb_flush)
510 kvm_flush_remote_tlbs(kvm);
511}
512
513static void kvm_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn,
514 struct mm_struct *mm,
515 unsigned long start,
516 unsigned long end)
517{
518 struct kvm *kvm = mmu_notifier_to_kvm(mn);
519
520 spin_lock(&kvm->mmu_lock);
521 /*
522 * This sequence increase will notify the kvm page fault that
523 * the page that is going to be mapped in the spte could have
524 * been freed.
525 */
526 kvm->mmu_notifier_seq++;
527 /*
528 * The above sequence increase must be visible before the
529 * below count decrease but both values are read by the kvm
530 * page fault under mmu_lock spinlock so we don't need to add
531 * a smb_wmb() here in between the two.
532 */
533 kvm->mmu_notifier_count--;
534 spin_unlock(&kvm->mmu_lock);
535
536 BUG_ON(kvm->mmu_notifier_count < 0);
537}
538
539static int kvm_mmu_notifier_clear_flush_young(struct mmu_notifier *mn,
540 struct mm_struct *mm,
541 unsigned long address)
542{
543 struct kvm *kvm = mmu_notifier_to_kvm(mn);
544 int young;
545
546 spin_lock(&kvm->mmu_lock);
547 young = kvm_age_hva(kvm, address);
548 spin_unlock(&kvm->mmu_lock);
549
550 if (young)
551 kvm_flush_remote_tlbs(kvm);
552
553 return young;
554}
555
556static const struct mmu_notifier_ops kvm_mmu_notifier_ops = {
557 .invalidate_page = kvm_mmu_notifier_invalidate_page,
558 .invalidate_range_start = kvm_mmu_notifier_invalidate_range_start,
559 .invalidate_range_end = kvm_mmu_notifier_invalidate_range_end,
560 .clear_flush_young = kvm_mmu_notifier_clear_flush_young,
561};
562#endif /* CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER */
563
Avi Kivityf17abe92007-02-21 19:28:04 +0200564static struct kvm *kvm_create_vm(void)
Avi Kivity6aa8b732006-12-10 02:21:36 -0800565{
Zhang Xiantaod19a9cd2007-11-18 18:43:45 +0800566 struct kvm *kvm = kvm_arch_create_vm();
Laurent Vivier5f94c172008-05-30 16:05:54 +0200567#ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
568 struct page *page;
569#endif
Avi Kivity6aa8b732006-12-10 02:21:36 -0800570
Zhang Xiantaod19a9cd2007-11-18 18:43:45 +0800571 if (IS_ERR(kvm))
572 goto out;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800573
Laurent Vivier5f94c172008-05-30 16:05:54 +0200574#ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
575 page = alloc_page(GFP_KERNEL | __GFP_ZERO);
576 if (!page) {
577 kfree(kvm);
578 return ERR_PTR(-ENOMEM);
579 }
580 kvm->coalesced_mmio_ring =
581 (struct kvm_coalesced_mmio_ring *)page_address(page);
582#endif
583
Andrea Arcangelie930bff2008-07-25 16:24:52 +0200584#if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
585 {
586 int err;
587 kvm->mmu_notifier.ops = &kvm_mmu_notifier_ops;
588 err = mmu_notifier_register(&kvm->mmu_notifier, current->mm);
589 if (err) {
590#ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
591 put_page(page);
592#endif
593 kfree(kvm);
594 return ERR_PTR(err);
595 }
596 }
597#endif
598
Avi Kivity6d4e4c42007-11-21 16:41:05 +0200599 kvm->mm = current->mm;
600 atomic_inc(&kvm->mm->mm_count);
Marcelo Tosattiaaee2c92007-12-20 19:18:26 -0500601 spin_lock_init(&kvm->mmu_lock);
Eddie Dong74906342007-06-19 18:05:03 +0300602 kvm_io_bus_init(&kvm->pio_bus);
Shaohua Li11ec2802007-07-23 14:51:37 +0800603 mutex_init(&kvm->lock);
Gregory Haskins2eeb2e92007-05-31 14:08:53 -0400604 kvm_io_bus_init(&kvm->mmio_bus);
Izik Eidus72dc67a2008-02-10 18:04:15 +0200605 init_rwsem(&kvm->slots_lock);
Izik Eidusd39f13b2008-03-30 16:01:25 +0300606 atomic_set(&kvm->users_count, 1);
Rusty Russell5e58cfe2007-07-23 17:08:21 +1000607 spin_lock(&kvm_lock);
608 list_add(&kvm->vm_list, &vm_list);
609 spin_unlock(&kvm_lock);
Laurent Vivier5f94c172008-05-30 16:05:54 +0200610#ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
611 kvm_coalesced_mmio_init(kvm);
612#endif
Zhang Xiantaod19a9cd2007-11-18 18:43:45 +0800613out:
Avi Kivityf17abe92007-02-21 19:28:04 +0200614 return kvm;
615}
616
Avi Kivity6aa8b732006-12-10 02:21:36 -0800617/*
618 * Free any memory in @free but not in @dont.
619 */
620static void kvm_free_physmem_slot(struct kvm_memory_slot *free,
621 struct kvm_memory_slot *dont)
622{
Izik Eidus290fc382007-09-27 14:11:22 +0200623 if (!dont || free->rmap != dont->rmap)
624 vfree(free->rmap);
Avi Kivity6aa8b732006-12-10 02:21:36 -0800625
626 if (!dont || free->dirty_bitmap != dont->dirty_bitmap)
627 vfree(free->dirty_bitmap);
628
Marcelo Tosatti05da4552008-02-23 11:44:30 -0300629 if (!dont || free->lpage_info != dont->lpage_info)
630 vfree(free->lpage_info);
631
Avi Kivity6aa8b732006-12-10 02:21:36 -0800632 free->npages = 0;
Al Viro8b6d44c2007-02-09 16:38:40 +0000633 free->dirty_bitmap = NULL;
Anthony Liguori8d4e1282007-10-18 09:59:34 -0500634 free->rmap = NULL;
Marcelo Tosatti05da4552008-02-23 11:44:30 -0300635 free->lpage_info = NULL;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800636}
637
Zhang Xiantaod19a9cd2007-11-18 18:43:45 +0800638void kvm_free_physmem(struct kvm *kvm)
Avi Kivity6aa8b732006-12-10 02:21:36 -0800639{
640 int i;
641
642 for (i = 0; i < kvm->nmemslots; ++i)
Al Viro8b6d44c2007-02-09 16:38:40 +0000643 kvm_free_physmem_slot(&kvm->memslots[i], NULL);
Avi Kivity6aa8b732006-12-10 02:21:36 -0800644}
645
Avi Kivityf17abe92007-02-21 19:28:04 +0200646static void kvm_destroy_vm(struct kvm *kvm)
647{
Avi Kivity6d4e4c42007-11-21 16:41:05 +0200648 struct mm_struct *mm = kvm->mm;
649
Avi Kivity133de902007-02-12 00:54:44 -0800650 spin_lock(&kvm_lock);
651 list_del(&kvm->vm_list);
652 spin_unlock(&kvm_lock);
Eddie Dong74906342007-06-19 18:05:03 +0300653 kvm_io_bus_destroy(&kvm->pio_bus);
Gregory Haskins2eeb2e92007-05-31 14:08:53 -0400654 kvm_io_bus_destroy(&kvm->mmio_bus);
Laurent Vivier5f94c172008-05-30 16:05:54 +0200655#ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
656 if (kvm->coalesced_mmio_ring != NULL)
657 free_page((unsigned long)kvm->coalesced_mmio_ring);
658#endif
Andrea Arcangelie930bff2008-07-25 16:24:52 +0200659#if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
660 mmu_notifier_unregister(&kvm->mmu_notifier, kvm->mm);
661#endif
Zhang Xiantaod19a9cd2007-11-18 18:43:45 +0800662 kvm_arch_destroy_vm(kvm);
Avi Kivity6d4e4c42007-11-21 16:41:05 +0200663 mmdrop(mm);
Avi Kivityf17abe92007-02-21 19:28:04 +0200664}
665
Izik Eidusd39f13b2008-03-30 16:01:25 +0300666void kvm_get_kvm(struct kvm *kvm)
667{
668 atomic_inc(&kvm->users_count);
669}
670EXPORT_SYMBOL_GPL(kvm_get_kvm);
671
672void kvm_put_kvm(struct kvm *kvm)
673{
674 if (atomic_dec_and_test(&kvm->users_count))
675 kvm_destroy_vm(kvm);
676}
677EXPORT_SYMBOL_GPL(kvm_put_kvm);
678
679
Avi Kivityf17abe92007-02-21 19:28:04 +0200680static int kvm_vm_release(struct inode *inode, struct file *filp)
681{
682 struct kvm *kvm = filp->private_data;
683
Izik Eidusd39f13b2008-03-30 16:01:25 +0300684 kvm_put_kvm(kvm);
Avi Kivity6aa8b732006-12-10 02:21:36 -0800685 return 0;
686}
687
Avi Kivity6aa8b732006-12-10 02:21:36 -0800688/*
Avi Kivity6aa8b732006-12-10 02:21:36 -0800689 * Allocate some memory and give it an address in the guest physical address
690 * space.
691 *
692 * Discontiguous memory is allowed, mostly for framebuffers.
Sheng Yangf78e0e22007-10-29 09:40:42 +0800693 *
Marcelo Tosatti10589a42007-12-20 19:18:22 -0500694 * Must be called holding mmap_sem for write.
Avi Kivity6aa8b732006-12-10 02:21:36 -0800695 */
Sheng Yangf78e0e22007-10-29 09:40:42 +0800696int __kvm_set_memory_region(struct kvm *kvm,
697 struct kvm_userspace_memory_region *mem,
698 int user_alloc)
Avi Kivity6aa8b732006-12-10 02:21:36 -0800699{
700 int r;
701 gfn_t base_gfn;
702 unsigned long npages;
703 unsigned long i;
704 struct kvm_memory_slot *memslot;
705 struct kvm_memory_slot old, new;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800706
707 r = -EINVAL;
708 /* General sanity checks */
709 if (mem->memory_size & (PAGE_SIZE - 1))
710 goto out;
711 if (mem->guest_phys_addr & (PAGE_SIZE - 1))
712 goto out;
Izik Eiduse0d62c72007-10-24 23:57:46 +0200713 if (mem->slot >= KVM_MEMORY_SLOTS + KVM_PRIVATE_MEM_SLOTS)
Avi Kivity6aa8b732006-12-10 02:21:36 -0800714 goto out;
715 if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr)
716 goto out;
717
718 memslot = &kvm->memslots[mem->slot];
719 base_gfn = mem->guest_phys_addr >> PAGE_SHIFT;
720 npages = mem->memory_size >> PAGE_SHIFT;
721
722 if (!npages)
723 mem->flags &= ~KVM_MEM_LOG_DIRTY_PAGES;
724
Avi Kivity6aa8b732006-12-10 02:21:36 -0800725 new = old = *memslot;
726
727 new.base_gfn = base_gfn;
728 new.npages = npages;
729 new.flags = mem->flags;
730
731 /* Disallow changing a memory slot's size. */
732 r = -EINVAL;
733 if (npages && old.npages && npages != old.npages)
Sheng Yangf78e0e22007-10-29 09:40:42 +0800734 goto out_free;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800735
736 /* Check for overlaps */
737 r = -EEXIST;
738 for (i = 0; i < KVM_MEMORY_SLOTS; ++i) {
739 struct kvm_memory_slot *s = &kvm->memslots[i];
740
741 if (s == memslot)
742 continue;
743 if (!((base_gfn + npages <= s->base_gfn) ||
744 (base_gfn >= s->base_gfn + s->npages)))
Sheng Yangf78e0e22007-10-29 09:40:42 +0800745 goto out_free;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800746 }
Avi Kivity6aa8b732006-12-10 02:21:36 -0800747
Avi Kivity6aa8b732006-12-10 02:21:36 -0800748 /* Free page dirty bitmap if unneeded */
749 if (!(new.flags & KVM_MEM_LOG_DIRTY_PAGES))
Al Viro8b6d44c2007-02-09 16:38:40 +0000750 new.dirty_bitmap = NULL;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800751
752 r = -ENOMEM;
753
754 /* Allocate if a slot is being created */
Carsten Otteeff01142008-06-27 15:05:31 +0200755#ifndef CONFIG_S390
Anthony Liguori8d4e1282007-10-18 09:59:34 -0500756 if (npages && !new.rmap) {
Mike Dayd77c26f2007-10-08 09:02:08 -0400757 new.rmap = vmalloc(npages * sizeof(struct page *));
Izik Eidus290fc382007-09-27 14:11:22 +0200758
759 if (!new.rmap)
Sheng Yangf78e0e22007-10-29 09:40:42 +0800760 goto out_free;
Izik Eidus290fc382007-09-27 14:11:22 +0200761
Izik Eidus290fc382007-09-27 14:11:22 +0200762 memset(new.rmap, 0, npages * sizeof(*new.rmap));
Anthony Liguori8d4e1282007-10-18 09:59:34 -0500763
Izik Eidus80b14b52007-10-25 11:54:04 +0200764 new.user_alloc = user_alloc;
Andrea Arcangeli604b38a2008-07-25 16:32:03 +0200765 /*
766 * hva_to_rmmap() serialzies with the mmu_lock and to be
767 * safe it has to ignore memslots with !user_alloc &&
768 * !userspace_addr.
769 */
770 if (user_alloc)
771 new.userspace_addr = mem->userspace_addr;
772 else
773 new.userspace_addr = 0;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800774 }
Marcelo Tosatti05da4552008-02-23 11:44:30 -0300775 if (npages && !new.lpage_info) {
776 int largepages = npages / KVM_PAGES_PER_HPAGE;
777 if (npages % KVM_PAGES_PER_HPAGE)
778 largepages++;
779 if (base_gfn % KVM_PAGES_PER_HPAGE)
780 largepages++;
781
782 new.lpage_info = vmalloc(largepages * sizeof(*new.lpage_info));
783
784 if (!new.lpage_info)
785 goto out_free;
786
787 memset(new.lpage_info, 0, largepages * sizeof(*new.lpage_info));
788
789 if (base_gfn % KVM_PAGES_PER_HPAGE)
790 new.lpage_info[0].write_count = 1;
791 if ((base_gfn+npages) % KVM_PAGES_PER_HPAGE)
792 new.lpage_info[largepages-1].write_count = 1;
793 }
Avi Kivity6aa8b732006-12-10 02:21:36 -0800794
795 /* Allocate page dirty bitmap if needed */
796 if ((new.flags & KVM_MEM_LOG_DIRTY_PAGES) && !new.dirty_bitmap) {
797 unsigned dirty_bytes = ALIGN(npages, BITS_PER_LONG) / 8;
798
799 new.dirty_bitmap = vmalloc(dirty_bytes);
800 if (!new.dirty_bitmap)
Sheng Yangf78e0e22007-10-29 09:40:42 +0800801 goto out_free;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800802 memset(new.dirty_bitmap, 0, dirty_bytes);
803 }
Carsten Otteeff01142008-06-27 15:05:31 +0200804#endif /* not defined CONFIG_S390 */
Avi Kivity6aa8b732006-12-10 02:21:36 -0800805
Marcelo Tosatti34d4cb82008-07-10 20:49:31 -0300806 if (!npages)
807 kvm_arch_flush_shadow(kvm);
808
Andrea Arcangeli604b38a2008-07-25 16:32:03 +0200809 spin_lock(&kvm->mmu_lock);
810 if (mem->slot >= kvm->nmemslots)
811 kvm->nmemslots = mem->slot + 1;
812
Avi Kivity6aa8b732006-12-10 02:21:36 -0800813 *memslot = new;
Andrea Arcangeli604b38a2008-07-25 16:32:03 +0200814 spin_unlock(&kvm->mmu_lock);
Avi Kivity6aa8b732006-12-10 02:21:36 -0800815
Zhang Xiantao0de10342007-11-20 16:25:04 +0800816 r = kvm_arch_set_memory_region(kvm, mem, old, user_alloc);
817 if (r) {
Andrea Arcangeli604b38a2008-07-25 16:32:03 +0200818 spin_lock(&kvm->mmu_lock);
Zhang Xiantao0de10342007-11-20 16:25:04 +0800819 *memslot = old;
Andrea Arcangeli604b38a2008-07-25 16:32:03 +0200820 spin_unlock(&kvm->mmu_lock);
Zhang Xiantao0de10342007-11-20 16:25:04 +0800821 goto out_free;
Zhang Xiantao3ad82a72007-11-20 13:11:38 +0800822 }
823
Avi Kivity6aa8b732006-12-10 02:21:36 -0800824 kvm_free_physmem_slot(&old, &new);
Xiantao Zhang8a98f662008-10-06 13:47:38 +0800825#ifdef CONFIG_DMAR
Ben-Ami Yassour62c476c2008-09-14 03:48:28 +0300826 /* map the pages in iommu page table */
827 r = kvm_iommu_map_pages(kvm, base_gfn, npages);
828 if (r)
829 goto out;
Xiantao Zhang8a98f662008-10-06 13:47:38 +0800830#endif
Avi Kivity6aa8b732006-12-10 02:21:36 -0800831 return 0;
832
Sheng Yangf78e0e22007-10-29 09:40:42 +0800833out_free:
Avi Kivity6aa8b732006-12-10 02:21:36 -0800834 kvm_free_physmem_slot(&new, &old);
835out:
836 return r;
Izik Eidus210c7c42007-10-24 23:52:57 +0200837
838}
Sheng Yangf78e0e22007-10-29 09:40:42 +0800839EXPORT_SYMBOL_GPL(__kvm_set_memory_region);
840
841int kvm_set_memory_region(struct kvm *kvm,
842 struct kvm_userspace_memory_region *mem,
843 int user_alloc)
844{
845 int r;
846
Izik Eidus72dc67a2008-02-10 18:04:15 +0200847 down_write(&kvm->slots_lock);
Sheng Yangf78e0e22007-10-29 09:40:42 +0800848 r = __kvm_set_memory_region(kvm, mem, user_alloc);
Izik Eidus72dc67a2008-02-10 18:04:15 +0200849 up_write(&kvm->slots_lock);
Sheng Yangf78e0e22007-10-29 09:40:42 +0800850 return r;
851}
Izik Eidus210c7c42007-10-24 23:52:57 +0200852EXPORT_SYMBOL_GPL(kvm_set_memory_region);
853
Carsten Otte1fe779f2007-10-29 16:08:35 +0100854int kvm_vm_ioctl_set_memory_region(struct kvm *kvm,
855 struct
856 kvm_userspace_memory_region *mem,
857 int user_alloc)
Izik Eidus210c7c42007-10-24 23:52:57 +0200858{
Izik Eiduse0d62c72007-10-24 23:57:46 +0200859 if (mem->slot >= KVM_MEMORY_SLOTS)
860 return -EINVAL;
Izik Eidus210c7c42007-10-24 23:52:57 +0200861 return kvm_set_memory_region(kvm, mem, user_alloc);
Avi Kivity6aa8b732006-12-10 02:21:36 -0800862}
863
Zhang Xiantao5bb064d2007-11-18 20:29:43 +0800864int kvm_get_dirty_log(struct kvm *kvm,
865 struct kvm_dirty_log *log, int *is_dirty)
Avi Kivity6aa8b732006-12-10 02:21:36 -0800866{
867 struct kvm_memory_slot *memslot;
868 int r, i;
869 int n;
870 unsigned long any = 0;
871
Avi Kivity6aa8b732006-12-10 02:21:36 -0800872 r = -EINVAL;
873 if (log->slot >= KVM_MEMORY_SLOTS)
874 goto out;
875
876 memslot = &kvm->memslots[log->slot];
877 r = -ENOENT;
878 if (!memslot->dirty_bitmap)
879 goto out;
880
Uri Lublincd1a4a92007-02-22 16:43:09 +0200881 n = ALIGN(memslot->npages, BITS_PER_LONG) / 8;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800882
Uri Lublincd1a4a92007-02-22 16:43:09 +0200883 for (i = 0; !any && i < n/sizeof(long); ++i)
Avi Kivity6aa8b732006-12-10 02:21:36 -0800884 any = memslot->dirty_bitmap[i];
885
886 r = -EFAULT;
887 if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n))
888 goto out;
889
Zhang Xiantao5bb064d2007-11-18 20:29:43 +0800890 if (any)
891 *is_dirty = 1;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800892
893 r = 0;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800894out:
Avi Kivity6aa8b732006-12-10 02:21:36 -0800895 return r;
896}
897
Izik Eiduscea7bb22007-10-17 19:17:48 +0200898int is_error_page(struct page *page)
899{
900 return page == bad_page;
901}
902EXPORT_SYMBOL_GPL(is_error_page);
903
Anthony Liguori35149e22008-04-02 14:46:56 -0500904int is_error_pfn(pfn_t pfn)
905{
906 return pfn == bad_pfn;
907}
908EXPORT_SYMBOL_GPL(is_error_pfn);
909
Izik Eidusf9d46eb2007-11-11 22:02:22 +0200910static inline unsigned long bad_hva(void)
911{
912 return PAGE_OFFSET;
913}
914
915int kvm_is_error_hva(unsigned long addr)
916{
917 return addr == bad_hva();
918}
919EXPORT_SYMBOL_GPL(kvm_is_error_hva);
920
Avi Kivitye8207542007-03-30 16:54:30 +0300921static struct kvm_memory_slot *__gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
Avi Kivity6aa8b732006-12-10 02:21:36 -0800922{
923 int i;
924
925 for (i = 0; i < kvm->nmemslots; ++i) {
926 struct kvm_memory_slot *memslot = &kvm->memslots[i];
927
928 if (gfn >= memslot->base_gfn
929 && gfn < memslot->base_gfn + memslot->npages)
930 return memslot;
931 }
Al Viro8b6d44c2007-02-09 16:38:40 +0000932 return NULL;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800933}
Avi Kivitye8207542007-03-30 16:54:30 +0300934
935struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
936{
937 gfn = unalias_gfn(kvm, gfn);
938 return __gfn_to_memslot(kvm, gfn);
939}
Avi Kivity6aa8b732006-12-10 02:21:36 -0800940
Izik Eiduse0d62c72007-10-24 23:57:46 +0200941int kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn)
942{
943 int i;
944
945 gfn = unalias_gfn(kvm, gfn);
946 for (i = 0; i < KVM_MEMORY_SLOTS; ++i) {
947 struct kvm_memory_slot *memslot = &kvm->memslots[i];
948
949 if (gfn >= memslot->base_gfn
950 && gfn < memslot->base_gfn + memslot->npages)
951 return 1;
952 }
953 return 0;
954}
955EXPORT_SYMBOL_GPL(kvm_is_visible_gfn);
956
Marcelo Tosatti05da4552008-02-23 11:44:30 -0300957unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn)
Izik Eidus539cb662007-11-11 22:05:04 +0200958{
959 struct kvm_memory_slot *slot;
960
961 gfn = unalias_gfn(kvm, gfn);
962 slot = __gfn_to_memslot(kvm, gfn);
963 if (!slot)
964 return bad_hva();
965 return (slot->userspace_addr + (gfn - slot->base_gfn) * PAGE_SIZE);
966}
Sheng Yang0d150292008-04-25 21:44:50 +0800967EXPORT_SYMBOL_GPL(gfn_to_hva);
Izik Eidus539cb662007-11-11 22:05:04 +0200968
Anthony Liguori35149e22008-04-02 14:46:56 -0500969pfn_t gfn_to_pfn(struct kvm *kvm, gfn_t gfn)
Avi Kivity954bbbc22007-03-30 14:02:32 +0300970{
Anthony Liguori8d4e1282007-10-18 09:59:34 -0500971 struct page *page[1];
Izik Eidus539cb662007-11-11 22:05:04 +0200972 unsigned long addr;
Anthony Liguori8d4e1282007-10-18 09:59:34 -0500973 int npages;
Anthony Liguori2e2e3732008-04-30 15:37:07 -0500974 pfn_t pfn;
Avi Kivity954bbbc22007-03-30 14:02:32 +0300975
Avi Kivity60395222007-10-21 11:03:36 +0200976 might_sleep();
977
Izik Eidus539cb662007-11-11 22:05:04 +0200978 addr = gfn_to_hva(kvm, gfn);
979 if (kvm_is_error_hva(addr)) {
Izik Eidus8a7ae052007-10-18 11:09:33 +0200980 get_page(bad_page);
Anthony Liguori35149e22008-04-02 14:46:56 -0500981 return page_to_pfn(bad_page);
Izik Eidus8a7ae052007-10-18 11:09:33 +0200982 }
Izik Eidus8a7ae052007-10-18 11:09:33 +0200983
Marcelo Tosatti4c2155c2008-09-16 20:54:47 -0300984 npages = get_user_pages_fast(addr, 1, 1, page);
Izik Eidus539cb662007-11-11 22:05:04 +0200985
Anthony Liguori2e2e3732008-04-30 15:37:07 -0500986 if (unlikely(npages != 1)) {
987 struct vm_area_struct *vma;
Anthony Liguori8d4e1282007-10-18 09:59:34 -0500988
Marcelo Tosatti4c2155c2008-09-16 20:54:47 -0300989 down_read(&current->mm->mmap_sem);
Anthony Liguori2e2e3732008-04-30 15:37:07 -0500990 vma = find_vma(current->mm, addr);
Marcelo Tosatti4c2155c2008-09-16 20:54:47 -0300991
Anthony Liguori2e2e3732008-04-30 15:37:07 -0500992 if (vma == NULL || addr < vma->vm_start ||
993 !(vma->vm_flags & VM_PFNMAP)) {
Marcelo Tosatti4c2155c2008-09-16 20:54:47 -0300994 up_read(&current->mm->mmap_sem);
Anthony Liguori2e2e3732008-04-30 15:37:07 -0500995 get_page(bad_page);
996 return page_to_pfn(bad_page);
997 }
998
999 pfn = ((addr - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
Marcelo Tosatti4c2155c2008-09-16 20:54:47 -03001000 up_read(&current->mm->mmap_sem);
Xiantao Zhangc77fb9d2008-09-27 10:55:40 +08001001 BUG_ON(!kvm_is_mmio_pfn(pfn));
Anthony Liguori2e2e3732008-04-30 15:37:07 -05001002 } else
1003 pfn = page_to_pfn(page[0]);
1004
1005 return pfn;
Anthony Liguori35149e22008-04-02 14:46:56 -05001006}
1007
1008EXPORT_SYMBOL_GPL(gfn_to_pfn);
1009
1010struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn)
1011{
Anthony Liguori2e2e3732008-04-30 15:37:07 -05001012 pfn_t pfn;
1013
1014 pfn = gfn_to_pfn(kvm, gfn);
Xiantao Zhangc77fb9d2008-09-27 10:55:40 +08001015 if (!kvm_is_mmio_pfn(pfn))
Anthony Liguori2e2e3732008-04-30 15:37:07 -05001016 return pfn_to_page(pfn);
1017
Xiantao Zhangc77fb9d2008-09-27 10:55:40 +08001018 WARN_ON(kvm_is_mmio_pfn(pfn));
Anthony Liguori2e2e3732008-04-30 15:37:07 -05001019
1020 get_page(bad_page);
1021 return bad_page;
Avi Kivity954bbbc22007-03-30 14:02:32 +03001022}
Anthony Liguoriaab61cc2007-10-29 15:15:20 -05001023
Avi Kivity954bbbc22007-03-30 14:02:32 +03001024EXPORT_SYMBOL_GPL(gfn_to_page);
1025
Izik Eidusb4231d62007-11-20 11:49:33 +02001026void kvm_release_page_clean(struct page *page)
1027{
Anthony Liguori35149e22008-04-02 14:46:56 -05001028 kvm_release_pfn_clean(page_to_pfn(page));
Izik Eidusb4231d62007-11-20 11:49:33 +02001029}
1030EXPORT_SYMBOL_GPL(kvm_release_page_clean);
1031
Anthony Liguori35149e22008-04-02 14:46:56 -05001032void kvm_release_pfn_clean(pfn_t pfn)
1033{
Xiantao Zhangc77fb9d2008-09-27 10:55:40 +08001034 if (!kvm_is_mmio_pfn(pfn))
Anthony Liguori2e2e3732008-04-30 15:37:07 -05001035 put_page(pfn_to_page(pfn));
Anthony Liguori35149e22008-04-02 14:46:56 -05001036}
1037EXPORT_SYMBOL_GPL(kvm_release_pfn_clean);
1038
Izik Eidusb4231d62007-11-20 11:49:33 +02001039void kvm_release_page_dirty(struct page *page)
Izik Eidus8a7ae052007-10-18 11:09:33 +02001040{
Anthony Liguori35149e22008-04-02 14:46:56 -05001041 kvm_release_pfn_dirty(page_to_pfn(page));
Izik Eidus8a7ae052007-10-18 11:09:33 +02001042}
Izik Eidusb4231d62007-11-20 11:49:33 +02001043EXPORT_SYMBOL_GPL(kvm_release_page_dirty);
Izik Eidus8a7ae052007-10-18 11:09:33 +02001044
Anthony Liguori35149e22008-04-02 14:46:56 -05001045void kvm_release_pfn_dirty(pfn_t pfn)
1046{
1047 kvm_set_pfn_dirty(pfn);
1048 kvm_release_pfn_clean(pfn);
1049}
1050EXPORT_SYMBOL_GPL(kvm_release_pfn_dirty);
1051
1052void kvm_set_page_dirty(struct page *page)
1053{
1054 kvm_set_pfn_dirty(page_to_pfn(page));
1055}
1056EXPORT_SYMBOL_GPL(kvm_set_page_dirty);
1057
1058void kvm_set_pfn_dirty(pfn_t pfn)
1059{
Xiantao Zhangc77fb9d2008-09-27 10:55:40 +08001060 if (!kvm_is_mmio_pfn(pfn)) {
Anthony Liguori2e2e3732008-04-30 15:37:07 -05001061 struct page *page = pfn_to_page(pfn);
1062 if (!PageReserved(page))
1063 SetPageDirty(page);
1064 }
Anthony Liguori35149e22008-04-02 14:46:56 -05001065}
1066EXPORT_SYMBOL_GPL(kvm_set_pfn_dirty);
1067
1068void kvm_set_pfn_accessed(pfn_t pfn)
1069{
Xiantao Zhangc77fb9d2008-09-27 10:55:40 +08001070 if (!kvm_is_mmio_pfn(pfn))
Anthony Liguori2e2e3732008-04-30 15:37:07 -05001071 mark_page_accessed(pfn_to_page(pfn));
Anthony Liguori35149e22008-04-02 14:46:56 -05001072}
1073EXPORT_SYMBOL_GPL(kvm_set_pfn_accessed);
1074
1075void kvm_get_pfn(pfn_t pfn)
1076{
Xiantao Zhangc77fb9d2008-09-27 10:55:40 +08001077 if (!kvm_is_mmio_pfn(pfn))
Anthony Liguori2e2e3732008-04-30 15:37:07 -05001078 get_page(pfn_to_page(pfn));
Anthony Liguori35149e22008-04-02 14:46:56 -05001079}
1080EXPORT_SYMBOL_GPL(kvm_get_pfn);
1081
Izik Eidus195aefd2007-10-01 22:14:18 +02001082static int next_segment(unsigned long len, int offset)
1083{
1084 if (len > PAGE_SIZE - offset)
1085 return PAGE_SIZE - offset;
1086 else
1087 return len;
1088}
1089
1090int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset,
1091 int len)
1092{
Izik Eiduse0506bc2007-11-11 22:10:22 +02001093 int r;
1094 unsigned long addr;
Izik Eidus195aefd2007-10-01 22:14:18 +02001095
Izik Eiduse0506bc2007-11-11 22:10:22 +02001096 addr = gfn_to_hva(kvm, gfn);
1097 if (kvm_is_error_hva(addr))
Izik Eidus195aefd2007-10-01 22:14:18 +02001098 return -EFAULT;
Izik Eiduse0506bc2007-11-11 22:10:22 +02001099 r = copy_from_user(data, (void __user *)addr + offset, len);
1100 if (r)
1101 return -EFAULT;
Izik Eidus195aefd2007-10-01 22:14:18 +02001102 return 0;
1103}
1104EXPORT_SYMBOL_GPL(kvm_read_guest_page);
1105
1106int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len)
1107{
1108 gfn_t gfn = gpa >> PAGE_SHIFT;
1109 int seg;
1110 int offset = offset_in_page(gpa);
1111 int ret;
1112
1113 while ((seg = next_segment(len, offset)) != 0) {
1114 ret = kvm_read_guest_page(kvm, gfn, data, offset, seg);
1115 if (ret < 0)
1116 return ret;
1117 offset = 0;
1118 len -= seg;
1119 data += seg;
1120 ++gfn;
1121 }
1122 return 0;
1123}
1124EXPORT_SYMBOL_GPL(kvm_read_guest);
1125
Marcelo Tosatti7ec54582007-12-20 19:18:23 -05001126int kvm_read_guest_atomic(struct kvm *kvm, gpa_t gpa, void *data,
1127 unsigned long len)
1128{
1129 int r;
1130 unsigned long addr;
1131 gfn_t gfn = gpa >> PAGE_SHIFT;
1132 int offset = offset_in_page(gpa);
1133
1134 addr = gfn_to_hva(kvm, gfn);
1135 if (kvm_is_error_hva(addr))
1136 return -EFAULT;
Andrea Arcangeli0aac03f2008-01-30 19:57:35 +01001137 pagefault_disable();
Marcelo Tosatti7ec54582007-12-20 19:18:23 -05001138 r = __copy_from_user_inatomic(data, (void __user *)addr + offset, len);
Andrea Arcangeli0aac03f2008-01-30 19:57:35 +01001139 pagefault_enable();
Marcelo Tosatti7ec54582007-12-20 19:18:23 -05001140 if (r)
1141 return -EFAULT;
1142 return 0;
1143}
1144EXPORT_SYMBOL(kvm_read_guest_atomic);
1145
Izik Eidus195aefd2007-10-01 22:14:18 +02001146int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn, const void *data,
1147 int offset, int len)
1148{
Izik Eiduse0506bc2007-11-11 22:10:22 +02001149 int r;
1150 unsigned long addr;
Izik Eidus195aefd2007-10-01 22:14:18 +02001151
Izik Eiduse0506bc2007-11-11 22:10:22 +02001152 addr = gfn_to_hva(kvm, gfn);
1153 if (kvm_is_error_hva(addr))
Izik Eidus195aefd2007-10-01 22:14:18 +02001154 return -EFAULT;
Izik Eiduse0506bc2007-11-11 22:10:22 +02001155 r = copy_to_user((void __user *)addr + offset, data, len);
1156 if (r)
1157 return -EFAULT;
Izik Eidus195aefd2007-10-01 22:14:18 +02001158 mark_page_dirty(kvm, gfn);
1159 return 0;
1160}
1161EXPORT_SYMBOL_GPL(kvm_write_guest_page);
1162
1163int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data,
1164 unsigned long len)
1165{
1166 gfn_t gfn = gpa >> PAGE_SHIFT;
1167 int seg;
1168 int offset = offset_in_page(gpa);
1169 int ret;
1170
1171 while ((seg = next_segment(len, offset)) != 0) {
1172 ret = kvm_write_guest_page(kvm, gfn, data, offset, seg);
1173 if (ret < 0)
1174 return ret;
1175 offset = 0;
1176 len -= seg;
1177 data += seg;
1178 ++gfn;
1179 }
1180 return 0;
1181}
1182
1183int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len)
1184{
Izik Eidus3e021bf2007-11-19 11:16:57 +02001185 return kvm_write_guest_page(kvm, gfn, empty_zero_page, offset, len);
Izik Eidus195aefd2007-10-01 22:14:18 +02001186}
1187EXPORT_SYMBOL_GPL(kvm_clear_guest_page);
1188
1189int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len)
1190{
1191 gfn_t gfn = gpa >> PAGE_SHIFT;
1192 int seg;
1193 int offset = offset_in_page(gpa);
1194 int ret;
1195
1196 while ((seg = next_segment(len, offset)) != 0) {
1197 ret = kvm_clear_guest_page(kvm, gfn, offset, seg);
1198 if (ret < 0)
1199 return ret;
1200 offset = 0;
1201 len -= seg;
1202 ++gfn;
1203 }
1204 return 0;
1205}
1206EXPORT_SYMBOL_GPL(kvm_clear_guest);
1207
Avi Kivity6aa8b732006-12-10 02:21:36 -08001208void mark_page_dirty(struct kvm *kvm, gfn_t gfn)
1209{
Nguyen Anh Quynh31389942007-06-05 10:35:19 +03001210 struct kvm_memory_slot *memslot;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001211
Uri Lublin3b6fff12007-10-30 10:42:09 +02001212 gfn = unalias_gfn(kvm, gfn);
Rusty Russell7e9d6192007-07-31 20:41:14 +10001213 memslot = __gfn_to_memslot(kvm, gfn);
1214 if (memslot && memslot->dirty_bitmap) {
1215 unsigned long rel_gfn = gfn - memslot->base_gfn;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001216
Rusty Russell7e9d6192007-07-31 20:41:14 +10001217 /* avoid RMW */
1218 if (!test_bit(rel_gfn, memslot->dirty_bitmap))
1219 set_bit(rel_gfn, memslot->dirty_bitmap);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001220 }
1221}
1222
Eddie Dongb6958ce2007-07-18 12:15:21 +03001223/*
1224 * The vCPU has executed a HLT instruction with in-kernel mode enabled.
1225 */
Hollis Blanchard8776e512007-10-31 17:24:24 -05001226void kvm_vcpu_block(struct kvm_vcpu *vcpu)
Eddie Dongb6958ce2007-07-18 12:15:21 +03001227{
Marcelo Tosattie5c239c2008-05-08 19:47:01 -03001228 DEFINE_WAIT(wait);
Eddie Dongb6958ce2007-07-18 12:15:21 +03001229
Marcelo Tosattie5c239c2008-05-08 19:47:01 -03001230 for (;;) {
1231 prepare_to_wait(&vcpu->wq, &wait, TASK_INTERRUPTIBLE);
Eddie Dongb6958ce2007-07-18 12:15:21 +03001232
Marcelo Tosattid7690172008-09-08 15:23:48 -03001233 if (kvm_cpu_has_interrupt(vcpu) ||
1234 kvm_cpu_has_pending_timer(vcpu) ||
1235 kvm_arch_vcpu_runnable(vcpu)) {
1236 set_bit(KVM_REQ_UNHALT, &vcpu->requests);
Marcelo Tosattie5c239c2008-05-08 19:47:01 -03001237 break;
Marcelo Tosattid7690172008-09-08 15:23:48 -03001238 }
Marcelo Tosattie5c239c2008-05-08 19:47:01 -03001239 if (signal_pending(current))
1240 break;
1241
Eddie Dongb6958ce2007-07-18 12:15:21 +03001242 vcpu_put(vcpu);
1243 schedule();
1244 vcpu_load(vcpu);
1245 }
1246
Marcelo Tosattie5c239c2008-05-08 19:47:01 -03001247 finish_wait(&vcpu->wq, &wait);
Eddie Dongb6958ce2007-07-18 12:15:21 +03001248}
1249
Avi Kivity6aa8b732006-12-10 02:21:36 -08001250void kvm_resched(struct kvm_vcpu *vcpu)
1251{
Yaozu Dong3fca0362007-04-25 16:49:19 +03001252 if (!need_resched())
1253 return;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001254 cond_resched();
Avi Kivity6aa8b732006-12-10 02:21:36 -08001255}
1256EXPORT_SYMBOL_GPL(kvm_resched);
1257
npiggin@suse.dee4a533a2007-12-05 18:15:52 +11001258static int kvm_vcpu_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
Avi Kivity9a2bb7f2007-02-22 12:58:31 +02001259{
1260 struct kvm_vcpu *vcpu = vma->vm_file->private_data;
Avi Kivity9a2bb7f2007-02-22 12:58:31 +02001261 struct page *page;
1262
npiggin@suse.dee4a533a2007-12-05 18:15:52 +11001263 if (vmf->pgoff == 0)
Avi Kivity039576c2007-03-20 12:46:50 +02001264 page = virt_to_page(vcpu->run);
Avi Kivity09566762008-01-23 18:14:23 +02001265#ifdef CONFIG_X86
npiggin@suse.dee4a533a2007-12-05 18:15:52 +11001266 else if (vmf->pgoff == KVM_PIO_PAGE_OFFSET)
Zhang Xiantaoad312c72007-12-13 23:50:52 +08001267 page = virt_to_page(vcpu->arch.pio_data);
Avi Kivity09566762008-01-23 18:14:23 +02001268#endif
Laurent Vivier5f94c172008-05-30 16:05:54 +02001269#ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
1270 else if (vmf->pgoff == KVM_COALESCED_MMIO_PAGE_OFFSET)
1271 page = virt_to_page(vcpu->kvm->coalesced_mmio_ring);
1272#endif
Avi Kivity039576c2007-03-20 12:46:50 +02001273 else
npiggin@suse.dee4a533a2007-12-05 18:15:52 +11001274 return VM_FAULT_SIGBUS;
Avi Kivity9a2bb7f2007-02-22 12:58:31 +02001275 get_page(page);
npiggin@suse.dee4a533a2007-12-05 18:15:52 +11001276 vmf->page = page;
1277 return 0;
Avi Kivity9a2bb7f2007-02-22 12:58:31 +02001278}
1279
1280static struct vm_operations_struct kvm_vcpu_vm_ops = {
npiggin@suse.dee4a533a2007-12-05 18:15:52 +11001281 .fault = kvm_vcpu_fault,
Avi Kivity9a2bb7f2007-02-22 12:58:31 +02001282};
1283
1284static int kvm_vcpu_mmap(struct file *file, struct vm_area_struct *vma)
1285{
1286 vma->vm_ops = &kvm_vcpu_vm_ops;
1287 return 0;
1288}
1289
Avi Kivitybccf2152007-02-21 18:04:26 +02001290static int kvm_vcpu_release(struct inode *inode, struct file *filp)
1291{
1292 struct kvm_vcpu *vcpu = filp->private_data;
1293
Al Viro66c0b392008-04-19 20:33:56 +01001294 kvm_put_kvm(vcpu->kvm);
Avi Kivitybccf2152007-02-21 18:04:26 +02001295 return 0;
1296}
1297
Jan Engelhardt5c502742008-01-22 20:46:14 +01001298static const struct file_operations kvm_vcpu_fops = {
Avi Kivitybccf2152007-02-21 18:04:26 +02001299 .release = kvm_vcpu_release,
1300 .unlocked_ioctl = kvm_vcpu_ioctl,
1301 .compat_ioctl = kvm_vcpu_ioctl,
Avi Kivity9a2bb7f2007-02-22 12:58:31 +02001302 .mmap = kvm_vcpu_mmap,
Avi Kivitybccf2152007-02-21 18:04:26 +02001303};
1304
1305/*
1306 * Allocates an inode for the vcpu.
1307 */
1308static int create_vcpu_fd(struct kvm_vcpu *vcpu)
1309{
Ulrich Drepper7d9dbca2008-07-23 21:29:22 -07001310 int fd = anon_inode_getfd("kvm-vcpu", &kvm_vcpu_fops, vcpu, 0);
Al Viro2030a422008-02-23 06:46:49 -05001311 if (fd < 0)
Al Viro66c0b392008-04-19 20:33:56 +01001312 kvm_put_kvm(vcpu->kvm);
Avi Kivitybccf2152007-02-21 18:04:26 +02001313 return fd;
Avi Kivitybccf2152007-02-21 18:04:26 +02001314}
1315
Avi Kivityc5ea7662007-02-20 18:41:05 +02001316/*
1317 * Creates some virtual cpus. Good luck creating more than one.
1318 */
1319static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, int n)
1320{
1321 int r;
1322 struct kvm_vcpu *vcpu;
1323
Avi Kivityc5ea7662007-02-20 18:41:05 +02001324 if (!valid_vcpu(n))
Rusty Russellfb3f0f52007-07-27 17:16:56 +10001325 return -EINVAL;
Avi Kivityc5ea7662007-02-20 18:41:05 +02001326
Zhang Xiantaoe9b11c12007-11-14 20:38:21 +08001327 vcpu = kvm_arch_vcpu_create(kvm, n);
Rusty Russellfb3f0f52007-07-27 17:16:56 +10001328 if (IS_ERR(vcpu))
1329 return PTR_ERR(vcpu);
Avi Kivityc5ea7662007-02-20 18:41:05 +02001330
Avi Kivity15ad7142007-07-11 18:17:21 +03001331 preempt_notifier_init(&vcpu->preempt_notifier, &kvm_preempt_ops);
1332
Avi Kivity26e52152007-11-20 15:30:24 +02001333 r = kvm_arch_vcpu_setup(vcpu);
1334 if (r)
Glauber Costa7d8fece2008-09-17 23:16:59 -03001335 return r;
Avi Kivity26e52152007-11-20 15:30:24 +02001336
Shaohua Li11ec2802007-07-23 14:51:37 +08001337 mutex_lock(&kvm->lock);
Rusty Russellfb3f0f52007-07-27 17:16:56 +10001338 if (kvm->vcpus[n]) {
1339 r = -EEXIST;
Zhang Xiantaoe9b11c12007-11-14 20:38:21 +08001340 goto vcpu_destroy;
Rusty Russellfb3f0f52007-07-27 17:16:56 +10001341 }
1342 kvm->vcpus[n] = vcpu;
Shaohua Li11ec2802007-07-23 14:51:37 +08001343 mutex_unlock(&kvm->lock);
Rusty Russellfb3f0f52007-07-27 17:16:56 +10001344
1345 /* Now it's all set up, let userspace reach it */
Al Viro66c0b392008-04-19 20:33:56 +01001346 kvm_get_kvm(kvm);
Avi Kivitybccf2152007-02-21 18:04:26 +02001347 r = create_vcpu_fd(vcpu);
1348 if (r < 0)
Rusty Russellfb3f0f52007-07-27 17:16:56 +10001349 goto unlink;
Avi Kivitybccf2152007-02-21 18:04:26 +02001350 return r;
Avi Kivityc5ea7662007-02-20 18:41:05 +02001351
Rusty Russellfb3f0f52007-07-27 17:16:56 +10001352unlink:
Shaohua Li11ec2802007-07-23 14:51:37 +08001353 mutex_lock(&kvm->lock);
Rusty Russellfb3f0f52007-07-27 17:16:56 +10001354 kvm->vcpus[n] = NULL;
Zhang Xiantaoe9b11c12007-11-14 20:38:21 +08001355vcpu_destroy:
Glauber Costa7d8fece2008-09-17 23:16:59 -03001356 mutex_unlock(&kvm->lock);
Hollis Blanchardd40ccc62007-11-19 14:04:43 -06001357 kvm_arch_vcpu_destroy(vcpu);
Avi Kivityc5ea7662007-02-20 18:41:05 +02001358 return r;
1359}
1360
Avi Kivity1961d272007-03-05 19:46:05 +02001361static int kvm_vcpu_ioctl_set_sigmask(struct kvm_vcpu *vcpu, sigset_t *sigset)
1362{
1363 if (sigset) {
1364 sigdelsetmask(sigset, sigmask(SIGKILL)|sigmask(SIGSTOP));
1365 vcpu->sigset_active = 1;
1366 vcpu->sigset = *sigset;
1367 } else
1368 vcpu->sigset_active = 0;
1369 return 0;
1370}
1371
Avi Kivitybccf2152007-02-21 18:04:26 +02001372static long kvm_vcpu_ioctl(struct file *filp,
1373 unsigned int ioctl, unsigned long arg)
Avi Kivity6aa8b732006-12-10 02:21:36 -08001374{
Avi Kivitybccf2152007-02-21 18:04:26 +02001375 struct kvm_vcpu *vcpu = filp->private_data;
Al Viro2f366982007-02-09 16:38:35 +00001376 void __user *argp = (void __user *)arg;
Carsten Otte313a3dc2007-10-11 19:16:52 +02001377 int r;
Dave Hansenfa3795a2008-08-11 10:01:46 -07001378 struct kvm_fpu *fpu = NULL;
1379 struct kvm_sregs *kvm_sregs = NULL;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001380
Avi Kivity6d4e4c42007-11-21 16:41:05 +02001381 if (vcpu->kvm->mm != current->mm)
1382 return -EIO;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001383 switch (ioctl) {
Avi Kivity9a2bb7f2007-02-22 12:58:31 +02001384 case KVM_RUN:
Avi Kivityf0fe5102007-03-07 13:11:17 +02001385 r = -EINVAL;
1386 if (arg)
1387 goto out;
Hollis Blanchardb6c7a5d2007-11-01 14:16:10 -05001388 r = kvm_arch_vcpu_ioctl_run(vcpu, vcpu->run);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001389 break;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001390 case KVM_GET_REGS: {
Xiantao Zhang3e4bb3a2008-02-25 18:52:20 +08001391 struct kvm_regs *kvm_regs;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001392
Xiantao Zhang3e4bb3a2008-02-25 18:52:20 +08001393 r = -ENOMEM;
1394 kvm_regs = kzalloc(sizeof(struct kvm_regs), GFP_KERNEL);
1395 if (!kvm_regs)
1396 goto out;
1397 r = kvm_arch_vcpu_ioctl_get_regs(vcpu, kvm_regs);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001398 if (r)
Xiantao Zhang3e4bb3a2008-02-25 18:52:20 +08001399 goto out_free1;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001400 r = -EFAULT;
Xiantao Zhang3e4bb3a2008-02-25 18:52:20 +08001401 if (copy_to_user(argp, kvm_regs, sizeof(struct kvm_regs)))
1402 goto out_free1;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001403 r = 0;
Xiantao Zhang3e4bb3a2008-02-25 18:52:20 +08001404out_free1:
1405 kfree(kvm_regs);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001406 break;
1407 }
1408 case KVM_SET_REGS: {
Xiantao Zhang3e4bb3a2008-02-25 18:52:20 +08001409 struct kvm_regs *kvm_regs;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001410
Xiantao Zhang3e4bb3a2008-02-25 18:52:20 +08001411 r = -ENOMEM;
1412 kvm_regs = kzalloc(sizeof(struct kvm_regs), GFP_KERNEL);
1413 if (!kvm_regs)
1414 goto out;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001415 r = -EFAULT;
Xiantao Zhang3e4bb3a2008-02-25 18:52:20 +08001416 if (copy_from_user(kvm_regs, argp, sizeof(struct kvm_regs)))
1417 goto out_free2;
1418 r = kvm_arch_vcpu_ioctl_set_regs(vcpu, kvm_regs);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001419 if (r)
Xiantao Zhang3e4bb3a2008-02-25 18:52:20 +08001420 goto out_free2;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001421 r = 0;
Xiantao Zhang3e4bb3a2008-02-25 18:52:20 +08001422out_free2:
1423 kfree(kvm_regs);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001424 break;
1425 }
1426 case KVM_GET_SREGS: {
Dave Hansenfa3795a2008-08-11 10:01:46 -07001427 kvm_sregs = kzalloc(sizeof(struct kvm_sregs), GFP_KERNEL);
1428 r = -ENOMEM;
1429 if (!kvm_sregs)
1430 goto out;
1431 r = kvm_arch_vcpu_ioctl_get_sregs(vcpu, kvm_sregs);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001432 if (r)
1433 goto out;
1434 r = -EFAULT;
Dave Hansenfa3795a2008-08-11 10:01:46 -07001435 if (copy_to_user(argp, kvm_sregs, sizeof(struct kvm_sregs)))
Avi Kivity6aa8b732006-12-10 02:21:36 -08001436 goto out;
1437 r = 0;
1438 break;
1439 }
1440 case KVM_SET_SREGS: {
Dave Hansenfa3795a2008-08-11 10:01:46 -07001441 kvm_sregs = kmalloc(sizeof(struct kvm_sregs), GFP_KERNEL);
1442 r = -ENOMEM;
1443 if (!kvm_sregs)
Avi Kivity6aa8b732006-12-10 02:21:36 -08001444 goto out;
Dave Hansenfa3795a2008-08-11 10:01:46 -07001445 r = -EFAULT;
1446 if (copy_from_user(kvm_sregs, argp, sizeof(struct kvm_sregs)))
1447 goto out;
1448 r = kvm_arch_vcpu_ioctl_set_sregs(vcpu, kvm_sregs);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001449 if (r)
1450 goto out;
1451 r = 0;
1452 break;
1453 }
Marcelo Tosatti62d9f0d2008-04-11 13:24:45 -03001454 case KVM_GET_MP_STATE: {
1455 struct kvm_mp_state mp_state;
1456
1457 r = kvm_arch_vcpu_ioctl_get_mpstate(vcpu, &mp_state);
1458 if (r)
1459 goto out;
1460 r = -EFAULT;
1461 if (copy_to_user(argp, &mp_state, sizeof mp_state))
1462 goto out;
1463 r = 0;
1464 break;
1465 }
1466 case KVM_SET_MP_STATE: {
1467 struct kvm_mp_state mp_state;
1468
1469 r = -EFAULT;
1470 if (copy_from_user(&mp_state, argp, sizeof mp_state))
1471 goto out;
1472 r = kvm_arch_vcpu_ioctl_set_mpstate(vcpu, &mp_state);
1473 if (r)
1474 goto out;
1475 r = 0;
1476 break;
1477 }
Avi Kivity6aa8b732006-12-10 02:21:36 -08001478 case KVM_TRANSLATE: {
1479 struct kvm_translation tr;
1480
1481 r = -EFAULT;
Al Viro2f366982007-02-09 16:38:35 +00001482 if (copy_from_user(&tr, argp, sizeof tr))
Avi Kivity6aa8b732006-12-10 02:21:36 -08001483 goto out;
Zhang Xiantao8b006792007-11-16 13:05:55 +08001484 r = kvm_arch_vcpu_ioctl_translate(vcpu, &tr);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001485 if (r)
1486 goto out;
1487 r = -EFAULT;
Al Viro2f366982007-02-09 16:38:35 +00001488 if (copy_to_user(argp, &tr, sizeof tr))
Avi Kivity6aa8b732006-12-10 02:21:36 -08001489 goto out;
1490 r = 0;
1491 break;
1492 }
Avi Kivity6aa8b732006-12-10 02:21:36 -08001493 case KVM_DEBUG_GUEST: {
1494 struct kvm_debug_guest dbg;
1495
1496 r = -EFAULT;
Al Viro2f366982007-02-09 16:38:35 +00001497 if (copy_from_user(&dbg, argp, sizeof dbg))
Avi Kivity6aa8b732006-12-10 02:21:36 -08001498 goto out;
Hollis Blanchardb6c7a5d2007-11-01 14:16:10 -05001499 r = kvm_arch_vcpu_ioctl_debug_guest(vcpu, &dbg);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001500 if (r)
1501 goto out;
1502 r = 0;
1503 break;
1504 }
Avi Kivity1961d272007-03-05 19:46:05 +02001505 case KVM_SET_SIGNAL_MASK: {
1506 struct kvm_signal_mask __user *sigmask_arg = argp;
1507 struct kvm_signal_mask kvm_sigmask;
1508 sigset_t sigset, *p;
1509
1510 p = NULL;
1511 if (argp) {
1512 r = -EFAULT;
1513 if (copy_from_user(&kvm_sigmask, argp,
1514 sizeof kvm_sigmask))
1515 goto out;
1516 r = -EINVAL;
1517 if (kvm_sigmask.len != sizeof sigset)
1518 goto out;
1519 r = -EFAULT;
1520 if (copy_from_user(&sigset, sigmask_arg->sigset,
1521 sizeof sigset))
1522 goto out;
1523 p = &sigset;
1524 }
1525 r = kvm_vcpu_ioctl_set_sigmask(vcpu, &sigset);
1526 break;
1527 }
Avi Kivityb8836732007-04-01 16:34:31 +03001528 case KVM_GET_FPU: {
Dave Hansenfa3795a2008-08-11 10:01:46 -07001529 fpu = kzalloc(sizeof(struct kvm_fpu), GFP_KERNEL);
1530 r = -ENOMEM;
1531 if (!fpu)
1532 goto out;
1533 r = kvm_arch_vcpu_ioctl_get_fpu(vcpu, fpu);
Avi Kivityb8836732007-04-01 16:34:31 +03001534 if (r)
1535 goto out;
1536 r = -EFAULT;
Dave Hansenfa3795a2008-08-11 10:01:46 -07001537 if (copy_to_user(argp, fpu, sizeof(struct kvm_fpu)))
Avi Kivityb8836732007-04-01 16:34:31 +03001538 goto out;
1539 r = 0;
1540 break;
1541 }
1542 case KVM_SET_FPU: {
Dave Hansenfa3795a2008-08-11 10:01:46 -07001543 fpu = kmalloc(sizeof(struct kvm_fpu), GFP_KERNEL);
1544 r = -ENOMEM;
1545 if (!fpu)
Avi Kivityb8836732007-04-01 16:34:31 +03001546 goto out;
Dave Hansenfa3795a2008-08-11 10:01:46 -07001547 r = -EFAULT;
1548 if (copy_from_user(fpu, argp, sizeof(struct kvm_fpu)))
1549 goto out;
1550 r = kvm_arch_vcpu_ioctl_set_fpu(vcpu, fpu);
Avi Kivityb8836732007-04-01 16:34:31 +03001551 if (r)
1552 goto out;
1553 r = 0;
1554 break;
1555 }
Avi Kivitybccf2152007-02-21 18:04:26 +02001556 default:
Carsten Otte313a3dc2007-10-11 19:16:52 +02001557 r = kvm_arch_vcpu_ioctl(filp, ioctl, arg);
Avi Kivitybccf2152007-02-21 18:04:26 +02001558 }
1559out:
Dave Hansenfa3795a2008-08-11 10:01:46 -07001560 kfree(fpu);
1561 kfree(kvm_sregs);
Avi Kivitybccf2152007-02-21 18:04:26 +02001562 return r;
1563}
1564
1565static long kvm_vm_ioctl(struct file *filp,
1566 unsigned int ioctl, unsigned long arg)
1567{
1568 struct kvm *kvm = filp->private_data;
1569 void __user *argp = (void __user *)arg;
Carsten Otte1fe779f2007-10-29 16:08:35 +01001570 int r;
Avi Kivitybccf2152007-02-21 18:04:26 +02001571
Avi Kivity6d4e4c42007-11-21 16:41:05 +02001572 if (kvm->mm != current->mm)
1573 return -EIO;
Avi Kivitybccf2152007-02-21 18:04:26 +02001574 switch (ioctl) {
1575 case KVM_CREATE_VCPU:
1576 r = kvm_vm_ioctl_create_vcpu(kvm, arg);
1577 if (r < 0)
1578 goto out;
1579 break;
Izik Eidus6fc138d2007-10-09 19:20:39 +02001580 case KVM_SET_USER_MEMORY_REGION: {
1581 struct kvm_userspace_memory_region kvm_userspace_mem;
1582
1583 r = -EFAULT;
1584 if (copy_from_user(&kvm_userspace_mem, argp,
1585 sizeof kvm_userspace_mem))
1586 goto out;
1587
1588 r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem, 1);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001589 if (r)
1590 goto out;
1591 break;
1592 }
1593 case KVM_GET_DIRTY_LOG: {
1594 struct kvm_dirty_log log;
1595
1596 r = -EFAULT;
Al Viro2f366982007-02-09 16:38:35 +00001597 if (copy_from_user(&log, argp, sizeof log))
Avi Kivity6aa8b732006-12-10 02:21:36 -08001598 goto out;
Avi Kivity2c6f5df2007-02-20 18:27:58 +02001599 r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001600 if (r)
1601 goto out;
1602 break;
1603 }
Laurent Vivier5f94c172008-05-30 16:05:54 +02001604#ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
1605 case KVM_REGISTER_COALESCED_MMIO: {
1606 struct kvm_coalesced_mmio_zone zone;
1607 r = -EFAULT;
1608 if (copy_from_user(&zone, argp, sizeof zone))
1609 goto out;
1610 r = -ENXIO;
1611 r = kvm_vm_ioctl_register_coalesced_mmio(kvm, &zone);
1612 if (r)
1613 goto out;
1614 r = 0;
1615 break;
1616 }
1617 case KVM_UNREGISTER_COALESCED_MMIO: {
1618 struct kvm_coalesced_mmio_zone zone;
1619 r = -EFAULT;
1620 if (copy_from_user(&zone, argp, sizeof zone))
1621 goto out;
1622 r = -ENXIO;
1623 r = kvm_vm_ioctl_unregister_coalesced_mmio(kvm, &zone);
1624 if (r)
1625 goto out;
1626 r = 0;
1627 break;
1628 }
1629#endif
Xiantao Zhang8a98f662008-10-06 13:47:38 +08001630#ifdef KVM_CAP_DEVICE_ASSIGNMENT
1631 case KVM_ASSIGN_PCI_DEVICE: {
1632 struct kvm_assigned_pci_dev assigned_dev;
1633
1634 r = -EFAULT;
1635 if (copy_from_user(&assigned_dev, argp, sizeof assigned_dev))
1636 goto out;
1637 r = kvm_vm_ioctl_assign_device(kvm, &assigned_dev);
1638 if (r)
1639 goto out;
1640 break;
1641 }
1642 case KVM_ASSIGN_IRQ: {
1643 struct kvm_assigned_irq assigned_irq;
1644
1645 r = -EFAULT;
1646 if (copy_from_user(&assigned_irq, argp, sizeof assigned_irq))
1647 goto out;
1648 r = kvm_vm_ioctl_assign_irq(kvm, &assigned_irq);
1649 if (r)
1650 goto out;
1651 break;
1652 }
1653#endif
Avi Kivityf17abe92007-02-21 19:28:04 +02001654 default:
Carsten Otte1fe779f2007-10-29 16:08:35 +01001655 r = kvm_arch_vm_ioctl(filp, ioctl, arg);
Avi Kivityf17abe92007-02-21 19:28:04 +02001656 }
1657out:
1658 return r;
1659}
1660
npiggin@suse.dee4a533a2007-12-05 18:15:52 +11001661static int kvm_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
Avi Kivityf17abe92007-02-21 19:28:04 +02001662{
Marcelo Tosatti777b3f42008-09-16 20:54:46 -03001663 struct page *page[1];
1664 unsigned long addr;
1665 int npages;
1666 gfn_t gfn = vmf->pgoff;
Avi Kivityf17abe92007-02-21 19:28:04 +02001667 struct kvm *kvm = vma->vm_file->private_data;
Avi Kivityf17abe92007-02-21 19:28:04 +02001668
Marcelo Tosatti777b3f42008-09-16 20:54:46 -03001669 addr = gfn_to_hva(kvm, gfn);
1670 if (kvm_is_error_hva(addr))
npiggin@suse.dee4a533a2007-12-05 18:15:52 +11001671 return VM_FAULT_SIGBUS;
Marcelo Tosatti777b3f42008-09-16 20:54:46 -03001672
1673 npages = get_user_pages(current, current->mm, addr, 1, 1, 0, page,
1674 NULL);
1675 if (unlikely(npages != 1))
npiggin@suse.dee4a533a2007-12-05 18:15:52 +11001676 return VM_FAULT_SIGBUS;
Marcelo Tosatti777b3f42008-09-16 20:54:46 -03001677
1678 vmf->page = page[0];
npiggin@suse.dee4a533a2007-12-05 18:15:52 +11001679 return 0;
Avi Kivityf17abe92007-02-21 19:28:04 +02001680}
1681
1682static struct vm_operations_struct kvm_vm_vm_ops = {
npiggin@suse.dee4a533a2007-12-05 18:15:52 +11001683 .fault = kvm_vm_fault,
Avi Kivityf17abe92007-02-21 19:28:04 +02001684};
1685
1686static int kvm_vm_mmap(struct file *file, struct vm_area_struct *vma)
1687{
1688 vma->vm_ops = &kvm_vm_vm_ops;
1689 return 0;
1690}
1691
Jan Engelhardt5c502742008-01-22 20:46:14 +01001692static const struct file_operations kvm_vm_fops = {
Avi Kivityf17abe92007-02-21 19:28:04 +02001693 .release = kvm_vm_release,
1694 .unlocked_ioctl = kvm_vm_ioctl,
1695 .compat_ioctl = kvm_vm_ioctl,
1696 .mmap = kvm_vm_mmap,
1697};
1698
1699static int kvm_dev_ioctl_create_vm(void)
1700{
Al Viro2030a422008-02-23 06:46:49 -05001701 int fd;
Avi Kivityf17abe92007-02-21 19:28:04 +02001702 struct kvm *kvm;
1703
Avi Kivityf17abe92007-02-21 19:28:04 +02001704 kvm = kvm_create_vm();
Avi Kivityd6d28162007-06-28 08:38:16 -04001705 if (IS_ERR(kvm))
1706 return PTR_ERR(kvm);
Ulrich Drepper7d9dbca2008-07-23 21:29:22 -07001707 fd = anon_inode_getfd("kvm-vm", &kvm_vm_fops, kvm, 0);
Al Viro2030a422008-02-23 06:46:49 -05001708 if (fd < 0)
Al Viro66c0b392008-04-19 20:33:56 +01001709 kvm_put_kvm(kvm);
Avi Kivityf17abe92007-02-21 19:28:04 +02001710
Avi Kivityf17abe92007-02-21 19:28:04 +02001711 return fd;
Avi Kivityf17abe92007-02-21 19:28:04 +02001712}
1713
1714static long kvm_dev_ioctl(struct file *filp,
1715 unsigned int ioctl, unsigned long arg)
1716{
Avi Kivity07c45a32007-03-07 13:05:38 +02001717 long r = -EINVAL;
Avi Kivityf17abe92007-02-21 19:28:04 +02001718
1719 switch (ioctl) {
1720 case KVM_GET_API_VERSION:
Avi Kivityf0fe5102007-03-07 13:11:17 +02001721 r = -EINVAL;
1722 if (arg)
1723 goto out;
Avi Kivityf17abe92007-02-21 19:28:04 +02001724 r = KVM_API_VERSION;
1725 break;
1726 case KVM_CREATE_VM:
Avi Kivityf0fe5102007-03-07 13:11:17 +02001727 r = -EINVAL;
1728 if (arg)
1729 goto out;
Avi Kivityf17abe92007-02-21 19:28:04 +02001730 r = kvm_dev_ioctl_create_vm();
1731 break;
Zhang Xiantao018d00d2007-11-15 23:07:47 +08001732 case KVM_CHECK_EXTENSION:
Christian Borntraeger1e1c65e2008-04-21 13:48:24 +02001733 r = kvm_dev_ioctl_check_extension(arg);
Avi Kivity5d308f42007-03-01 17:56:20 +02001734 break;
Avi Kivity07c45a32007-03-07 13:05:38 +02001735 case KVM_GET_VCPU_MMAP_SIZE:
1736 r = -EINVAL;
1737 if (arg)
1738 goto out;
Avi Kivityadb1ff42008-01-24 15:13:08 +02001739 r = PAGE_SIZE; /* struct kvm_run */
1740#ifdef CONFIG_X86
1741 r += PAGE_SIZE; /* pio data page */
1742#endif
Laurent Vivier5f94c172008-05-30 16:05:54 +02001743#ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
1744 r += PAGE_SIZE; /* coalesced mmio ring page */
1745#endif
Avi Kivity07c45a32007-03-07 13:05:38 +02001746 break;
Feng(Eric) Liud4c9ff22008-04-10 08:47:53 -04001747 case KVM_TRACE_ENABLE:
1748 case KVM_TRACE_PAUSE:
1749 case KVM_TRACE_DISABLE:
1750 r = kvm_trace_ioctl(ioctl, arg);
1751 break;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001752 default:
Carsten Otte043405e2007-10-10 17:16:19 +02001753 return kvm_arch_dev_ioctl(filp, ioctl, arg);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001754 }
1755out:
1756 return r;
1757}
1758
Avi Kivity6aa8b732006-12-10 02:21:36 -08001759static struct file_operations kvm_chardev_ops = {
Avi Kivity6aa8b732006-12-10 02:21:36 -08001760 .unlocked_ioctl = kvm_dev_ioctl,
1761 .compat_ioctl = kvm_dev_ioctl,
Avi Kivity6aa8b732006-12-10 02:21:36 -08001762};
1763
1764static struct miscdevice kvm_dev = {
Avi Kivitybbe44322007-03-04 13:27:36 +02001765 KVM_MINOR,
Avi Kivity6aa8b732006-12-10 02:21:36 -08001766 "kvm",
1767 &kvm_chardev_ops,
1768};
1769
Avi Kivity1b6c0162007-05-24 13:03:52 +03001770static void hardware_enable(void *junk)
1771{
1772 int cpu = raw_smp_processor_id();
1773
1774 if (cpu_isset(cpu, cpus_hardware_enabled))
1775 return;
1776 cpu_set(cpu, cpus_hardware_enabled);
Zhang Xiantaoe9b11c12007-11-14 20:38:21 +08001777 kvm_arch_hardware_enable(NULL);
Avi Kivity1b6c0162007-05-24 13:03:52 +03001778}
1779
1780static void hardware_disable(void *junk)
1781{
1782 int cpu = raw_smp_processor_id();
1783
1784 if (!cpu_isset(cpu, cpus_hardware_enabled))
1785 return;
1786 cpu_clear(cpu, cpus_hardware_enabled);
Zhang Xiantaoe9b11c12007-11-14 20:38:21 +08001787 kvm_arch_hardware_disable(NULL);
Avi Kivity1b6c0162007-05-24 13:03:52 +03001788}
1789
Avi Kivity774c47f2007-02-12 00:54:47 -08001790static int kvm_cpu_hotplug(struct notifier_block *notifier, unsigned long val,
1791 void *v)
1792{
1793 int cpu = (long)v;
1794
Avi Kivity1a6f4d72007-11-11 18:37:32 +02001795 val &= ~CPU_TASKS_FROZEN;
Avi Kivity774c47f2007-02-12 00:54:47 -08001796 switch (val) {
Avi Kivitycec9ad22007-05-24 13:11:41 +03001797 case CPU_DYING:
Avi Kivity6ec8a852007-08-19 15:57:26 +03001798 printk(KERN_INFO "kvm: disabling virtualization on CPU%d\n",
1799 cpu);
1800 hardware_disable(NULL);
1801 break;
Avi Kivity774c47f2007-02-12 00:54:47 -08001802 case CPU_UP_CANCELED:
Jeremy Katz43934a32007-02-19 14:37:46 +02001803 printk(KERN_INFO "kvm: disabling virtualization on CPU%d\n",
1804 cpu);
Jens Axboe8691e5a2008-06-06 11:18:06 +02001805 smp_call_function_single(cpu, hardware_disable, NULL, 1);
Avi Kivity774c47f2007-02-12 00:54:47 -08001806 break;
Jeremy Katz43934a32007-02-19 14:37:46 +02001807 case CPU_ONLINE:
1808 printk(KERN_INFO "kvm: enabling virtualization on CPU%d\n",
1809 cpu);
Jens Axboe8691e5a2008-06-06 11:18:06 +02001810 smp_call_function_single(cpu, hardware_enable, NULL, 1);
Avi Kivity774c47f2007-02-12 00:54:47 -08001811 break;
1812 }
1813 return NOTIFY_OK;
1814}
1815
Avi Kivity4ecac3f2008-05-13 13:23:38 +03001816
1817asmlinkage void kvm_handle_fault_on_reboot(void)
1818{
1819 if (kvm_rebooting)
1820 /* spin while reset goes on */
1821 while (true)
1822 ;
1823 /* Fault while not rebooting. We want the trace. */
1824 BUG();
1825}
1826EXPORT_SYMBOL_GPL(kvm_handle_fault_on_reboot);
1827
Rusty Russell9a2b85c2007-07-17 23:17:55 +10001828static int kvm_reboot(struct notifier_block *notifier, unsigned long val,
Mike Dayd77c26f2007-10-08 09:02:08 -04001829 void *v)
Rusty Russell9a2b85c2007-07-17 23:17:55 +10001830{
1831 if (val == SYS_RESTART) {
1832 /*
1833 * Some (well, at least mine) BIOSes hang on reboot if
1834 * in vmx root mode.
1835 */
1836 printk(KERN_INFO "kvm: exiting hardware virtualization\n");
Avi Kivity4ecac3f2008-05-13 13:23:38 +03001837 kvm_rebooting = true;
Jens Axboe15c8b6c2008-05-09 09:39:44 +02001838 on_each_cpu(hardware_disable, NULL, 1);
Rusty Russell9a2b85c2007-07-17 23:17:55 +10001839 }
1840 return NOTIFY_OK;
1841}
1842
1843static struct notifier_block kvm_reboot_notifier = {
1844 .notifier_call = kvm_reboot,
1845 .priority = 0,
1846};
1847
Gregory Haskins2eeb2e92007-05-31 14:08:53 -04001848void kvm_io_bus_init(struct kvm_io_bus *bus)
1849{
1850 memset(bus, 0, sizeof(*bus));
1851}
1852
1853void kvm_io_bus_destroy(struct kvm_io_bus *bus)
1854{
1855 int i;
1856
1857 for (i = 0; i < bus->dev_count; i++) {
1858 struct kvm_io_device *pos = bus->devs[i];
1859
1860 kvm_iodevice_destructor(pos);
1861 }
1862}
1863
Laurent Vivier92760492008-05-30 16:05:53 +02001864struct kvm_io_device *kvm_io_bus_find_dev(struct kvm_io_bus *bus,
1865 gpa_t addr, int len, int is_write)
Gregory Haskins2eeb2e92007-05-31 14:08:53 -04001866{
1867 int i;
1868
1869 for (i = 0; i < bus->dev_count; i++) {
1870 struct kvm_io_device *pos = bus->devs[i];
1871
Laurent Vivier92760492008-05-30 16:05:53 +02001872 if (pos->in_range(pos, addr, len, is_write))
Gregory Haskins2eeb2e92007-05-31 14:08:53 -04001873 return pos;
1874 }
1875
1876 return NULL;
1877}
1878
1879void kvm_io_bus_register_dev(struct kvm_io_bus *bus, struct kvm_io_device *dev)
1880{
1881 BUG_ON(bus->dev_count > (NR_IOBUS_DEVS-1));
1882
1883 bus->devs[bus->dev_count++] = dev;
1884}
1885
Avi Kivity774c47f2007-02-12 00:54:47 -08001886static struct notifier_block kvm_cpu_notifier = {
1887 .notifier_call = kvm_cpu_hotplug,
1888 .priority = 20, /* must be > scheduler priority */
1889};
1890
Christoph Hellwig8b88b092008-02-08 04:20:26 -08001891static int vm_stat_get(void *_offset, u64 *val)
Avi Kivityba1389b2007-11-18 16:24:12 +02001892{
1893 unsigned offset = (long)_offset;
Avi Kivityba1389b2007-11-18 16:24:12 +02001894 struct kvm *kvm;
1895
Christoph Hellwig8b88b092008-02-08 04:20:26 -08001896 *val = 0;
Avi Kivityba1389b2007-11-18 16:24:12 +02001897 spin_lock(&kvm_lock);
1898 list_for_each_entry(kvm, &vm_list, vm_list)
Christoph Hellwig8b88b092008-02-08 04:20:26 -08001899 *val += *(u32 *)((void *)kvm + offset);
Avi Kivityba1389b2007-11-18 16:24:12 +02001900 spin_unlock(&kvm_lock);
Christoph Hellwig8b88b092008-02-08 04:20:26 -08001901 return 0;
Avi Kivityba1389b2007-11-18 16:24:12 +02001902}
1903
1904DEFINE_SIMPLE_ATTRIBUTE(vm_stat_fops, vm_stat_get, NULL, "%llu\n");
1905
Christoph Hellwig8b88b092008-02-08 04:20:26 -08001906static int vcpu_stat_get(void *_offset, u64 *val)
Avi Kivity1165f5f2007-04-19 17:27:43 +03001907{
1908 unsigned offset = (long)_offset;
Avi Kivity1165f5f2007-04-19 17:27:43 +03001909 struct kvm *kvm;
1910 struct kvm_vcpu *vcpu;
1911 int i;
1912
Christoph Hellwig8b88b092008-02-08 04:20:26 -08001913 *val = 0;
Avi Kivity1165f5f2007-04-19 17:27:43 +03001914 spin_lock(&kvm_lock);
1915 list_for_each_entry(kvm, &vm_list, vm_list)
1916 for (i = 0; i < KVM_MAX_VCPUS; ++i) {
Rusty Russellfb3f0f52007-07-27 17:16:56 +10001917 vcpu = kvm->vcpus[i];
1918 if (vcpu)
Christoph Hellwig8b88b092008-02-08 04:20:26 -08001919 *val += *(u32 *)((void *)vcpu + offset);
Avi Kivity1165f5f2007-04-19 17:27:43 +03001920 }
1921 spin_unlock(&kvm_lock);
Christoph Hellwig8b88b092008-02-08 04:20:26 -08001922 return 0;
Avi Kivity1165f5f2007-04-19 17:27:43 +03001923}
1924
Avi Kivityba1389b2007-11-18 16:24:12 +02001925DEFINE_SIMPLE_ATTRIBUTE(vcpu_stat_fops, vcpu_stat_get, NULL, "%llu\n");
1926
1927static struct file_operations *stat_fops[] = {
1928 [KVM_STAT_VCPU] = &vcpu_stat_fops,
1929 [KVM_STAT_VM] = &vm_stat_fops,
1930};
Avi Kivity1165f5f2007-04-19 17:27:43 +03001931
Zhang Xiantaoa16b0432007-11-16 14:38:21 +08001932static void kvm_init_debug(void)
Avi Kivity6aa8b732006-12-10 02:21:36 -08001933{
1934 struct kvm_stats_debugfs_item *p;
1935
Hollis Blanchard76f7c872008-04-15 16:05:42 -05001936 kvm_debugfs_dir = debugfs_create_dir("kvm", NULL);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001937 for (p = debugfs_entries; p->name; ++p)
Hollis Blanchard76f7c872008-04-15 16:05:42 -05001938 p->dentry = debugfs_create_file(p->name, 0444, kvm_debugfs_dir,
Avi Kivity1165f5f2007-04-19 17:27:43 +03001939 (void *)(long)p->offset,
Avi Kivityba1389b2007-11-18 16:24:12 +02001940 stat_fops[p->kind]);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001941}
1942
1943static void kvm_exit_debug(void)
1944{
1945 struct kvm_stats_debugfs_item *p;
1946
1947 for (p = debugfs_entries; p->name; ++p)
1948 debugfs_remove(p->dentry);
Hollis Blanchard76f7c872008-04-15 16:05:42 -05001949 debugfs_remove(kvm_debugfs_dir);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001950}
1951
Avi Kivity59ae6c62007-02-12 00:54:48 -08001952static int kvm_suspend(struct sys_device *dev, pm_message_t state)
1953{
Avi Kivity4267c412007-05-24 13:09:41 +03001954 hardware_disable(NULL);
Avi Kivity59ae6c62007-02-12 00:54:48 -08001955 return 0;
1956}
1957
1958static int kvm_resume(struct sys_device *dev)
1959{
Avi Kivity4267c412007-05-24 13:09:41 +03001960 hardware_enable(NULL);
Avi Kivity59ae6c62007-02-12 00:54:48 -08001961 return 0;
1962}
1963
1964static struct sysdev_class kvm_sysdev_class = {
Kay Sieversaf5ca3f2007-12-20 02:09:39 +01001965 .name = "kvm",
Avi Kivity59ae6c62007-02-12 00:54:48 -08001966 .suspend = kvm_suspend,
1967 .resume = kvm_resume,
1968};
1969
1970static struct sys_device kvm_sysdev = {
1971 .id = 0,
1972 .cls = &kvm_sysdev_class,
1973};
1974
Izik Eiduscea7bb22007-10-17 19:17:48 +02001975struct page *bad_page;
Anthony Liguori35149e22008-04-02 14:46:56 -05001976pfn_t bad_pfn;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001977
Avi Kivity15ad7142007-07-11 18:17:21 +03001978static inline
1979struct kvm_vcpu *preempt_notifier_to_vcpu(struct preempt_notifier *pn)
1980{
1981 return container_of(pn, struct kvm_vcpu, preempt_notifier);
1982}
1983
1984static void kvm_sched_in(struct preempt_notifier *pn, int cpu)
1985{
1986 struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);
1987
Zhang Xiantaoe9b11c12007-11-14 20:38:21 +08001988 kvm_arch_vcpu_load(vcpu, cpu);
Avi Kivity15ad7142007-07-11 18:17:21 +03001989}
1990
1991static void kvm_sched_out(struct preempt_notifier *pn,
1992 struct task_struct *next)
1993{
1994 struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);
1995
Zhang Xiantaoe9b11c12007-11-14 20:38:21 +08001996 kvm_arch_vcpu_put(vcpu);
Avi Kivity15ad7142007-07-11 18:17:21 +03001997}
1998
Zhang Xiantaof8c16bb2007-11-14 20:40:21 +08001999int kvm_init(void *opaque, unsigned int vcpu_size,
Rusty Russellc16f8622007-07-30 21:12:19 +10002000 struct module *module)
Avi Kivity6aa8b732006-12-10 02:21:36 -08002001{
2002 int r;
Yang, Sheng002c7f72007-07-31 14:23:01 +03002003 int cpu;
Avi Kivity6aa8b732006-12-10 02:21:36 -08002004
Zhang Xiantaocb498ea2007-11-14 20:39:31 +08002005 kvm_init_debug();
2006
Zhang Xiantaof8c16bb2007-11-14 20:40:21 +08002007 r = kvm_arch_init(opaque);
2008 if (r)
Zhang Xiantaod23087842007-11-29 15:35:39 +08002009 goto out_fail;
Zhang Xiantaocb498ea2007-11-14 20:39:31 +08002010
2011 bad_page = alloc_page(GFP_KERNEL | __GFP_ZERO);
2012
2013 if (bad_page == NULL) {
2014 r = -ENOMEM;
2015 goto out;
2016 }
2017
Anthony Liguori35149e22008-04-02 14:46:56 -05002018 bad_pfn = page_to_pfn(bad_page);
2019
Zhang Xiantaoe9b11c12007-11-14 20:38:21 +08002020 r = kvm_arch_hardware_setup();
Avi Kivity6aa8b732006-12-10 02:21:36 -08002021 if (r < 0)
Zhang Xiantaod23087842007-11-29 15:35:39 +08002022 goto out_free_0;
Avi Kivity6aa8b732006-12-10 02:21:36 -08002023
Yang, Sheng002c7f72007-07-31 14:23:01 +03002024 for_each_online_cpu(cpu) {
2025 smp_call_function_single(cpu,
Zhang Xiantaoe9b11c12007-11-14 20:38:21 +08002026 kvm_arch_check_processor_compat,
Jens Axboe8691e5a2008-06-06 11:18:06 +02002027 &r, 1);
Yang, Sheng002c7f72007-07-31 14:23:01 +03002028 if (r < 0)
Zhang Xiantaod23087842007-11-29 15:35:39 +08002029 goto out_free_1;
Yang, Sheng002c7f72007-07-31 14:23:01 +03002030 }
2031
Jens Axboe15c8b6c2008-05-09 09:39:44 +02002032 on_each_cpu(hardware_enable, NULL, 1);
Avi Kivity774c47f2007-02-12 00:54:47 -08002033 r = register_cpu_notifier(&kvm_cpu_notifier);
2034 if (r)
Zhang Xiantaod23087842007-11-29 15:35:39 +08002035 goto out_free_2;
Avi Kivity6aa8b732006-12-10 02:21:36 -08002036 register_reboot_notifier(&kvm_reboot_notifier);
2037
Avi Kivity59ae6c62007-02-12 00:54:48 -08002038 r = sysdev_class_register(&kvm_sysdev_class);
2039 if (r)
Zhang Xiantaod23087842007-11-29 15:35:39 +08002040 goto out_free_3;
Avi Kivity59ae6c62007-02-12 00:54:48 -08002041
2042 r = sysdev_register(&kvm_sysdev);
2043 if (r)
Zhang Xiantaod23087842007-11-29 15:35:39 +08002044 goto out_free_4;
Avi Kivity59ae6c62007-02-12 00:54:48 -08002045
Rusty Russellc16f8622007-07-30 21:12:19 +10002046 /* A kmem cache lets us meet the alignment requirements of fx_save. */
2047 kvm_vcpu_cache = kmem_cache_create("kvm_vcpu", vcpu_size,
Joe Perches56919c52007-11-12 20:06:51 -08002048 __alignof__(struct kvm_vcpu),
2049 0, NULL);
Rusty Russellc16f8622007-07-30 21:12:19 +10002050 if (!kvm_vcpu_cache) {
2051 r = -ENOMEM;
Zhang Xiantaod23087842007-11-29 15:35:39 +08002052 goto out_free_5;
Rusty Russellc16f8622007-07-30 21:12:19 +10002053 }
2054
Avi Kivity6aa8b732006-12-10 02:21:36 -08002055 kvm_chardev_ops.owner = module;
2056
2057 r = misc_register(&kvm_dev);
2058 if (r) {
Mike Dayd77c26f2007-10-08 09:02:08 -04002059 printk(KERN_ERR "kvm: misc device register failed\n");
Avi Kivity6aa8b732006-12-10 02:21:36 -08002060 goto out_free;
2061 }
2062
Avi Kivity15ad7142007-07-11 18:17:21 +03002063 kvm_preempt_ops.sched_in = kvm_sched_in;
2064 kvm_preempt_ops.sched_out = kvm_sched_out;
2065
Avi Kivityc7addb92007-09-16 18:58:32 +02002066 return 0;
Avi Kivity6aa8b732006-12-10 02:21:36 -08002067
2068out_free:
Rusty Russellc16f8622007-07-30 21:12:19 +10002069 kmem_cache_destroy(kvm_vcpu_cache);
Zhang Xiantaod23087842007-11-29 15:35:39 +08002070out_free_5:
Avi Kivity59ae6c62007-02-12 00:54:48 -08002071 sysdev_unregister(&kvm_sysdev);
Zhang Xiantaod23087842007-11-29 15:35:39 +08002072out_free_4:
Avi Kivity59ae6c62007-02-12 00:54:48 -08002073 sysdev_class_unregister(&kvm_sysdev_class);
Zhang Xiantaod23087842007-11-29 15:35:39 +08002074out_free_3:
Avi Kivity6aa8b732006-12-10 02:21:36 -08002075 unregister_reboot_notifier(&kvm_reboot_notifier);
Avi Kivity774c47f2007-02-12 00:54:47 -08002076 unregister_cpu_notifier(&kvm_cpu_notifier);
Zhang Xiantaod23087842007-11-29 15:35:39 +08002077out_free_2:
Jens Axboe15c8b6c2008-05-09 09:39:44 +02002078 on_each_cpu(hardware_disable, NULL, 1);
Zhang Xiantaod23087842007-11-29 15:35:39 +08002079out_free_1:
Zhang Xiantaoe9b11c12007-11-14 20:38:21 +08002080 kvm_arch_hardware_unsetup();
Zhang Xiantaod23087842007-11-29 15:35:39 +08002081out_free_0:
2082 __free_page(bad_page);
Avi Kivityca45aaa2007-03-01 19:21:03 +02002083out:
Zhang Xiantaof8c16bb2007-11-14 20:40:21 +08002084 kvm_arch_exit();
Zhang Xiantaocb498ea2007-11-14 20:39:31 +08002085 kvm_exit_debug();
Zhang Xiantaod23087842007-11-29 15:35:39 +08002086out_fail:
Avi Kivity6aa8b732006-12-10 02:21:36 -08002087 return r;
2088}
Zhang Xiantaocb498ea2007-11-14 20:39:31 +08002089EXPORT_SYMBOL_GPL(kvm_init);
Avi Kivity6aa8b732006-12-10 02:21:36 -08002090
Zhang Xiantaocb498ea2007-11-14 20:39:31 +08002091void kvm_exit(void)
Avi Kivity6aa8b732006-12-10 02:21:36 -08002092{
Feng(Eric) Liud4c9ff22008-04-10 08:47:53 -04002093 kvm_trace_cleanup();
Avi Kivity6aa8b732006-12-10 02:21:36 -08002094 misc_deregister(&kvm_dev);
Rusty Russellc16f8622007-07-30 21:12:19 +10002095 kmem_cache_destroy(kvm_vcpu_cache);
Avi Kivity59ae6c62007-02-12 00:54:48 -08002096 sysdev_unregister(&kvm_sysdev);
2097 sysdev_class_unregister(&kvm_sysdev_class);
Avi Kivity6aa8b732006-12-10 02:21:36 -08002098 unregister_reboot_notifier(&kvm_reboot_notifier);
Avi Kivity59ae6c62007-02-12 00:54:48 -08002099 unregister_cpu_notifier(&kvm_cpu_notifier);
Jens Axboe15c8b6c2008-05-09 09:39:44 +02002100 on_each_cpu(hardware_disable, NULL, 1);
Zhang Xiantaoe9b11c12007-11-14 20:38:21 +08002101 kvm_arch_hardware_unsetup();
Zhang Xiantaof8c16bb2007-11-14 20:40:21 +08002102 kvm_arch_exit();
Avi Kivity6aa8b732006-12-10 02:21:36 -08002103 kvm_exit_debug();
Izik Eiduscea7bb22007-10-17 19:17:48 +02002104 __free_page(bad_page);
Avi Kivity6aa8b732006-12-10 02:21:36 -08002105}
Zhang Xiantaocb498ea2007-11-14 20:39:31 +08002106EXPORT_SYMBOL_GPL(kvm_exit);