blob: b2e12893e3f4d4b0f39d64ff733333171e3a0dfb [file] [log] [blame]
Avi Kivity6aa8b732006-12-10 02:21:36 -08001/*
2 * Kernel-based Virtual Machine driver for Linux
3 *
4 * This module enables machines with Intel VT-x extensions to run virtual
5 * machines without emulation or binary translation.
6 *
7 * Copyright (C) 2006 Qumranet, Inc.
8 *
9 * Authors:
10 * Avi Kivity <avi@qumranet.com>
11 * Yaniv Kamay <yaniv@qumranet.com>
12 *
13 * This work is licensed under the terms of the GNU GPL, version 2. See
14 * the COPYING file in the top-level directory.
15 *
16 */
17
Hollis Blancharde2174022007-12-03 15:30:24 -060018#include "iodev.h"
Avi Kivity6aa8b732006-12-10 02:21:36 -080019
Avi Kivityedf88412007-12-16 11:02:48 +020020#include <linux/kvm_host.h>
Avi Kivity6aa8b732006-12-10 02:21:36 -080021#include <linux/kvm.h>
22#include <linux/module.h>
23#include <linux/errno.h>
Avi Kivity6aa8b732006-12-10 02:21:36 -080024#include <linux/percpu.h>
25#include <linux/gfp.h>
Avi Kivity6aa8b732006-12-10 02:21:36 -080026#include <linux/mm.h>
27#include <linux/miscdevice.h>
28#include <linux/vmalloc.h>
Avi Kivity6aa8b732006-12-10 02:21:36 -080029#include <linux/reboot.h>
Avi Kivity6aa8b732006-12-10 02:21:36 -080030#include <linux/debugfs.h>
31#include <linux/highmem.h>
32#include <linux/file.h>
Avi Kivity59ae6c62007-02-12 00:54:48 -080033#include <linux/sysdev.h>
Avi Kivity774c47f2007-02-12 00:54:47 -080034#include <linux/cpu.h>
Alexey Dobriyane8edc6e2007-05-21 01:22:52 +040035#include <linux/sched.h>
Avi Kivityd9e368d2007-06-07 19:18:30 +030036#include <linux/cpumask.h>
37#include <linux/smp.h>
Avi Kivityd6d28162007-06-28 08:38:16 -040038#include <linux/anon_inodes.h>
Avi Kivity04d2cc72007-09-10 18:10:54 +030039#include <linux/profile.h>
Anthony Liguori7aa81cc2007-09-17 14:57:50 -050040#include <linux/kvm_para.h>
Izik Eidus6fc138d2007-10-09 19:20:39 +020041#include <linux/pagemap.h>
Anthony Liguori8d4e1282007-10-18 09:59:34 -050042#include <linux/mman.h>
Avi Kivity6aa8b732006-12-10 02:21:36 -080043
Avi Kivitye4956062007-06-28 14:15:57 -040044#include <asm/processor.h>
Avi Kivitye4956062007-06-28 14:15:57 -040045#include <asm/io.h>
46#include <asm/uaccess.h>
Izik Eidus3e021bf2007-11-19 11:16:57 +020047#include <asm/pgtable.h>
Avi Kivity6aa8b732006-12-10 02:21:36 -080048
49MODULE_AUTHOR("Qumranet");
50MODULE_LICENSE("GPL");
51
Zhang Xiantaoe9b11c12007-11-14 20:38:21 +080052DEFINE_SPINLOCK(kvm_lock);
53LIST_HEAD(vm_list);
Avi Kivity133de902007-02-12 00:54:44 -080054
Avi Kivity1b6c0162007-05-24 13:03:52 +030055static cpumask_t cpus_hardware_enabled;
56
Rusty Russellc16f8622007-07-30 21:12:19 +100057struct kmem_cache *kvm_vcpu_cache;
58EXPORT_SYMBOL_GPL(kvm_vcpu_cache);
Avi Kivity1165f5f2007-04-19 17:27:43 +030059
Avi Kivity15ad7142007-07-11 18:17:21 +030060static __read_mostly struct preempt_ops kvm_preempt_ops;
61
Avi Kivity6aa8b732006-12-10 02:21:36 -080062static struct dentry *debugfs_dir;
63
Avi Kivitybccf2152007-02-21 18:04:26 +020064static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl,
65 unsigned long arg);
66
James Morris5aacf0c2006-12-22 01:04:55 -080067static inline int valid_vcpu(int n)
68{
69 return likely(n >= 0 && n < KVM_MAX_VCPUS);
70}
71
Avi Kivity6aa8b732006-12-10 02:21:36 -080072/*
73 * Switches to specified vcpu, until a matching vcpu_put()
74 */
Carsten Otte313a3dc2007-10-11 19:16:52 +020075void vcpu_load(struct kvm_vcpu *vcpu)
Avi Kivity6aa8b732006-12-10 02:21:36 -080076{
Avi Kivity15ad7142007-07-11 18:17:21 +030077 int cpu;
78
Avi Kivitybccf2152007-02-21 18:04:26 +020079 mutex_lock(&vcpu->mutex);
Avi Kivity15ad7142007-07-11 18:17:21 +030080 cpu = get_cpu();
81 preempt_notifier_register(&vcpu->preempt_notifier);
Carsten Otte313a3dc2007-10-11 19:16:52 +020082 kvm_arch_vcpu_load(vcpu, cpu);
Avi Kivity15ad7142007-07-11 18:17:21 +030083 put_cpu();
Avi Kivitybccf2152007-02-21 18:04:26 +020084}
85
Carsten Otte313a3dc2007-10-11 19:16:52 +020086void vcpu_put(struct kvm_vcpu *vcpu)
Avi Kivity6aa8b732006-12-10 02:21:36 -080087{
Avi Kivity15ad7142007-07-11 18:17:21 +030088 preempt_disable();
Carsten Otte313a3dc2007-10-11 19:16:52 +020089 kvm_arch_vcpu_put(vcpu);
Avi Kivity15ad7142007-07-11 18:17:21 +030090 preempt_notifier_unregister(&vcpu->preempt_notifier);
91 preempt_enable();
Avi Kivity6aa8b732006-12-10 02:21:36 -080092 mutex_unlock(&vcpu->mutex);
93}
94
Avi Kivityd9e368d2007-06-07 19:18:30 +030095static void ack_flush(void *_completed)
96{
Avi Kivityd9e368d2007-06-07 19:18:30 +030097}
98
99void kvm_flush_remote_tlbs(struct kvm *kvm)
100{
Laurent Vivier49d3bd72007-10-22 16:33:07 +0200101 int i, cpu;
Avi Kivityd9e368d2007-06-07 19:18:30 +0300102 cpumask_t cpus;
103 struct kvm_vcpu *vcpu;
Avi Kivityd9e368d2007-06-07 19:18:30 +0300104
Avi Kivityd9e368d2007-06-07 19:18:30 +0300105 cpus_clear(cpus);
Rusty Russellfb3f0f52007-07-27 17:16:56 +1000106 for (i = 0; i < KVM_MAX_VCPUS; ++i) {
107 vcpu = kvm->vcpus[i];
108 if (!vcpu)
109 continue;
Avi Kivity3176bc32007-10-16 17:22:08 +0200110 if (test_and_set_bit(KVM_REQ_TLB_FLUSH, &vcpu->requests))
Avi Kivityd9e368d2007-06-07 19:18:30 +0300111 continue;
112 cpu = vcpu->cpu;
113 if (cpu != -1 && cpu != raw_smp_processor_id())
Laurent Vivier49d3bd72007-10-22 16:33:07 +0200114 cpu_set(cpu, cpus);
Avi Kivityd9e368d2007-06-07 19:18:30 +0300115 }
Avi Kivity0f74a242007-11-20 23:01:14 +0200116 if (cpus_empty(cpus))
117 return;
118 ++kvm->stat.remote_tlb_flush;
Laurent Vivier49d3bd72007-10-22 16:33:07 +0200119 smp_call_function_mask(cpus, ack_flush, NULL, 1);
Avi Kivityd9e368d2007-06-07 19:18:30 +0300120}
121
Rusty Russellfb3f0f52007-07-27 17:16:56 +1000122int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id)
123{
124 struct page *page;
125 int r;
126
127 mutex_init(&vcpu->mutex);
128 vcpu->cpu = -1;
Rusty Russellfb3f0f52007-07-27 17:16:56 +1000129 vcpu->kvm = kvm;
130 vcpu->vcpu_id = id;
Eddie Dongb6958ce2007-07-18 12:15:21 +0300131 init_waitqueue_head(&vcpu->wq);
Rusty Russellfb3f0f52007-07-27 17:16:56 +1000132
133 page = alloc_page(GFP_KERNEL | __GFP_ZERO);
134 if (!page) {
135 r = -ENOMEM;
136 goto fail;
137 }
138 vcpu->run = page_address(page);
139
Zhang Xiantaoe9b11c12007-11-14 20:38:21 +0800140 r = kvm_arch_vcpu_init(vcpu);
Rusty Russellfb3f0f52007-07-27 17:16:56 +1000141 if (r < 0)
Zhang Xiantaoe9b11c12007-11-14 20:38:21 +0800142 goto fail_free_run;
Rusty Russellfb3f0f52007-07-27 17:16:56 +1000143 return 0;
144
Rusty Russellfb3f0f52007-07-27 17:16:56 +1000145fail_free_run:
146 free_page((unsigned long)vcpu->run);
147fail:
Rusty Russell76fafa52007-10-08 10:50:48 +1000148 return r;
Rusty Russellfb3f0f52007-07-27 17:16:56 +1000149}
150EXPORT_SYMBOL_GPL(kvm_vcpu_init);
151
152void kvm_vcpu_uninit(struct kvm_vcpu *vcpu)
153{
Zhang Xiantaoe9b11c12007-11-14 20:38:21 +0800154 kvm_arch_vcpu_uninit(vcpu);
Rusty Russellfb3f0f52007-07-27 17:16:56 +1000155 free_page((unsigned long)vcpu->run);
156}
157EXPORT_SYMBOL_GPL(kvm_vcpu_uninit);
158
Avi Kivityf17abe92007-02-21 19:28:04 +0200159static struct kvm *kvm_create_vm(void)
Avi Kivity6aa8b732006-12-10 02:21:36 -0800160{
Zhang Xiantaod19a9cd2007-11-18 18:43:45 +0800161 struct kvm *kvm = kvm_arch_create_vm();
Avi Kivity6aa8b732006-12-10 02:21:36 -0800162
Zhang Xiantaod19a9cd2007-11-18 18:43:45 +0800163 if (IS_ERR(kvm))
164 goto out;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800165
Avi Kivity6d4e4c42007-11-21 16:41:05 +0200166 kvm->mm = current->mm;
167 atomic_inc(&kvm->mm->mm_count);
Marcelo Tosattiaaee2c92007-12-20 19:18:26 -0500168 spin_lock_init(&kvm->mmu_lock);
Eddie Dong74906342007-06-19 18:05:03 +0300169 kvm_io_bus_init(&kvm->pio_bus);
Shaohua Li11ec2802007-07-23 14:51:37 +0800170 mutex_init(&kvm->lock);
Gregory Haskins2eeb2e92007-05-31 14:08:53 -0400171 kvm_io_bus_init(&kvm->mmio_bus);
Izik Eidus72dc67a2008-02-10 18:04:15 +0200172 init_rwsem(&kvm->slots_lock);
Rusty Russell5e58cfe2007-07-23 17:08:21 +1000173 spin_lock(&kvm_lock);
174 list_add(&kvm->vm_list, &vm_list);
175 spin_unlock(&kvm_lock);
Zhang Xiantaod19a9cd2007-11-18 18:43:45 +0800176out:
Avi Kivityf17abe92007-02-21 19:28:04 +0200177 return kvm;
178}
179
Avi Kivity6aa8b732006-12-10 02:21:36 -0800180/*
181 * Free any memory in @free but not in @dont.
182 */
183static void kvm_free_physmem_slot(struct kvm_memory_slot *free,
184 struct kvm_memory_slot *dont)
185{
Izik Eidus290fc382007-09-27 14:11:22 +0200186 if (!dont || free->rmap != dont->rmap)
187 vfree(free->rmap);
Avi Kivity6aa8b732006-12-10 02:21:36 -0800188
189 if (!dont || free->dirty_bitmap != dont->dirty_bitmap)
190 vfree(free->dirty_bitmap);
191
Avi Kivity6aa8b732006-12-10 02:21:36 -0800192 free->npages = 0;
Al Viro8b6d44c2007-02-09 16:38:40 +0000193 free->dirty_bitmap = NULL;
Anthony Liguori8d4e1282007-10-18 09:59:34 -0500194 free->rmap = NULL;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800195}
196
Zhang Xiantaod19a9cd2007-11-18 18:43:45 +0800197void kvm_free_physmem(struct kvm *kvm)
Avi Kivity6aa8b732006-12-10 02:21:36 -0800198{
199 int i;
200
201 for (i = 0; i < kvm->nmemslots; ++i)
Al Viro8b6d44c2007-02-09 16:38:40 +0000202 kvm_free_physmem_slot(&kvm->memslots[i], NULL);
Avi Kivity6aa8b732006-12-10 02:21:36 -0800203}
204
Avi Kivityf17abe92007-02-21 19:28:04 +0200205static void kvm_destroy_vm(struct kvm *kvm)
206{
Avi Kivity6d4e4c42007-11-21 16:41:05 +0200207 struct mm_struct *mm = kvm->mm;
208
Avi Kivity133de902007-02-12 00:54:44 -0800209 spin_lock(&kvm_lock);
210 list_del(&kvm->vm_list);
211 spin_unlock(&kvm_lock);
Eddie Dong74906342007-06-19 18:05:03 +0300212 kvm_io_bus_destroy(&kvm->pio_bus);
Gregory Haskins2eeb2e92007-05-31 14:08:53 -0400213 kvm_io_bus_destroy(&kvm->mmio_bus);
Zhang Xiantaod19a9cd2007-11-18 18:43:45 +0800214 kvm_arch_destroy_vm(kvm);
Avi Kivity6d4e4c42007-11-21 16:41:05 +0200215 mmdrop(mm);
Avi Kivityf17abe92007-02-21 19:28:04 +0200216}
217
218static int kvm_vm_release(struct inode *inode, struct file *filp)
219{
220 struct kvm *kvm = filp->private_data;
221
222 kvm_destroy_vm(kvm);
Avi Kivity6aa8b732006-12-10 02:21:36 -0800223 return 0;
224}
225
Avi Kivity6aa8b732006-12-10 02:21:36 -0800226/*
Avi Kivity6aa8b732006-12-10 02:21:36 -0800227 * Allocate some memory and give it an address in the guest physical address
228 * space.
229 *
230 * Discontiguous memory is allowed, mostly for framebuffers.
Sheng Yangf78e0e22007-10-29 09:40:42 +0800231 *
Marcelo Tosatti10589a42007-12-20 19:18:22 -0500232 * Must be called holding mmap_sem for write.
Avi Kivity6aa8b732006-12-10 02:21:36 -0800233 */
Sheng Yangf78e0e22007-10-29 09:40:42 +0800234int __kvm_set_memory_region(struct kvm *kvm,
235 struct kvm_userspace_memory_region *mem,
236 int user_alloc)
Avi Kivity6aa8b732006-12-10 02:21:36 -0800237{
238 int r;
239 gfn_t base_gfn;
240 unsigned long npages;
241 unsigned long i;
242 struct kvm_memory_slot *memslot;
243 struct kvm_memory_slot old, new;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800244
245 r = -EINVAL;
246 /* General sanity checks */
247 if (mem->memory_size & (PAGE_SIZE - 1))
248 goto out;
249 if (mem->guest_phys_addr & (PAGE_SIZE - 1))
250 goto out;
Izik Eiduse0d62c72007-10-24 23:57:46 +0200251 if (mem->slot >= KVM_MEMORY_SLOTS + KVM_PRIVATE_MEM_SLOTS)
Avi Kivity6aa8b732006-12-10 02:21:36 -0800252 goto out;
253 if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr)
254 goto out;
255
256 memslot = &kvm->memslots[mem->slot];
257 base_gfn = mem->guest_phys_addr >> PAGE_SHIFT;
258 npages = mem->memory_size >> PAGE_SHIFT;
259
260 if (!npages)
261 mem->flags &= ~KVM_MEM_LOG_DIRTY_PAGES;
262
Avi Kivity6aa8b732006-12-10 02:21:36 -0800263 new = old = *memslot;
264
265 new.base_gfn = base_gfn;
266 new.npages = npages;
267 new.flags = mem->flags;
268
269 /* Disallow changing a memory slot's size. */
270 r = -EINVAL;
271 if (npages && old.npages && npages != old.npages)
Sheng Yangf78e0e22007-10-29 09:40:42 +0800272 goto out_free;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800273
274 /* Check for overlaps */
275 r = -EEXIST;
276 for (i = 0; i < KVM_MEMORY_SLOTS; ++i) {
277 struct kvm_memory_slot *s = &kvm->memslots[i];
278
279 if (s == memslot)
280 continue;
281 if (!((base_gfn + npages <= s->base_gfn) ||
282 (base_gfn >= s->base_gfn + s->npages)))
Sheng Yangf78e0e22007-10-29 09:40:42 +0800283 goto out_free;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800284 }
Avi Kivity6aa8b732006-12-10 02:21:36 -0800285
Avi Kivity6aa8b732006-12-10 02:21:36 -0800286 /* Free page dirty bitmap if unneeded */
287 if (!(new.flags & KVM_MEM_LOG_DIRTY_PAGES))
Al Viro8b6d44c2007-02-09 16:38:40 +0000288 new.dirty_bitmap = NULL;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800289
290 r = -ENOMEM;
291
292 /* Allocate if a slot is being created */
Anthony Liguori8d4e1282007-10-18 09:59:34 -0500293 if (npages && !new.rmap) {
Mike Dayd77c26f2007-10-08 09:02:08 -0400294 new.rmap = vmalloc(npages * sizeof(struct page *));
Izik Eidus290fc382007-09-27 14:11:22 +0200295
296 if (!new.rmap)
Sheng Yangf78e0e22007-10-29 09:40:42 +0800297 goto out_free;
Izik Eidus290fc382007-09-27 14:11:22 +0200298
Izik Eidus290fc382007-09-27 14:11:22 +0200299 memset(new.rmap, 0, npages * sizeof(*new.rmap));
Anthony Liguori8d4e1282007-10-18 09:59:34 -0500300
Izik Eidus80b14b52007-10-25 11:54:04 +0200301 new.user_alloc = user_alloc;
Zhang Xiantao0de10342007-11-20 16:25:04 +0800302 new.userspace_addr = mem->userspace_addr;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800303 }
304
305 /* Allocate page dirty bitmap if needed */
306 if ((new.flags & KVM_MEM_LOG_DIRTY_PAGES) && !new.dirty_bitmap) {
307 unsigned dirty_bytes = ALIGN(npages, BITS_PER_LONG) / 8;
308
309 new.dirty_bitmap = vmalloc(dirty_bytes);
310 if (!new.dirty_bitmap)
Sheng Yangf78e0e22007-10-29 09:40:42 +0800311 goto out_free;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800312 memset(new.dirty_bitmap, 0, dirty_bytes);
313 }
314
Avi Kivity6aa8b732006-12-10 02:21:36 -0800315 if (mem->slot >= kvm->nmemslots)
316 kvm->nmemslots = mem->slot + 1;
317
318 *memslot = new;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800319
Zhang Xiantao0de10342007-11-20 16:25:04 +0800320 r = kvm_arch_set_memory_region(kvm, mem, old, user_alloc);
321 if (r) {
322 *memslot = old;
323 goto out_free;
Zhang Xiantao3ad82a72007-11-20 13:11:38 +0800324 }
325
Avi Kivity6aa8b732006-12-10 02:21:36 -0800326 kvm_free_physmem_slot(&old, &new);
327 return 0;
328
Sheng Yangf78e0e22007-10-29 09:40:42 +0800329out_free:
Avi Kivity6aa8b732006-12-10 02:21:36 -0800330 kvm_free_physmem_slot(&new, &old);
331out:
332 return r;
Izik Eidus210c7c42007-10-24 23:52:57 +0200333
334}
Sheng Yangf78e0e22007-10-29 09:40:42 +0800335EXPORT_SYMBOL_GPL(__kvm_set_memory_region);
336
337int kvm_set_memory_region(struct kvm *kvm,
338 struct kvm_userspace_memory_region *mem,
339 int user_alloc)
340{
341 int r;
342
Izik Eidus72dc67a2008-02-10 18:04:15 +0200343 down_write(&kvm->slots_lock);
Sheng Yangf78e0e22007-10-29 09:40:42 +0800344 r = __kvm_set_memory_region(kvm, mem, user_alloc);
Izik Eidus72dc67a2008-02-10 18:04:15 +0200345 up_write(&kvm->slots_lock);
Sheng Yangf78e0e22007-10-29 09:40:42 +0800346 return r;
347}
Izik Eidus210c7c42007-10-24 23:52:57 +0200348EXPORT_SYMBOL_GPL(kvm_set_memory_region);
349
Carsten Otte1fe779f2007-10-29 16:08:35 +0100350int kvm_vm_ioctl_set_memory_region(struct kvm *kvm,
351 struct
352 kvm_userspace_memory_region *mem,
353 int user_alloc)
Izik Eidus210c7c42007-10-24 23:52:57 +0200354{
Izik Eiduse0d62c72007-10-24 23:57:46 +0200355 if (mem->slot >= KVM_MEMORY_SLOTS)
356 return -EINVAL;
Izik Eidus210c7c42007-10-24 23:52:57 +0200357 return kvm_set_memory_region(kvm, mem, user_alloc);
Avi Kivity6aa8b732006-12-10 02:21:36 -0800358}
359
Zhang Xiantao5bb064d2007-11-18 20:29:43 +0800360int kvm_get_dirty_log(struct kvm *kvm,
361 struct kvm_dirty_log *log, int *is_dirty)
Avi Kivity6aa8b732006-12-10 02:21:36 -0800362{
363 struct kvm_memory_slot *memslot;
364 int r, i;
365 int n;
366 unsigned long any = 0;
367
Avi Kivity6aa8b732006-12-10 02:21:36 -0800368 r = -EINVAL;
369 if (log->slot >= KVM_MEMORY_SLOTS)
370 goto out;
371
372 memslot = &kvm->memslots[log->slot];
373 r = -ENOENT;
374 if (!memslot->dirty_bitmap)
375 goto out;
376
Uri Lublincd1a4a92007-02-22 16:43:09 +0200377 n = ALIGN(memslot->npages, BITS_PER_LONG) / 8;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800378
Uri Lublincd1a4a92007-02-22 16:43:09 +0200379 for (i = 0; !any && i < n/sizeof(long); ++i)
Avi Kivity6aa8b732006-12-10 02:21:36 -0800380 any = memslot->dirty_bitmap[i];
381
382 r = -EFAULT;
383 if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n))
384 goto out;
385
Zhang Xiantao5bb064d2007-11-18 20:29:43 +0800386 if (any)
387 *is_dirty = 1;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800388
389 r = 0;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800390out:
Avi Kivity6aa8b732006-12-10 02:21:36 -0800391 return r;
392}
393
Izik Eiduscea7bb22007-10-17 19:17:48 +0200394int is_error_page(struct page *page)
395{
396 return page == bad_page;
397}
398EXPORT_SYMBOL_GPL(is_error_page);
399
Izik Eidusf9d46eb2007-11-11 22:02:22 +0200400static inline unsigned long bad_hva(void)
401{
402 return PAGE_OFFSET;
403}
404
405int kvm_is_error_hva(unsigned long addr)
406{
407 return addr == bad_hva();
408}
409EXPORT_SYMBOL_GPL(kvm_is_error_hva);
410
Avi Kivitye8207542007-03-30 16:54:30 +0300411static struct kvm_memory_slot *__gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
Avi Kivity6aa8b732006-12-10 02:21:36 -0800412{
413 int i;
414
415 for (i = 0; i < kvm->nmemslots; ++i) {
416 struct kvm_memory_slot *memslot = &kvm->memslots[i];
417
418 if (gfn >= memslot->base_gfn
419 && gfn < memslot->base_gfn + memslot->npages)
420 return memslot;
421 }
Al Viro8b6d44c2007-02-09 16:38:40 +0000422 return NULL;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800423}
Avi Kivitye8207542007-03-30 16:54:30 +0300424
425struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
426{
427 gfn = unalias_gfn(kvm, gfn);
428 return __gfn_to_memslot(kvm, gfn);
429}
Avi Kivity6aa8b732006-12-10 02:21:36 -0800430
Izik Eiduse0d62c72007-10-24 23:57:46 +0200431int kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn)
432{
433 int i;
434
435 gfn = unalias_gfn(kvm, gfn);
436 for (i = 0; i < KVM_MEMORY_SLOTS; ++i) {
437 struct kvm_memory_slot *memslot = &kvm->memslots[i];
438
439 if (gfn >= memslot->base_gfn
440 && gfn < memslot->base_gfn + memslot->npages)
441 return 1;
442 }
443 return 0;
444}
445EXPORT_SYMBOL_GPL(kvm_is_visible_gfn);
446
Izik Eidus539cb662007-11-11 22:05:04 +0200447static unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn)
448{
449 struct kvm_memory_slot *slot;
450
451 gfn = unalias_gfn(kvm, gfn);
452 slot = __gfn_to_memslot(kvm, gfn);
453 if (!slot)
454 return bad_hva();
455 return (slot->userspace_addr + (gfn - slot->base_gfn) * PAGE_SIZE);
456}
457
Anthony Liguoriaab61cc2007-10-29 15:15:20 -0500458/*
459 * Requires current->mm->mmap_sem to be held
460 */
Marcelo Tosatti10589a42007-12-20 19:18:22 -0500461struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn)
Avi Kivity954bbbc2007-03-30 14:02:32 +0300462{
Anthony Liguori8d4e1282007-10-18 09:59:34 -0500463 struct page *page[1];
Izik Eidus539cb662007-11-11 22:05:04 +0200464 unsigned long addr;
Anthony Liguori8d4e1282007-10-18 09:59:34 -0500465 int npages;
Avi Kivity954bbbc2007-03-30 14:02:32 +0300466
Avi Kivity60395222007-10-21 11:03:36 +0200467 might_sleep();
468
Izik Eidus539cb662007-11-11 22:05:04 +0200469 addr = gfn_to_hva(kvm, gfn);
470 if (kvm_is_error_hva(addr)) {
Izik Eidus8a7ae052007-10-18 11:09:33 +0200471 get_page(bad_page);
Izik Eiduscea7bb22007-10-17 19:17:48 +0200472 return bad_page;
Izik Eidus8a7ae052007-10-18 11:09:33 +0200473 }
Izik Eidus8a7ae052007-10-18 11:09:33 +0200474
Izik Eidus539cb662007-11-11 22:05:04 +0200475 npages = get_user_pages(current, current->mm, addr, 1, 1, 1, page,
476 NULL);
477
Anthony Liguori8d4e1282007-10-18 09:59:34 -0500478 if (npages != 1) {
479 get_page(bad_page);
480 return bad_page;
Izik Eidus8a7ae052007-10-18 11:09:33 +0200481 }
Anthony Liguori8d4e1282007-10-18 09:59:34 -0500482
483 return page[0];
Avi Kivity954bbbc2007-03-30 14:02:32 +0300484}
Anthony Liguoriaab61cc2007-10-29 15:15:20 -0500485
Avi Kivity954bbbc2007-03-30 14:02:32 +0300486EXPORT_SYMBOL_GPL(gfn_to_page);
487
Izik Eidusb4231d62007-11-20 11:49:33 +0200488void kvm_release_page_clean(struct page *page)
489{
490 put_page(page);
491}
492EXPORT_SYMBOL_GPL(kvm_release_page_clean);
493
494void kvm_release_page_dirty(struct page *page)
Izik Eidus8a7ae052007-10-18 11:09:33 +0200495{
496 if (!PageReserved(page))
497 SetPageDirty(page);
498 put_page(page);
499}
Izik Eidusb4231d62007-11-20 11:49:33 +0200500EXPORT_SYMBOL_GPL(kvm_release_page_dirty);
Izik Eidus8a7ae052007-10-18 11:09:33 +0200501
Izik Eidus195aefd2007-10-01 22:14:18 +0200502static int next_segment(unsigned long len, int offset)
503{
504 if (len > PAGE_SIZE - offset)
505 return PAGE_SIZE - offset;
506 else
507 return len;
508}
509
510int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset,
511 int len)
512{
Izik Eiduse0506bc2007-11-11 22:10:22 +0200513 int r;
514 unsigned long addr;
Izik Eidus195aefd2007-10-01 22:14:18 +0200515
Izik Eiduse0506bc2007-11-11 22:10:22 +0200516 addr = gfn_to_hva(kvm, gfn);
517 if (kvm_is_error_hva(addr))
Izik Eidus195aefd2007-10-01 22:14:18 +0200518 return -EFAULT;
Izik Eiduse0506bc2007-11-11 22:10:22 +0200519 r = copy_from_user(data, (void __user *)addr + offset, len);
520 if (r)
521 return -EFAULT;
Izik Eidus195aefd2007-10-01 22:14:18 +0200522 return 0;
523}
524EXPORT_SYMBOL_GPL(kvm_read_guest_page);
525
526int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len)
527{
528 gfn_t gfn = gpa >> PAGE_SHIFT;
529 int seg;
530 int offset = offset_in_page(gpa);
531 int ret;
532
533 while ((seg = next_segment(len, offset)) != 0) {
534 ret = kvm_read_guest_page(kvm, gfn, data, offset, seg);
535 if (ret < 0)
536 return ret;
537 offset = 0;
538 len -= seg;
539 data += seg;
540 ++gfn;
541 }
542 return 0;
543}
544EXPORT_SYMBOL_GPL(kvm_read_guest);
545
Marcelo Tosatti7ec54582007-12-20 19:18:23 -0500546int kvm_read_guest_atomic(struct kvm *kvm, gpa_t gpa, void *data,
547 unsigned long len)
548{
549 int r;
550 unsigned long addr;
551 gfn_t gfn = gpa >> PAGE_SHIFT;
552 int offset = offset_in_page(gpa);
553
554 addr = gfn_to_hva(kvm, gfn);
555 if (kvm_is_error_hva(addr))
556 return -EFAULT;
Marcelo Tosatti7ec54582007-12-20 19:18:23 -0500557 r = __copy_from_user_inatomic(data, (void __user *)addr + offset, len);
Marcelo Tosatti7ec54582007-12-20 19:18:23 -0500558 if (r)
559 return -EFAULT;
560 return 0;
561}
562EXPORT_SYMBOL(kvm_read_guest_atomic);
563
Izik Eidus195aefd2007-10-01 22:14:18 +0200564int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn, const void *data,
565 int offset, int len)
566{
Izik Eiduse0506bc2007-11-11 22:10:22 +0200567 int r;
568 unsigned long addr;
Izik Eidus195aefd2007-10-01 22:14:18 +0200569
Izik Eiduse0506bc2007-11-11 22:10:22 +0200570 addr = gfn_to_hva(kvm, gfn);
571 if (kvm_is_error_hva(addr))
Izik Eidus195aefd2007-10-01 22:14:18 +0200572 return -EFAULT;
Izik Eiduse0506bc2007-11-11 22:10:22 +0200573 r = copy_to_user((void __user *)addr + offset, data, len);
574 if (r)
575 return -EFAULT;
Izik Eidus195aefd2007-10-01 22:14:18 +0200576 mark_page_dirty(kvm, gfn);
577 return 0;
578}
579EXPORT_SYMBOL_GPL(kvm_write_guest_page);
580
581int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data,
582 unsigned long len)
583{
584 gfn_t gfn = gpa >> PAGE_SHIFT;
585 int seg;
586 int offset = offset_in_page(gpa);
587 int ret;
588
589 while ((seg = next_segment(len, offset)) != 0) {
590 ret = kvm_write_guest_page(kvm, gfn, data, offset, seg);
591 if (ret < 0)
592 return ret;
593 offset = 0;
594 len -= seg;
595 data += seg;
596 ++gfn;
597 }
598 return 0;
599}
600
601int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len)
602{
Izik Eidus3e021bf2007-11-19 11:16:57 +0200603 return kvm_write_guest_page(kvm, gfn, empty_zero_page, offset, len);
Izik Eidus195aefd2007-10-01 22:14:18 +0200604}
605EXPORT_SYMBOL_GPL(kvm_clear_guest_page);
606
607int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len)
608{
609 gfn_t gfn = gpa >> PAGE_SHIFT;
610 int seg;
611 int offset = offset_in_page(gpa);
612 int ret;
613
614 while ((seg = next_segment(len, offset)) != 0) {
615 ret = kvm_clear_guest_page(kvm, gfn, offset, seg);
616 if (ret < 0)
617 return ret;
618 offset = 0;
619 len -= seg;
620 ++gfn;
621 }
622 return 0;
623}
624EXPORT_SYMBOL_GPL(kvm_clear_guest);
625
Avi Kivity6aa8b732006-12-10 02:21:36 -0800626void mark_page_dirty(struct kvm *kvm, gfn_t gfn)
627{
Nguyen Anh Quynh31389942007-06-05 10:35:19 +0300628 struct kvm_memory_slot *memslot;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800629
Uri Lublin3b6fff12007-10-30 10:42:09 +0200630 gfn = unalias_gfn(kvm, gfn);
Rusty Russell7e9d6192007-07-31 20:41:14 +1000631 memslot = __gfn_to_memslot(kvm, gfn);
632 if (memslot && memslot->dirty_bitmap) {
633 unsigned long rel_gfn = gfn - memslot->base_gfn;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800634
Rusty Russell7e9d6192007-07-31 20:41:14 +1000635 /* avoid RMW */
636 if (!test_bit(rel_gfn, memslot->dirty_bitmap))
637 set_bit(rel_gfn, memslot->dirty_bitmap);
Avi Kivity6aa8b732006-12-10 02:21:36 -0800638 }
639}
640
Eddie Dongb6958ce2007-07-18 12:15:21 +0300641/*
642 * The vCPU has executed a HLT instruction with in-kernel mode enabled.
643 */
Hollis Blanchard8776e512007-10-31 17:24:24 -0500644void kvm_vcpu_block(struct kvm_vcpu *vcpu)
Eddie Dongb6958ce2007-07-18 12:15:21 +0300645{
646 DECLARE_WAITQUEUE(wait, current);
647
648 add_wait_queue(&vcpu->wq, &wait);
649
650 /*
651 * We will block until either an interrupt or a signal wakes us up
652 */
He, Qingc5ec1532007-09-03 17:07:41 +0300653 while (!kvm_cpu_has_interrupt(vcpu)
654 && !signal_pending(current)
Hollis Blanchard53e0aa72007-12-03 16:15:26 -0600655 && !kvm_arch_vcpu_runnable(vcpu)) {
Eddie Dongb6958ce2007-07-18 12:15:21 +0300656 set_current_state(TASK_INTERRUPTIBLE);
657 vcpu_put(vcpu);
658 schedule();
659 vcpu_load(vcpu);
660 }
661
He, Qingc5ec1532007-09-03 17:07:41 +0300662 __set_current_state(TASK_RUNNING);
Eddie Dongb6958ce2007-07-18 12:15:21 +0300663 remove_wait_queue(&vcpu->wq, &wait);
Eddie Dongb6958ce2007-07-18 12:15:21 +0300664}
665
Avi Kivity6aa8b732006-12-10 02:21:36 -0800666void kvm_resched(struct kvm_vcpu *vcpu)
667{
Yaozu Dong3fca0362007-04-25 16:49:19 +0300668 if (!need_resched())
669 return;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800670 cond_resched();
Avi Kivity6aa8b732006-12-10 02:21:36 -0800671}
672EXPORT_SYMBOL_GPL(kvm_resched);
673
npiggin@suse.dee4a533a2007-12-05 18:15:52 +1100674static int kvm_vcpu_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
Avi Kivity9a2bb7f2007-02-22 12:58:31 +0200675{
676 struct kvm_vcpu *vcpu = vma->vm_file->private_data;
Avi Kivity9a2bb7f2007-02-22 12:58:31 +0200677 struct page *page;
678
npiggin@suse.dee4a533a2007-12-05 18:15:52 +1100679 if (vmf->pgoff == 0)
Avi Kivity039576c2007-03-20 12:46:50 +0200680 page = virt_to_page(vcpu->run);
npiggin@suse.dee4a533a2007-12-05 18:15:52 +1100681 else if (vmf->pgoff == KVM_PIO_PAGE_OFFSET)
Zhang Xiantaoad312c72007-12-13 23:50:52 +0800682 page = virt_to_page(vcpu->arch.pio_data);
Avi Kivity039576c2007-03-20 12:46:50 +0200683 else
npiggin@suse.dee4a533a2007-12-05 18:15:52 +1100684 return VM_FAULT_SIGBUS;
Avi Kivity9a2bb7f2007-02-22 12:58:31 +0200685 get_page(page);
npiggin@suse.dee4a533a2007-12-05 18:15:52 +1100686 vmf->page = page;
687 return 0;
Avi Kivity9a2bb7f2007-02-22 12:58:31 +0200688}
689
690static struct vm_operations_struct kvm_vcpu_vm_ops = {
npiggin@suse.dee4a533a2007-12-05 18:15:52 +1100691 .fault = kvm_vcpu_fault,
Avi Kivity9a2bb7f2007-02-22 12:58:31 +0200692};
693
694static int kvm_vcpu_mmap(struct file *file, struct vm_area_struct *vma)
695{
696 vma->vm_ops = &kvm_vcpu_vm_ops;
697 return 0;
698}
699
Avi Kivitybccf2152007-02-21 18:04:26 +0200700static int kvm_vcpu_release(struct inode *inode, struct file *filp)
701{
702 struct kvm_vcpu *vcpu = filp->private_data;
703
704 fput(vcpu->kvm->filp);
705 return 0;
706}
707
708static struct file_operations kvm_vcpu_fops = {
709 .release = kvm_vcpu_release,
710 .unlocked_ioctl = kvm_vcpu_ioctl,
711 .compat_ioctl = kvm_vcpu_ioctl,
Avi Kivity9a2bb7f2007-02-22 12:58:31 +0200712 .mmap = kvm_vcpu_mmap,
Avi Kivitybccf2152007-02-21 18:04:26 +0200713};
714
715/*
716 * Allocates an inode for the vcpu.
717 */
718static int create_vcpu_fd(struct kvm_vcpu *vcpu)
719{
720 int fd, r;
721 struct inode *inode;
722 struct file *file;
723
Avi Kivityd6d28162007-06-28 08:38:16 -0400724 r = anon_inode_getfd(&fd, &inode, &file,
725 "kvm-vcpu", &kvm_vcpu_fops, vcpu);
726 if (r)
727 return r;
Avi Kivitybccf2152007-02-21 18:04:26 +0200728 atomic_inc(&vcpu->kvm->filp->f_count);
Avi Kivitybccf2152007-02-21 18:04:26 +0200729 return fd;
Avi Kivitybccf2152007-02-21 18:04:26 +0200730}
731
Avi Kivityc5ea7662007-02-20 18:41:05 +0200732/*
733 * Creates some virtual cpus. Good luck creating more than one.
734 */
735static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, int n)
736{
737 int r;
738 struct kvm_vcpu *vcpu;
739
Avi Kivityc5ea7662007-02-20 18:41:05 +0200740 if (!valid_vcpu(n))
Rusty Russellfb3f0f52007-07-27 17:16:56 +1000741 return -EINVAL;
Avi Kivityc5ea7662007-02-20 18:41:05 +0200742
Zhang Xiantaoe9b11c12007-11-14 20:38:21 +0800743 vcpu = kvm_arch_vcpu_create(kvm, n);
Rusty Russellfb3f0f52007-07-27 17:16:56 +1000744 if (IS_ERR(vcpu))
745 return PTR_ERR(vcpu);
Avi Kivityc5ea7662007-02-20 18:41:05 +0200746
Avi Kivity15ad7142007-07-11 18:17:21 +0300747 preempt_notifier_init(&vcpu->preempt_notifier, &kvm_preempt_ops);
748
Avi Kivity26e52152007-11-20 15:30:24 +0200749 r = kvm_arch_vcpu_setup(vcpu);
750 if (r)
751 goto vcpu_destroy;
752
Shaohua Li11ec2802007-07-23 14:51:37 +0800753 mutex_lock(&kvm->lock);
Rusty Russellfb3f0f52007-07-27 17:16:56 +1000754 if (kvm->vcpus[n]) {
755 r = -EEXIST;
Shaohua Li11ec2802007-07-23 14:51:37 +0800756 mutex_unlock(&kvm->lock);
Zhang Xiantaoe9b11c12007-11-14 20:38:21 +0800757 goto vcpu_destroy;
Rusty Russellfb3f0f52007-07-27 17:16:56 +1000758 }
759 kvm->vcpus[n] = vcpu;
Shaohua Li11ec2802007-07-23 14:51:37 +0800760 mutex_unlock(&kvm->lock);
Rusty Russellfb3f0f52007-07-27 17:16:56 +1000761
762 /* Now it's all set up, let userspace reach it */
Avi Kivitybccf2152007-02-21 18:04:26 +0200763 r = create_vcpu_fd(vcpu);
764 if (r < 0)
Rusty Russellfb3f0f52007-07-27 17:16:56 +1000765 goto unlink;
Avi Kivitybccf2152007-02-21 18:04:26 +0200766 return r;
Avi Kivityc5ea7662007-02-20 18:41:05 +0200767
Rusty Russellfb3f0f52007-07-27 17:16:56 +1000768unlink:
Shaohua Li11ec2802007-07-23 14:51:37 +0800769 mutex_lock(&kvm->lock);
Rusty Russellfb3f0f52007-07-27 17:16:56 +1000770 kvm->vcpus[n] = NULL;
Shaohua Li11ec2802007-07-23 14:51:37 +0800771 mutex_unlock(&kvm->lock);
Zhang Xiantaoe9b11c12007-11-14 20:38:21 +0800772vcpu_destroy:
Hollis Blanchardd40ccc62007-11-19 14:04:43 -0600773 kvm_arch_vcpu_destroy(vcpu);
Avi Kivityc5ea7662007-02-20 18:41:05 +0200774 return r;
775}
776
Avi Kivity1961d272007-03-05 19:46:05 +0200777static int kvm_vcpu_ioctl_set_sigmask(struct kvm_vcpu *vcpu, sigset_t *sigset)
778{
779 if (sigset) {
780 sigdelsetmask(sigset, sigmask(SIGKILL)|sigmask(SIGSTOP));
781 vcpu->sigset_active = 1;
782 vcpu->sigset = *sigset;
783 } else
784 vcpu->sigset_active = 0;
785 return 0;
786}
787
Avi Kivitybccf2152007-02-21 18:04:26 +0200788static long kvm_vcpu_ioctl(struct file *filp,
789 unsigned int ioctl, unsigned long arg)
Avi Kivity6aa8b732006-12-10 02:21:36 -0800790{
Avi Kivitybccf2152007-02-21 18:04:26 +0200791 struct kvm_vcpu *vcpu = filp->private_data;
Al Viro2f3669872007-02-09 16:38:35 +0000792 void __user *argp = (void __user *)arg;
Carsten Otte313a3dc2007-10-11 19:16:52 +0200793 int r;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800794
Avi Kivity6d4e4c42007-11-21 16:41:05 +0200795 if (vcpu->kvm->mm != current->mm)
796 return -EIO;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800797 switch (ioctl) {
Avi Kivity9a2bb7f2007-02-22 12:58:31 +0200798 case KVM_RUN:
Avi Kivityf0fe5102007-03-07 13:11:17 +0200799 r = -EINVAL;
800 if (arg)
801 goto out;
Hollis Blanchardb6c7a5d2007-11-01 14:16:10 -0500802 r = kvm_arch_vcpu_ioctl_run(vcpu, vcpu->run);
Avi Kivity6aa8b732006-12-10 02:21:36 -0800803 break;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800804 case KVM_GET_REGS: {
805 struct kvm_regs kvm_regs;
806
Avi Kivitybccf2152007-02-21 18:04:26 +0200807 memset(&kvm_regs, 0, sizeof kvm_regs);
Hollis Blanchardb6c7a5d2007-11-01 14:16:10 -0500808 r = kvm_arch_vcpu_ioctl_get_regs(vcpu, &kvm_regs);
Avi Kivity6aa8b732006-12-10 02:21:36 -0800809 if (r)
810 goto out;
811 r = -EFAULT;
Al Viro2f3669872007-02-09 16:38:35 +0000812 if (copy_to_user(argp, &kvm_regs, sizeof kvm_regs))
Avi Kivity6aa8b732006-12-10 02:21:36 -0800813 goto out;
814 r = 0;
815 break;
816 }
817 case KVM_SET_REGS: {
818 struct kvm_regs kvm_regs;
819
820 r = -EFAULT;
Al Viro2f3669872007-02-09 16:38:35 +0000821 if (copy_from_user(&kvm_regs, argp, sizeof kvm_regs))
Avi Kivity6aa8b732006-12-10 02:21:36 -0800822 goto out;
Hollis Blanchardb6c7a5d2007-11-01 14:16:10 -0500823 r = kvm_arch_vcpu_ioctl_set_regs(vcpu, &kvm_regs);
Avi Kivity6aa8b732006-12-10 02:21:36 -0800824 if (r)
825 goto out;
826 r = 0;
827 break;
828 }
829 case KVM_GET_SREGS: {
830 struct kvm_sregs kvm_sregs;
831
Avi Kivitybccf2152007-02-21 18:04:26 +0200832 memset(&kvm_sregs, 0, sizeof kvm_sregs);
Hollis Blanchardb6c7a5d2007-11-01 14:16:10 -0500833 r = kvm_arch_vcpu_ioctl_get_sregs(vcpu, &kvm_sregs);
Avi Kivity6aa8b732006-12-10 02:21:36 -0800834 if (r)
835 goto out;
836 r = -EFAULT;
Al Viro2f3669872007-02-09 16:38:35 +0000837 if (copy_to_user(argp, &kvm_sregs, sizeof kvm_sregs))
Avi Kivity6aa8b732006-12-10 02:21:36 -0800838 goto out;
839 r = 0;
840 break;
841 }
842 case KVM_SET_SREGS: {
843 struct kvm_sregs kvm_sregs;
844
845 r = -EFAULT;
Al Viro2f3669872007-02-09 16:38:35 +0000846 if (copy_from_user(&kvm_sregs, argp, sizeof kvm_sregs))
Avi Kivity6aa8b732006-12-10 02:21:36 -0800847 goto out;
Hollis Blanchardb6c7a5d2007-11-01 14:16:10 -0500848 r = kvm_arch_vcpu_ioctl_set_sregs(vcpu, &kvm_sregs);
Avi Kivity6aa8b732006-12-10 02:21:36 -0800849 if (r)
850 goto out;
851 r = 0;
852 break;
853 }
854 case KVM_TRANSLATE: {
855 struct kvm_translation tr;
856
857 r = -EFAULT;
Al Viro2f3669872007-02-09 16:38:35 +0000858 if (copy_from_user(&tr, argp, sizeof tr))
Avi Kivity6aa8b732006-12-10 02:21:36 -0800859 goto out;
Zhang Xiantao8b006792007-11-16 13:05:55 +0800860 r = kvm_arch_vcpu_ioctl_translate(vcpu, &tr);
Avi Kivity6aa8b732006-12-10 02:21:36 -0800861 if (r)
862 goto out;
863 r = -EFAULT;
Al Viro2f3669872007-02-09 16:38:35 +0000864 if (copy_to_user(argp, &tr, sizeof tr))
Avi Kivity6aa8b732006-12-10 02:21:36 -0800865 goto out;
866 r = 0;
867 break;
868 }
Avi Kivity6aa8b732006-12-10 02:21:36 -0800869 case KVM_DEBUG_GUEST: {
870 struct kvm_debug_guest dbg;
871
872 r = -EFAULT;
Al Viro2f3669872007-02-09 16:38:35 +0000873 if (copy_from_user(&dbg, argp, sizeof dbg))
Avi Kivity6aa8b732006-12-10 02:21:36 -0800874 goto out;
Hollis Blanchardb6c7a5d2007-11-01 14:16:10 -0500875 r = kvm_arch_vcpu_ioctl_debug_guest(vcpu, &dbg);
Avi Kivity6aa8b732006-12-10 02:21:36 -0800876 if (r)
877 goto out;
878 r = 0;
879 break;
880 }
Avi Kivity1961d272007-03-05 19:46:05 +0200881 case KVM_SET_SIGNAL_MASK: {
882 struct kvm_signal_mask __user *sigmask_arg = argp;
883 struct kvm_signal_mask kvm_sigmask;
884 sigset_t sigset, *p;
885
886 p = NULL;
887 if (argp) {
888 r = -EFAULT;
889 if (copy_from_user(&kvm_sigmask, argp,
890 sizeof kvm_sigmask))
891 goto out;
892 r = -EINVAL;
893 if (kvm_sigmask.len != sizeof sigset)
894 goto out;
895 r = -EFAULT;
896 if (copy_from_user(&sigset, sigmask_arg->sigset,
897 sizeof sigset))
898 goto out;
899 p = &sigset;
900 }
901 r = kvm_vcpu_ioctl_set_sigmask(vcpu, &sigset);
902 break;
903 }
Avi Kivityb8836732007-04-01 16:34:31 +0300904 case KVM_GET_FPU: {
905 struct kvm_fpu fpu;
906
907 memset(&fpu, 0, sizeof fpu);
Hollis Blanchardd0752062007-10-31 17:24:25 -0500908 r = kvm_arch_vcpu_ioctl_get_fpu(vcpu, &fpu);
Avi Kivityb8836732007-04-01 16:34:31 +0300909 if (r)
910 goto out;
911 r = -EFAULT;
912 if (copy_to_user(argp, &fpu, sizeof fpu))
913 goto out;
914 r = 0;
915 break;
916 }
917 case KVM_SET_FPU: {
918 struct kvm_fpu fpu;
919
920 r = -EFAULT;
921 if (copy_from_user(&fpu, argp, sizeof fpu))
922 goto out;
Hollis Blanchardd0752062007-10-31 17:24:25 -0500923 r = kvm_arch_vcpu_ioctl_set_fpu(vcpu, &fpu);
Avi Kivityb8836732007-04-01 16:34:31 +0300924 if (r)
925 goto out;
926 r = 0;
927 break;
928 }
Avi Kivitybccf2152007-02-21 18:04:26 +0200929 default:
Carsten Otte313a3dc2007-10-11 19:16:52 +0200930 r = kvm_arch_vcpu_ioctl(filp, ioctl, arg);
Avi Kivitybccf2152007-02-21 18:04:26 +0200931 }
932out:
933 return r;
934}
935
936static long kvm_vm_ioctl(struct file *filp,
937 unsigned int ioctl, unsigned long arg)
938{
939 struct kvm *kvm = filp->private_data;
940 void __user *argp = (void __user *)arg;
Carsten Otte1fe779f2007-10-29 16:08:35 +0100941 int r;
Avi Kivitybccf2152007-02-21 18:04:26 +0200942
Avi Kivity6d4e4c42007-11-21 16:41:05 +0200943 if (kvm->mm != current->mm)
944 return -EIO;
Avi Kivitybccf2152007-02-21 18:04:26 +0200945 switch (ioctl) {
946 case KVM_CREATE_VCPU:
947 r = kvm_vm_ioctl_create_vcpu(kvm, arg);
948 if (r < 0)
949 goto out;
950 break;
Izik Eidus6fc138d2007-10-09 19:20:39 +0200951 case KVM_SET_USER_MEMORY_REGION: {
952 struct kvm_userspace_memory_region kvm_userspace_mem;
953
954 r = -EFAULT;
955 if (copy_from_user(&kvm_userspace_mem, argp,
956 sizeof kvm_userspace_mem))
957 goto out;
958
959 r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem, 1);
Avi Kivity6aa8b732006-12-10 02:21:36 -0800960 if (r)
961 goto out;
962 break;
963 }
964 case KVM_GET_DIRTY_LOG: {
965 struct kvm_dirty_log log;
966
967 r = -EFAULT;
Al Viro2f3669872007-02-09 16:38:35 +0000968 if (copy_from_user(&log, argp, sizeof log))
Avi Kivity6aa8b732006-12-10 02:21:36 -0800969 goto out;
Avi Kivity2c6f5df2007-02-20 18:27:58 +0200970 r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
Avi Kivity6aa8b732006-12-10 02:21:36 -0800971 if (r)
972 goto out;
973 break;
974 }
Avi Kivityf17abe92007-02-21 19:28:04 +0200975 default:
Carsten Otte1fe779f2007-10-29 16:08:35 +0100976 r = kvm_arch_vm_ioctl(filp, ioctl, arg);
Avi Kivityf17abe92007-02-21 19:28:04 +0200977 }
978out:
979 return r;
980}
981
npiggin@suse.dee4a533a2007-12-05 18:15:52 +1100982static int kvm_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
Avi Kivityf17abe92007-02-21 19:28:04 +0200983{
984 struct kvm *kvm = vma->vm_file->private_data;
Avi Kivityf17abe92007-02-21 19:28:04 +0200985 struct page *page;
986
npiggin@suse.dee4a533a2007-12-05 18:15:52 +1100987 if (!kvm_is_visible_gfn(kvm, vmf->pgoff))
988 return VM_FAULT_SIGBUS;
Marcelo Tosatti10589a42007-12-20 19:18:22 -0500989 page = gfn_to_page(kvm, vmf->pgoff);
Izik Eidus8a7ae052007-10-18 11:09:33 +0200990 if (is_error_page(page)) {
Izik Eidusb4231d62007-11-20 11:49:33 +0200991 kvm_release_page_clean(page);
npiggin@suse.dee4a533a2007-12-05 18:15:52 +1100992 return VM_FAULT_SIGBUS;
Izik Eidus8a7ae052007-10-18 11:09:33 +0200993 }
npiggin@suse.dee4a533a2007-12-05 18:15:52 +1100994 vmf->page = page;
995 return 0;
Avi Kivityf17abe92007-02-21 19:28:04 +0200996}
997
998static struct vm_operations_struct kvm_vm_vm_ops = {
npiggin@suse.dee4a533a2007-12-05 18:15:52 +1100999 .fault = kvm_vm_fault,
Avi Kivityf17abe92007-02-21 19:28:04 +02001000};
1001
1002static int kvm_vm_mmap(struct file *file, struct vm_area_struct *vma)
1003{
1004 vma->vm_ops = &kvm_vm_vm_ops;
1005 return 0;
1006}
1007
1008static struct file_operations kvm_vm_fops = {
1009 .release = kvm_vm_release,
1010 .unlocked_ioctl = kvm_vm_ioctl,
1011 .compat_ioctl = kvm_vm_ioctl,
1012 .mmap = kvm_vm_mmap,
1013};
1014
1015static int kvm_dev_ioctl_create_vm(void)
1016{
1017 int fd, r;
1018 struct inode *inode;
1019 struct file *file;
1020 struct kvm *kvm;
1021
Avi Kivityf17abe92007-02-21 19:28:04 +02001022 kvm = kvm_create_vm();
Avi Kivityd6d28162007-06-28 08:38:16 -04001023 if (IS_ERR(kvm))
1024 return PTR_ERR(kvm);
1025 r = anon_inode_getfd(&fd, &inode, &file, "kvm-vm", &kvm_vm_fops, kvm);
1026 if (r) {
1027 kvm_destroy_vm(kvm);
1028 return r;
Avi Kivityf17abe92007-02-21 19:28:04 +02001029 }
1030
Avi Kivitybccf2152007-02-21 18:04:26 +02001031 kvm->filp = file;
Avi Kivityf17abe92007-02-21 19:28:04 +02001032
Avi Kivityf17abe92007-02-21 19:28:04 +02001033 return fd;
Avi Kivityf17abe92007-02-21 19:28:04 +02001034}
1035
1036static long kvm_dev_ioctl(struct file *filp,
1037 unsigned int ioctl, unsigned long arg)
1038{
1039 void __user *argp = (void __user *)arg;
Avi Kivity07c45a32007-03-07 13:05:38 +02001040 long r = -EINVAL;
Avi Kivityf17abe92007-02-21 19:28:04 +02001041
1042 switch (ioctl) {
1043 case KVM_GET_API_VERSION:
Avi Kivityf0fe5102007-03-07 13:11:17 +02001044 r = -EINVAL;
1045 if (arg)
1046 goto out;
Avi Kivityf17abe92007-02-21 19:28:04 +02001047 r = KVM_API_VERSION;
1048 break;
1049 case KVM_CREATE_VM:
Avi Kivityf0fe5102007-03-07 13:11:17 +02001050 r = -EINVAL;
1051 if (arg)
1052 goto out;
Avi Kivityf17abe92007-02-21 19:28:04 +02001053 r = kvm_dev_ioctl_create_vm();
1054 break;
Zhang Xiantao018d00d2007-11-15 23:07:47 +08001055 case KVM_CHECK_EXTENSION:
1056 r = kvm_dev_ioctl_check_extension((long)argp);
Avi Kivity5d308f42007-03-01 17:56:20 +02001057 break;
Avi Kivity07c45a32007-03-07 13:05:38 +02001058 case KVM_GET_VCPU_MMAP_SIZE:
1059 r = -EINVAL;
1060 if (arg)
1061 goto out;
Avi Kivity039576c2007-03-20 12:46:50 +02001062 r = 2 * PAGE_SIZE;
Avi Kivity07c45a32007-03-07 13:05:38 +02001063 break;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001064 default:
Carsten Otte043405e2007-10-10 17:16:19 +02001065 return kvm_arch_dev_ioctl(filp, ioctl, arg);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001066 }
1067out:
1068 return r;
1069}
1070
Avi Kivity6aa8b732006-12-10 02:21:36 -08001071static struct file_operations kvm_chardev_ops = {
Avi Kivity6aa8b732006-12-10 02:21:36 -08001072 .unlocked_ioctl = kvm_dev_ioctl,
1073 .compat_ioctl = kvm_dev_ioctl,
Avi Kivity6aa8b732006-12-10 02:21:36 -08001074};
1075
1076static struct miscdevice kvm_dev = {
Avi Kivitybbe44322007-03-04 13:27:36 +02001077 KVM_MINOR,
Avi Kivity6aa8b732006-12-10 02:21:36 -08001078 "kvm",
1079 &kvm_chardev_ops,
1080};
1081
Avi Kivity1b6c0162007-05-24 13:03:52 +03001082static void hardware_enable(void *junk)
1083{
1084 int cpu = raw_smp_processor_id();
1085
1086 if (cpu_isset(cpu, cpus_hardware_enabled))
1087 return;
1088 cpu_set(cpu, cpus_hardware_enabled);
Zhang Xiantaoe9b11c12007-11-14 20:38:21 +08001089 kvm_arch_hardware_enable(NULL);
Avi Kivity1b6c0162007-05-24 13:03:52 +03001090}
1091
1092static void hardware_disable(void *junk)
1093{
1094 int cpu = raw_smp_processor_id();
1095
1096 if (!cpu_isset(cpu, cpus_hardware_enabled))
1097 return;
1098 cpu_clear(cpu, cpus_hardware_enabled);
1099 decache_vcpus_on_cpu(cpu);
Zhang Xiantaoe9b11c12007-11-14 20:38:21 +08001100 kvm_arch_hardware_disable(NULL);
Avi Kivity1b6c0162007-05-24 13:03:52 +03001101}
1102
Avi Kivity774c47f2007-02-12 00:54:47 -08001103static int kvm_cpu_hotplug(struct notifier_block *notifier, unsigned long val,
1104 void *v)
1105{
1106 int cpu = (long)v;
1107
Avi Kivity1a6f4d72007-11-11 18:37:32 +02001108 val &= ~CPU_TASKS_FROZEN;
Avi Kivity774c47f2007-02-12 00:54:47 -08001109 switch (val) {
Avi Kivitycec9ad22007-05-24 13:11:41 +03001110 case CPU_DYING:
Avi Kivity6ec8a8562007-08-19 15:57:26 +03001111 printk(KERN_INFO "kvm: disabling virtualization on CPU%d\n",
1112 cpu);
1113 hardware_disable(NULL);
1114 break;
Avi Kivity774c47f2007-02-12 00:54:47 -08001115 case CPU_UP_CANCELED:
Jeremy Katz43934a32007-02-19 14:37:46 +02001116 printk(KERN_INFO "kvm: disabling virtualization on CPU%d\n",
1117 cpu);
Avi Kivity1b6c0162007-05-24 13:03:52 +03001118 smp_call_function_single(cpu, hardware_disable, NULL, 0, 1);
Avi Kivity774c47f2007-02-12 00:54:47 -08001119 break;
Jeremy Katz43934a32007-02-19 14:37:46 +02001120 case CPU_ONLINE:
1121 printk(KERN_INFO "kvm: enabling virtualization on CPU%d\n",
1122 cpu);
Avi Kivity1b6c0162007-05-24 13:03:52 +03001123 smp_call_function_single(cpu, hardware_enable, NULL, 0, 1);
Avi Kivity774c47f2007-02-12 00:54:47 -08001124 break;
1125 }
1126 return NOTIFY_OK;
1127}
1128
Rusty Russell9a2b85c2007-07-17 23:17:55 +10001129static int kvm_reboot(struct notifier_block *notifier, unsigned long val,
Mike Dayd77c26f2007-10-08 09:02:08 -04001130 void *v)
Rusty Russell9a2b85c2007-07-17 23:17:55 +10001131{
1132 if (val == SYS_RESTART) {
1133 /*
1134 * Some (well, at least mine) BIOSes hang on reboot if
1135 * in vmx root mode.
1136 */
1137 printk(KERN_INFO "kvm: exiting hardware virtualization\n");
1138 on_each_cpu(hardware_disable, NULL, 0, 1);
1139 }
1140 return NOTIFY_OK;
1141}
1142
1143static struct notifier_block kvm_reboot_notifier = {
1144 .notifier_call = kvm_reboot,
1145 .priority = 0,
1146};
1147
Gregory Haskins2eeb2e92007-05-31 14:08:53 -04001148void kvm_io_bus_init(struct kvm_io_bus *bus)
1149{
1150 memset(bus, 0, sizeof(*bus));
1151}
1152
1153void kvm_io_bus_destroy(struct kvm_io_bus *bus)
1154{
1155 int i;
1156
1157 for (i = 0; i < bus->dev_count; i++) {
1158 struct kvm_io_device *pos = bus->devs[i];
1159
1160 kvm_iodevice_destructor(pos);
1161 }
1162}
1163
1164struct kvm_io_device *kvm_io_bus_find_dev(struct kvm_io_bus *bus, gpa_t addr)
1165{
1166 int i;
1167
1168 for (i = 0; i < bus->dev_count; i++) {
1169 struct kvm_io_device *pos = bus->devs[i];
1170
1171 if (pos->in_range(pos, addr))
1172 return pos;
1173 }
1174
1175 return NULL;
1176}
1177
1178void kvm_io_bus_register_dev(struct kvm_io_bus *bus, struct kvm_io_device *dev)
1179{
1180 BUG_ON(bus->dev_count > (NR_IOBUS_DEVS-1));
1181
1182 bus->devs[bus->dev_count++] = dev;
1183}
1184
Avi Kivity774c47f2007-02-12 00:54:47 -08001185static struct notifier_block kvm_cpu_notifier = {
1186 .notifier_call = kvm_cpu_hotplug,
1187 .priority = 20, /* must be > scheduler priority */
1188};
1189
Christoph Hellwig8b88b092008-02-08 04:20:26 -08001190static int vm_stat_get(void *_offset, u64 *val)
Avi Kivityba1389b2007-11-18 16:24:12 +02001191{
1192 unsigned offset = (long)_offset;
Avi Kivityba1389b2007-11-18 16:24:12 +02001193 struct kvm *kvm;
1194
Christoph Hellwig8b88b092008-02-08 04:20:26 -08001195 *val = 0;
Avi Kivityba1389b2007-11-18 16:24:12 +02001196 spin_lock(&kvm_lock);
1197 list_for_each_entry(kvm, &vm_list, vm_list)
Christoph Hellwig8b88b092008-02-08 04:20:26 -08001198 *val += *(u32 *)((void *)kvm + offset);
Avi Kivityba1389b2007-11-18 16:24:12 +02001199 spin_unlock(&kvm_lock);
Christoph Hellwig8b88b092008-02-08 04:20:26 -08001200 return 0;
Avi Kivityba1389b2007-11-18 16:24:12 +02001201}
1202
1203DEFINE_SIMPLE_ATTRIBUTE(vm_stat_fops, vm_stat_get, NULL, "%llu\n");
1204
Christoph Hellwig8b88b092008-02-08 04:20:26 -08001205static int vcpu_stat_get(void *_offset, u64 *val)
Avi Kivity1165f5f2007-04-19 17:27:43 +03001206{
1207 unsigned offset = (long)_offset;
Avi Kivity1165f5f2007-04-19 17:27:43 +03001208 struct kvm *kvm;
1209 struct kvm_vcpu *vcpu;
1210 int i;
1211
Christoph Hellwig8b88b092008-02-08 04:20:26 -08001212 *val = 0;
Avi Kivity1165f5f2007-04-19 17:27:43 +03001213 spin_lock(&kvm_lock);
1214 list_for_each_entry(kvm, &vm_list, vm_list)
1215 for (i = 0; i < KVM_MAX_VCPUS; ++i) {
Rusty Russellfb3f0f52007-07-27 17:16:56 +10001216 vcpu = kvm->vcpus[i];
1217 if (vcpu)
Christoph Hellwig8b88b092008-02-08 04:20:26 -08001218 *val += *(u32 *)((void *)vcpu + offset);
Avi Kivity1165f5f2007-04-19 17:27:43 +03001219 }
1220 spin_unlock(&kvm_lock);
Christoph Hellwig8b88b092008-02-08 04:20:26 -08001221 return 0;
Avi Kivity1165f5f2007-04-19 17:27:43 +03001222}
1223
Avi Kivityba1389b2007-11-18 16:24:12 +02001224DEFINE_SIMPLE_ATTRIBUTE(vcpu_stat_fops, vcpu_stat_get, NULL, "%llu\n");
1225
1226static struct file_operations *stat_fops[] = {
1227 [KVM_STAT_VCPU] = &vcpu_stat_fops,
1228 [KVM_STAT_VM] = &vm_stat_fops,
1229};
Avi Kivity1165f5f2007-04-19 17:27:43 +03001230
Zhang Xiantaoa16b0432007-11-16 14:38:21 +08001231static void kvm_init_debug(void)
Avi Kivity6aa8b732006-12-10 02:21:36 -08001232{
1233 struct kvm_stats_debugfs_item *p;
1234
Al Viro8b6d44c2007-02-09 16:38:40 +00001235 debugfs_dir = debugfs_create_dir("kvm", NULL);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001236 for (p = debugfs_entries; p->name; ++p)
Avi Kivity1165f5f2007-04-19 17:27:43 +03001237 p->dentry = debugfs_create_file(p->name, 0444, debugfs_dir,
1238 (void *)(long)p->offset,
Avi Kivityba1389b2007-11-18 16:24:12 +02001239 stat_fops[p->kind]);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001240}
1241
1242static void kvm_exit_debug(void)
1243{
1244 struct kvm_stats_debugfs_item *p;
1245
1246 for (p = debugfs_entries; p->name; ++p)
1247 debugfs_remove(p->dentry);
1248 debugfs_remove(debugfs_dir);
1249}
1250
Avi Kivity59ae6c62007-02-12 00:54:48 -08001251static int kvm_suspend(struct sys_device *dev, pm_message_t state)
1252{
Avi Kivity4267c412007-05-24 13:09:41 +03001253 hardware_disable(NULL);
Avi Kivity59ae6c62007-02-12 00:54:48 -08001254 return 0;
1255}
1256
1257static int kvm_resume(struct sys_device *dev)
1258{
Avi Kivity4267c412007-05-24 13:09:41 +03001259 hardware_enable(NULL);
Avi Kivity59ae6c62007-02-12 00:54:48 -08001260 return 0;
1261}
1262
1263static struct sysdev_class kvm_sysdev_class = {
Kay Sieversaf5ca3f2007-12-20 02:09:39 +01001264 .name = "kvm",
Avi Kivity59ae6c62007-02-12 00:54:48 -08001265 .suspend = kvm_suspend,
1266 .resume = kvm_resume,
1267};
1268
1269static struct sys_device kvm_sysdev = {
1270 .id = 0,
1271 .cls = &kvm_sysdev_class,
1272};
1273
Izik Eiduscea7bb22007-10-17 19:17:48 +02001274struct page *bad_page;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001275
Avi Kivity15ad7142007-07-11 18:17:21 +03001276static inline
1277struct kvm_vcpu *preempt_notifier_to_vcpu(struct preempt_notifier *pn)
1278{
1279 return container_of(pn, struct kvm_vcpu, preempt_notifier);
1280}
1281
1282static void kvm_sched_in(struct preempt_notifier *pn, int cpu)
1283{
1284 struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);
1285
Zhang Xiantaoe9b11c12007-11-14 20:38:21 +08001286 kvm_arch_vcpu_load(vcpu, cpu);
Avi Kivity15ad7142007-07-11 18:17:21 +03001287}
1288
1289static void kvm_sched_out(struct preempt_notifier *pn,
1290 struct task_struct *next)
1291{
1292 struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);
1293
Zhang Xiantaoe9b11c12007-11-14 20:38:21 +08001294 kvm_arch_vcpu_put(vcpu);
Avi Kivity15ad7142007-07-11 18:17:21 +03001295}
1296
Zhang Xiantaof8c16bb2007-11-14 20:40:21 +08001297int kvm_init(void *opaque, unsigned int vcpu_size,
Rusty Russellc16f8622007-07-30 21:12:19 +10001298 struct module *module)
Avi Kivity6aa8b732006-12-10 02:21:36 -08001299{
1300 int r;
Yang, Sheng002c7f72007-07-31 14:23:01 +03001301 int cpu;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001302
Zhang Xiantaocb498ea2007-11-14 20:39:31 +08001303 kvm_init_debug();
1304
Zhang Xiantaof8c16bb2007-11-14 20:40:21 +08001305 r = kvm_arch_init(opaque);
1306 if (r)
Zhang Xiantaod23087842007-11-29 15:35:39 +08001307 goto out_fail;
Zhang Xiantaocb498ea2007-11-14 20:39:31 +08001308
1309 bad_page = alloc_page(GFP_KERNEL | __GFP_ZERO);
1310
1311 if (bad_page == NULL) {
1312 r = -ENOMEM;
1313 goto out;
1314 }
1315
Zhang Xiantaoe9b11c12007-11-14 20:38:21 +08001316 r = kvm_arch_hardware_setup();
Avi Kivity6aa8b732006-12-10 02:21:36 -08001317 if (r < 0)
Zhang Xiantaod23087842007-11-29 15:35:39 +08001318 goto out_free_0;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001319
Yang, Sheng002c7f72007-07-31 14:23:01 +03001320 for_each_online_cpu(cpu) {
1321 smp_call_function_single(cpu,
Zhang Xiantaoe9b11c12007-11-14 20:38:21 +08001322 kvm_arch_check_processor_compat,
Yang, Sheng002c7f72007-07-31 14:23:01 +03001323 &r, 0, 1);
1324 if (r < 0)
Zhang Xiantaod23087842007-11-29 15:35:39 +08001325 goto out_free_1;
Yang, Sheng002c7f72007-07-31 14:23:01 +03001326 }
1327
Avi Kivity1b6c0162007-05-24 13:03:52 +03001328 on_each_cpu(hardware_enable, NULL, 0, 1);
Avi Kivity774c47f2007-02-12 00:54:47 -08001329 r = register_cpu_notifier(&kvm_cpu_notifier);
1330 if (r)
Zhang Xiantaod23087842007-11-29 15:35:39 +08001331 goto out_free_2;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001332 register_reboot_notifier(&kvm_reboot_notifier);
1333
Avi Kivity59ae6c62007-02-12 00:54:48 -08001334 r = sysdev_class_register(&kvm_sysdev_class);
1335 if (r)
Zhang Xiantaod23087842007-11-29 15:35:39 +08001336 goto out_free_3;
Avi Kivity59ae6c62007-02-12 00:54:48 -08001337
1338 r = sysdev_register(&kvm_sysdev);
1339 if (r)
Zhang Xiantaod23087842007-11-29 15:35:39 +08001340 goto out_free_4;
Avi Kivity59ae6c62007-02-12 00:54:48 -08001341
Rusty Russellc16f8622007-07-30 21:12:19 +10001342 /* A kmem cache lets us meet the alignment requirements of fx_save. */
1343 kvm_vcpu_cache = kmem_cache_create("kvm_vcpu", vcpu_size,
Joe Perches56919c52007-11-12 20:06:51 -08001344 __alignof__(struct kvm_vcpu),
1345 0, NULL);
Rusty Russellc16f8622007-07-30 21:12:19 +10001346 if (!kvm_vcpu_cache) {
1347 r = -ENOMEM;
Zhang Xiantaod23087842007-11-29 15:35:39 +08001348 goto out_free_5;
Rusty Russellc16f8622007-07-30 21:12:19 +10001349 }
1350
Avi Kivity6aa8b732006-12-10 02:21:36 -08001351 kvm_chardev_ops.owner = module;
1352
1353 r = misc_register(&kvm_dev);
1354 if (r) {
Mike Dayd77c26f2007-10-08 09:02:08 -04001355 printk(KERN_ERR "kvm: misc device register failed\n");
Avi Kivity6aa8b732006-12-10 02:21:36 -08001356 goto out_free;
1357 }
1358
Avi Kivity15ad7142007-07-11 18:17:21 +03001359 kvm_preempt_ops.sched_in = kvm_sched_in;
1360 kvm_preempt_ops.sched_out = kvm_sched_out;
1361
Avi Kivityc7addb92007-09-16 18:58:32 +02001362 return 0;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001363
1364out_free:
Rusty Russellc16f8622007-07-30 21:12:19 +10001365 kmem_cache_destroy(kvm_vcpu_cache);
Zhang Xiantaod23087842007-11-29 15:35:39 +08001366out_free_5:
Avi Kivity59ae6c62007-02-12 00:54:48 -08001367 sysdev_unregister(&kvm_sysdev);
Zhang Xiantaod23087842007-11-29 15:35:39 +08001368out_free_4:
Avi Kivity59ae6c62007-02-12 00:54:48 -08001369 sysdev_class_unregister(&kvm_sysdev_class);
Zhang Xiantaod23087842007-11-29 15:35:39 +08001370out_free_3:
Avi Kivity6aa8b732006-12-10 02:21:36 -08001371 unregister_reboot_notifier(&kvm_reboot_notifier);
Avi Kivity774c47f2007-02-12 00:54:47 -08001372 unregister_cpu_notifier(&kvm_cpu_notifier);
Zhang Xiantaod23087842007-11-29 15:35:39 +08001373out_free_2:
Avi Kivity1b6c0162007-05-24 13:03:52 +03001374 on_each_cpu(hardware_disable, NULL, 0, 1);
Zhang Xiantaod23087842007-11-29 15:35:39 +08001375out_free_1:
Zhang Xiantaoe9b11c12007-11-14 20:38:21 +08001376 kvm_arch_hardware_unsetup();
Zhang Xiantaod23087842007-11-29 15:35:39 +08001377out_free_0:
1378 __free_page(bad_page);
Avi Kivityca45aaa2007-03-01 19:21:03 +02001379out:
Zhang Xiantaof8c16bb2007-11-14 20:40:21 +08001380 kvm_arch_exit();
Zhang Xiantaocb498ea2007-11-14 20:39:31 +08001381 kvm_exit_debug();
Zhang Xiantaod23087842007-11-29 15:35:39 +08001382out_fail:
Avi Kivity6aa8b732006-12-10 02:21:36 -08001383 return r;
1384}
Zhang Xiantaocb498ea2007-11-14 20:39:31 +08001385EXPORT_SYMBOL_GPL(kvm_init);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001386
Zhang Xiantaocb498ea2007-11-14 20:39:31 +08001387void kvm_exit(void)
Avi Kivity6aa8b732006-12-10 02:21:36 -08001388{
1389 misc_deregister(&kvm_dev);
Rusty Russellc16f8622007-07-30 21:12:19 +10001390 kmem_cache_destroy(kvm_vcpu_cache);
Avi Kivity59ae6c62007-02-12 00:54:48 -08001391 sysdev_unregister(&kvm_sysdev);
1392 sysdev_class_unregister(&kvm_sysdev_class);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001393 unregister_reboot_notifier(&kvm_reboot_notifier);
Avi Kivity59ae6c62007-02-12 00:54:48 -08001394 unregister_cpu_notifier(&kvm_cpu_notifier);
Avi Kivity1b6c0162007-05-24 13:03:52 +03001395 on_each_cpu(hardware_disable, NULL, 0, 1);
Zhang Xiantaoe9b11c12007-11-14 20:38:21 +08001396 kvm_arch_hardware_unsetup();
Zhang Xiantaof8c16bb2007-11-14 20:40:21 +08001397 kvm_arch_exit();
Avi Kivity6aa8b732006-12-10 02:21:36 -08001398 kvm_exit_debug();
Izik Eiduscea7bb22007-10-17 19:17:48 +02001399 __free_page(bad_page);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001400}
Zhang Xiantaocb498ea2007-11-14 20:39:31 +08001401EXPORT_SYMBOL_GPL(kvm_exit);